From a0758dfa1afd5b04773ba3b3b17ac71d22054821 Mon Sep 17 00:00:00 2001
From: felix <felix.von.s@posteo.de>
Date: Wed, 5 Aug 2015 22:40:46 +0200
Subject: [PATCH 0001/1696] [filmon] new extractor

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/filmon.py     | 144 +++++++++++++++++++++++++++++
 2 files changed, 145 insertions(+)
 create mode 100644 youtube_dl/extractor/filmon.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 578359a5e..c9b9ebd23 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -287,6 +287,7 @@ from .fc2 import (
     FC2EmbedIE,
 )
 from .fczenit import FczenitIE
+from .filmon import FilmOnIE, FilmOnVODIE
 from .firstpost import FirstpostIE
 from .firsttv import FirstTVIE
 from .fivemin import FiveMinIE
diff --git a/youtube_dl/extractor/filmon.py b/youtube_dl/extractor/filmon.py
new file mode 100644
index 000000000..987792fec
--- /dev/null
+++ b/youtube_dl/extractor/filmon.py
@@ -0,0 +1,144 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import qualities
+from ..compat import compat_urllib_request
+
+
+_QUALITY = qualities(('low', 'high'))
+
+
+class FilmOnIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?filmon\.com/(?:tv|channel)/(?P<id>[a-z0-9-]+)'
+    _TESTS = [{
+        'url': 'https://www.filmon.com/channel/filmon-sports',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.filmon.com/tv/2894',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        channel_id = self._match_id(url)
+
+        request = compat_urllib_request.Request('https://www.filmon.com/channel/%s' % (channel_id))
+        request.add_header('X-Requested-With', 'XMLHttpRequest')
+        channel_info = self._download_json(request, channel_id)
+        now_playing = channel_info['now_playing']
+
+        thumbnails = []
+        for thumb in now_playing.get('images', ()):
+            if thumb['type'] != '2':
+                continue
+            thumbnails.append({
+                'url': thumb['url'],
+                'width': int(thumb['width']),
+                'height': int(thumb['height']),
+            })
+
+        formats = []
+
+        for stream in channel_info['streams']:
+            formats.append({
+                'format_id': str(stream['id']),
+                # this is an m3u8 stream, but we are deliberately not using _extract_m3u8_formats
+                # because 0) it doesn't have bitrate variants anyway, and 1) the ids generated
+                # by that method are highly unstable (because the bitrate is variable)
+                'url': stream['url'],
+                'resolution': stream['name'],
+                'format_note': 'expires after %u seconds' % int(stream['watch-timeout']),
+                'ext': 'mp4',
+                'quality': _QUALITY(stream['quality']),
+                'preference': int(stream['watch-timeout']),
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': str(channel_info['id']),
+            'display_id': channel_info['alias'],
+            'formats': formats,
+            # XXX: use the channel description (channel_info['description'])?
+            'uploader_id': channel_info['alias'],
+            'uploader': channel_info['title'], # XXX: kinda stretching it...
+            'title': now_playing.get('programme_name') or channel_info['title'],
+            'description': now_playing.get('programme_description'),
+            'thumbnails': thumbnails,
+            'is_live': True,
+        }
+
+
+class FilmOnVODIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?filmon\.com/vod/view/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.filmon.com/vod/view/24869-0-plan-9-from-outer-space',
+        'info_dict': {
+            'id': '24869',
+            'ext': 'mp4',
+            'title': 'Plan 9 From Outer Space',
+            'description': 'Dead human, zombies and vampires',
+        },
+    }, {
+        'url': 'https://www.filmon.com/vod/view/2825-1-popeye-series-1',
+        'info_dict': {
+            'id': '2825',
+            'title': 'Popeye Series 1',
+        },
+        'playlist_count': 8,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        result = self._download_json('https://www.filmon.com/api/vod/movie?id=%s' % (video_id), video_id)
+        if result['code'] != 200:
+            raise ExtractorError('FilmOn said: %s' % (result['reason']), expected=True)
+
+        response = result['response']
+
+        if response.get('episodes'):
+            return {
+                '_type': 'playlist',
+                'id': video_id,
+                'title': response['title'],
+                'entries': [{
+                    '_type': 'url',
+                    'url': 'https://www.filmon.com/vod/view/%s' % (ep),
+                } for ep in response['episodes']]
+            }
+
+        formats = []
+        for (id, stream) in response['streams'].items():
+            formats.append({
+                'format_id': id,
+                'url': stream['url'],
+                'resolution': stream['name'],
+                'format_note': 'expires after %u seconds' % int(stream['watch-timeout']),
+                'ext': 'mp4',
+                'quality': _QUALITY(stream['quality']),
+                'preference': int(stream['watch-timeout']),
+            })
+        self._sort_formats(formats)
+
+        poster = response['poster']
+        thumbnails = [{
+            'id': 'poster',
+            'url': poster['url'],
+            'width': poster['width'],
+            'height': poster['height'],
+        }]
+        for (id, thumb) in poster['thumbs'].items():
+            thumbnails.append({
+                'id': id,
+                'url': thumb['url'],
+                'width': thumb['width'],
+                'height': thumb['height'],
+            })
+
+        return {
+            'id': video_id,
+            'title': response['title'],
+            'formats': formats,
+            'description': response['description'],
+            'thumbnails': thumbnails,
+        }

From ed06da4e7b274fd444a6ada23ba9bb4c559761d3 Mon Sep 17 00:00:00 2001
From: sh!zeeg <shizeeque@gmail.com>
Date: Thu, 5 Jan 2017 04:52:42 +0300
Subject: [PATCH 0002/1696] [freesound] Fix extraction and extended (closes
 #11602)

---
 youtube_dl/extractor/freesound.py | 55 ++++++++++++++++++++++++++++---
 1 file changed, 50 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/freesound.py b/youtube_dl/extractor/freesound.py
index 5ff62af2a..f0b2400cf 100644
--- a/youtube_dl/extractor/freesound.py
+++ b/youtube_dl/extractor/freesound.py
@@ -3,6 +3,15 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    float_or_none,
+    get_element_by_class,
+    get_element_by_id,
+    int_or_none,
+    parse_filesize,
+    unified_strdate,
+)
 
 
 class FreesoundIE(InfoExtractor):
@@ -23,17 +32,53 @@ class FreesoundIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         music_id = mobj.group('id')
         webpage = self._download_webpage(url, music_id)
-        title = self._html_search_regex(
-            r'<div id="single_sample_header">.*?<a href="#">(.+?)</a>',
-            webpage, 'music title', flags=re.DOTALL)
+
+        audio_url = self._og_search_property('audio', webpage, 'song url')
+        title = self._og_search_property('audio:title', webpage, 'song title')
+        duration = float_or_none(get_element_by_class('duration', webpage), scale=1000)
+        tags = get_element_by_class('tags', webpage)
+        sound_info = get_element_by_id('sound_information_box', webpage)
+        release_date = get_element_by_id('sound_date', webpage)
+
         description = self._html_search_regex(
             r'<div id="sound_description">(.*?)</div>', webpage, 'description',
             fatal=False, flags=re.DOTALL)
 
+        download_count = int_or_none(self._html_search_regex(
+            r'Downloaded.*>(\d+)<', webpage, 'downloaded', fatal=False))
+
+        filesize = float_or_none(parse_filesize(self._search_regex(
+            r'Filesize</dt><dd>(.*)</dd>', sound_info, 'file size (approx)', fatal=False)))
+
+        if release_date:
+            release_date = unified_strdate(release_date.replace('th', ''))
+
+        bitdepth = self._html_search_regex(
+            r'Bitdepth</dt><dd>(.*)</dd>', sound_info, 'Bitdepth', fatal=False)
+
+        channels = self._html_search_regex(
+            r'Channels</dt><dd>(.*)</dd>', sound_info, 'Channels info', fatal=False)
+
+        formats = [{
+            'url': audio_url,
+            'id': music_id,
+            'format_id': self._og_search_property('audio:type', webpage, 'audio format', fatal=False),
+            'format_note': '{0} {1} {2}'.format(determine_ext(audio_url), bitdepth, channels),
+            'filesize_approx': filesize,
+            'asr': int_or_none(self._html_search_regex(
+                r'Samplerate</dt><dd>(\d+).*</dd>',
+                sound_info, 'samplerate', fatal=False)),
+        }]
+
         return {
             'id': music_id,
             'title': title,
-            'url': self._og_search_property('audio', webpage, 'music url'),
-            'uploader': self._og_search_property('audio:artist', webpage, 'music uploader'),
+            'uploader': self._og_search_property('audio:artist', webpage, 'music uploader', fatal=False),
             'description': description,
+            'duration': duration,
+            'tags': [self._html_search_regex(r'>(.*)</a>', t, 'tag', fatal=False)
+                     for t in tags.split('\n') if t.strip()],
+            'formats': formats,
+            'release_date': release_date,
+            'likes_count': download_count,
         }

From cb655f34fbbd741f18e22cb8ec0cae1c4c3bfebe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 12 Jan 2017 22:39:45 +0700
Subject: [PATCH 0003/1696] [utils] Add more date formats

---
 test/test_utils.py  | 3 +++
 youtube_dl/utils.py | 6 ++++++
 2 files changed, 9 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 3092db5c1..e99bf794e 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -295,6 +295,9 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(unified_strdate('27.02.2016 17:30'), '20160227')
         self.assertEqual(unified_strdate('UNKNOWN DATE FORMAT'), None)
         self.assertEqual(unified_strdate('Feb 7, 2016 at 6:35 pm'), '20160207')
+        self.assertEqual(unified_strdate('July 15th, 2013'), '20130715')
+        self.assertEqual(unified_strdate('September 1st, 2013'), '20130901')
+        self.assertEqual(unified_strdate('Sep 2nd, 2013'), '20130902')
 
     def test_unified_timestamps(self):
         self.assertEqual(unified_timestamp('December 21, 2010'), 1292889600)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 39dd6c49f..12863e74a 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -128,7 +128,13 @@ DATE_FORMATS = (
     '%d %B %Y',
     '%d %b %Y',
     '%B %d %Y',
+    '%B %dst %Y',
+    '%B %dnd %Y',
+    '%B %dth %Y',
     '%b %d %Y',
+    '%b %dst %Y',
+    '%b %dnd %Y',
+    '%b %dth %Y',
     '%b %dst %Y %I:%M',
     '%b %dnd %Y %I:%M',
     '%b %dth %Y %I:%M',

From 3a407e707ac96bc082fd82325e916802a3b55d36 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 12 Jan 2017 23:03:53 +0700
Subject: [PATCH 0004/1696] [freesound] Improve and remove unrelated metadata
 (closes #11608)

---
 youtube_dl/extractor/freesound.py | 75 +++++++++++++++----------------
 1 file changed, 35 insertions(+), 40 deletions(-)

diff --git a/youtube_dl/extractor/freesound.py b/youtube_dl/extractor/freesound.py
index f0b2400cf..138b6bc58 100644
--- a/youtube_dl/extractor/freesound.py
+++ b/youtube_dl/extractor/freesound.py
@@ -4,18 +4,15 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
-    determine_ext,
     float_or_none,
     get_element_by_class,
     get_element_by_id,
-    int_or_none,
-    parse_filesize,
     unified_strdate,
 )
 
 
 class FreesoundIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?freesound\.org/people/([^/]+)/sounds/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?freesound\.org/people/[^/]+/sounds/(?P<id>[^/]+)'
     _TEST = {
         'url': 'http://www.freesound.org/people/miklovan/sounds/194503/',
         'md5': '12280ceb42c81f19a515c745eae07650',
@@ -23,62 +20,60 @@ class FreesoundIE(InfoExtractor):
             'id': '194503',
             'ext': 'mp3',
             'title': 'gulls in the city.wav',
-            'uploader': 'miklovan',
             'description': 'the sounds of seagulls in the city',
+            'duration': 130.233,
+            'uploader': 'miklovan',
+            'upload_date': '20130715',
+            'tags': list,
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        music_id = mobj.group('id')
-        webpage = self._download_webpage(url, music_id)
+        audio_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, audio_id)
 
         audio_url = self._og_search_property('audio', webpage, 'song url')
         title = self._og_search_property('audio:title', webpage, 'song title')
-        duration = float_or_none(get_element_by_class('duration', webpage), scale=1000)
-        tags = get_element_by_class('tags', webpage)
-        sound_info = get_element_by_id('sound_information_box', webpage)
-        release_date = get_element_by_id('sound_date', webpage)
 
         description = self._html_search_regex(
-            r'<div id="sound_description">(.*?)</div>', webpage, 'description',
-            fatal=False, flags=re.DOTALL)
+            r'(?s)id=["\']sound_description["\'][^>]*>(.+?)</div>',
+            webpage, 'description', fatal=False)
 
-        download_count = int_or_none(self._html_search_regex(
-            r'Downloaded.*>(\d+)<', webpage, 'downloaded', fatal=False))
+        duration = float_or_none(
+            get_element_by_class('duration', webpage), scale=1000)
 
-        filesize = float_or_none(parse_filesize(self._search_regex(
-            r'Filesize</dt><dd>(.*)</dd>', sound_info, 'file size (approx)', fatal=False)))
-
-        if release_date:
-            release_date = unified_strdate(release_date.replace('th', ''))
-
-        bitdepth = self._html_search_regex(
-            r'Bitdepth</dt><dd>(.*)</dd>', sound_info, 'Bitdepth', fatal=False)
+        upload_date = unified_strdate(get_element_by_id('sound_date', webpage))
+        uploader = self._og_search_property(
+            'audio:artist', webpage, 'uploader', fatal=False)
 
         channels = self._html_search_regex(
-            r'Channels</dt><dd>(.*)</dd>', sound_info, 'Channels info', fatal=False)
+            r'Channels</dt><dd>(.+?)</dd>', webpage,
+            'channels info', fatal=False)
+
+        tags_str = get_element_by_class('tags', webpage)
+        tags = re.findall(r'<a[^>]+>([^<]+)', tags_str) if tags_str else None
+
+        audio_urls = [audio_url]
+
+        LQ_FORMAT = '-lq.mp3'
+        if LQ_FORMAT in audio_url:
+            audio_urls.append(audio_url.replace(LQ_FORMAT, '-hq.mp3'))
 
         formats = [{
-            'url': audio_url,
-            'id': music_id,
-            'format_id': self._og_search_property('audio:type', webpage, 'audio format', fatal=False),
-            'format_note': '{0} {1} {2}'.format(determine_ext(audio_url), bitdepth, channels),
-            'filesize_approx': filesize,
-            'asr': int_or_none(self._html_search_regex(
-                r'Samplerate</dt><dd>(\d+).*</dd>',
-                sound_info, 'samplerate', fatal=False)),
-        }]
+            'url': format_url,
+            'format_note': channels,
+            'quality': quality,
+        } for quality, format_url in enumerate(audio_urls)]
+        self._sort_formats(formats)
 
         return {
-            'id': music_id,
+            'id': audio_id,
             'title': title,
-            'uploader': self._og_search_property('audio:artist', webpage, 'music uploader', fatal=False),
             'description': description,
             'duration': duration,
-            'tags': [self._html_search_regex(r'>(.*)</a>', t, 'tag', fatal=False)
-                     for t in tags.split('\n') if t.strip()],
+            'uploader': uploader,
+            'upload_date': upload_date,
+            'tags': tags,
             'formats': formats,
-            'release_date': release_date,
-            'likes_count': download_count,
         }

From c4251b9aaa9a69e7f7b55197b3907e52b17150d4 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 13 Jan 2017 10:08:51 +0100
Subject: [PATCH 0005/1696] [common] add possibility to customize akamai
 manifest host

---
 youtube_dl/extractor/common.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 6fa7c334e..dce8c7d0d 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1967,10 +1967,13 @@ class InfoExtractor(object):
                 entries.append(media_info)
         return entries
 
-    def _extract_akamai_formats(self, manifest_url, video_id):
+    def _extract_akamai_formats(self, manifest_url, video_id, hosts={}):
         formats = []
         hdcore_sign = 'hdcore=3.7.0'
-        f4m_url = re.sub(r'(https?://.+?)/i/', r'\1/z/', manifest_url).replace('/master.m3u8', '/manifest.f4m')
+        f4m_url = re.sub(r'(https?://[^/+])/i/', r'\1/z/', manifest_url).replace('/master.m3u8', '/manifest.f4m')
+        hds_host = hosts.get('hds')
+        if hds_host:
+            f4m_url = re.sub(r'(https?://)[^/]+', r'\1' + hds_host, f4m_url)
         if 'hdcore=' not in f4m_url:
             f4m_url += ('&' if '?' in f4m_url else '?') + hdcore_sign
         f4m_formats = self._extract_f4m_formats(
@@ -1978,7 +1981,10 @@ class InfoExtractor(object):
         for entry in f4m_formats:
             entry.update({'extra_param_to_segment_url': hdcore_sign})
         formats.extend(f4m_formats)
-        m3u8_url = re.sub(r'(https?://.+?)/z/', r'\1/i/', manifest_url).replace('/manifest.f4m', '/master.m3u8')
+        m3u8_url = re.sub(r'(https?://[^/]+)/z/', r'\1/i/', manifest_url).replace('/manifest.f4m', '/master.m3u8')
+        hls_host = hosts.get('hls')
+        if hls_host:
+            m3u8_url = re.sub(r'(https?://)[^/]+', r'\1' + hls_host, m3u8_url)
         formats.extend(self._extract_m3u8_formats(
             m3u8_url, video_id, 'mp4', 'm3u8_native',
             m3u8_id='hls', fatal=False))

From 1f393a324191591d895bafc1e4c756951f368b3c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 13 Jan 2017 10:19:53 +0100
Subject: [PATCH 0006/1696] [tv4] improve extraction(closes #11698)

- remove check for requires_subscription
- extract more formats
- extract subtitles
---
 youtube_dl/extractor/tv4.py | 49 +++++++++++++++++++++++--------------
 1 file changed, 31 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/tv4.py b/youtube_dl/extractor/tv4.py
index 29f62b970..ad79db92b 100644
--- a/youtube_dl/extractor/tv4.py
+++ b/youtube_dl/extractor/tv4.py
@@ -4,11 +4,10 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
-    ExtractorError,
     int_or_none,
     parse_iso8601,
     try_get,
-    update_url_query,
+    determine_ext,
 )
 
 
@@ -28,7 +27,7 @@ class TV4IE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://www.tv4.se/kalla-fakta/klipp/kalla-fakta-5-english-subtitles-2491650',
-            'md5': '909d6454b87b10a25aa04c4bdd416a9b',
+            'md5': 'cb837212f342d77cec06e6dad190e96d',
             'info_dict': {
                 'id': '2491650',
                 'ext': 'mp4',
@@ -40,7 +39,7 @@ class TV4IE(InfoExtractor):
         },
         {
             'url': 'http://www.tv4play.se/iframe/video/3054113',
-            'md5': '77f851c55139ffe0ebd41b6a5552489b',
+            'md5': 'cb837212f342d77cec06e6dad190e96d',
             'info_dict': {
                 'id': '3054113',
                 'ext': 'mp4',
@@ -75,11 +74,10 @@ class TV4IE(InfoExtractor):
         # If is_geo_restricted is true, it doesn't necessarily mean we can't download it
         if info.get('is_geo_restricted'):
             self.report_warning('This content might not be available in your country due to licensing restrictions.')
-        if info.get('requires_subscription'):
-            raise ExtractorError('This content requires subscription.', expected=True)
 
         title = info['title']
 
+        subtitles = {}
         formats = []
         # http formats are linked with unresolvable host
         for kind in ('hls', ''):
@@ -87,26 +85,41 @@ class TV4IE(InfoExtractor):
                 'https://prima.tv4play.se/api/web/asset/%s/play.json' % video_id,
                 video_id, 'Downloading sources JSON', query={
                     'protocol': kind,
-                    'videoFormat': 'MP4+WEBVTTS+WEBVTT',
+                    'videoFormat': 'MP4+WEBVTT',
                 })
-            item = try_get(data, lambda x: x['playback']['items']['item'], dict)
-            manifest_url = item.get('url')
-            if not isinstance(manifest_url, compat_str):
+            items = try_get(data, lambda x: x['playback']['items']['item'])
+            if not items:
                 continue
-            if kind == 'hls':
-                formats.extend(self._extract_m3u8_formats(
-                    manifest_url, video_id, 'mp4', entry_protocol='m3u8_native',
-                    m3u8_id=kind, fatal=False))
-            else:
-                formats.extend(self._extract_f4m_formats(
-                    update_url_query(manifest_url, {'hdcore': '3.8.0'}),
-                    video_id, f4m_id='hds', fatal=False))
+            if isinstance(items, dict):
+                items = [items]
+            for item in items:
+                manifest_url = item.get('url')
+                if not isinstance(manifest_url, compat_str):
+                    continue
+                ext = determine_ext(manifest_url)
+                if ext == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        manifest_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                        m3u8_id=kind, fatal=False))
+                elif ext == 'f4m':
+                    formats.extend(self._extract_akamai_formats(
+                        manifest_url, video_id, {
+                            'hls': 'tv4play-i.akamaihd.net',
+                        }))
+                elif ext == 'webvtt':
+                    subtitles = self._merge_subtitles(
+                        subtitles, {
+                            'sv': [{
+                                'url': manifest_url,
+                                'ext': 'vtt',
+                            }]})
         self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': title,
             'formats': formats,
+            'subtitles': subtitles,
             'description': info.get('description'),
             'timestamp': parse_iso8601(info.get('broadcast_date_time')),
             'duration': int_or_none(info.get('duration')),

From 06e9363b7a21acf6a592780a706b0fdd6b5a2d4e Mon Sep 17 00:00:00 2001
From: Vijay Singh <sudovijay@users.noreply.github.com>
Date: Sun, 8 Jan 2017 22:27:28 +0530
Subject: [PATCH 0007/1696] [openload] Fix extraction (closes #10408)

Just a minor fix for openload
---
 youtube_dl/extractor/openload.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 2ce9f3826..3d4ad7dca 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -64,16 +64,17 @@ class OpenloadIE(InfoExtractor):
             raise ExtractorError('File not found', expected=True)
 
         ol_id = self._search_regex(
-            '<span[^>]+id="[a-zA-Z0-9]+x"[^>]*>([0-9]+)</span>',
+            '<span[^>]+id="[^"]+"[^>]*>([0-9]+)</span>',
             webpage, 'openload ID')
 
-        first_two_chars = int(float(ol_id[0:][:2]))
+        first_three_chars = int(float(ol_id[0:][:3]))
+        fifth_char = int(float(ol_id[3:5]))
         urlcode = ''
-        num = 2
+        num = 5
 
         while num < len(ol_id):
-            urlcode += compat_chr(int(float(ol_id[num:][:3])) -
-                                  first_two_chars * int(float(ol_id[num + 3:][:2])))
+            urlcode += compat_chr(int(float(ol_id[num:][:3])) +
+                                  first_three_chars - fifth_char * int(float(ol_id[num + 3:][:2])))
             num += 5
 
         video_url = 'https://openload.co/stream/' + urlcode

From fb6a59205e3dc5bb1d37d50ac1161314c0d66cf1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Jan 2017 23:55:55 +0700
Subject: [PATCH 0008/1696] [mixcloud] Fix extraction (closes #11674)

---
 youtube_dl/extractor/mixcloud.py | 22 +++++-----------------
 1 file changed, 5 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 4ba2310fd..a24b3165a 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -16,7 +16,6 @@ from ..utils import (
     clean_html,
     ExtractorError,
     OnDemandPagedList,
-    parse_count,
     str_to_int,
 )
 
@@ -36,7 +35,6 @@ class MixcloudIE(InfoExtractor):
             'uploader_id': 'dholbach',
             'thumbnail': r're:https?://.*\.jpg',
             'view_count': int,
-            'like_count': int,
         },
     }, {
         'url': 'http://www.mixcloud.com/gillespeterson/caribou-7-inch-vinyl-mix-chat/',
@@ -49,7 +47,6 @@ class MixcloudIE(InfoExtractor):
             'uploader_id': 'gillespeterson',
             'thumbnail': 're:https?://.*',
             'view_count': int,
-            'like_count': int,
         },
     }, {
         'url': 'https://beta.mixcloud.com/RedLightRadio/nosedrip-15-red-light-radio-01-18-2016/',
@@ -89,26 +86,18 @@ class MixcloudIE(InfoExtractor):
 
         song_url = play_info['stream_url']
 
-        PREFIX = (
-            r'm-play-on-spacebar[^>]+'
-            r'(?:\s+[a-zA-Z0-9-]+(?:="[^"]+")?)*?\s+')
-        title = self._html_search_regex(
-            PREFIX + r'm-title="([^"]+)"', webpage, 'title')
+        title = self._html_search_regex(r'm-title="([^"]+)"', webpage, 'title')
         thumbnail = self._proto_relative_url(self._html_search_regex(
-            PREFIX + r'm-thumbnail-url="([^"]+)"', webpage, 'thumbnail',
-            fatal=False))
+            r'm-thumbnail-url="([^"]+)"', webpage, 'thumbnail', fatal=False))
         uploader = self._html_search_regex(
-            PREFIX + r'm-owner-name="([^"]+)"',
-            webpage, 'uploader', fatal=False)
+            r'm-owner-name="([^"]+)"', webpage, 'uploader', fatal=False)
         uploader_id = self._search_regex(
             r'\s+"profile": "([^"]+)",', webpage, 'uploader id', fatal=False)
         description = self._og_search_description(webpage)
-        like_count = parse_count(self._search_regex(
-            r'\bbutton-favorite[^>]+>.*?<span[^>]+class=["\']toggle-number[^>]+>\s*([^<]+)',
-            webpage, 'like count', default=None))
         view_count = str_to_int(self._search_regex(
             [r'<meta itemprop="interactionCount" content="UserPlays:([0-9]+)"',
-             r'/listeners/?">([0-9,.]+)</a>'],
+             r'/listeners/?">([0-9,.]+)</a>',
+             r'm-tooltip=["\']([\d,.]+) plays'],
             webpage, 'play count', default=None))
 
         return {
@@ -120,7 +109,6 @@ class MixcloudIE(InfoExtractor):
             'uploader': uploader,
             'uploader_id': uploader_id,
             'view_count': view_count,
-            'like_count': like_count,
         }
 
 

From 9837cb7507e0635755082a7fd2e748c4106fefc4 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 13 Jan 2017 23:02:50 +0100
Subject: [PATCH 0009/1696] [ooyala] add support for videos with
 embedToken(#11684)

---
 youtube_dl/extractor/generic.py |  9 ++++++++-
 youtube_dl/extractor/ooyala.py  | 14 +++++++++++---
 2 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 86dc79307..ac29ec600 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1939,7 +1939,14 @@ class GenericIE(InfoExtractor):
                 re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P<ec>.{32})[\'"]\)', webpage) or
                 re.search(r'data-ooyala-video-id\s*=\s*[\'"](?P<ec>.{32})[\'"]', webpage))
         if mobj is not None:
-            return OoyalaIE._build_url_result(smuggle_url(mobj.group('ec'), {'domain': url}))
+            embed_token = self._search_regex(
+                r'embedToken[\'"]?\s*:\s*[\'"]([^\'"]+)',
+                webpage, 'ooyala embed token', default=None)
+            return OoyalaIE._build_url_result(smuggle_url(
+                mobj.group('ec'), {
+                    'domain': url,
+                    'embed_token': embed_token,
+                }))
 
         # Look for multiple Ooyala embeds on SBN network websites
         mobj = re.search(r'SBN\.VideoLinkset\.entryGroup\((\[.*?\])', webpage)
diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index c2807d0f6..f00cf745b 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -18,7 +18,7 @@ class OoyalaBaseIE(InfoExtractor):
     _CONTENT_TREE_BASE = _PLAYER_BASE + 'player_api/v1/content_tree/'
     _AUTHORIZATION_URL_TEMPLATE = _PLAYER_BASE + 'sas/player_api/v2/authorization/embed_code/%s/%s?'
 
-    def _extract(self, content_tree_url, video_id, domain='example.org', supportedformats=None):
+    def _extract(self, content_tree_url, video_id, domain='example.org', supportedformats=None, embed_token=None):
         content_tree = self._download_json(content_tree_url, video_id)['content_tree']
         metadata = content_tree[list(content_tree)[0]]
         embed_code = metadata['embed_code']
@@ -29,7 +29,8 @@ class OoyalaBaseIE(InfoExtractor):
             self._AUTHORIZATION_URL_TEMPLATE % (pcode, embed_code) +
             compat_urllib_parse_urlencode({
                 'domain': domain,
-                'supportedFormats': supportedformats or 'mp4,rtmp,m3u8,hds',
+                'supportedFormats': supportedformats or 'mp4,rtmp,m3u8,hds,dash,smooth',
+                'embedToken': embed_token,
             }), video_id)
 
         cur_auth_data = auth_data['authorization_data'][embed_code]
@@ -52,6 +53,12 @@ class OoyalaBaseIE(InfoExtractor):
                 elif delivery_type == 'hds' or ext == 'f4m':
                     formats.extend(self._extract_f4m_formats(
                         s_url + '?hdcore=3.7.0', embed_code, f4m_id='hds', fatal=False))
+                elif delivery_type == 'hds' or ext == 'mpd':
+                    formats.extend(self._extract_mpd_formats(
+                        s_url, embed_code, mpd_id='dash', fatal=False))
+                elif delivery_type == 'smooth':
+                    self._extract_ism_formats(
+                        s_url, embed_code, ism_id='mss', fatal=False)
                 elif ext == 'smil':
                     formats.extend(self._extract_smil_formats(
                         s_url, embed_code, fatal=False))
@@ -146,8 +153,9 @@ class OoyalaIE(OoyalaBaseIE):
         embed_code = self._match_id(url)
         domain = smuggled_data.get('domain')
         supportedformats = smuggled_data.get('supportedformats')
+        embed_token = smuggled_data.get('embed_token')
         content_tree_url = self._CONTENT_TREE_BASE + 'embed_code/%s/%s' % (embed_code, embed_code)
-        return self._extract(content_tree_url, embed_code, domain, supportedformats)
+        return self._extract(content_tree_url, embed_code, domain, supportedformats, embed_token)
 
 
 class OoyalaExternalIE(OoyalaBaseIE):

From 5e8eebb6009ac3e9f7dfc803d8561174d207c1a2 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 13 Jan 2017 23:06:07 +0100
Subject: [PATCH 0010/1696] [mitele] extract dash formats

---
 youtube_dl/extractor/mitele.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index 8984d3b8d..79e0b8ada 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -190,7 +190,7 @@ class MiTeleIE(InfoExtractor):
         return {
             '_type': 'url_transparent',
             # for some reason only HLS is supported
-            'url': smuggle_url('ooyala:' + embedCode, {'supportedformats': 'm3u8'}),
+            'url': smuggle_url('ooyala:' + embedCode, {'supportedformats': 'm3u8,dash'}),
             'id': video_id,
             'title': title,
             'description': description,

From adf063dad1792f0c9c680d13ccd984b4ad60ac29 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Jan 2017 06:17:03 +0700
Subject: [PATCH 0011/1696] [mtv,cc,cmt,spike] Improve and refactor - Eliminate
 _transform_rtmp_url * Generalize triforce mgid extraction + [cmt] Add support
 for full-episodes (closes #11623)

---
 youtube_dl/extractor/cmt.py           | 25 ++++++------
 youtube_dl/extractor/comedycentral.py | 17 +-------
 youtube_dl/extractor/mtv.py           | 58 ++++++++++++++++++---------
 youtube_dl/extractor/spike.py         |  2 +-
 4 files changed, 54 insertions(+), 48 deletions(-)

diff --git a/youtube_dl/extractor/cmt.py b/youtube_dl/extractor/cmt.py
index 7d3e9b0c9..6302b8d9c 100644
--- a/youtube_dl/extractor/cmt.py
+++ b/youtube_dl/extractor/cmt.py
@@ -1,13 +1,11 @@
 from __future__ import unicode_literals
 
 from .mtv import MTVIE
-from ..utils import ExtractorError
 
 
 class CMTIE(MTVIE):
     IE_NAME = 'cmt.com'
-    _VALID_URL = r'https?://(?:www\.)?cmt\.com/(?:videos|shows)/(?:[^/]+/)*(?P<videoid>\d+)'
-    _FEED_URL = 'http://www.cmt.com/sitewide/apps/player/embed/rss/'
+    _VALID_URL = r'https?://(?:www\.)?cmt\.com/(?:videos|shows|full-episodes)/(?P<id>[^/]+)'
 
     _TESTS = [{
         'url': 'http://www.cmt.com/videos/garth-brooks/989124/the-call-featuring-trisha-yearwood.jhtml#artist=30061',
@@ -35,15 +33,16 @@ class CMTIE(MTVIE):
         'only_matching': True,
     }]
 
-    @classmethod
-    def _transform_rtmp_url(cls, rtmp_video_url):
-        if 'error_not_available.swf' in rtmp_video_url:
-            raise ExtractorError(
-                '%s said: video is not available' % cls.IE_NAME, expected=True)
-
-        return super(CMTIE, cls)._transform_rtmp_url(rtmp_video_url)
-
     def _extract_mgid(self, webpage):
-        return self._search_regex(
+        mgid = self._search_regex(
             r'MTVN\.VIDEO\.contentUri\s*=\s*([\'"])(?P<mgid>.+?)\1',
-            webpage, 'mgid', group='mgid')
+            webpage, 'mgid', group='mgid', default=None)
+        if not mgid:
+            mgid = self._extract_triforce_mgid(webpage)
+        return mgid
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        mgid = self._extract_mgid(webpage)
+        return self.url_result('http://media.mtvnservices.com/embed/%s' % mgid)
diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 816e0bfb6..4cac29415 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -48,17 +48,8 @@ class ComedyCentralFullEpisodesIE(MTVServicesInfoExtractor):
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
         webpage = self._download_webpage(url, playlist_id)
-
-        feed_json = self._search_regex(r'var triforceManifestFeed\s*=\s*(\{.+?\});\n', webpage, 'triforce feeed')
-        feed = self._parse_json(feed_json, playlist_id)
-        zones = feed['manifest']['zones']
-
-        video_zone = zones['t2_lc_promo1']
-        feed = self._download_json(video_zone['feed'], playlist_id)
-        mgid = feed['result']['data']['id']
-
+        mgid = self._extract_triforce_mgid(webpage, data_zone='t2_lc_promo1')
         videos_info = self._get_videos_info(mgid)
-
         return videos_info
 
 
@@ -94,12 +85,6 @@ class ToshIE(MTVServicesInfoExtractor):
         'only_matching': True,
     }]
 
-    @classmethod
-    def _transform_rtmp_url(cls, rtmp_video_url):
-        new_urls = super(ToshIE, cls)._transform_rtmp_url(rtmp_video_url)
-        new_urls['rtmp'] = rtmp_video_url.replace('viacomccstrm', 'viacommtvstrm')
-        return new_urls
-
 
 class ComedyCentralTVIE(MTVServicesInfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?comedycentral\.tv/(?:staffeln|shows)/(?P<id>[^/?#&]+)'
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 5250db212..00a980c7d 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -13,11 +13,11 @@ from ..utils import (
     fix_xml_ampersands,
     float_or_none,
     HEADRequest,
-    NO_DEFAULT,
     RegexNotFoundError,
     sanitized_Request,
     strip_or_none,
     timeconvert,
+    try_get,
     unescapeHTML,
     update_url_query,
     url_basename,
@@ -42,15 +42,6 @@ class MTVServicesInfoExtractor(InfoExtractor):
         # Remove the templates, like &device={device}
         return re.sub(r'&[^=]*?={.*?}(?=(&|$))', '', url)
 
-    # This was originally implemented for ComedyCentral, but it also works here
-    @classmethod
-    def _transform_rtmp_url(cls, rtmp_video_url):
-        m = re.match(r'^rtmpe?://.*?/(?P<finalid>gsp\..+?/.*)$', rtmp_video_url)
-        if not m:
-            return {'rtmp': rtmp_video_url}
-        base = 'http://viacommtvstrmfs.fplive.net/'
-        return {'http': base + m.group('finalid')}
-
     def _get_feed_url(self, uri):
         return self._FEED_URL
 
@@ -91,22 +82,28 @@ class MTVServicesInfoExtractor(InfoExtractor):
             if rendition.get('method') == 'hls':
                 hls_url = rendition.find('./src').text
                 formats.extend(self._extract_m3u8_formats(
-                    hls_url, video_id, ext='mp4', entry_protocol='m3u8_native'))
+                    hls_url, video_id, ext='mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls'))
             else:
                 # fms
                 try:
                     _, _, ext = rendition.attrib['type'].partition('/')
                     rtmp_video_url = rendition.find('./src').text
+                    if 'error_not_available.swf' in rtmp_video_url:
+                        raise ExtractorError(
+                            '%s said: video is not available' % self.IE_NAME,
+                            expected=True)
                     if rtmp_video_url.endswith('siteunavail.png'):
                         continue
-                    new_urls = self._transform_rtmp_url(rtmp_video_url)
                     formats.extend([{
-                        'ext': 'flv' if new_url.startswith('rtmp') else ext,
-                        'url': new_url,
-                        'format_id': '-'.join(filter(None, [kind, rendition.get('bitrate')])),
+                        'ext': 'flv' if rtmp_video_url.startswith('rtmp') else ext,
+                        'url': rtmp_video_url,
+                        'format_id': '-'.join(filter(None, [
+                            'rtmp' if rtmp_video_url.startswith('rtmp') else None,
+                            rendition.get('bitrate')])),
                         'width': int(rendition.get('width')),
                         'height': int(rendition.get('height')),
-                    } for kind, new_url in new_urls.items()])
+                    }])
                 except (KeyError, TypeError):
                     raise ExtractorError('Invalid rendition field.')
         self._sort_formats(formats)
@@ -212,7 +209,28 @@ class MTVServicesInfoExtractor(InfoExtractor):
             [self._get_video_info(item, use_hls) for item in idoc.findall('.//item')],
             playlist_title=title, playlist_description=description)
 
-    def _extract_mgid(self, webpage, default=NO_DEFAULT):
+    def _extract_triforce_mgid(self, webpage, data_zone=None, video_id=None):
+        triforce_feed = self._parse_json(self._search_regex(
+            r'triforceManifestFeed\s*=\s*(\{.+?\});\n', webpage,
+            'triforce feed', default='{}'), video_id, fatal=False)
+
+        data_zone = self._search_regex(
+            r'data-zone=(["\'])(?P<zone>.+?_lc_promo.*?)\1', webpage,
+            'data zone', default=data_zone, group='zone')
+
+        feed_url = try_get(
+            triforce_feed, lambda x: x['manifest']['zones'][data_zone]['feed'],
+            compat_str)
+        if not feed_url:
+            return
+
+        feed = self._download_json(feed_url, video_id, fatal=False)
+        if not feed:
+            return
+
+        return try_get(feed, lambda x: x['result']['data']['id'], compat_str)
+
+    def _extract_mgid(self, webpage):
         try:
             # the url can be http://media.mtvnservices.com/fb/{mgid}.swf
             # or http://media.mtvnservices.com/{mgid}
@@ -232,7 +250,11 @@ class MTVServicesInfoExtractor(InfoExtractor):
             sm4_embed = self._html_search_meta(
                 'sm4:video:embed', webpage, 'sm4 embed', default='')
             mgid = self._search_regex(
-                r'embed/(mgid:.+?)["\'&?/]', sm4_embed, 'mgid', default=default)
+                r'embed/(mgid:.+?)["\'&?/]', sm4_embed, 'mgid', default=None)
+
+        if not mgid:
+            mgid = self._extract_triforce_mgid(webpage)
+
         return mgid
 
     def _real_extract(self, url):
diff --git a/youtube_dl/extractor/spike.py b/youtube_dl/extractor/spike.py
index abfee3ece..c59896a17 100644
--- a/youtube_dl/extractor/spike.py
+++ b/youtube_dl/extractor/spike.py
@@ -46,7 +46,7 @@ class SpikeIE(MTVServicesInfoExtractor):
     _CUSTOM_URL_REGEX = re.compile(r'spikenetworkapp://([^/]+/[-a-fA-F0-9]+)')
 
     def _extract_mgid(self, webpage):
-        mgid = super(SpikeIE, self)._extract_mgid(webpage, default=None)
+        mgid = super(SpikeIE, self)._extract_mgid(webpage)
         if mgid is None:
             url_parts = self._search_regex(self._CUSTOM_URL_REGEX, webpage, 'episode_id')
             video_type, episode_id = url_parts.split('/', 1)

From e54fc0524ebf7e3ec02fbd22f00fce466c952791 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Jan 2017 06:23:24 +0700
Subject: [PATCH 0012/1696] [cmt] Add support for video-clips

---
 youtube_dl/extractor/cmt.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cmt.py b/youtube_dl/extractor/cmt.py
index 6302b8d9c..f6b794fb3 100644
--- a/youtube_dl/extractor/cmt.py
+++ b/youtube_dl/extractor/cmt.py
@@ -5,7 +5,7 @@ from .mtv import MTVIE
 
 class CMTIE(MTVIE):
     IE_NAME = 'cmt.com'
-    _VALID_URL = r'https?://(?:www\.)?cmt\.com/(?:videos|shows|full-episodes)/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?cmt\.com/(?:videos|shows|full-episodes|video-clips)/(?P<id>[^/]+)'
 
     _TESTS = [{
         'url': 'http://www.cmt.com/videos/garth-brooks/989124/the-call-featuring-trisha-yearwood.jhtml#artist=30061',
@@ -31,6 +31,12 @@ class CMTIE(MTVIE):
     }, {
         'url': 'http://www.cmt.com/shows/party-down-south/party-down-south-ep-407-gone-girl/1738172/playlist/#id=1738172',
         'only_matching': True,
+    }, {
+        'url': 'http://www.cmt.com/full-episodes/537qb3/nashville-the-wayfaring-stranger-season-5-ep-501',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.cmt.com/video-clips/t9e4ci/nashville-juliette-in-2-minutes',
+        'only_matching': True,
     }]
 
     def _extract_mgid(self, webpage):

From 4f66c16f337f3b2250d369b56bc31cfd7de06f89 Mon Sep 17 00:00:00 2001
From: Jakub Wilk <jwilk@jwilk.net>
Date: Sat, 14 Jan 2017 00:26:11 +0100
Subject: [PATCH 0013/1696] [brightcove:legacy] Fix misplaced backslash in a
 regexp

---
 youtube_dl/extractor/brightcove.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index aa2923ccf..2e56d1df9 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -204,7 +204,7 @@ class BrightcoveLegacyIE(InfoExtractor):
         #   // build Brightcove <object /> XML
         # }
         m = re.search(
-            r'''(?x)customBC.\createVideo\(
+            r'''(?x)customBC\.createVideo\(
                 .*?                                                  # skipping width and height
                 ["\'](?P<playerID>\d+)["\']\s*,\s*                   # playerID
                 ["\'](?P<playerKey>AQ[^"\']{48})[^"\']*["\']\s*,\s*  # playerKey begins with AQ and is 50 characters

From 0b94510cd00d50ddda74ba0079f856650f24680e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Jan 2017 07:27:20 +0700
Subject: [PATCH 0014/1696] [ChangeLog] Actualize

---
 ChangeLog | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index f1e234507..0106a7ae8 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,24 @@
+version <unreleased>
+
+Core
++ [common] Add ability to customize akamai manifest host
++ [utils] Add more date formats
+
+Extractors
+- [mtv] Eliminate _transform_rtmp_url
+* [mtv] Generalize triforce mgid extraction
++ [cmt] Add support for full episodes and video clips (#11623)
++ [mitele] Extract DASH formats
++ [ooyala] Add support for videos with embedToken (#11684)
+* [mixcloud] Fix extraction (#11674)
+* [openload] Fix extraction (#10408)
+* [tv4] Improve extraction (#11698)
+* [freesound] Fix and improve extraction (#11602)
++ [nick] Add support for beta.nick.com (#11655)
+* [mtv,cc] Use HLS by default with native HLS downloader (#11641)
+* [mtv] Fix non-HLS extraction
+
+
 version 2017.01.10
 
 Extractors

From 5d4c7daa49b8ff83aa6fb13b183f47d4427c6513 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Jan 2017 07:31:07 +0700
Subject: [PATCH 0015/1696] release 2017.01.14

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 6a4c25680..a7bf2b90c 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.01.10*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.01.10**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.01.14*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.01.14**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.01.10
+[debug] youtube-dl version 2017.01.14
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 0106a7ae8..dba18d39b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.01.14
 
 Core
 + [common] Add ability to customize akamai manifest host
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 214124722..17c6f9eb2 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.01.10'
+__version__ = '2017.01.14'

From abe8cb763fd43ee2db09c73965f38db7db02559e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Jan 2017 08:30:00 +0700
Subject: [PATCH 0016/1696] [cbc] Improve playlist support (closes #11704)

---
 youtube_dl/extractor/cbc.py | 55 +++++++++++++++++++++++--------------
 1 file changed, 34 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/cbc.py b/youtube_dl/extractor/cbc.py
index 7c76ceac8..a291685bf 100644
--- a/youtube_dl/extractor/cbc.py
+++ b/youtube_dl/extractor/cbc.py
@@ -90,36 +90,49 @@ class CBCIE(InfoExtractor):
             },
         }],
         'skip': 'Geo-restricted to Canada',
+    }, {
+        # multiple CBC.APP.Caffeine.initInstance(...)
+        'url': 'http://www.cbc.ca/news/canada/calgary/dog-indoor-exercise-winter-1.3928238',
+        'info_dict': {
+            'title': 'Keep Rover active during the deep freeze with doggie pushups and other fun indoor tasks',
+            'id': 'dog-indoor-exercise-winter-1.3928238',
+        },
+        'playlist_mincount': 6,
     }]
 
     @classmethod
     def suitable(cls, url):
         return False if CBCPlayerIE.suitable(url) else super(CBCIE, cls).suitable(url)
 
+    def _extract_player_init(self, player_init, display_id):
+        player_info = self._parse_json(player_init, display_id, js_to_json)
+        media_id = player_info.get('mediaId')
+        if not media_id:
+            clip_id = player_info['clipId']
+            feed = self._download_json(
+                'http://tpfeed.cbc.ca/f/ExhSPC/vms_5akSXx4Ng_Zn?byCustomValue={:mpsReleases}{%s}' % clip_id,
+                clip_id, fatal=False)
+            if feed:
+                media_id = try_get(feed, lambda x: x['entries'][0]['guid'], compat_str)
+            if not media_id:
+                media_id = self._download_json(
+                    'http://feed.theplatform.com/f/h9dtGB/punlNGjMlc1F?fields=id&byContent=byReleases%3DbyId%253D' + clip_id,
+                    clip_id)['entries'][0]['id'].split('/')[-1]
+        return self.url_result('cbcplayer:%s' % media_id, 'CBCPlayer', media_id)
+
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
-        player_init = self._search_regex(
-            r'CBC\.APP\.Caffeine\.initInstance\(({.+?})\);', webpage, 'player init',
-            default=None)
-        if player_init:
-            player_info = self._parse_json(player_init, display_id, js_to_json)
-            media_id = player_info.get('mediaId')
-            if not media_id:
-                clip_id = player_info['clipId']
-                feed = self._download_json(
-                    'http://tpfeed.cbc.ca/f/ExhSPC/vms_5akSXx4Ng_Zn?byCustomValue={:mpsReleases}{%s}' % clip_id,
-                    clip_id, fatal=False)
-                if feed:
-                    media_id = try_get(feed, lambda x: x['entries'][0]['guid'], compat_str)
-                if not media_id:
-                    media_id = self._download_json(
-                        'http://feed.theplatform.com/f/h9dtGB/punlNGjMlc1F?fields=id&byContent=byReleases%3DbyId%253D' + clip_id,
-                        clip_id)['entries'][0]['id'].split('/')[-1]
-            return self.url_result('cbcplayer:%s' % media_id, 'CBCPlayer', media_id)
-        else:
-            entries = [self.url_result('cbcplayer:%s' % media_id, 'CBCPlayer', media_id) for media_id in re.findall(r'<iframe[^>]+src="[^"]+?mediaId=(\d+)"', webpage)]
-            return self.playlist_result(entries)
+        entries = [
+            self._extract_player_init(player_init, display_id)
+            for player_init in re.findall(r'CBC\.APP\.Caffeine\.initInstance\(({.+?})\);', webpage)]
+        entries.extend([
+            self.url_result('cbcplayer:%s' % media_id, 'CBCPlayer', media_id)
+            for media_id in re.findall(r'<iframe[^>]+src="[^"]+?mediaId=(\d+)"', webpage)])
+        return self.playlist_result(
+            entries, display_id,
+            self._og_search_title(webpage, fatal=False),
+            self._og_search_description(webpage))
 
 
 class CBCPlayerIE(InfoExtractor):

From 8854f3fe782e48f4b145eacf58cca533a9f9b199 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Jan 2017 08:30:00 +0700
Subject: [PATCH 0017/1696] [README.md] Clarify newline format in cookies
 section (closes #11709)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 905c1b73f..a606346b2 100644
--- a/README.md
+++ b/README.md
@@ -841,7 +841,7 @@ Use the `--cookies` option, for example `--cookies /path/to/cookies/file.txt`.
 
 In order to extract cookies from browser use any conforming browser extension for exporting cookies. For example, [cookies.txt](https://chrome.google.com/webstore/detail/cookiestxt/njabckikapfpffapmjgojcnbfjonfjfg) (for Chrome) or [Export Cookies](https://addons.mozilla.org/en-US/firefox/addon/export-cookies/) (for Firefox).
 
-Note that the cookies file must be in Mozilla/Netscape format and the first line of the cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in the cookies file and convert newlines if necessary to correspond with your OS, namely `CRLF` (`\r\n`) for Windows, `LF` (`\n`) for Linux and `CR` (`\r`) for Mac OS. `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format.
+Note that the cookies file must be in Mozilla/Netscape format and the first line of the cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in the cookies file and convert newlines if necessary to correspond with your OS, namely `CRLF` (`\r\n`) for Windows and `LF` (`\n`) for Unix and Unix-like systems (Linux, Mac OS, etc.). `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format.
 
 Passing cookies to youtube-dl is a good way to workaround login when a particular extractor does not implement it explicitly. Another use case is working around [CAPTCHA](https://en.wikipedia.org/wiki/CAPTCHA) some websites require you to solve in particular cases in order to get access (e.g. YouTube, CloudFlare).
 

From 99d537a5e08499e20c3507c3f84048feacf77522 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 14 Jan 2017 07:12:31 +0100
Subject: [PATCH 0018/1696] [ooyala] fix typo

---
 youtube_dl/extractor/ooyala.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index f00cf745b..84be2b1e3 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -53,7 +53,7 @@ class OoyalaBaseIE(InfoExtractor):
                 elif delivery_type == 'hds' or ext == 'f4m':
                     formats.extend(self._extract_f4m_formats(
                         s_url + '?hdcore=3.7.0', embed_code, f4m_id='hds', fatal=False))
-                elif delivery_type == 'hds' or ext == 'mpd':
+                elif delivery_type == 'dash' or ext == 'mpd':
                     formats.extend(self._extract_mpd_formats(
                         s_url, embed_code, mpd_id='dash', fatal=False))
                 elif delivery_type == 'smooth':

From b80e2ebc8daa1ec30396cfa69836f1d96d23028f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Jan 2017 18:27:22 +0700
Subject: [PATCH 0019/1696] [dramafever] Add support for URLs with language
 code (#11714)

---
 youtube_dl/extractor/dramafever.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 1edd8e7bd..bcd9fe2a0 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -66,7 +66,7 @@ class DramaFeverBaseIE(AMPIE):
 
 class DramaFeverIE(DramaFeverBaseIE):
     IE_NAME = 'dramafever'
-    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+/[0-9]+)(?:/|$)'
+    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/(?:[^/]+/)?drama/(?P<id>[0-9]+/[0-9]+)(?:/|$)'
     _TESTS = [{
         'url': 'http://www.dramafever.com/drama/4512/1/Cooking_with_Shin/',
         'info_dict': {
@@ -103,6 +103,9 @@ class DramaFeverIE(DramaFeverBaseIE):
             # m3u8 download
             'skip_download': True,
         },
+    }, {
+        'url': 'https://www.dramafever.com/zh-cn/drama/4972/15/Doctor_Romantic/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -148,7 +151,7 @@ class DramaFeverIE(DramaFeverBaseIE):
 
 class DramaFeverSeriesIE(DramaFeverBaseIE):
     IE_NAME = 'dramafever:series'
-    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+)(?:/(?:(?!\d+(?:/|$)).+)?)?$'
+    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/(?:[^/]+/)?drama/(?P<id>[0-9]+)(?:/(?:(?!\d+(?:/|$)).+)?)?$'
     _TESTS = [{
         'url': 'http://www.dramafever.com/drama/4512/Cooking_with_Shin/',
         'info_dict': {

From 621a2800ca259399c0c010a1cbc2c56aee90228c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Jan 2017 04:42:05 +0700
Subject: [PATCH 0020/1696] [vevo] Improve geo restriction detection

---
 youtube_dl/extractor/vevo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index d82261e5e..f0a8075fb 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -206,7 +206,7 @@ class VevoIE(VevoBaseIE):
             note='Retrieving oauth token',
             errnote='Unable to retrieve oauth token')
 
-        if 'THIS PAGE IS CURRENTLY UNAVAILABLE IN YOUR REGION' in webpage:
+        if re.search(r'(?i)THIS PAGE IS CURRENTLY UNAVAILABLE IN YOUR REGION', webpage):
             self.raise_geo_restricted(
                 '%s said: This page is currently unavailable in your region' % self.IE_NAME)
 

From cd55c6ccd7b9cd0c48d475330c40f382eb0bc625 Mon Sep 17 00:00:00 2001
From: sh!zeeg <shizeeque@gmail.com>
Date: Wed, 4 Jan 2017 01:51:08 +0300
Subject: [PATCH 0021/1696] [beam:live] Add extractor

---
 youtube_dl/extractor/beampro.py    | 82 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  1 +
 2 files changed, 83 insertions(+)
 create mode 100644 youtube_dl/extractor/beampro.py

diff --git a/youtube_dl/extractor/beampro.py b/youtube_dl/extractor/beampro.py
new file mode 100644
index 000000000..dc0a2b4af
--- /dev/null
+++ b/youtube_dl/extractor/beampro.py
@@ -0,0 +1,82 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    clean_html,
+    compat_str,
+    int_or_none,
+    parse_iso8601,
+    try_get,
+)
+
+
+class BeamProLiveIE(InfoExtractor):
+    IE_NAME = 'Beam:live'
+    _VALID_URL = r'https?://(?:\w+.)?beam.pro/(?P<id>[^?]+)$'
+    _API_CHANNEL = 'https://beam.pro/api/v1/channels/{0}'
+    _API_MANIFEST = 'https://beam.pro/api/v1/channels/{0}/manifest.m3u8'
+    _RATINGS = {'family': 0, 'teen': 13, '18+': 18}
+
+    _TEST = {
+        'url': 'http://www.beam.pro/niterhayven',
+        'info_dict': {
+            'id': '261562',
+            'ext': 'mp4',
+            'uploader': 'niterhayven',
+            'timestamp': 1483477281,
+            'age_limit': 18,
+            'title': 'Introducing The Witcher 3 //  The Grind Starts Now!',
+            'thumbnail': r're:https://.*\.jpg$',
+            'upload_date': '20170103',
+            'uploader_id': 373396,
+            'description': 'md5:0b161ac080f15fe05d18a07adb44a74d',
+            'is_live': True,
+        },
+        'skip': 'niterhayven is offline',
+        'params': {
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        channel_id = self._match_id(url)
+        chan_data = self._download_json(self._API_CHANNEL.format(channel_id), channel_id)
+
+        if not chan_data.get('online'):
+            raise ExtractorError('{0} is offline'.format(channel_id), expected=True)
+
+        formats = self._extract_m3u8_formats(
+            self._API_MANIFEST.format(chan_data.get('id')), channel_id, ext='mp4')
+
+        self._sort_formats(formats)
+        info = {}
+        info['formats'] = formats
+        if chan_data:
+            info.update(self._extract_info(chan_data))
+        if not info.get('title'):
+            info['title'] = self._live_title(channel_id)
+        if not info.get('id'):  # barely possible but just in case
+            info['id'] = compat_str(abs(hash(channel_id)) % (10 ** 8))
+
+        return info
+
+    def _extract_info(self, info):
+        thumbnail = try_get(info, lambda x: x['thumbnail']['url'], compat_str)
+        username = try_get(info, lambda x: x['user']['url'], compat_str)
+        video_id = compat_str(info['id']) if info.get('id') else None
+        rating = info.get('audience')
+
+        return {
+            'id': video_id,
+            'title': info.get('name'),
+            'description': clean_html(info.get('description')),
+            'age_limit': self._RATINGS[rating] if rating in self._RATINGS else None,
+            'is_live': True if info.get('online') else False,
+            'timestamp': parse_iso8601(info.get('updatedAt')),
+            'uploader': info.get('token') or username,
+            'uploader_id': int_or_none(info.get('userId')),
+            'view_count': int_or_none(info.get('viewersTotal')),
+            'thumbnail': thumbnail,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 5ba8efb0e..9d0610d21 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -88,6 +88,7 @@ from .bbc import (
     BBCCoUkPlaylistIE,
     BBCIE,
 )
+from .beampro import BeamProLiveIE
 from .beeg import BeegIE
 from .behindkink import BehindKinkIE
 from .bellmedia import BellMediaIE

From af62de104f33ebf8b473b3f7935451077fa56ee9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Jan 2017 06:07:35 +0700
Subject: [PATCH 0022/1696] [beam:live] Improve and simplify (#10702, closes
 #11596)

---
 youtube_dl/extractor/beampro.py | 73 +++++++++++++++------------------
 1 file changed, 32 insertions(+), 41 deletions(-)

diff --git a/youtube_dl/extractor/beampro.py b/youtube_dl/extractor/beampro.py
index dc0a2b4af..f3a9e3278 100644
--- a/youtube_dl/extractor/beampro.py
+++ b/youtube_dl/extractor/beampro.py
@@ -14,25 +14,23 @@ from ..utils import (
 
 class BeamProLiveIE(InfoExtractor):
     IE_NAME = 'Beam:live'
-    _VALID_URL = r'https?://(?:\w+.)?beam.pro/(?P<id>[^?]+)$'
-    _API_CHANNEL = 'https://beam.pro/api/v1/channels/{0}'
-    _API_MANIFEST = 'https://beam.pro/api/v1/channels/{0}/manifest.m3u8'
+    _VALID_URL = r'https?://(?:\w+\.)?beam\.pro/(?P<id>[^/?#&]+)'
     _RATINGS = {'family': 0, 'teen': 13, '18+': 18}
-
     _TEST = {
         'url': 'http://www.beam.pro/niterhayven',
         'info_dict': {
             'id': '261562',
             'ext': 'mp4',
-            'uploader': 'niterhayven',
-            'timestamp': 1483477281,
-            'age_limit': 18,
             'title': 'Introducing The Witcher 3 //  The Grind Starts Now!',
-            'thumbnail': r're:https://.*\.jpg$',
-            'upload_date': '20170103',
-            'uploader_id': 373396,
             'description': 'md5:0b161ac080f15fe05d18a07adb44a74d',
+            'thumbnail': r're:https://.*\.jpg$',
+            'timestamp': 1483477281,
+            'upload_date': '20170103',
+            'uploader': 'niterhayven',
+            'uploader_id': '373396',
+            'age_limit': 18,
             'is_live': True,
+            'view_count': int,
         },
         'skip': 'niterhayven is offline',
         'params': {
@@ -41,42 +39,35 @@ class BeamProLiveIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        channel_id = self._match_id(url)
-        chan_data = self._download_json(self._API_CHANNEL.format(channel_id), channel_id)
+        channel_name = self._match_id(url)
 
-        if not chan_data.get('online'):
-            raise ExtractorError('{0} is offline'.format(channel_id), expected=True)
+        chan = self._download_json(
+            'https://beam.pro/api/v1/channels/%s' % channel_name, channel_name)
+
+        if chan.get('online') is False:
+            raise ExtractorError(
+                '{0} is offline'.format(channel_name), expected=True)
+
+        channel_id = chan['id']
 
         formats = self._extract_m3u8_formats(
-            self._API_MANIFEST.format(chan_data.get('id')), channel_id, ext='mp4')
-
+            'https://beam.pro/api/v1/channels/%s/manifest.m3u8' % channel_id,
+            channel_name, ext='mp4', m3u8_id='hls', fatal=False)
         self._sort_formats(formats)
-        info = {}
-        info['formats'] = formats
-        if chan_data:
-            info.update(self._extract_info(chan_data))
-        if not info.get('title'):
-            info['title'] = self._live_title(channel_id)
-        if not info.get('id'):  # barely possible but just in case
-            info['id'] = compat_str(abs(hash(channel_id)) % (10 ** 8))
 
-        return info
-
-    def _extract_info(self, info):
-        thumbnail = try_get(info, lambda x: x['thumbnail']['url'], compat_str)
-        username = try_get(info, lambda x: x['user']['url'], compat_str)
-        video_id = compat_str(info['id']) if info.get('id') else None
-        rating = info.get('audience')
+        user_id = chan.get('userId') or try_get(chan, lambda x: x['user']['id'])
 
         return {
-            'id': video_id,
-            'title': info.get('name'),
-            'description': clean_html(info.get('description')),
-            'age_limit': self._RATINGS[rating] if rating in self._RATINGS else None,
-            'is_live': True if info.get('online') else False,
-            'timestamp': parse_iso8601(info.get('updatedAt')),
-            'uploader': info.get('token') or username,
-            'uploader_id': int_or_none(info.get('userId')),
-            'view_count': int_or_none(info.get('viewersTotal')),
-            'thumbnail': thumbnail,
+            'id': compat_str(chan.get('id') or channel_name),
+            'title': self._live_title(chan.get('name') or channel_name),
+            'description': clean_html(chan.get('description')),
+            'thumbnail': try_get(chan, lambda x: x['thumbnail']['url'], compat_str),
+            'timestamp': parse_iso8601(chan.get('updatedAt')),
+            'uploader': chan.get('token') or try_get(
+                chan, lambda x: x['user']['username'], compat_str),
+            'uploader_id': compat_str(user_id) if user_id else None,
+            'age_limit': self._RATINGS.get(chan.get('audience')),
+            'is_live': True,
+            'view_count': int_or_none(chan.get('viewersTotal')),
+            'formats': formats,
         }

From 6f0be937473c5d5f60cd8e712287fcee844093d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Jan 2017 06:09:32 +0700
Subject: [PATCH 0023/1696] [YoutubeDL] Improve protocol auto determining
 (closes #11720)

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 5d654f55f..41d9a63ee 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1363,7 +1363,7 @@ class YoutubeDL(object):
                 format['ext'] = determine_ext(format['url']).lower()
             # Automatically determine protocol if missing (useful for format
             # selection purposes)
-            if 'protocol' not in format:
+            if format.get('protocol') is None:
                 format['protocol'] = determine_protocol(format)
             # Add HTTP headers, so that external programs can use them from the
             # json output

From a7acf868a55b3d734bef564e3392020f18c20422 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Jan 2017 10:34:39 +0700
Subject: [PATCH 0024/1696] [yourupload] Fix extraction (closes #11601)

---
 youtube_dl/extractor/yourupload.py | 49 +++++++++++++-----------------
 1 file changed, 21 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/yourupload.py b/youtube_dl/extractor/yourupload.py
index 4ce327845..9fa772838 100644
--- a/youtube_dl/extractor/yourupload.py
+++ b/youtube_dl/extractor/yourupload.py
@@ -2,44 +2,37 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import urljoin
 
 
 class YourUploadIE(InfoExtractor):
-    _VALID_URL = r'''(?x)https?://(?:www\.)?
-        (?:yourupload\.com/watch|
-           embed\.yourupload\.com|
-           embed\.yucache\.net
-        )/(?P<id>[A-Za-z0-9]+)
-        '''
-    _TESTS = [
-        {
-            'url': 'http://yourupload.com/watch/14i14h',
-            'md5': '5e2c63385454c557f97c4c4131a393cd',
-            'info_dict': {
-                'id': '14i14h',
-                'ext': 'mp4',
-                'title': 'BigBuckBunny_320x180.mp4',
-                'thumbnail': r're:^https?://.*\.jpe?g',
-            }
-        },
-        {
-            'url': 'http://embed.yourupload.com/14i14h',
-            'only_matching': True,
-        },
-        {
-            'url': 'http://embed.yucache.net/14i14h?client_file_id=803349',
-            'only_matching': True,
-        },
-    ]
+    _VALID_URL = r'https?://(?:www\.)?(?:yourupload\.com/(?:watch|embed)|embed\.yourupload\.com)/(?P<id>[A-Za-z0-9]+)'
+    _TESTS = [{
+        'url': 'http://yourupload.com/watch/14i14h',
+        'md5': '5e2c63385454c557f97c4c4131a393cd',
+        'info_dict': {
+            'id': '14i14h',
+            'ext': 'mp4',
+            'title': 'BigBuckBunny_320x180.mp4',
+            'thumbnail': r're:^https?://.*\.jpe?g',
+        }
+    }, {
+        'url': 'http://www.yourupload.com/embed/14i14h',
+        'only_matching': True,
+    }, {
+        'url': 'http://embed.yourupload.com/14i14h',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        embed_url = 'http://embed.yucache.net/{0:}'.format(video_id)
+        embed_url = 'http://www.yourupload.com/embed/%s' % video_id
+
         webpage = self._download_webpage(embed_url, video_id)
 
         title = self._og_search_title(webpage)
-        video_url = self._og_search_video_url(webpage)
+        video_url = urljoin(embed_url, self._og_search_video_url(webpage))
         thumbnail = self._og_search_thumbnail(webpage, default=None)
 
         return {

From 8e4988f1a21184839dcd23d7133c250a43c5ea58 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 15 Jan 2017 22:10:57 +0800
Subject: [PATCH 0025/1696] [niconico] Remove codes for downloading anonymously

Apparently Niconico now blocks playing without an account

Closes #11170
---
 youtube_dl/extractor/niconico.py | 27 +++++++--------------------
 1 file changed, 7 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index a104e33f8..7e6c594c8 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -7,7 +7,6 @@ import datetime
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse_urlencode,
     compat_urlparse,
 )
 from ..utils import (
@@ -40,6 +39,7 @@ class NiconicoIE(InfoExtractor):
             'description': '(c) copyright 2008, Blender Foundation / www.bigbuckbunny.org',
             'duration': 33,
         },
+        'skip': 'Requires an account',
     }, {
         # File downloaded with and without credentials are different, so omit
         # the md5 field
@@ -55,6 +55,7 @@ class NiconicoIE(InfoExtractor):
             'timestamp': 1304065916,
             'duration': 209,
         },
+        'skip': 'Requires an account',
     }, {
         # 'video exists but is marked as "deleted"
         # md5 is unstable
@@ -65,9 +66,10 @@ class NiconicoIE(InfoExtractor):
             'description': 'deleted',
             'title': 'ドラえもんエターナル第3話「決戦第3新東京市」＜前編＞',
             'upload_date': '20071224',
-            'timestamp': 1198527840,  # timestamp field has different value if logged in
+            'timestamp': int,  # timestamp field has different value if logged in
             'duration': 304,
         },
+        'skip': 'Requires an account',
     }, {
         'url': 'http://www.nicovideo.jp/watch/so22543406',
         'info_dict': {
@@ -79,7 +81,8 @@ class NiconicoIE(InfoExtractor):
             'upload_date': '20140104',
             'uploader': 'アニメロチャンネル',
             'uploader_id': '312',
-        }
+        },
+        'skip': 'The viewing period of the video you were searching for has expired.',
     }]
 
     _VALID_URL = r'https?://(?:www\.|secure\.)?nicovideo\.jp/watch/(?P<id>(?:[a-z]{2})?[0-9]+)'
@@ -134,23 +137,7 @@ class NiconicoIE(InfoExtractor):
                 'http://flapi.nicovideo.jp/api/getflv/' + video_id + '?as3=1',
                 video_id, 'Downloading flv info')
         else:
-            # Get external player info
-            ext_player_info = self._download_webpage(
-                'http://ext.nicovideo.jp/thumb_watch/' + video_id, video_id)
-            thumb_play_key = self._search_regex(
-                r'\'thumbPlayKey\'\s*:\s*\'(.*?)\'', ext_player_info, 'thumbPlayKey')
-
-            # Get flv info
-            flv_info_data = compat_urllib_parse_urlencode({
-                'k': thumb_play_key,
-                'v': video_id
-            })
-            flv_info_request = sanitized_Request(
-                'http://ext.nicovideo.jp/thumb_watch', flv_info_data,
-                {'Content-Type': 'application/x-www-form-urlencoded'})
-            flv_info_webpage = self._download_webpage(
-                flv_info_request, video_id,
-                note='Downloading flv info', errnote='Unable to download flv info')
+            raise ExtractorError('Niconico videos now require logging in', expected=True)
 
         flv_info = compat_urlparse.parse_qs(flv_info_webpage)
         if 'url' not in flv_info:

From dcae7b3fdc6e6812e78c8dba96d671ccf0ab068e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 15 Jan 2017 22:51:54 +0800
Subject: [PATCH 0026/1696] [niconico] Allow login via cookies

Some codes are borrowed from #7968, which is by @jlhg

Closes #7968
---
 ChangeLog                        |  5 +++++
 youtube_dl/extractor/niconico.py | 18 +++++++-----------
 2 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index dba18d39b..029d13426 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,8 @@
+version <unreleased>
+
+Extractors
++ [niconico] Support login via cookies (#7968)
+
 version 2017.01.14
 
 Core
diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 7e6c594c8..8baac23e4 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -87,8 +87,6 @@ class NiconicoIE(InfoExtractor):
 
     _VALID_URL = r'https?://(?:www\.|secure\.)?nicovideo\.jp/watch/(?P<id>(?:[a-z]{2})?[0-9]+)'
     _NETRC_MACHINE = 'niconico'
-    # Determine whether the downloader used authentication to download video
-    _AUTHENTICATED = False
 
     def _real_initialize(self):
         self._login()
@@ -112,8 +110,6 @@ class NiconicoIE(InfoExtractor):
         if re.search(r'(?i)<h1 class="mb8p4">Log in error</h1>', login_results) is not None:
             self._downloader.report_warning('unable to log in: bad username or password')
             return False
-        # Successful login
-        self._AUTHENTICATED = True
         return True
 
     def _real_extract(self, url):
@@ -131,19 +127,19 @@ class NiconicoIE(InfoExtractor):
             'http://ext.nicovideo.jp/api/getthumbinfo/' + video_id, video_id,
             note='Downloading video info page')
 
-        if self._AUTHENTICATED:
-            # Get flv info
-            flv_info_webpage = self._download_webpage(
-                'http://flapi.nicovideo.jp/api/getflv/' + video_id + '?as3=1',
-                video_id, 'Downloading flv info')
-        else:
-            raise ExtractorError('Niconico videos now require logging in', expected=True)
+        # Get flv info
+        flv_info_webpage = self._download_webpage(
+            'http://flapi.nicovideo.jp/api/getflv/' + video_id + '?as3=1',
+            video_id, 'Downloading flv info')
 
         flv_info = compat_urlparse.parse_qs(flv_info_webpage)
         if 'url' not in flv_info:
             if 'deleted' in flv_info:
                 raise ExtractorError('The video has been deleted.',
                                      expected=True)
+            elif 'closed' in flv_info:
+                raise ExtractorError('Niconico videos now require logging in',
+                                     expected=True)
             else:
                 raise ExtractorError('Unable to find video URL')
 

From 16e2c8f7710bffb462921dbc93adfa6274bd9334 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 16 Jan 2017 00:06:52 +0800
Subject: [PATCH 0027/1696] [brightcove] Recognize another player ID

Closes #11688
---
 ChangeLog                          |  1 +
 youtube_dl/extractor/brightcove.py |  2 +-
 youtube_dl/extractor/generic.py    | 20 ++++++++++++++++++++
 3 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 029d13426..2e0ddd4f6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
++ [brightcove] Recognize another player ID pattern (#11688)
 + [niconico] Support login via cookies (#7968)
 
 version 2017.01.14
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 2e56d1df9..5c6e99da1 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -179,7 +179,7 @@ class BrightcoveLegacyIE(InfoExtractor):
 
         params = {}
 
-        playerID = find_param('playerID')
+        playerID = find_param('playerID') or find_param('playerId')
         if playerID is None:
             raise ExtractorError('Cannot find player ID')
         params['playerID'] = playerID
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ac29ec600..a3ac7d26b 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -422,6 +422,26 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,  # m3u8 download
             },
         },
+        {
+            # Brightcove with alternative playerID key
+            'url': 'http://www.nature.com/nmeth/journal/v9/n7/fig_tab/nmeth.2062_SV1.html',
+            'info_dict': {
+                'id': 'nmeth.2062_SV1',
+                'title': 'Simultaneous multiview imaging of the Drosophila syncytial blastoderm : Quantitative high-speed imaging of entire developing embryos with simultaneous multiview light-sheet microscopy : Nature Methods : Nature Research',
+            },
+            'playlist': [{
+                'info_dict': {
+                    'id': '2228375078001',
+                    'ext': 'mp4',
+                    'title': 'nmeth.2062-sv1',
+                    'description': 'nmeth.2062-sv1',
+                    'timestamp': 1363357591,
+                    'upload_date': '20130315',
+                    'uploader': 'Nature Publishing Group',
+                    'uploader_id': '1964492299001',
+                },
+            }],
+        },
         # ooyala video
         {
             'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',

From 906420cae37ee3c2f48d23c3a4fa0543a66947d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 16 Jan 2017 21:54:47 +0700
Subject: [PATCH 0028/1696] [limelight] Improve and make more robust (closes
 #11737) + Add support for direct http for videos hosted on video.llnw.net *
 Check handmade http URLs

---
 youtube_dl/extractor/limelight.py | 28 ++++++++++++++++++++--------
 1 file changed, 20 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index 905a0e85f..e635f3c4d 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -59,14 +59,26 @@ class LimelightBaseIE(InfoExtractor):
                     format_id = 'rtmp'
                     if stream.get('videoBitRate'):
                         format_id += '-%d' % int_or_none(stream['videoBitRate'])
-                    http_url = 'http://cpl.delvenetworks.com/' + rtmp.group('playpath')[4:]
-                    urls.append(http_url)
-                    http_fmt = fmt.copy()
-                    http_fmt.update({
-                        'url': http_url,
-                        'format_id': format_id.replace('rtmp', 'http'),
-                    })
-                    formats.append(http_fmt)
+                    http_format_id = format_id.replace('rtmp', 'http')
+
+                    CDN_HOSTS = (
+                        ('delvenetworks.com', 'cpl.delvenetworks.com'),
+                        ('video.llnw.net', 's2.content.video.llnw.net'),
+                    )
+                    for cdn_host, http_host in CDN_HOSTS:
+                        if cdn_host not in rtmp.group('host').lower():
+                            continue
+                        http_url = 'http://%s/%s' % (http_host, rtmp.group('playpath')[4:])
+                        urls.append(http_url)
+                        if self._is_valid_url(http_url, video_id, http_format_id):
+                            http_fmt = fmt.copy()
+                            http_fmt.update({
+                                'url': http_url,
+                                'format_id': http_format_id,
+                            })
+                            formats.append(http_fmt)
+                            break
+
                     fmt.update({
                         'url': rtmp.group('url'),
                         'play_path': rtmp.group('playpath'),

From 0ce8c66fb05fefbe51ac1eca8d3ddbd561b38a54 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 16 Jan 2017 22:07:12 +0700
Subject: [PATCH 0029/1696] [options] Include custom conf in final argv (closes
 #11741)

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 0eb4924b6..0b8c1671d 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -867,7 +867,7 @@ def parseOpts(overrideArguments=None):
             if '--ignore-config' not in system_conf:
                 user_conf = _readUserConf()
 
-        argv = system_conf + user_conf + command_line_conf
+        argv = system_conf + user_conf + custom_conf + command_line_conf
         opts, args = parser.parse_args(argv)
         if opts.verbose:
             for conf_label, conf in (

From 79fc8496c6ab423d591f9ed1a41358d038242bbb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 16 Jan 2017 23:31:50 +0700
Subject: [PATCH 0030/1696] [xiami] Improve extraction (closes #11699) * Relax
 _VALID_URLs * Improve track metadata extraction

---
 youtube_dl/extractor/xiami.py | 53 +++++++++++++++++++++++++++--------
 1 file changed, 41 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/xiami.py b/youtube_dl/extractor/xiami.py
index 86abef257..d017e03de 100644
--- a/youtube_dl/extractor/xiami.py
+++ b/youtube_dl/extractor/xiami.py
@@ -16,7 +16,9 @@ class XiamiBaseIE(InfoExtractor):
         return webpage
 
     def _extract_track(self, track, track_id=None):
-        title = track['title']
+        track_name = track.get('songName') or track.get('name') or track['subName']
+        artist = track.get('artist') or track.get('artist_name') or track.get('singers')
+        title = '%s - %s' % (artist, track_name) if artist else track_name
         track_url = self._decrypt(track['location'])
 
         subtitles = {}
@@ -31,9 +33,10 @@ class XiamiBaseIE(InfoExtractor):
             'thumbnail': track.get('pic') or track.get('album_pic'),
             'duration': int_or_none(track.get('length')),
             'creator': track.get('artist', '').split(';')[0],
-            'track': title,
-            'album': track.get('album_name'),
-            'artist': track.get('artist'),
+            'track': track_name,
+            'track_number': int_or_none(track.get('track')),
+            'album': track.get('album_name') or track.get('title'),
+            'artist': artist,
             'subtitles': subtitles,
         }
 
@@ -68,14 +71,14 @@ class XiamiBaseIE(InfoExtractor):
 class XiamiSongIE(XiamiBaseIE):
     IE_NAME = 'xiami:song'
     IE_DESC = '虾米音乐'
-    _VALID_URL = r'https?://(?:www\.)?xiami\.com/song/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?xiami\.com/song/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://www.xiami.com/song/1775610518',
         'md5': '521dd6bea40fd5c9c69f913c232cb57e',
         'info_dict': {
             'id': '1775610518',
             'ext': 'mp3',
-            'title': 'Woman',
+            'title': 'HONNE - Woman',
             'thumbnail': r're:http://img\.xiami\.net/images/album/.*\.jpg',
             'duration': 265,
             'creator': 'HONNE',
@@ -95,7 +98,7 @@ class XiamiSongIE(XiamiBaseIE):
         'info_dict': {
             'id': '1775256504',
             'ext': 'mp3',
-            'title': '悟空',
+            'title': '戴荃 - 悟空',
             'thumbnail': r're:http://img\.xiami\.net/images/album/.*\.jpg',
             'duration': 200,
             'creator': '戴荃',
@@ -109,6 +112,26 @@ class XiamiSongIE(XiamiBaseIE):
             },
         },
         'skip': 'Georestricted',
+    }, {
+        'url': 'http://www.xiami.com/song/1775953850',
+        'info_dict': {
+            'id': '1775953850',
+            'ext': 'mp3',
+            'title': 'До Скону - Чума Пожирает Землю',
+            'thumbnail': r're:http://img\.xiami\.net/images/album/.*\.jpg',
+            'duration': 683,
+            'creator': 'До Скону',
+            'track': 'Чума Пожирает Землю',
+            'track_number': 7,
+            'album': 'Ад',
+            'artist': 'До Скону',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://www.xiami.com/song/xLHGwgd07a1',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -124,7 +147,7 @@ class XiamiPlaylistBaseIE(XiamiBaseIE):
 class XiamiAlbumIE(XiamiPlaylistBaseIE):
     IE_NAME = 'xiami:album'
     IE_DESC = '虾米音乐 - 专辑'
-    _VALID_URL = r'https?://(?:www\.)?xiami\.com/album/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?xiami\.com/album/(?P<id>[^/?#&]+)'
     _TYPE = '1'
     _TESTS = [{
         'url': 'http://www.xiami.com/album/2100300444',
@@ -136,28 +159,34 @@ class XiamiAlbumIE(XiamiPlaylistBaseIE):
     }, {
         'url': 'http://www.xiami.com/album/512288?spm=a1z1s.6843761.1110925389.6.hhE9p9',
         'only_matching': True,
+    }, {
+        'url': 'http://www.xiami.com/album/URVDji2a506',
+        'only_matching': True,
     }]
 
 
 class XiamiArtistIE(XiamiPlaylistBaseIE):
     IE_NAME = 'xiami:artist'
     IE_DESC = '虾米音乐 - 歌手'
-    _VALID_URL = r'https?://(?:www\.)?xiami\.com/artist/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?xiami\.com/artist/(?P<id>[^/?#&]+)'
     _TYPE = '2'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.xiami.com/artist/2132?spm=0.0.0.0.dKaScp',
         'info_dict': {
             'id': '2132',
         },
         'playlist_count': 20,
         'skip': 'Georestricted',
-    }
+    }, {
+        'url': 'http://www.xiami.com/artist/bC5Tk2K6eb99',
+        'only_matching': True,
+    }]
 
 
 class XiamiCollectionIE(XiamiPlaylistBaseIE):
     IE_NAME = 'xiami:collection'
     IE_DESC = '虾米音乐 - 精选集'
-    _VALID_URL = r'https?://(?:www\.)?xiami\.com/collect/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?xiami\.com/collect/(?P<id>[^/?#&]+)'
     _TYPE = '3'
     _TEST = {
         'url': 'http://www.xiami.com/collect/156527391?spm=a1z1s.2943601.6856193.12.4jpBnr',

From ddd53c392e0b3d3d2c62ba28117a9b07702c5bd8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 16 Jan 2017 23:42:04 +0700
Subject: [PATCH 0031/1696] [ChangeLog] Actualize

---
 ChangeLog | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 2e0ddd4f6..ee59e120c 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,8 +1,22 @@
 version <unreleased>
 
+Core
+* [options] Apply custom config to final composite configuration (#11741)
+* [YoutubeDL] Improve protocol auto determining (#11720)
+
 Extractors
+* [xiami] Relax URL regular expressions
+* [xiami] Improve track metadata extraction (#11699)
++ [limelight] Check hand-make direct HTTP links
++ [limelight] Add support for direct HTTP links at video.llnw.net (#11737)
 + [brightcove] Recognize another player ID pattern (#11688)
 + [niconico] Support login via cookies (#7968)
+* [yourupload] Fix extraction (#11601)
++ [beam:live] Add support for beam.pro live streams (#10702, #11596)
+* [vevo] Improve geo restriction detection
++ [dramafever] Add support for URLs with language code (#11714)
+* [cbc] Improve playlist support (#11704)
+
 
 version 2017.01.14
 

From c1c2fe2045911c310fd5d2eda7bbb53ad581d250 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 16 Jan 2017 23:44:04 +0700
Subject: [PATCH 0032/1696] release 2017.01.16

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 1 +
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index a7bf2b90c..c04f6246a 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.01.14*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.01.14**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.01.16*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.01.16**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.01.14
+[debug] youtube-dl version 2017.01.16
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index ee59e120c..f6d73f982 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.01.16
 
 Core
 * [options] Apply custom config to final composite configuration (#11741)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 0f6c4ec0c..a3c76d5db 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -86,6 +86,7 @@
  - **bbc.co.uk:article**: BBC articles
  - **bbc.co.uk:iplayer:playlist**
  - **bbc.co.uk:playlist**
+ - **Beam:live**
  - **Beatport**
  - **Beeg**
  - **BehindKink**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 17c6f9eb2..c20718dd6 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.01.14'
+__version__ = '2017.01.16'

From c0bd51c090d617811f5e405294dce06f5871d717 Mon Sep 17 00:00:00 2001
From: Kagami Hiiragi <kagami@genshiken.org>
Date: Mon, 16 Jan 2017 22:19:52 +0300
Subject: [PATCH 0033/1696] [naver] Support tv.naver.com links

---
 youtube_dl/extractor/naver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/naver.py b/youtube_dl/extractor/naver.py
index 055070ff5..aba0a9a70 100644
--- a/youtube_dl/extractor/naver.py
+++ b/youtube_dl/extractor/naver.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class NaverIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:m\.)?tvcast\.naver\.com/v/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:m\.)?tv(?:cast)?\.naver\.com/v/(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://tvcast.naver.com/v/81652',

From 8a5f0a6357746d293f7330e40a3cf5823b1b626d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Jan 2017 21:19:57 +0700
Subject: [PATCH 0034/1696] [naver] Update tests for #11743

---
 youtube_dl/extractor/naver.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/naver.py b/youtube_dl/extractor/naver.py
index aba0a9a70..e8131333f 100644
--- a/youtube_dl/extractor/naver.py
+++ b/youtube_dl/extractor/naver.py
@@ -15,7 +15,7 @@ class NaverIE(InfoExtractor):
     _VALID_URL = r'https?://(?:m\.)?tv(?:cast)?\.naver\.com/v/(?P<id>\d+)'
 
     _TESTS = [{
-        'url': 'http://tvcast.naver.com/v/81652',
+        'url': 'http://tv.naver.com/v/81652',
         'info_dict': {
             'id': '81652',
             'ext': 'mp4',
@@ -24,7 +24,7 @@ class NaverIE(InfoExtractor):
             'upload_date': '20130903',
         },
     }, {
-        'url': 'http://tvcast.naver.com/v/395837',
+        'url': 'http://tv.naver.com/v/395837',
         'md5': '638ed4c12012c458fefcddfd01f173cd',
         'info_dict': {
             'id': '395837',
@@ -34,6 +34,9 @@ class NaverIE(InfoExtractor):
             'upload_date': '20150519',
         },
         'skip': 'Georestricted',
+    }, {
+        'url': 'http://tvcast.naver.com/v/81652',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 136078966b2047b21e9784060cebdc893c643ee9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Jan 2017 23:14:07 +0700
Subject: [PATCH 0035/1696] [imdb] Extend _VALID_URL (closes #11744)

---
 youtube_dl/extractor/imdb.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py
index f0fc8d49a..f95c00c73 100644
--- a/youtube_dl/extractor/imdb.py
+++ b/youtube_dl/extractor/imdb.py
@@ -13,7 +13,7 @@ from ..utils import (
 class ImdbIE(InfoExtractor):
     IE_NAME = 'imdb'
     IE_DESC = 'Internet Movie Database trailers'
-    _VALID_URL = r'https?://(?:www|m)\.imdb\.com/(?:video/[^/]+/|title/tt\d+.*?#lb-)vi(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www|m)\.imdb\.com/(?:video/[^/]+/|title/tt\d+.*?#lb-|videoplayer/)vi(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.imdb.com/video/imdb/vi2524815897',
@@ -32,6 +32,9 @@ class ImdbIE(InfoExtractor):
     }, {
         'url': 'http://www.imdb.com/title/tt1667889/#lb-vi2524815897',
         'only_matching': True,
+    }, {
+        'url': 'http://www.imdb.com/videoplayer/vi1562949145',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 4e44598547b02d42aa628506245c40c3d633814e Mon Sep 17 00:00:00 2001
From: Alex Seiler <seileralex@gmail.com>
Date: Mon, 9 Jan 2017 21:19:55 +0100
Subject: [PATCH 0036/1696] [20min] Fix extraction

---
 youtube_dl/extractor/twentymin.py | 37 ++++++++++++++++++++++++++-----
 1 file changed, 31 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/twentymin.py b/youtube_dl/extractor/twentymin.py
index b721ecb0a..68d5a0cb5 100644
--- a/youtube_dl/extractor/twentymin.py
+++ b/youtube_dl/extractor/twentymin.py
@@ -13,10 +13,10 @@ class TwentyMinutenIE(InfoExtractor):
     _TESTS = [{
         # regular video
         'url': 'http://www.20min.ch/videotv/?vid=469148&cid=2',
-        'md5': 'b52d6bc6ea6398e6a38f12cfd418149c',
+        'md5': 'e7264320db31eed8c38364150c12496e',
         'info_dict': {
             'id': '469148',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': '85 000 Franken für 15 perfekte Minuten',
             'description': 'Was die Besucher vom Silvesterzauber erwarten können. (Video: Alice Grosjean/Murat Temel)',
             'thumbnail': 'http://thumbnails.20min-tv.ch/server063/469148/frame-72-469148.jpg'
@@ -34,17 +34,29 @@ class TwentyMinutenIE(InfoExtractor):
             'thumbnail': 'http://www.20min.ch/images/content/2/2/0/22050469/10/teaserbreit.jpg'
         },
         'skip': '"This video is no longer available" is shown both on the web page and in the downloaded file.',
+    }, {
+        # news article with video
+        'url': 'http://www.20min.ch/schweiz/news/story/So-kommen-Sie-bei-Eis-und-Schnee-sicher-an-27032552',
+        'md5': '372917ba85ed969e176d287ae54b2f94',
+        'info_dict': {
+            'id': '523629',
+            'display_id': 'So-kommen-Sie-bei-Eis-und-Schnee-sicher-an-27032552',
+            'ext': 'mp4',
+            'title': 'So kommen Sie bei Eis und Schnee sicher an',
+            'description': 'Schneegestöber und Glatteis führten in den letzten Tagen zu zahlreichen Strassenunfällen. Ein Experte erklärt, worauf man nun beim Autofahren achten muss.',
+            'thumbnail': 'http://www.20min.ch/images/content/2/7/0/27032552/83/teaserbreit.jpg',
+        }
     }, {
         # YouTube embed
         'url': 'http://www.20min.ch/ro/sports/football/story/Il-marque-une-bicyclette-de-plus-de-30-metres--21115184',
-        'md5': 'cec64d59aa01c0ed9dbba9cf639dd82f',
+        'md5': 'e7e237fd98da2a3cc1422ce683df234d',
         'info_dict': {
             'id': 'ivM7A7SpDOs',
             'ext': 'mp4',
             'title': 'GOLAZO DE CHILENA DE JAVI GÓMEZ, FINALISTA AL BALÓN DE CLM 2016',
             'description': 'md5:903c92fbf2b2f66c09de514bc25e9f5a',
             'upload_date': '20160424',
-            'uploader': 'RTVCM Castilla-La Mancha',
+            'uploader': 'CMM Castilla-La Mancha Media',
             'uploader_id': 'RTVCM',
         },
         'add_ie': ['Youtube'],
@@ -77,18 +89,31 @@ class TwentyMinutenIE(InfoExtractor):
                 r'^20 [Mm]inuten.*? -', '', self._og_search_title(webpage)), ' - News')
 
         if not video_id:
+            params = self._html_search_regex(
+                r'<iframe[^>]+src="(?:https?:)?//www\.20min\.ch/videoplayer/videoplayer\.html\?params=(.+?[^"])"',
+                webpage, '20min embed URL')
             video_id = self._search_regex(
-                r'"file\d?"\s*,\s*\"(\d+)', webpage, 'video id')
+                r'.*videoId@(\d+)',
+                params, 'Video Id')
 
         description = self._html_search_meta(
             'description', webpage, 'description')
         thumbnail = self._og_search_thumbnail(webpage)
 
+        formats = []
+        format_preferences = [('sd', ''), ('hd', 'h')]
+        for format_id, url_extension in format_preferences:
+            format_url = 'http://podcast.20min-tv.ch/podcast/20min/%s%s.mp4' % (video_id, url_extension)
+            formats.append({
+                'format_id': format_id,
+                'url': format_url,
+            })
+
         return {
             'id': video_id,
             'display_id': display_id,
-            'url': 'http://speed.20min-tv.ch/%sm.flv' % video_id,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
+            'formats': formats,
         }

From 538b17a09c6546d58babc5eb4a3abc08dcff2d89 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 18 Jan 2017 22:05:11 +0700
Subject: [PATCH 0037/1696] [20min] Improve

---
 youtube_dl/extractor/twentymin.py | 122 ++++++++++++------------------
 1 file changed, 47 insertions(+), 75 deletions(-)

diff --git a/youtube_dl/extractor/twentymin.py b/youtube_dl/extractor/twentymin.py
index 68d5a0cb5..4fd1aa4bf 100644
--- a/youtube_dl/extractor/twentymin.py
+++ b/youtube_dl/extractor/twentymin.py
@@ -4,116 +4,88 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import remove_end
+from ..utils import (
+    int_or_none,
+    try_get,
+)
 
 
 class TwentyMinutenIE(InfoExtractor):
     IE_NAME = '20min'
-    _VALID_URL = r'https?://(?:www\.)?20min\.ch/(?:videotv/*\?.*\bvid=(?P<id>\d+)|(?:[^/]+/)*(?P<display_id>[^/#?]+))'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:www\.)?20min\.ch/
+                        (?:
+                            videotv/*\?.*?\bvid=|
+                            videoplayer/videoplayer\.html\?.*?\bvideoId@
+                        )
+                        (?P<id>\d+)
+                    '''
     _TESTS = [{
-        # regular video
         'url': 'http://www.20min.ch/videotv/?vid=469148&cid=2',
         'md5': 'e7264320db31eed8c38364150c12496e',
         'info_dict': {
             'id': '469148',
             'ext': 'mp4',
             'title': '85 000 Franken für 15 perfekte Minuten',
-            'description': 'Was die Besucher vom Silvesterzauber erwarten können. (Video: Alice Grosjean/Murat Temel)',
-            'thumbnail': 'http://thumbnails.20min-tv.ch/server063/469148/frame-72-469148.jpg'
-        }
-    }, {
-        # news article with video
-        'url': 'http://www.20min.ch/schweiz/news/story/-Wir-muessen-mutig-nach-vorne-schauen--22050469',
-        'md5': 'cd4cbb99b94130cff423e967cd275e5e',
-        'info_dict': {
-            'id': '469408',
-            'display_id': '-Wir-muessen-mutig-nach-vorne-schauen--22050469',
-            'ext': 'flv',
-            'title': '«Wir müssen mutig nach vorne schauen»',
-            'description': 'Kein Land sei innovativer als die Schweiz, sagte Johann Schneider-Ammann in seiner Neujahrsansprache. Das Land müsse aber seine Hausaufgaben machen.',
-            'thumbnail': 'http://www.20min.ch/images/content/2/2/0/22050469/10/teaserbreit.jpg'
+            'thumbnail': r're:https?://.*\.jpg$',
         },
-        'skip': '"This video is no longer available" is shown both on the web page and in the downloaded file.',
     }, {
-        # news article with video
-        'url': 'http://www.20min.ch/schweiz/news/story/So-kommen-Sie-bei-Eis-und-Schnee-sicher-an-27032552',
-        'md5': '372917ba85ed969e176d287ae54b2f94',
+        'url': 'http://www.20min.ch/videoplayer/videoplayer.html?params=client@twentyDE|videoId@523629',
         'info_dict': {
             'id': '523629',
-            'display_id': 'So-kommen-Sie-bei-Eis-und-Schnee-sicher-an-27032552',
             'ext': 'mp4',
             'title': 'So kommen Sie bei Eis und Schnee sicher an',
-            'description': 'Schneegestöber und Glatteis führten in den letzten Tagen zu zahlreichen Strassenunfällen. Ein Experte erklärt, worauf man nun beim Autofahren achten muss.',
-            'thumbnail': 'http://www.20min.ch/images/content/2/7/0/27032552/83/teaserbreit.jpg',
-        }
-    }, {
-        # YouTube embed
-        'url': 'http://www.20min.ch/ro/sports/football/story/Il-marque-une-bicyclette-de-plus-de-30-metres--21115184',
-        'md5': 'e7e237fd98da2a3cc1422ce683df234d',
-        'info_dict': {
-            'id': 'ivM7A7SpDOs',
-            'ext': 'mp4',
-            'title': 'GOLAZO DE CHILENA DE JAVI GÓMEZ, FINALISTA AL BALÓN DE CLM 2016',
-            'description': 'md5:903c92fbf2b2f66c09de514bc25e9f5a',
-            'upload_date': '20160424',
-            'uploader': 'CMM Castilla-La Mancha Media',
-            'uploader_id': 'RTVCM',
+            'description': 'md5:117c212f64b25e3d95747e5276863f7d',
+            'thumbnail': r're:https?://.*\.jpg$',
+        },
+        'params': {
+            'skip_download': True,
         },
-        'add_ie': ['Youtube'],
     }, {
         'url': 'http://www.20min.ch/videotv/?cid=44&vid=468738',
         'only_matching': True,
-    }, {
-        'url': 'http://www.20min.ch/ro/sortir/cinema/story/Grandir-au-bahut--c-est-dur-18927411',
-        'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_urls(webpage):
+        return [m.group('url') for m in re.finditer(
+            r'<iframe[^>]+src=(["\'])(?P<url>(?:https?://)?(?:www\.)?20min\.ch/videoplayer/videoplayer.html\?.*?\bvideoId@\d+.*?)\1',
+            webpage)]
+
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        display_id = mobj.group('display_id') or video_id
+        video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, display_id)
+        video = self._download_json(
+            'http://api.20min.ch/video/%s/show' % video_id,
+            video_id)['content']
 
-        youtube_url = self._html_search_regex(
-            r'<iframe[^>]+src="((?:https?:)?//www\.youtube\.com/embed/[^"]+)"',
-            webpage, 'YouTube embed URL', default=None)
-        if youtube_url is not None:
-            return self.url_result(youtube_url, 'Youtube')
+        title = video['title']
 
-        title = self._html_search_regex(
-            r'<h1>.*?<span>(.+?)</span></h1>',
-            webpage, 'title', default=None)
-        if not title:
-            title = remove_end(re.sub(
-                r'^20 [Mm]inuten.*? -', '', self._og_search_title(webpage)), ' - News')
+        formats = [{
+            'format_id': format_id,
+            'url': 'http://podcast.20min-tv.ch/podcast/20min/%s%s.mp4' % (video_id, p),
+            'quality': quality,
+        } for quality, (format_id, p) in enumerate([('sd', ''), ('hd', 'h')])]
+        self._sort_formats(formats)
 
-        if not video_id:
-            params = self._html_search_regex(
-                r'<iframe[^>]+src="(?:https?:)?//www\.20min\.ch/videoplayer/videoplayer\.html\?params=(.+?[^"])"',
-                webpage, '20min embed URL')
-            video_id = self._search_regex(
-                r'.*videoId@(\d+)',
-                params, 'Video Id')
+        description = video.get('lead')
+        thumbnail = video.get('thumbnail')
 
-        description = self._html_search_meta(
-            'description', webpage, 'description')
-        thumbnail = self._og_search_thumbnail(webpage)
+        def extract_count(kind):
+            return try_get(
+                video,
+                lambda x: int_or_none(x['communityobject']['thumbs_%s' % kind]))
 
-        formats = []
-        format_preferences = [('sd', ''), ('hd', 'h')]
-        for format_id, url_extension in format_preferences:
-            format_url = 'http://podcast.20min-tv.ch/podcast/20min/%s%s.mp4' % (video_id, url_extension)
-            formats.append({
-                'format_id': format_id,
-                'url': format_url,
-            })
+        like_count = extract_count('up')
+        dislike_count = extract_count('down')
 
         return {
             'id': video_id,
-            'display_id': display_id,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
+            'like_count': like_count,
+            'dislike_count': dislike_count,
             'formats': formats,
         }

From b687c85eab942553e925256ad10de693227ba553 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 18 Jan 2017 22:08:31 +0700
Subject: [PATCH 0038/1696] [extractor/generic] Add support for 20 minuten
 embeds (closes #11683, closes #11751)

---
 youtube_dl/extractor/generic.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a3ac7d26b..154545df7 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -78,6 +78,7 @@ from .vbox7 import Vbox7IE
 from .dbtv import DBTVIE
 from .piksel import PikselIE
 from .videa import VideaIE
+from .twentymin import TwentyMinutenIE
 
 
 class GenericIE(InfoExtractor):
@@ -1468,6 +1469,20 @@ class GenericIE(InfoExtractor):
             },
             'playlist_mincount': 2,
         },
+        {
+            # 20 minuten embed
+            'url': 'http://www.20min.ch/schweiz/news/story/So-kommen-Sie-bei-Eis-und-Schnee-sicher-an-27032552',
+            'info_dict': {
+                'id': '523629',
+                'ext': 'mp4',
+                'title': 'So kommen Sie bei Eis und Schnee sicher an',
+                'description': 'md5:117c212f64b25e3d95747e5276863f7d',
+            },
+            'params': {
+                'skip_download': True,
+            },
+            'add_ie': [TwentyMinutenIE.ie_key()],
+        }
         # {
         #     # TODO: find another test
         #     # http://schema.org/VideoObject
@@ -2421,6 +2436,12 @@ class GenericIE(InfoExtractor):
         if videa_urls:
             return _playlist_from_matches(videa_urls, ie=VideaIE.ie_key())
 
+        # Look for 20 minuten embeds
+        twentymin_urls = TwentyMinutenIE._extract_urls(webpage)
+        if twentymin_urls:
+            return _playlist_from_matches(
+                twentymin_urls, ie=TwentyMinutenIE.ie_key())
+
         # Looking for http://schema.org/VideoObject
         json_ld = self._search_json_ld(
             webpage, video_id, default={}, expected_type='VideoObject')

From aaf2b7c57a3d2dc9ba12f1aa401cba088e114916 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 18 Jan 2017 22:20:11 +0700
Subject: [PATCH 0039/1696] [canalplus] Add fallback for video id (closes
 #11764)

---
 youtube_dl/extractor/canalplus.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 10cf165bc..b3f76a7b1 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -107,7 +107,7 @@ class CanalplusIE(InfoExtractor):
             [r'<canal:player[^>]+?videoId=(["\'])(?P<id>\d+)',
              r'id=["\']canal_video_player(?P<id>\d+)',
              r'data-video=["\'](?P<id>\d+)'],
-            webpage, 'video id', group='id')
+            webpage, 'video id', default=mobj.group('vid'), group='id')
 
         info_url = self._VIDEO_INFO_TEMPLATE % (site_id, video_id)
         video_data = self._download_json(info_url, video_id, 'Downloading video JSON')

From baa3e1845b26d9756642325bbb0d58e22025b2ec Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 18 Jan 2017 17:00:15 +0100
Subject: [PATCH 0040/1696] [bilibili] fix extraction(closes #11077)

---
 youtube_dl/extractor/bilibili.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 5051934ef..85ea5e6ee 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -34,8 +34,8 @@ class BiliBiliIE(InfoExtractor):
         },
     }
 
-    _APP_KEY = '6f90a59ac58a4123'
-    _BILIBILI_KEY = '0bfd84cc3940035173f35e6777508326'
+    _APP_KEY = '84956560bc028eb7'
+    _BILIBILI_KEY = '94aba54af9065f71de72f5508f1cd42e'
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 460f61fac42592eb273b7d58efc314cc83687b8b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 18 Jan 2017 23:06:46 +0700
Subject: [PATCH 0041/1696] [ChangeLog] Actualize

---
 ChangeLog | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index f6d73f982..994895edc 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,13 @@
+version <unreleased>
+
+Extractors
+* [bilibili] Fix extraction (#11077)
++ [canalplus] Add fallback for video id (#11764)
+* [20min] Fix extraction (#11683, #11751)
+* [imdb] Extend URL regular expression (#11744)
++ [naver] Add support for tv.naver.com links (#11743)
+
+
 version 2017.01.16
 
 Core

From 1560baacc677c43c1007acfc89b8190f81a59684 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 18 Jan 2017 23:10:00 +0700
Subject: [PATCH 0042/1696] release 2017.01.18

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index c04f6246a..38cb13a33 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.01.16*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.01.16**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.01.18*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.01.18**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.01.16
+[debug] youtube-dl version 2017.01.18
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 994895edc..5aa4e3c6b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.01.18
 
 Extractors
 * [bilibili] Fix extraction (#11077)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index c20718dd6..669f60f65 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.01.16'
+__version__ = '2017.01.18'

From f1e70fc2ff6f1536873ed73ffc9bff63653fd5ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 18 Jan 2017 23:34:11 +0700
Subject: [PATCH 0043/1696] [mtv] Relax triforce feed regex (closes #11766)

---
 youtube_dl/extractor/mtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 00a980c7d..e48ea2481 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -211,7 +211,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
 
     def _extract_triforce_mgid(self, webpage, data_zone=None, video_id=None):
         triforce_feed = self._parse_json(self._search_regex(
-            r'triforceManifestFeed\s*=\s*(\{.+?\});\n', webpage,
+            r'triforceManifestFeed\s*=\s*({.+?})\s*;\s*\n', webpage,
             'triforce feed', default='{}'), video_id, fatal=False)
 
         data_zone = self._search_regex(

From eb3f008c9e686f38c50511004d5c9a51b2e8cdd2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Jan 2017 04:49:31 +0700
Subject: [PATCH 0044/1696] [uol] Fix extraction (closes #11770)

---
 youtube_dl/extractor/uol.py | 27 +++++++++++++++++++++------
 1 file changed, 21 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/uol.py b/youtube_dl/extractor/uol.py
index c27c64387..e67083004 100644
--- a/youtube_dl/extractor/uol.py
+++ b/youtube_dl/extractor/uol.py
@@ -84,12 +84,27 @@ class UOLIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        if not video_id.isdigit():
-            embed_page = self._download_webpage('https://jsuol.com.br/c/tv/uol/embed/?params=[embed,%s]' % video_id, video_id)
-            video_id = self._search_regex(r'mediaId=(\d+)', embed_page, 'media id')
+        media_id = None
+
+        if video_id.isdigit():
+            media_id = video_id
+
+        if not media_id:
+            embed_page = self._download_webpage(
+                'https://jsuol.com.br/c/tv/uol/embed/?params=[embed,%s]' % video_id,
+                video_id, 'Downloading embed page', fatal=False)
+            if embed_page:
+                media_id = self._search_regex(
+                    (r'uol\.com\.br/(\d+)', r'mediaId=(\d+)'),
+                    embed_page, 'media id', default=None)
+
+        if not media_id:
+            webpage = self._download_webpage(url, video_id)
+            media_id = self._search_regex(r'mediaId=(\d+)', webpage, 'media id')
+
         video_data = self._download_json(
-            'http://mais.uol.com.br/apiuol/v3/player/getMedia/%s.json' % video_id,
-            video_id)['item']
+            'http://mais.uol.com.br/apiuol/v3/player/getMedia/%s.json' % media_id,
+            media_id)['item']
         title = video_data['title']
 
         query = {
@@ -118,7 +133,7 @@ class UOLIE(InfoExtractor):
             tags.append(tag_description)
 
         return {
-            'id': video_id,
+            'id': media_id,
             'title': title,
             'description': clean_html(video_data.get('desMedia')),
             'thumbnail': video_data.get('thumbnail'),

From cccd70a2752ad079ed560e42ff085adcabebaac2 Mon Sep 17 00:00:00 2001
From: james mike dupont <jamesmikedupont@gmail.com>
Date: Thu, 19 Jan 2017 04:18:13 -0500
Subject: [PATCH 0045/1696] untie

---
 youtube_dl/extractor/flipagram.py | 2 +-
 youtube_dl/extractor/vimeo.py     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/flipagram.py b/youtube_dl/extractor/flipagram.py
index 1902a2393..b7be40f1b 100644
--- a/youtube_dl/extractor/flipagram.py
+++ b/youtube_dl/extractor/flipagram.py
@@ -81,7 +81,7 @@ class FlipagramIE(InfoExtractor):
             'filesize': int_or_none(cover.get('size')),
         } for cover in flipagram.get('covers', []) if cover.get('url')]
 
-        # Note that this only retrieves comments that are initally loaded.
+        # Note that this only retrieves comments that are initially loaded.
         # For videos with large amounts of comments, most won't be retrieved.
         comments = []
         for comment in video_data.get('comments', {}).get(video_id, {}).get('items', []):
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 2e98b0e6f..add753635 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -338,7 +338,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             'expected_warnings': ['Unable to download JSON metadata'],
         },
         {
-            # redirects to ondemand extractor and should be passed throught it
+            # redirects to ondemand extractor and should be passed through it
             # for successful extraction
             'url': 'https://vimeo.com/73445910',
             'info_dict': {

From 1fe84be0f3b36822af804db6cf7c06a1ac5ac688 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Jan 2017 00:47:04 +0700
Subject: [PATCH 0046/1696] [1tv] Add support for hls (closes #11786)

---
 youtube_dl/extractor/firsttv.py | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/firsttv.py b/youtube_dl/extractor/firsttv.py
index c6fb67057..081c71842 100644
--- a/youtube_dl/extractor/firsttv.py
+++ b/youtube_dl/extractor/firsttv.py
@@ -86,18 +86,43 @@ class FirstTVIE(InfoExtractor):
             title = item['title']
             quality = qualities(QUALITIES)
             formats = []
+            path = None
             for f in item.get('mbr', []):
                 src = f.get('src')
                 if not src or not isinstance(src, compat_str):
                     continue
                 tbr = int_or_none(self._search_regex(
                     r'_(\d{3,})\.mp4', src, 'tbr', default=None))
+                if not path:
+                    path = self._search_regex(
+                        r'//[^/]+/(.+?)_\d+\.mp4', src,
+                        'm3u8 path', default=None)
                 formats.append({
                     'url': src,
                     'format_id': f.get('name'),
                     'tbr': tbr,
-                    'quality': quality(f.get('name')),
+                    'source_preference': quality(f.get('name')),
                 })
+            # m3u8 URL format is reverse engineered from [1] (search for
+            # master.m3u8). dashEdges (that is currently balancer-vod.1tv.ru)
+            # is taken from [2].
+            # 1. http://static.1tv.ru/player/eump1tv-current/eump-1tv.all.min.js?rnd=9097422834:formatted
+            # 2. http://static.1tv.ru/player/eump1tv-config/config-main.js?rnd=9097422834
+            if not path and len(formats) == 1:
+                path = self._search_regex(
+                    r'//[^/]+/(.+?$)', formats[0]['url'],
+                    'm3u8 path', default=None)
+            if path:
+                if len(formats) == 1:
+                    m3u8_path = ','
+                else:
+                    tbrs = [compat_str(t) for t in sorted(f['tbr'] for f in formats)]
+                    m3u8_path = '_,%s,%s' % (','.join(tbrs), '.mp4')
+                formats.extend(self._extract_m3u8_formats(
+                    'http://balancer-vod.1tv.ru/%s%s.urlset/master.m3u8'
+                    % (path, m3u8_path),
+                    display_id, 'mp4',
+                    entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
             self._sort_formats(formats)
 
             thumbnail = item.get('poster') or self._og_search_thumbnail(webpage)

From d77ac737900eede5e1508b9822e71c8595fe0879 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 20 Jan 2017 21:59:24 +0800
Subject: [PATCH 0047/1696] [ustream] Add UstreamIE._extract_url()

Ref: #11547
---
 youtube_dl/extractor/generic.py | 8 ++++----
 youtube_dl/extractor/ustream.py | 7 +++++++
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 154545df7..a7c104845 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -79,6 +79,7 @@ from .dbtv import DBTVIE
 from .piksel import PikselIE
 from .videa import VideaIE
 from .twentymin import TwentyMinutenIE
+from .ustream import UstreamIE
 
 
 class GenericIE(InfoExtractor):
@@ -2112,10 +2113,9 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'), 'TED')
 
         # Look for embedded Ustream videos
-        mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>http://www\.ustream\.tv/embed/.+?)\1', webpage)
-        if mobj is not None:
-            return self.url_result(mobj.group('url'), 'Ustream')
+        ustream_url = UstreamIE._extract_url(webpage)
+        if ustream_url:
+            return self.url_result(ustream_url, UstreamIE.ie_key())
 
         # Look for embedded arte.tv player
         mobj = re.search(
diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index 0c06bf36b..5737d4d16 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -69,6 +69,13 @@ class UstreamIE(InfoExtractor):
         },
     }]
 
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>http://www\.ustream\.tv/embed/.+?)\1', webpage)
+        if mobj is not None:
+            return mobj.group('url')
+
     def _get_stream_info(self, url, video_id, app_id_ver, extra_note=None):
         def num_to_hex(n):
             return hex(n)[2:]

From 4447fb23320b9214ab3188717794d00b18887617 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 20 Jan 2017 22:11:43 +0800
Subject: [PATCH 0048/1696] [cspan] Support Ustream embedded videos

Closes #11547
---
 ChangeLog                     |  6 ++++++
 youtube_dl/extractor/cspan.py | 19 +++++++++++++++++++
 2 files changed, 25 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 5aa4e3c6b..217971ec6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
++ [cspan] Support Ustream embedded videos (#11547)
+
+
 version 2017.01.18
 
 Extractors
diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index 7e5d4f227..92a827a4b 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -12,6 +12,7 @@ from ..utils import (
     ExtractorError,
 )
 from .senateisvp import SenateISVPIE
+from .ustream import UstreamIE
 
 
 class CSpanIE(InfoExtractor):
@@ -57,12 +58,30 @@ class CSpanIE(InfoExtractor):
         'params': {
             'skip_download': True,  # m3u8 downloads
         }
+    }, {
+        # Ustream embedded video
+        'url': 'https://www.c-span.org/video/?114917-1/armed-services',
+        'info_dict': {
+            'id': '58428542',
+            'ext': 'flv',
+            'title': 'USHR07 Armed Services Committee',
+            'description': 'hsas00-2118-20150204-1000et-07\n\n\nUSHR07 Armed Services Committee',
+            'timestamp': 1423060374,
+            'upload_date': '20150204',
+            'uploader': 'HouseCommittee',
+            'uploader_id': '12987475',
+        },
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         video_type = None
         webpage = self._download_webpage(url, video_id)
+
+        ustream_url = UstreamIE._extract_url(webpage)
+        if ustream_url:
+            return self.url_result(ustream_url, UstreamIE.ie_key())
+
         # We first look for clipid, because clipprog always appears before
         patterns = [r'id=\'clip(%s)\'\s*value=\'([0-9]+)\'' % t for t in ('id', 'prog')]
         results = list(filter(None, (re.search(p, webpage) for p in patterns)))

From 972efe60c3fdaff83f9b8e7a637ee81f4c27bb64 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 20 Jan 2017 22:13:54 +0800
Subject: [PATCH 0049/1696] [generic] Remove a dead test

The web page does not contain a video anymore

Ref: #2694, #2696
---
 youtube_dl/extractor/generic.py | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a7c104845..40201f311 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -589,17 +589,6 @@ class GenericIE(InfoExtractor):
                 'description': 'md5:8145d19d320ff3e52f28401f4c4283b9',
             }
         },
-        # Embedded Ustream video
-        {
-            'url': 'http://www.american.edu/spa/pti/nsa-privacy-janus-2014.cfm',
-            'md5': '27b99cdb639c9b12a79bca876a073417',
-            'info_dict': {
-                'id': '45734260',
-                'ext': 'flv',
-                'uploader': 'AU SPA:  The NSA and Privacy',
-                'title': 'NSA and Privacy Forum Debate featuring General Hayden and Barton Gellman'
-            }
-        },
         # nowvideo embed hidden behind percent encoding
         {
             'url': 'http://www.waoanime.tv/the-super-dimension-fortress-macross-episode-1/',

From f3c21cb7a7e2d8685f466368e3142739077498cf Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 20 Jan 2017 22:25:20 +0800
Subject: [PATCH 0050/1696] [cspan] Fix _TESTS

---
 youtube_dl/extractor/cspan.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index 92a827a4b..d4576160b 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -23,14 +23,13 @@ class CSpanIE(InfoExtractor):
         'md5': '94b29a4f131ff03d23471dd6f60b6a1d',
         'info_dict': {
             'id': '315139',
-            'ext': 'mp4',
             'title': 'Attorney General Eric Holder on Voting Rights Act Decision',
-            'description': 'Attorney General Eric Holder speaks to reporters following the Supreme Court decision in [Shelby County v. Holder], in which the court ruled that the preclearance provisions of the Voting Rights Act could not be enforced.',
         },
+        'playlist_mincount': 2,
         'skip': 'Regularly fails on travis, for unknown reasons',
     }, {
         'url': 'http://www.c-span.org/video/?c4486943/cspan-international-health-care-models',
-        'md5': '8e5fbfabe6ad0f89f3012a7943c1287b',
+        # md5 is unstable
         'info_dict': {
             'id': 'c4486943',
             'ext': 'mp4',
@@ -39,14 +38,11 @@ class CSpanIE(InfoExtractor):
         }
     }, {
         'url': 'http://www.c-span.org/video/?318608-1/gm-ignition-switch-recall',
-        'md5': '2ae5051559169baadba13fc35345ae74',
         'info_dict': {
             'id': '342759',
-            'ext': 'mp4',
             'title': 'General Motors Ignition Switch Recall',
-            'duration': 14848,
-            'description': 'md5:118081aedd24bf1d3b68b3803344e7f3'
         },
+        'playlist_mincount': 6,
     }, {
         # Video from senate.gov
         'url': 'http://www.c-span.org/video/?104517-1/immigration-reforms-needed-protect-skilled-american-workers',

From f4ec8dce481564589419e4dffc45437211daa13f Mon Sep 17 00:00:00 2001
From: Iulian Onofrei <revolter@users.noreply.github.com>
Date: Fri, 20 Jan 2017 18:25:04 +0200
Subject: [PATCH 0051/1696] Update README.md (#11787)

Add audio format argument dependency warning
---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 0b8c1671d..0d2ce8d15 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -751,7 +751,7 @@ def parseOpts(overrideArguments=None):
         help='Convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
     postproc.add_option(
         '--audio-format', metavar='FORMAT', dest='audioformat', default='best',
-        help='Specify audio format: "best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; "%default" by default')
+        help='Specify audio format: "best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; "%default" by default; No effect without -x')
     postproc.add_option(
         '--audio-quality', metavar='QUALITY',
         dest='audioquality', default='5',

From 12afdc2ad617dedfd7d60654b8c57b99604332ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Jan 2017 18:10:32 +0700
Subject: [PATCH 0052/1696] [youtube] Extract episode metadata (closes #9695,
 closes #11774)

---
 youtube_dl/extractor/youtube.py | 37 +++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e6b840735..63597dd16 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -864,6 +864,30 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            # YouTube Red video with episode data
+            'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
+            'info_dict': {
+                'id': 'iqKdEhx-dD4',
+                'ext': 'mp4',
+                'title': 'Isolation - Mind Field (Ep 1)',
+                'description': 'md5:3a72f23c086a1496c9e2c54a25fa0822',
+                'upload_date': '20170118',
+                'uploader': 'Vsauce',
+                'uploader_id': 'Vsauce',
+                'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
+                'license': 'Standard YouTube License',
+                'series': 'Mind Field',
+                'season_number': 1,
+                'episode_number': 1,
+            },
+            'params': {
+                'skip_download': True,
+            },
+            'expected_warnings': [
+                'Skipping DASH manifest',
+            ],
+        },
         {
             # itag 212
             'url': '1t24XAntNCY',
@@ -1454,6 +1478,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         else:
             video_alt_title = video_creator = None
 
+        m_episode = re.search(
+            r'<div[^>]+id="watch7-headline"[^>]*>\s*<span[^>]*>.*?>(?P<series>[^<]+)</a></b>\s*S(?P<season>\d+)\s*•\s*E(?P<episode>\d+)</span>',
+            video_webpage)
+        if m_episode:
+            series = m_episode.group('series')
+            season_number = int(m_episode.group('season'))
+            episode_number = int(m_episode.group('episode'))
+        else:
+            series = season_number = episode_number = None
+
         m_cat_container = self._search_regex(
             r'(?s)<h4[^>]*>\s*Category\s*</h4>\s*<ul[^>]*>(.*?)</ul>',
             video_webpage, 'categories', default=None)
@@ -1743,6 +1777,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'is_live': is_live,
             'start_time': start_time,
             'end_time': end_time,
+            'series': series,
+            'season_number': season_number,
+            'episode_number': episode_number,
         }
 
 

From 04a3d4d23472ffa4a482d8ebf2d8fdbb3e974327 Mon Sep 17 00:00:00 2001
From: ha shao <mozbugbox@yahoo.com.au>
Date: Sat, 21 Jan 2017 15:47:39 +0800
Subject: [PATCH 0053/1696] [vimeo:channel] Extract videos' titles for playlist
 entries

---
 youtube_dl/extractor/vimeo.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index add753635..a6bbd4c05 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -730,12 +730,12 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
             # Try extracting href first since not all videos are available via
             # short https://vimeo.com/id URL (e.g. https://vimeo.com/channels/tributes/6213729)
             clips = re.findall(
-                r'id="clip_(\d+)"[^>]*>\s*<a[^>]+href="(/(?:[^/]+/)*\1)', webpage)
+                r'id="clip_(\d+)"[^>]*>\s*<a[^>]+href="(/(?:[^/]+/)*\1)(?:[^>]+\btitle="([^"]+)")?', webpage)
             if clips:
-                for video_id, video_url in clips:
+                for video_id, video_url, video_title in clips:
                     yield self.url_result(
                         compat_urlparse.urljoin(base_url, video_url),
-                        VimeoIE.ie_key(), video_id=video_id)
+                        VimeoIE.ie_key(), video_id=video_id, video_title=video_title)
             # More relaxed fallback
             else:
                 for video_id in re.findall(r'id=["\']clip_(\d+)', webpage):

From 7c20b7484cc91a4818a98ca8d5b7ef94d5c38fb8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 22 Jan 2017 02:06:34 +0800
Subject: [PATCH 0054/1696] [nextmedia] Support redirected URLs

---
 ChangeLog                         |  1 +
 youtube_dl/extractor/nextmedia.py | 13 ++++++++++++-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 217971ec6..00c8a063f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
++ [nextmedia] Support redirected URLs
 + [cspan] Support Ustream embedded videos (#11547)
 
 
diff --git a/youtube_dl/extractor/nextmedia.py b/youtube_dl/extractor/nextmedia.py
index c900f232a..626ed8b49 100644
--- a/youtube_dl/extractor/nextmedia.py
+++ b/youtube_dl/extractor/nextmedia.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import parse_iso8601
 
 
@@ -30,6 +31,12 @@ class NextMediaIE(InfoExtractor):
         return self._extract_from_nextmedia_page(news_id, url, page)
 
     def _extract_from_nextmedia_page(self, news_id, url, page):
+        redirection_url = self._search_regex(
+            r'window\.location\.href\s*=\s*([\'"])(?P<url>(?!\1).+)\1',
+            page, 'redirection URL', default=None, group='url')
+        if redirection_url:
+            return self.url_result(compat_urlparse.urljoin(url, redirection_url))
+
         title = self._fetch_title(page)
         video_url = self._search_regex(self._URL_PATTERN, page, 'video url')
 
@@ -93,7 +100,7 @@ class NextMediaActionNewsIE(NextMediaIE):
 
 class AppleDailyIE(NextMediaIE):
     IE_DESC = '臺灣蘋果日報'
-    _VALID_URL = r'https?://(www|ent)\.appledaily\.com\.tw/(?:animation|appledaily|enews|realtimenews|actionnews)/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
+    _VALID_URL = r'https?://(www|ent)\.appledaily\.com\.tw/[^/]+/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
     _TESTS = [{
         'url': 'http://ent.appledaily.com.tw/enews/article/entertainment/20150128/36354694',
         'md5': 'a843ab23d150977cc55ef94f1e2c1e4d',
@@ -157,6 +164,10 @@ class AppleDailyIE(NextMediaIE):
     }, {
         'url': 'http://www.appledaily.com.tw/actionnews/appledaily/7/20161003/960588/',
         'only_matching': True,
+    }, {
+        # Redirected from http://ent.appledaily.com.tw/enews/article/entertainment/20150128/36354694
+        'url': 'http://ent.appledaily.com.tw/section/article/headline/20150128/36354694',
+        'only_matching': True,
     }]
 
     _URL_PATTERN = r'\{url: \'(.+)\'\}'

From e84495cd8d7bdb89bbfe233263bd8ad0b448f8cc Mon Sep 17 00:00:00 2001
From: Alex Seiler <seileralex@gmail.com>
Date: Sat, 21 Jan 2017 15:23:26 +0100
Subject: [PATCH 0055/1696] [azmedien] Add extractor (closes #11785)

---
 youtube_dl/extractor/azmedientv.py | 87 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  4 ++
 2 files changed, 91 insertions(+)
 create mode 100644 youtube_dl/extractor/azmedientv.py

diff --git a/youtube_dl/extractor/azmedientv.py b/youtube_dl/extractor/azmedientv.py
new file mode 100644
index 000000000..51d46fb94
--- /dev/null
+++ b/youtube_dl/extractor/azmedientv.py
@@ -0,0 +1,87 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from .kaltura import KalturaIE
+from ..utils import get_element_by_class
+
+
+class AZMedienTVIE(InfoExtractor):
+    IE_DESC = 'telezueri.ch, telebaern.tv and telem1.ch videos'
+    _VALID_URL = r'http://(?:www\.)?(?:telezueri\.ch|telebaern\.tv|telem1\.ch)/[0-9]+-show-[^/#]+(?:/[0-9]+-episode-[^/#]+(?:/[0-9]+-segment-(?:[^/#]+#)?|#)|#)(?P<id>[^#]+)'
+
+    _TESTS = [{
+        # URL with 'segment'
+        'url': 'http://www.telezueri.ch/62-show-zuerinews/13772-episode-sonntag-18-dezember-2016/32419-segment-massenabweisungen-beim-hiltl-club-wegen-pelzboom',
+        'md5': 'fda85ada1299cee517a622bfbc5f6b66',
+        'info_dict': {
+            'id': '1_2444peh4',
+            'ext': 'mov',
+            'title': 'Massenabweisungen beim Hiltl Club wegen Pelzboom',
+            'description': 'md5:9ea9dd1b159ad65b36ddcf7f0d7c76a8',
+            'uploader_id': 'TeleZ?ri',
+            'upload_date': '20161218',
+            'timestamp': 1482084490,
+        }
+    }, {
+        # URL with 'segment' and fragment:
+        'url': 'http://www.telebaern.tv/118-show-news/14240-episode-dienstag-17-januar-2017/33666-segment-achtung-gefahr#zu-wenig-pflegerinnen-und-pfleger',
+        'only_matching': True
+    }, {
+        # URL with 'episode' and fragment:
+        'url': 'http://www.telem1.ch/47-show-sonntalk/13986-episode-soldaten-fuer-grenzschutz-energiestrategie-obama-bilanz#soldaten-fuer-grenzschutz-energiestrategie-obama-bilanz',
+        'only_matching': True
+    }, {
+        # URL with 'show' and fragment:
+        'url': 'http://www.telezueri.ch/66-show-sonntalk#burka-plakate-trump-putin-china-besuch',
+        'only_matching': True
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        kaltura_partner_id = self._html_search_regex(
+            r'<script[^>]+src=["\']https?://www\.kaltura\.com/.*/partner_id/([0-9]+)',
+            webpage, 'Kaltura partner ID')
+        kaltura_entry_id = self._html_search_regex(
+            r'<a[^>]+data-id=["\'](.*?)["\'][^>]+data-slug=["\']%s' % video_id,
+            webpage, 'Kaltura entry ID')
+
+        return self.url_result(
+            'kaltura:%s:%s' % (kaltura_partner_id, kaltura_entry_id),
+            ie=KalturaIE.ie_key())
+
+
+class AZMedienTVShowIE(InfoExtractor):
+    IE_DESC = 'telezueri.ch, telebaern.tv and telem1.ch shows'
+    _VALID_URL = r'http://(?:www\.)?(?:telezueri\.ch|telebaern\.tv|telem1\.ch)/(?P<id>[0-9]+-show-[^/#]+(?:/[0-9]+-episode-[^/#]+)?)$'
+
+    _TESTS = [{
+        # URL with 'episode':
+        'url': 'http://www.telebaern.tv/118-show-news/13735-episode-donnerstag-15-dezember-2016',
+        'info_dict': {
+            'id': '118-show-news/13735-episode-donnerstag-15-dezember-2016',
+            'title': 'News',
+        },
+        'playlist_count': 9,
+    }, {
+        # URL with 'show' only:
+        'url': 'http://www.telezueri.ch/86-show-talktaeglich',
+        'only_matching': True
+    }]
+
+    def _real_extract(self, url):
+        show_id = self._match_id(url)
+        webpage = self._download_webpage(url, show_id)
+
+        title = get_element_by_class('title-block-cell', webpage)
+        if title:
+            title = title.strip()
+
+        entries = [self.url_result(m.group('url'), ie=AZMedienTVIE.ie_key()) for m in re.finditer(
+            r'<a href=["\']#["\'][^>]+data-real=["\'](?P<url>.+?)["\']', webpage)]
+
+        return self.playlist_result(
+            entries, show_id, title)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 9d0610d21..4cfb3c70f 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -77,6 +77,10 @@ from .awaan import (
     AWAANLiveIE,
     AWAANSeasonIE,
 )
+from .azmedientv import (
+    AZMedienTVIE,
+    AZMedienTVShowIE,
+)
 from .azubu import AzubuIE, AzubuLiveIE
 from .baidu import BaiduVideoIE
 from .bambuser import BambuserIE, BambuserChannelIE

From 94629e537f2f6ed80b19e3863456f9ba8073af36 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Jan 2017 02:15:20 +0700
Subject: [PATCH 0056/1696] [azmedien] Improve (closes #11784)

---
 youtube_dl/extractor/azmedien.py   | 132 +++++++++++++++++++++++++++++
 youtube_dl/extractor/azmedientv.py |  87 -------------------
 youtube_dl/extractor/extractors.py |   6 +-
 3 files changed, 135 insertions(+), 90 deletions(-)
 create mode 100644 youtube_dl/extractor/azmedien.py
 delete mode 100644 youtube_dl/extractor/azmedientv.py

diff --git a/youtube_dl/extractor/azmedien.py b/youtube_dl/extractor/azmedien.py
new file mode 100644
index 000000000..059dc6e4b
--- /dev/null
+++ b/youtube_dl/extractor/azmedien.py
@@ -0,0 +1,132 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from .kaltura import KalturaIE
+from ..utils import (
+    get_element_by_class,
+    strip_or_none,
+)
+
+
+class AZMedienBaseIE(InfoExtractor):
+    def _kaltura_video(self, partner_id, entry_id):
+        return self.url_result(
+            'kaltura:%s:%s' % (partner_id, entry_id), ie=KalturaIE.ie_key(),
+            video_id=entry_id)
+
+
+class AZMedienIE(AZMedienBaseIE):
+    IE_DESC = 'AZ Medien videos'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:www\.)?
+                        (?:
+                            telezueri\.ch|
+                            telebaern\.tv|
+                            telem1\.ch
+                        )/
+                        [0-9]+-show-[^/\#]+
+                        (?:
+                            /[0-9]+-episode-[^/\#]+
+                            (?:
+                                /[0-9]+-segment-(?:[^/\#]+\#)?|
+                                \#
+                            )|
+                            \#
+                        )
+                        (?P<id>[^\#]+)
+                    '''
+
+    _TESTS = [{
+        # URL with 'segment'
+        'url': 'http://www.telezueri.ch/62-show-zuerinews/13772-episode-sonntag-18-dezember-2016/32419-segment-massenabweisungen-beim-hiltl-club-wegen-pelzboom',
+        'info_dict': {
+            'id': '1_2444peh4',
+            'ext': 'mov',
+            'title': 'Massenabweisungen beim Hiltl Club wegen Pelzboom',
+            'description': 'md5:9ea9dd1b159ad65b36ddcf7f0d7c76a8',
+            'uploader_id': 'TeleZ?ri',
+            'upload_date': '20161218',
+            'timestamp': 1482084490,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        # URL with 'segment' and fragment:
+        'url': 'http://www.telebaern.tv/118-show-news/14240-episode-dienstag-17-januar-2017/33666-segment-achtung-gefahr#zu-wenig-pflegerinnen-und-pfleger',
+        'only_matching': True
+    }, {
+        # URL with 'episode' and fragment:
+        'url': 'http://www.telem1.ch/47-show-sonntalk/13986-episode-soldaten-fuer-grenzschutz-energiestrategie-obama-bilanz#soldaten-fuer-grenzschutz-energiestrategie-obama-bilanz',
+        'only_matching': True
+    }, {
+        # URL with 'show' and fragment:
+        'url': 'http://www.telezueri.ch/66-show-sonntalk#burka-plakate-trump-putin-china-besuch',
+        'only_matching': True
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        partner_id = self._search_regex(
+            r'<script[^>]+src=["\'](?:https?:)?//(?:[^/]+\.)?kaltura\.com(?:/[^/]+)*/(?:p|partner_id)/([0-9]+)',
+            webpage, 'kaltura partner id')
+        entry_id = self._html_search_regex(
+            r'<a[^>]+data-id=(["\'])(?P<id>(?:(?!\1).)+)\1[^>]+data-slug=["\']%s'
+            % re.escape(video_id), webpage, 'kaltura entry id', group='id')
+
+        return self._kaltura_video(partner_id, entry_id)
+
+
+class AZMedienShowIE(AZMedienBaseIE):
+    IE_DESC = 'AZ Medien shows'
+    _VALID_URL = r'https?://(?:www\.)?(?:telezueri\.ch|telebaern\.tv|telem1\.ch)/(?P<id>[0-9]+-show-[^/#]+(?:/[0-9]+-episode-[^/#]+)?)$'
+
+    _TESTS = [{
+        # URL with 'episode'
+        'url': 'http://www.telebaern.tv/118-show-news/13735-episode-donnerstag-15-dezember-2016',
+        'info_dict': {
+            'id': '118-show-news/13735-episode-donnerstag-15-dezember-2016',
+            'title': 'News - Donnerstag, 15. Dezember 2016',
+        },
+        'playlist_count': 9,
+    }, {
+        # URL with 'show' only
+        'url': 'http://www.telezueri.ch/86-show-talktaeglich',
+        'only_matching': True
+    }]
+
+    def _real_extract(self, url):
+        show_id = self._match_id(url)
+        webpage = self._download_webpage(url, show_id)
+
+        entries = []
+
+        partner_id = self._search_regex(
+            r'src=["\'](?:https?:)?//(?:[^/]+\.)kaltura\.com/(?:[^/]+/)*(?:p|partner_id)/(\d+)',
+            webpage, 'kaltura partner id', default=None)
+
+        if partner_id:
+            entries = [
+                self._kaltura_video(partner_id, m.group('id'))
+                for m in re.finditer(
+                    r'data-id=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage)]
+
+        if not entries:
+            entries = [
+                self.url_result(m.group('url'), ie=AZMedienIE.ie_key())
+                for m in re.finditer(
+                    r'<a[^>]+data-real=(["\'])(?P<url>http.+?)\1', webpage)]
+
+        title = self._search_regex(
+            r'episodeShareTitle\s*=\s*(["\'])(?P<title>(?:(?!\1).)+)\1',
+            webpage, 'title',
+            default=strip_or_none(get_element_by_class(
+                'title-block-cell', webpage)), group='title')
+
+        return self.playlist_result(entries, show_id, title)
diff --git a/youtube_dl/extractor/azmedientv.py b/youtube_dl/extractor/azmedientv.py
deleted file mode 100644
index 51d46fb94..000000000
--- a/youtube_dl/extractor/azmedientv.py
+++ /dev/null
@@ -1,87 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from .kaltura import KalturaIE
-from ..utils import get_element_by_class
-
-
-class AZMedienTVIE(InfoExtractor):
-    IE_DESC = 'telezueri.ch, telebaern.tv and telem1.ch videos'
-    _VALID_URL = r'http://(?:www\.)?(?:telezueri\.ch|telebaern\.tv|telem1\.ch)/[0-9]+-show-[^/#]+(?:/[0-9]+-episode-[^/#]+(?:/[0-9]+-segment-(?:[^/#]+#)?|#)|#)(?P<id>[^#]+)'
-
-    _TESTS = [{
-        # URL with 'segment'
-        'url': 'http://www.telezueri.ch/62-show-zuerinews/13772-episode-sonntag-18-dezember-2016/32419-segment-massenabweisungen-beim-hiltl-club-wegen-pelzboom',
-        'md5': 'fda85ada1299cee517a622bfbc5f6b66',
-        'info_dict': {
-            'id': '1_2444peh4',
-            'ext': 'mov',
-            'title': 'Massenabweisungen beim Hiltl Club wegen Pelzboom',
-            'description': 'md5:9ea9dd1b159ad65b36ddcf7f0d7c76a8',
-            'uploader_id': 'TeleZ?ri',
-            'upload_date': '20161218',
-            'timestamp': 1482084490,
-        }
-    }, {
-        # URL with 'segment' and fragment:
-        'url': 'http://www.telebaern.tv/118-show-news/14240-episode-dienstag-17-januar-2017/33666-segment-achtung-gefahr#zu-wenig-pflegerinnen-und-pfleger',
-        'only_matching': True
-    }, {
-        # URL with 'episode' and fragment:
-        'url': 'http://www.telem1.ch/47-show-sonntalk/13986-episode-soldaten-fuer-grenzschutz-energiestrategie-obama-bilanz#soldaten-fuer-grenzschutz-energiestrategie-obama-bilanz',
-        'only_matching': True
-    }, {
-        # URL with 'show' and fragment:
-        'url': 'http://www.telezueri.ch/66-show-sonntalk#burka-plakate-trump-putin-china-besuch',
-        'only_matching': True
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        kaltura_partner_id = self._html_search_regex(
-            r'<script[^>]+src=["\']https?://www\.kaltura\.com/.*/partner_id/([0-9]+)',
-            webpage, 'Kaltura partner ID')
-        kaltura_entry_id = self._html_search_regex(
-            r'<a[^>]+data-id=["\'](.*?)["\'][^>]+data-slug=["\']%s' % video_id,
-            webpage, 'Kaltura entry ID')
-
-        return self.url_result(
-            'kaltura:%s:%s' % (kaltura_partner_id, kaltura_entry_id),
-            ie=KalturaIE.ie_key())
-
-
-class AZMedienTVShowIE(InfoExtractor):
-    IE_DESC = 'telezueri.ch, telebaern.tv and telem1.ch shows'
-    _VALID_URL = r'http://(?:www\.)?(?:telezueri\.ch|telebaern\.tv|telem1\.ch)/(?P<id>[0-9]+-show-[^/#]+(?:/[0-9]+-episode-[^/#]+)?)$'
-
-    _TESTS = [{
-        # URL with 'episode':
-        'url': 'http://www.telebaern.tv/118-show-news/13735-episode-donnerstag-15-dezember-2016',
-        'info_dict': {
-            'id': '118-show-news/13735-episode-donnerstag-15-dezember-2016',
-            'title': 'News',
-        },
-        'playlist_count': 9,
-    }, {
-        # URL with 'show' only:
-        'url': 'http://www.telezueri.ch/86-show-talktaeglich',
-        'only_matching': True
-    }]
-
-    def _real_extract(self, url):
-        show_id = self._match_id(url)
-        webpage = self._download_webpage(url, show_id)
-
-        title = get_element_by_class('title-block-cell', webpage)
-        if title:
-            title = title.strip()
-
-        entries = [self.url_result(m.group('url'), ie=AZMedienTVIE.ie_key()) for m in re.finditer(
-            r'<a href=["\']#["\'][^>]+data-real=["\'](?P<url>.+?)["\']', webpage)]
-
-        return self.playlist_result(
-            entries, show_id, title)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 4cfb3c70f..de5f94738 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -77,9 +77,9 @@ from .awaan import (
     AWAANLiveIE,
     AWAANSeasonIE,
 )
-from .azmedientv import (
-    AZMedienTVIE,
-    AZMedienTVShowIE,
+from .azmedien import (
+    AZMedienIE,
+    AZMedienShowIE,
 )
 from .azubu import AzubuIE, AzubuLiveIE
 from .baidu import BaiduVideoIE

From 42697bab3c4d65a232054d5d5482cc177da12c72 Mon Sep 17 00:00:00 2001
From: einstein95 <einstein95@users.noreply.github.com>
Date: Sun, 22 Jan 2017 02:00:38 +1300
Subject: [PATCH 0057/1696] [chaturbate] Fix extraction

---
 youtube_dl/extractor/chaturbate.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/chaturbate.py b/youtube_dl/extractor/chaturbate.py
index 29a8820d5..1c2f065df 100644
--- a/youtube_dl/extractor/chaturbate.py
+++ b/youtube_dl/extractor/chaturbate.py
@@ -1,5 +1,7 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import ExtractorError
 
@@ -31,30 +33,32 @@ class ChaturbateIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        m3u8_url = self._search_regex(
-            r'src=(["\'])(?P<url>http.+?\.m3u8.*?)\1', webpage,
-            'playlist', default=None, group='url')
+        m3u8_urls = re.findall(
+            r'var hlsSource.+? = (["\'])(?P<url>http.+?\.m3u8)', webpage)
 
-        if not m3u8_url:
+        if not m3u8_urls:
             error = self._search_regex(
                 [r'<span[^>]+class=(["\'])desc_span\1[^>]*>(?P<error>[^<]+)</span>',
                  r'<div[^>]+id=(["\'])defchat\1[^>]*>\s*<p><strong>(?P<error>[^<]+)<'],
                 webpage, 'error', group='error', default=None)
             if not error:
-                if any(p not in webpage for p in (
+                if any(p in webpage for p in (
                         self._ROOM_OFFLINE, 'offline_tipping', 'tip_offline')):
                     error = self._ROOM_OFFLINE
             if error:
                 raise ExtractorError(error, expected=True)
             raise ExtractorError('Unable to find stream URL')
 
-        formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
+        formats = []
+        for m3u8_url in m3u8_urls:
+            formats.append(self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')[0])
+
         self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': self._live_title(video_id),
-            'thumbnail': 'https://cdn-s.highwebmedia.com/uHK3McUtGCG3SMFcd4ZJsRv8/roomimage/%s.jpg' % video_id,
+            'thumbnail': 'https://roomimg.stream.highwebmedia.com/ri/%s.jpg' % video_id,
             'age_limit': self._rta_search(webpage),
             'is_live': True,
             'formats': formats,

From a243abb80d5fdaacc502bc5a2b5cb20d0766e93a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Jan 2017 03:00:10 +0700
Subject: [PATCH 0058/1696] [chaturbate] Improve (closes #11797)

---
 youtube_dl/extractor/chaturbate.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/chaturbate.py b/youtube_dl/extractor/chaturbate.py
index 1c2f065df..8fbc91c1f 100644
--- a/youtube_dl/extractor/chaturbate.py
+++ b/youtube_dl/extractor/chaturbate.py
@@ -33,10 +33,10 @@ class ChaturbateIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        m3u8_urls = re.findall(
-            r'var hlsSource.+? = (["\'])(?P<url>http.+?\.m3u8)', webpage)
+        m3u8_formats = [(m.group('id').lower(), m.group('url')) for m in re.finditer(
+            r'hlsSource(?P<id>.+?)\s*=\s*(?P<q>["\'])(?P<url>http.+?)(?P=q)', webpage)]
 
-        if not m3u8_urls:
+        if not m3u8_formats:
             error = self._search_regex(
                 [r'<span[^>]+class=(["\'])desc_span\1[^>]*>(?P<error>[^<]+)</span>',
                  r'<div[^>]+id=(["\'])defchat\1[^>]*>\s*<p><strong>(?P<error>[^<]+)<'],
@@ -50,9 +50,12 @@ class ChaturbateIE(InfoExtractor):
             raise ExtractorError('Unable to find stream URL')
 
         formats = []
-        for m3u8_url in m3u8_urls:
-            formats.append(self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')[0])
-
+        for m3u8_id, m3u8_url in m3u8_formats:
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, ext='mp4',
+                # ffmpeg skips segments for fast m3u8
+                preference=-10 if m3u8_id == 'fast' else None,
+                m3u8_id=m3u8_id, fatal=False, live=True))
         self._sort_formats(formats)
 
         return {

From 8d1fbe0cb20fdfab8487bb478c2a002f12c1a5d9 Mon Sep 17 00:00:00 2001
From: einstein95 <einstein95@users.noreply.github.com>
Date: Sat, 21 Jan 2017 20:02:55 +1300
Subject: [PATCH 0059/1696] [pornflip] Add extractor (closes #11556)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/pornflip.py   | 59 ++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+)
 create mode 100644 youtube_dl/extractor/pornflip.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index de5f94738..cfddf5b92 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -724,6 +724,7 @@ from .polskieradio import (
 )
 from .porn91 import Porn91IE
 from .porncom import PornComIE
+from .pornflip import PornFlipIE
 from .pornhd import PornHdIE
 from .pornhub import (
     PornHubIE,
diff --git a/youtube_dl/extractor/pornflip.py b/youtube_dl/extractor/pornflip.py
new file mode 100644
index 000000000..b6077f7cb
--- /dev/null
+++ b/youtube_dl/extractor/pornflip.py
@@ -0,0 +1,59 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_parse_qs,
+)
+from ..utils import (
+    int_or_none,
+    try_get,
+    RegexNotFoundError,
+)
+
+
+class PornFlipIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?pornflip\.com/v/(?P<id>[0-9A-Za-z]{11})'
+    _TEST = {
+        'url': 'https://www.pornflip.com/v/wz7DfNhMmep',
+        'md5': '98c46639849145ae1fd77af532a9278c',
+        'info_dict': {
+            'id': 'wz7DfNhMmep',
+            'ext': 'mp4',
+            'title': '2 Amateurs swallow make his dream cumshots true',
+            'uploader': 'figifoto',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'age_limit': 18,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        uploader = self._html_search_regex(
+            r'<span class="name">\s+<a class="ajax" href=".+>\s+<strong>([^<]+)<', webpage, 'uploader', fatal=False)
+        flashvars = compat_parse_qs(self._html_search_regex(
+            r'<embed.+?flashvars="([^"]+)"',
+            webpage, 'flashvars'))
+        title = flashvars['video_vars[title]'][0]
+        thumbnail = try_get(flashvars, lambda x: x['video_vars[big_thumb]'][0])
+        formats = []
+        for k, v in flashvars.items():
+            height = self._search_regex(r'video_vars\[video_urls\]\[(\d+).+?\]', k, 'height', default=None)
+            if height:
+                url = v[0]
+                formats.append({
+                    'height': int_or_none(height),
+                    'url': url
+                })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'title': title,
+            'uploader': uploader,
+            'thumbnail': thumbnail,
+            'age_limit': 18,
+        }

From 271808b6b2bd75ec9bdf943a55dbc4737bfa6f81 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Jan 2017 03:43:27 +0700
Subject: [PATCH 0060/1696] [pornflip] Improve and extract dash formats (closes
 #11795)

---
 youtube_dl/extractor/pornflip.py | 79 ++++++++++++++++++++++----------
 1 file changed, 56 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/pornflip.py b/youtube_dl/extractor/pornflip.py
index b6077f7cb..a4a5d390e 100644
--- a/youtube_dl/extractor/pornflip.py
+++ b/youtube_dl/extractor/pornflip.py
@@ -4,56 +4,89 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..compat import (
     compat_parse_qs,
+    compat_str,
 )
 from ..utils import (
     int_or_none,
     try_get,
-    RegexNotFoundError,
+    unified_timestamp,
 )
 
 
 class PornFlipIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?pornflip\.com/v/(?P<id>[0-9A-Za-z]{11})'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?pornflip\.com/(?:v|embed)/(?P<id>[0-9A-Za-z]{11})'
+    _TESTS = [{
         'url': 'https://www.pornflip.com/v/wz7DfNhMmep',
         'md5': '98c46639849145ae1fd77af532a9278c',
         'info_dict': {
             'id': 'wz7DfNhMmep',
             'ext': 'mp4',
             'title': '2 Amateurs swallow make his dream cumshots true',
-            'uploader': 'figifoto',
             'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 112,
+            'timestamp': 1481655502,
+            'upload_date': '20161213',
+            'uploader_id': '106786',
+            'uploader': 'figifoto',
+            'view_count': int,
             'age_limit': 18,
         }
-    }
+    }, {
+        'url': 'https://www.pornflip.com/embed/wz7DfNhMmep',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        uploader = self._html_search_regex(
-            r'<span class="name">\s+<a class="ajax" href=".+>\s+<strong>([^<]+)<', webpage, 'uploader', fatal=False)
-        flashvars = compat_parse_qs(self._html_search_regex(
-            r'<embed.+?flashvars="([^"]+)"',
-            webpage, 'flashvars'))
-        title = flashvars['video_vars[title]'][0]
-        thumbnail = try_get(flashvars, lambda x: x['video_vars[big_thumb]'][0])
-        formats = []
-        for k, v in flashvars.items():
-            height = self._search_regex(r'video_vars\[video_urls\]\[(\d+).+?\]', k, 'height', default=None)
-            if height:
-                url = v[0]
-                formats.append({
-                    'height': int_or_none(height),
-                    'url': url
-                })
 
+        webpage = self._download_webpage(
+            'https://www.pornflip.com/v/%s' % video_id, video_id)
+
+        flashvars = compat_parse_qs(self._search_regex(
+            r'<embed[^>]+flashvars=(["\'])(?P<flashvars>(?:(?!\1).)+)\1',
+            webpage, 'flashvars', group='flashvars'))
+
+        title = flashvars['video_vars[title]'][0]
+
+        def flashvar(kind):
+            return try_get(
+                flashvars, lambda x: x['video_vars[%s]' % kind][0], compat_str)
+
+        formats = []
+        for key, value in flashvars.items():
+            if not (value and isinstance(value, list)):
+                continue
+            format_url = value[0]
+            if key == 'video_vars[hds_manifest]':
+                formats.extend(self._extract_mpd_formats(
+                    format_url, video_id, mpd_id='dash', fatal=False))
+                continue
+            height = self._search_regex(
+                r'video_vars\[video_urls\]\[(\d+)', key, 'height', default=None)
+            if not height:
+                continue
+            formats.append({
+                'url': format_url,
+                'format_id': 'http-%s' % height,
+                'height': int_or_none(height),
+            })
         self._sort_formats(formats)
 
+        uploader = self._html_search_regex(
+            (r'<span[^>]+class="name"[^>]*>\s*<a[^>]+>\s*<strong>(?P<uploader>[^<]+)',
+             r'<meta[^>]+content=(["\'])[^>]*\buploaded by (?P<uploader>.+?)\1'),
+            webpage, 'uploader', fatal=False, group='uploader')
+
         return {
             'id': video_id,
             'formats': formats,
             'title': title,
+            'thumbnail': flashvar('big_thumb'),
+            'duration': int_or_none(flashvar('duration')),
+            'timestamp': unified_timestamp(self._html_search_meta(
+                'uploadDate', webpage, 'timestamp')),
+            'uploader_id': flashvar('author_id'),
             'uploader': uploader,
-            'thumbnail': thumbnail,
+            'view_count': int_or_none(flashvar('views')),
             'age_limit': 18,
         }

From 6c031a35f31717cc1a535d5d808b94967b841a93 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Jan 2017 18:57:15 +0700
Subject: [PATCH 0061/1696] [ChangeLog] Actualize

---
 ChangeLog | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 00c8a063f..a814b934c 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,8 +1,16 @@
 version <unreleased>
 
 Extractors
++ [pornflip] Add support for pornflip.com (#11556, #11795)
+* [chaturbate] Fix extraction (#11797, #11802)
++ [azmedien] Add support for AZ Medien sites (#11784, #11785)
 + [nextmedia] Support redirected URLs
++ [vimeo:channel] Extract videos' titles for playlist entries (#11796)
++ [youtube] Extract episode metadata (#9695, #11774)
 + [cspan] Support Ustream embedded videos (#11547)
++ [1tv] Add support for HLS videos (#11786)
+* [uol] Fix extraction (#11770)
+* [mtv] Relax triforce feed regular expression (#11766)
 
 
 version 2017.01.18

From 9d5b29c881f679b1d4270326af4ba6f657807011 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Jan 2017 18:59:04 +0700
Subject: [PATCH 0062/1696] release 2017.01.22

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 README.md                 | 2 +-
 docs/supportedsites.md    | 3 +++
 youtube_dl/version.py     | 2 +-
 5 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 38cb13a33..30cc27c7b 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.01.18*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.01.18**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.01.22*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.01.22**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.01.18
+[debug] youtube-dl version 2017.01.22
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index a814b934c..beea17e54 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.01.22
 
 Extractors
 + [pornflip] Add support for pornflip.com (#11556, #11795)
diff --git a/README.md b/README.md
index a606346b2..4f677d0cc 100644
--- a/README.md
+++ b/README.md
@@ -374,7 +374,7 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
                                      avprobe)
     --audio-format FORMAT            Specify audio format: "best", "aac",
                                      "vorbis", "mp3", "m4a", "opus", or "wav";
-                                     "best" by default
+                                     "best" by default; No effect without -x
     --audio-quality QUALITY          Specify ffmpeg/avconv audio quality, insert
                                      a value between 0 (better) and 9 (worse)
                                      for VBR or a specific bitrate like 128K
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index a3c76d5db..b906d443a 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -74,6 +74,8 @@
  - **awaan:live**
  - **awaan:season**
  - **awaan:video**
+ - **AZMedien**: AZ Medien videos
+ - **AZMedienShow**: AZ Medien shows
  - **Azubu**
  - **AzubuLive**
  - **BaiduVideo**: 百度视频
@@ -572,6 +574,7 @@
  - **PolskieRadio**
  - **PolskieRadioCategory**
  - **PornCom**
+ - **PornFlip**
  - **PornHd**
  - **PornHub**: PornHub and Thumbzilla
  - **PornHubPlaylist**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 669f60f65..9466c9637 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.01.18'
+__version__ = '2017.01.22'

From 30dda24de304dd53fc63dfb5bf4672c2ec747014 Mon Sep 17 00:00:00 2001
From: Gaetan Gilbert <gaetan.gilbert@ens-lyon.fr>
Date: Sun, 22 Jan 2017 20:27:38 +0100
Subject: [PATCH 0063/1696] [chirbit] Extract uploader

---
 youtube_dl/extractor/chirbit.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/chirbit.py b/youtube_dl/extractor/chirbit.py
index f35df143a..4815b34be 100644
--- a/youtube_dl/extractor/chirbit.py
+++ b/youtube_dl/extractor/chirbit.py
@@ -19,6 +19,7 @@ class ChirbitIE(InfoExtractor):
             'title': 'md5:f542ea253f5255240be4da375c6a5d7e',
             'description': 'md5:f24a4e22a71763e32da5fed59e47c770',
             'duration': 306,
+            'uploader': 'Gerryaudio',
         },
         'params': {
             'skip_download': True,
@@ -54,6 +55,9 @@ class ChirbitIE(InfoExtractor):
         duration = parse_duration(self._search_regex(
             r'class=["\']c-length["\'][^>]*>([^<]+)',
             webpage, 'duration', fatal=False))
+        uploader = self._search_regex(
+            r'id=["\']chirbit-username["\'][^>]*>([^<]+)',
+            webpage, 'uploader', fatal=False)
 
         return {
             'id': audio_id,
@@ -61,6 +65,7 @@ class ChirbitIE(InfoExtractor):
             'title': title,
             'description': description,
             'duration': duration,
+            'uploader': uploader,
         }
 
 

From a089545e036619a798aa19f33085f2b0b87a1b0a Mon Sep 17 00:00:00 2001
From: Alex Seiler <seileralex@gmail.com>
Date: Sun, 22 Jan 2017 20:30:29 +0100
Subject: [PATCH 0064/1696] [azmedien:show] Improve _VALID_URL

---
 youtube_dl/extractor/azmedien.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/azmedien.py b/youtube_dl/extractor/azmedien.py
index 059dc6e4b..a89f71c20 100644
--- a/youtube_dl/extractor/azmedien.py
+++ b/youtube_dl/extractor/azmedien.py
@@ -85,7 +85,20 @@ class AZMedienIE(AZMedienBaseIE):
 
 class AZMedienShowIE(AZMedienBaseIE):
     IE_DESC = 'AZ Medien shows'
-    _VALID_URL = r'https?://(?:www\.)?(?:telezueri\.ch|telebaern\.tv|telem1\.ch)/(?P<id>[0-9]+-show-[^/#]+(?:/[0-9]+-episode-[^/#]+)?)$'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:www\.)?
+                        (?:
+                            telezueri\.ch|
+                            telebaern\.tv|
+                            telem1\.ch
+                        )/
+                        (?P<id>[0-9]+-show-[^/\#]+
+                            (?:
+                                /[0-9]+-episode-[^/\#]+
+                            )?
+                        )$
+                    '''
 
     _TESTS = [{
         # URL with 'episode'

From 8bc0800d7cf24b17204f0fb3c6e76327ed8d527f Mon Sep 17 00:00:00 2001
From: Grzegorz P <Grzechooo@users.noreply.github.com>
Date: Sun, 22 Jan 2017 20:35:38 +0100
Subject: [PATCH 0065/1696] [youtube:playlist] Fix nonexistent/private playlist
 detection (closes #11604)

---
 youtube_dl/extractor/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 63597dd16..644653357 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1998,7 +1998,8 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
         url = self._TEMPLATE_URL % playlist_id
         page = self._download_webpage(url, playlist_id)
 
-        for match in re.findall(r'<div class="yt-alert-message">([^<]+)</div>', page):
+        # the yt-alert-message now has tabindex attribute (see https://github.com/rg3/youtube-dl/issues/11604)
+        for match in re.findall(r'<div class="yt-alert-message"[^>]*>([^<]+)</div>', page):
             match = match.strip()
             # Check if the playlist exists or is private
             if re.match(r'[^<]*(The|This) playlist (does not exist|is private)[^<]*', match):

From 4201ba13e674788c36ae69fbfbffc4b246717d6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Jan 2017 02:49:56 +0700
Subject: [PATCH 0066/1696] [youtube:playlist] Fix nonexistent/private playlist
 detection and skip private tests

---
 youtube_dl/extractor/youtube.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 644653357..5202beb3e 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1856,6 +1856,7 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
             'title': 'YDL_Empty_List',
         },
         'playlist_count': 0,
+        'skip': 'This playlist is private',
     }, {
         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
@@ -1887,6 +1888,7 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
         },
         'playlist_count': 2,
+        'skip': 'This playlist is private',
     }, {
         'note': 'embedded',
         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
@@ -2002,11 +2004,14 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
         for match in re.findall(r'<div class="yt-alert-message"[^>]*>([^<]+)</div>', page):
             match = match.strip()
             # Check if the playlist exists or is private
-            if re.match(r'[^<]*(The|This) playlist (does not exist|is private)[^<]*', match):
-                raise ExtractorError(
-                    'The playlist doesn\'t exist or is private, use --username or '
-                    '--netrc to access it.',
-                    expected=True)
+            mobj = re.match(r'[^<]*(?:The|This) playlist (?P<reason>does not exist|is private)[^<]*', match)
+            if mobj:
+                reason = mobj.group('reason')
+                message = 'This playlist %s' % reason
+                if 'private' in reason:
+                    message += ', use --username or --netrc to access it'
+                message += '.'
+                raise ExtractorError(message, expected=True)
             elif re.match(r'[^<]*Invalid parameters[^<]*', match):
                 raise ExtractorError(
                     'Invalid parameters. Maybe URL is incorrect.',

From 6d119c2a6bdd2a987ef2e7553b357bd4a3f18690 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Jan 2017 03:50:39 +0700
Subject: [PATCH 0067/1696] [24video] Fix extraction (closes #11811)

---
 youtube_dl/extractor/twentyfourvideo.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/twentyfourvideo.py b/youtube_dl/extractor/twentyfourvideo.py
index 1093a3829..a983ebf05 100644
--- a/youtube_dl/extractor/twentyfourvideo.py
+++ b/youtube_dl/extractor/twentyfourvideo.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class TwentyFourVideoIE(InfoExtractor):
     IE_NAME = '24video'
-    _VALID_URL = r'https?://(?:www\.)?24video\.(?:net|me|xxx)/(?:video/(?:view|xml)/|player/new24_play\.swf\?id=)(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?24video\.(?:net|me|xxx|sex)/(?:video/(?:view|xml)/|player/new24_play\.swf\?id=)(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.24video.net/video/view/1044982',
@@ -43,7 +43,7 @@ class TwentyFourVideoIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(
-            'http://www.24video.net/video/view/%s' % video_id, video_id)
+            'http://www.24video.sex/video/view/%s' % video_id, video_id)
 
         title = self._og_search_title(webpage)
         description = self._html_search_regex(
@@ -69,11 +69,11 @@ class TwentyFourVideoIE(InfoExtractor):
 
         # Sets some cookies
         self._download_xml(
-            r'http://www.24video.net/video/xml/%s?mode=init' % video_id,
+            r'http://www.24video.sex/video/xml/%s?mode=init' % video_id,
             video_id, 'Downloading init XML')
 
         video_xml = self._download_xml(
-            'http://www.24video.net/video/xml/%s?mode=play' % video_id,
+            'http://www.24video.sex/video/xml/%s?mode=play' % video_id,
             video_id, 'Downloading video XML')
 
         video = xpath_element(video_xml, './/video', 'video', fatal=True)

From 0c1c6f4b9f97375ffc68cbc9c7276838f7bf8514 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 23 Jan 2017 23:31:43 +0800
Subject: [PATCH 0068/1696] [utils] Add another date format seen in NextTV

---
 youtube_dl/utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 12863e74a..98acc2b45 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -143,6 +143,7 @@ DATE_FORMATS = (
     '%Y/%m/%d',
     '%Y/%m/%d %H:%M',
     '%Y/%m/%d %H:%M:%S',
+    '%Y-%m-%d %H:%M',
     '%Y-%m-%d %H:%M:%S',
     '%Y-%m-%d %H:%M:%S.%f',
     '%d.%m.%Y %H:%M',

From bc35ed3fb6fcae88d59fd440b505b9e1a7cf112e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 23 Jan 2017 23:33:30 +0800
Subject: [PATCH 0069/1696] =?UTF-8?q?[nextmedia]=20Add=20support=20for=20N?=
 =?UTF-8?q?extTV=20(=E5=A3=B9=E9=9B=BB=E8=A6=96)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 ChangeLog                          |  6 ++++
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/nextmedia.py  | 54 +++++++++++++++++++++++++++++-
 3 files changed, 60 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index beea17e54..ba2f5cffc 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
++ [nextmedia] Add support for NextTV (壹電視)
+
+
 version 2017.01.22
 
 Extractors
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index cfddf5b92..e23b5d0f6 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -598,6 +598,7 @@ from .nextmedia import (
     NextMediaIE,
     NextMediaActionNewsIE,
     AppleDailyIE,
+    NextTVIE,
 )
 from .nfb import NFBIE
 from .nfl import NFLIE
diff --git a/youtube_dl/extractor/nextmedia.py b/youtube_dl/extractor/nextmedia.py
index 626ed8b49..680f03aad 100644
--- a/youtube_dl/extractor/nextmedia.py
+++ b/youtube_dl/extractor/nextmedia.py
@@ -3,7 +3,14 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..compat import compat_urlparse
-from ..utils import parse_iso8601
+from ..utils import (
+    clean_html,
+    get_element_by_class,
+    int_or_none,
+    parse_iso8601,
+    remove_start,
+    unified_timestamp,
+)
 
 
 class NextMediaIE(InfoExtractor):
@@ -184,3 +191,48 @@ class AppleDailyIE(NextMediaIE):
 
     def _fetch_description(self, page):
         return self._html_search_meta('description', page, 'news description')
+
+
+class NextTVIE(InfoExtractor):
+    IE_DESC = '壹電視'
+    _VALID_URL = r'https?://(?:www\.)?nexttv\.com\.tw/(?:[^/]+/)+(?P<id>\d+)'
+
+    _TEST = {
+        'url': 'http://www.nexttv.com.tw/news/realtime/politics/11779671',
+        'info_dict': {
+            'id': '11779671',
+            'ext': 'mp4',
+            'title': '「超收稅」近4千億！　藍議員籲發消費券',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'timestamp': 1484825400,
+            'upload_date': '20170119',
+            'view_count': int,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(
+            r'<h1[^>]*>([^<]+)</h1>', webpage, 'title')
+
+        data = self._hidden_inputs(webpage)
+
+        video_url = data['ntt-vod-src-detailview']
+
+        date_str = get_element_by_class('date', webpage)
+        timestamp = unified_timestamp(date_str + '+0800') if date_str else None
+
+        view_count = int_or_none(remove_start(
+            clean_html(get_element_by_class('click', webpage)), '點閱：'))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': video_url,
+            'thumbnail': data.get('ntt-vod-img-src'),
+            'timestamp': timestamp,
+            'view_count': view_count,
+        }

From b494d6856c55bd351107fd7266f8ac2eeaee341f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 24 Jan 2017 02:50:49 +0700
Subject: [PATCH 0070/1696] [pluralsight] Fix extraction (closes #11820)

---
 youtube_dl/extractor/pluralsight.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 0ffd41ecd..5c798e874 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -157,13 +157,10 @@ class PluralsightIE(PluralsightBaseIE):
 
         display_id = '%s-%s' % (name, clip_id)
 
-        parsed_url = compat_urlparse.urlparse(url)
-
-        payload_url = compat_urlparse.urlunparse(parsed_url._replace(
-            netloc='app.pluralsight.com', path='player/api/v1/payload'))
-
         course = self._download_json(
-            payload_url, display_id, headers={'Referer': url})['payload']['course']
+            'https://app.pluralsight.com/player/user/api/v1/player/payload',
+            display_id, data=urlencode_postdata({'courseId': course_name}),
+            headers={'Referer': url})
 
         collection = course['modules']
 

From ee4c091ce5bb3732c3016410230f45f2283e5055 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 24 Jan 2017 02:56:19 +0700
Subject: [PATCH 0071/1696] [ChangeLog] Actualize

---
 ChangeLog | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index ba2f5cffc..406301549 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,11 @@
 version <unreleased>
 
 Extractors
+* [pluralsight] Fix extraction (#11820)
 + [nextmedia] Add support for NextTV (壹電視)
+* [24video] Fix extraction (#11811)
+* [youtube:playlist] Fix nonexistent and private playlist detection (#11604)
++ [chirbit] Extract uploader (#11809)
 
 
 version 2017.01.22

From c3a65c3de0667b8de4af8fdc8c1eb04a1498e104 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 24 Jan 2017 02:58:37 +0700
Subject: [PATCH 0072/1696] release 2017.01.24

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 1 +
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 30cc27c7b..f771d72c0 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.01.22*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.01.22**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.01.24*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.01.24**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.01.22
+[debug] youtube-dl version 2017.01.24
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 406301549..4bc30cff7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.01.24
 
 Extractors
 * [pluralsight] Fix extraction (#11820)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index b906d443a..2d28b3f72 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -485,6 +485,7 @@
  - **Newstube**
  - **NextMedia**: 蘋果日報
  - **NextMediaActionNews**: 蘋果日報 - 動新聞
+ - **NextTV**: 壹電視
  - **nfb**: National Film Board of Canada
  - **nfl.com**
  - **NhkVod**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 9466c9637..8a66c2fb9 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.01.22'
+__version__ = '2017.01.24'

From d61aa5eb37244a04caa09f1f238a4f81366c109b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 24 Jan 2017 22:46:40 +0700
Subject: [PATCH 0073/1696] [vimeo:review] Fix config URL extraction (closes
 #11821)

---
 youtube_dl/extractor/vimeo.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index a6bbd4c05..c12eeadd4 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -884,10 +884,14 @@ class VimeoReviewIE(VimeoBaseInfoExtractor):
 
     def _get_config_url(self, webpage_url, video_id, video_password_verified=False):
         webpage = self._download_webpage(webpage_url, video_id)
-        data = self._parse_json(self._search_regex(
-            r'window\s*=\s*_extend\(window,\s*({.+?})\);', webpage, 'data',
-            default=NO_DEFAULT if video_password_verified else '{}'), video_id)
-        config_url = data.get('vimeo_esi', {}).get('config', {}).get('configUrl')
+        config_url = self._html_search_regex(
+            r'data-config-url=(["\'])(?P<url>(?:(?!\1).)+)\1', webpage,
+            'config URL', default=None, group='url')
+        if not config_url:
+            data = self._parse_json(self._search_regex(
+                r'window\s*=\s*_extend\(window,\s*({.+?})\);', webpage, 'data',
+                default=NO_DEFAULT if video_password_verified else '{}'), video_id)
+            config_url = data.get('vimeo_esi', {}).get('config', {}).get('configUrl')
         if config_url is None:
             self._verify_video_password(webpage_url, video_id, webpage)
             config_url = self._get_config_url(

From 74af9c700d308e3638db0ff2e4510770f9daf31c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 24 Jan 2017 22:55:49 +0700
Subject: [PATCH 0074/1696] [konserthusetplay] Add support for hls formats
 (closes #11823)

---
 youtube_dl/extractor/konserthusetplay.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/konserthusetplay.py b/youtube_dl/extractor/konserthusetplay.py
index 55291c66f..7e6ea9696 100644
--- a/youtube_dl/extractor/konserthusetplay.py
+++ b/youtube_dl/extractor/konserthusetplay.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
+    determine_ext,
     float_or_none,
     int_or_none,
 )
@@ -42,12 +43,18 @@ class KonserthusetPlayIE(InfoExtractor):
         player_config = media['playerconfig']
         playlist = player_config['playlist']
 
-        source = next(f for f in playlist if f.get('bitrates'))
+        source = next(f for f in playlist if f.get('bitrates') or f.get('provider'))
 
         FORMAT_ID_REGEX = r'_([^_]+)_h264m\.mp4'
 
         formats = []
 
+        m3u8_url = source.get('url')
+        if m3u8_url and determine_ext(m3u8_url) == 'm3u8':
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                m3u8_id='hls', fatal=False))
+
         fallback_url = source.get('fallbackUrl')
         fallback_format_id = None
         if fallback_url:

From 23b35a634e06d9b92c9650b0d66a3d5d7eb03a54 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 24 Jan 2017 16:55:07 +0100
Subject: [PATCH 0075/1696] [crackle] improve extraction

- extract vtt subtitles
- extract multiple resolutions for thumbnails
- pass geo verification proxy headers
- add support for mobile urls
---
 youtube_dl/extractor/crackle.py | 53 ++++++++++++++++++++++++++-------
 1 file changed, 43 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/crackle.py b/youtube_dl/extractor/crackle.py
index 25c5e7d04..377fb45e9 100644
--- a/youtube_dl/extractor/crackle.py
+++ b/youtube_dl/extractor/crackle.py
@@ -6,7 +6,7 @@ from ..utils import int_or_none
 
 
 class CrackleIE(InfoExtractor):
-    _VALID_URL = r'(?:crackle:|https?://(?:www\.)?crackle\.com/(?:playlist/\d+/|(?:[^/]+/)+))(?P<id>\d+)'
+    _VALID_URL = r'(?:crackle:|https?://(?:(?:www|m)\.)?crackle\.com/(?:playlist/\d+/|(?:[^/]+/)+))(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.crackle.com/comedians-in-cars-getting-coffee/2498934',
         'info_dict': {
@@ -31,8 +31,32 @@ class CrackleIE(InfoExtractor):
         }
     }
 
+    _THUMBNAIL_RES = [
+        (120, 90),
+        (208, 156),
+        (220, 124),
+        (220, 220),
+        (240, 180),
+        (250, 141),
+        (315, 236),
+        (320, 180),
+        (360, 203),
+        (400, 300),
+        (421, 316),
+        (460, 330),
+        (460, 460),
+        (462, 260),
+        (480, 270),
+        (587, 330),
+        (640, 480),
+        (700, 330),
+        (700, 394),
+        (854, 480),
+        (1024, 1024),
+        (1920, 1080),
+    ]
+
     # extracted from http://legacyweb-us.crackle.com/flash/ReferrerRedirect.ashx
-    _THUMBNAIL_TEMPLATE = 'http://images-us-am.crackle.com/%stnl_1920x1080.jpg?ts=20140107233116?c=635333335057637614'
     _MEDIA_FILE_SLOTS = {
         'c544.flv': {
             'width': 544,
@@ -61,17 +85,25 @@ class CrackleIE(InfoExtractor):
 
         item = self._download_xml(
             'http://legacyweb-us.crackle.com/app/revamp/vidwallcache.aspx?flags=-1&fm=%s' % video_id,
-            video_id).find('i')
+            video_id, headers=self.geo_verification_headers()).find('i')
         title = item.attrib['t']
 
         subtitles = {}
         formats = self._extract_m3u8_formats(
             'http://content.uplynk.com/ext/%s/%s.m3u8' % (config_doc.attrib['strUplynkOwnerId'], video_id),
             video_id, 'mp4', m3u8_id='hls', fatal=None)
-        thumbnail = None
+        thumbnails = []
         path = item.attrib.get('p')
         if path:
-            thumbnail = self._THUMBNAIL_TEMPLATE % path
+            for width, height in self._THUMBNAIL_RES:
+                res = '%dx%d' % (width, height)
+                thumbnails.append({
+                    'id': res,
+                    'url': 'http://images-us-am.crackle.com/%stnl_%s.jpg' % (path, res),
+                    'width': width,
+                    'height': height,
+                    'resolution': res,
+                })
             http_base_url = 'http://ahttp.crackle.com/' + path
             for mfs_path, mfs_info in self._MEDIA_FILE_SLOTS.items():
                 formats.append({
@@ -86,10 +118,11 @@ class CrackleIE(InfoExtractor):
                 if locale and v:
                     if locale not in subtitles:
                         subtitles[locale] = []
-                    subtitles[locale] = [{
-                        'url': '%s/%s%s_%s.xml' % (config_doc.attrib['strSubtitleServer'], path, locale, v),
-                        'ext': 'ttml',
-                    }]
+                    for url_ext, ext in (('vtt', 'vtt'), ('xml', 'tt')):
+                        subtitles.setdefault(locale, []).append({
+                            'url': '%s/%s%s_%s.%s' % (config_doc.attrib['strSubtitleServer'], path, locale, v, url_ext),
+                            'ext': ext,
+                        })
         self._sort_formats(formats, ('width', 'height', 'tbr', 'format_id'))
 
         return {
@@ -100,7 +133,7 @@ class CrackleIE(InfoExtractor):
             'series': item.attrib.get('sn'),
             'season_number': int_or_none(item.attrib.get('se')),
             'episode_number': int_or_none(item.attrib.get('ep')),
-            'thumbnail': thumbnail,
+            'thumbnails': thumbnails,
             'subtitles': subtitles,
             'formats': formats,
         }

From af59bddc4e4a6c260e7966fe75d9d687c3b13b32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 24 Jan 2017 23:02:20 +0700
Subject: [PATCH 0076/1696] [konserthusetplay] Extract subtitles (#11823)

---
 youtube_dl/extractor/konserthusetplay.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/konserthusetplay.py b/youtube_dl/extractor/konserthusetplay.py
index 7e6ea9696..3ae2aa317 100644
--- a/youtube_dl/extractor/konserthusetplay.py
+++ b/youtube_dl/extractor/konserthusetplay.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     determine_ext,
     float_or_none,
@@ -104,6 +105,13 @@ class KonserthusetPlayIE(InfoExtractor):
         thumbnail = media.get('image')
         duration = float_or_none(media.get('duration'), 1000)
 
+        subtitles = {}
+        captions = source.get('captionsAvailableLanguages')
+        if isinstance(captions, dict):
+            for lang, subtitle_url in captions.items():
+                if lang != 'none' and isinstance(subtitle_url, compat_str):
+                    subtitles.setdefault(lang, []).append({'url': subtitle_url})
+
         return {
             'id': video_id,
             'title': title,
@@ -111,4 +119,5 @@ class KonserthusetPlayIE(InfoExtractor):
             'thumbnail': thumbnail,
             'duration': duration,
             'formats': formats,
+            'subtitles': subtitles,
         }

From c60089c0222433775dcc1305d85b42fc6158c8df Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 25 Jan 2017 07:38:17 +0100
Subject: [PATCH 0077/1696] [afreecatv:global] Add new extractor(closes #11807)

---
 youtube_dl/extractor/afreecatv.py  | 92 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  5 +-
 2 files changed, 96 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py
index 75b366993..4f6cdb8a2 100644
--- a/youtube_dl/extractor/afreecatv.py
+++ b/youtube_dl/extractor/afreecatv.py
@@ -18,6 +18,7 @@ from ..utils import (
 
 
 class AfreecaTVIE(InfoExtractor):
+    IE_NAME = 'afreecatv'
     IE_DESC = 'afreecatv.com'
     _VALID_URL = r'''(?x)
                     https?://
@@ -143,3 +144,94 @@ class AfreecaTVIE(InfoExtractor):
                 expected=True)
 
         return info
+
+
+class AfreecaTVGlobalIE(AfreecaTVIE):
+    IE_NAME = 'afreecatv:global'
+    _VALID_URL = r'https?://(?:www\.)?afreeca\.tv/(?P<channel_id>\d+)(?:/v/(?P<video_id>\d+))?'
+    _TESTS = [{
+        'url': 'http://afreeca.tv/36853014/v/58301',
+        'info_dict': {
+            'id': '58301',
+            'title': 'tryhard top100',
+            'uploader_id': '36853014',
+            'uploader': 'makgi Hearthstone Live!',
+        },
+        'playlist_count': 3,
+    }]
+
+    def _real_extract(self, url):
+        channel_id, video_id = re.match(self._VALID_URL, url).groups()
+        video_type = 'video' if video_id else 'live'
+        query = {
+            'pt': 'view',
+            'bid': channel_id,
+        }
+        if video_id:
+            query['vno'] = video_id
+        video_data = self._download_json(
+            'http://api.afreeca.tv/%s/view_%s.php' % (video_type, video_type),
+            video_id or channel_id, query=query)['channel']
+
+        if video_data.get('result') != 1:
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, video_data['remsg']))
+
+        title = video_data['title']
+
+        info = {
+            'thumbnail': video_data.get('thumb'),
+            'view_count': int_or_none(video_data.get('vcnt')),
+            'age_limit': int_or_none(video_data.get('grade')),
+            'uploader_id': channel_id,
+            'uploader': video_data.get('cname'),
+        }
+
+        if video_id:
+            entries = []
+            for i, f in enumerate(video_data.get('flist', [])):
+                video_key = self.parse_video_key(f.get('key', ''))
+                f_url = f.get('file')
+                if not video_key or not f_url:
+                    continue
+                entries.append({
+                    'id': '%s_%s' % (video_id, video_key.get('part', i + 1)),
+                    'title': title,
+                    'upload_date': video_key.get('upload_date'),
+                    'duration': int_or_none(f.get('length')),
+                    'url': f_url,
+                    'protocol': 'm3u8_native',
+                    'ext': 'mp4',
+                })
+
+            info.update({
+                'id': video_id,
+                'title': title,
+                'duration': int_or_none(video_data.get('length')),
+            })
+            if len(entries) > 1:
+                info['_type'] = 'multi_video'
+                info['entries'] = entries
+            elif len(entries) == 1:
+                i = entries[0].copy()
+                i.update(info)
+                info = i
+        else:
+            formats = []
+            for s in video_data.get('strm', []):
+                s_url = s.get('purl')
+                if not s_url:
+                    continue
+                # TODO: extract rtmp formats
+                if s.get('stype') == 'HLS':
+                    formats.extend(self._extract_m3u8_formats(
+                        s_url, channel_id, 'mp4', fatal=False))
+            self._sort_formats(formats)
+
+            info.update({
+                'id': channel_id,
+                'title': self._live_title(title),
+                'is_live': True,
+                'formats': formats,
+            })
+
+        return info
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index e23b5d0f6..f09b4cf2c 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -30,7 +30,10 @@ from .aenetworks import (
     AENetworksIE,
     HistoryTopicIE,
 )
-from .afreecatv import AfreecaTVIE
+from .afreecatv import (
+    AfreecaTVIE,
+    AfreecaTVGlobalIE,
+)
 from .airmozilla import AirMozillaIE
 from .aljazeera import AlJazeeraIE
 from .alphaporno import AlphaPornoIE

From b8a03b66601f6af9e6b4009cba634dac6e0d30e6 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 25 Jan 2017 07:39:11 +0100
Subject: [PATCH 0078/1696] [srgssr] fix rts video extraction(closes #11831)

---
 youtube_dl/extractor/srgssr.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/youtube_dl/extractor/srgssr.py b/youtube_dl/extractor/srgssr.py
index 47aa887cc..319a48a7a 100644
--- a/youtube_dl/extractor/srgssr.py
+++ b/youtube_dl/extractor/srgssr.py
@@ -48,9 +48,6 @@ class SRGSSRIE(InfoExtractor):
     def _real_extract(self, url):
         bu, media_type, media_id = re.match(self._VALID_URL, url).groups()
 
-        if bu == 'rts':
-            return self.url_result('rts:%s' % media_id, 'RTS')
-
         media_data = self.get_media_data(bu, media_type, media_id)
 
         metadata = media_data['AssetMetadatas']['AssetMetadata'][0]

From 17f8deeb481a7aa3079d7e11da2c255f893b9e8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 25 Jan 2017 23:27:22 +0700
Subject: [PATCH 0079/1696] [extractor/generic] Add support for openload embeds
 (closes #11536, closes #11812)

---
 youtube_dl/extractor/generic.py  | 7 +++++++
 youtube_dl/extractor/openload.py | 8 ++++++++
 2 files changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 40201f311..a23486620 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -80,6 +80,7 @@ from .piksel import PikselIE
 from .videa import VideaIE
 from .twentymin import TwentyMinutenIE
 from .ustream import UstreamIE
+from .openload import OpenloadIE
 
 
 class GenericIE(InfoExtractor):
@@ -2431,6 +2432,12 @@ class GenericIE(InfoExtractor):
             return _playlist_from_matches(
                 twentymin_urls, ie=TwentyMinutenIE.ie_key())
 
+        # Look for Openload embeds
+        openload_urls = OpenloadIE._extract_urls(webpage)
+        if openload_urls:
+            return _playlist_from_matches(
+                openload_urls, ie=OpenloadIE.ie_key())
+
         # Looking for http://schema.org/VideoObject
         json_ld = self._search_json_ld(
             webpage, video_id, default={}, expected_type='VideoObject')
diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 3d4ad7dca..4893ade5d 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import compat_chr
 from ..utils import (
@@ -56,6 +58,12 @@ class OpenloadIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_urls(webpage):
+        return re.findall(
+            r'<iframe[^>]+src=["\']((?:https?://)?(?:openload\.(?:co|io)|oload\.tv)/embed/[a-zA-Z0-9-_]+)',
+            webpage)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage('https://openload.co/embed/%s/' % video_id, video_id)

From c1fa3f46727ccbbb75389ce82753f2e63449ece6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 25 Jan 2017 23:28:45 +0700
Subject: [PATCH 0080/1696] [openload] Fallback video extension to mp4

---
 youtube_dl/extractor/openload.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 4893ade5d..32289d897 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -101,7 +101,7 @@ class OpenloadIE(InfoExtractor):
             'thumbnail': self._og_search_thumbnail(webpage, default=None),
             'url': video_url,
             # Seems all videos have extensions in their titles
-            'ext': determine_ext(title),
+            'ext': determine_ext(title, 'mp4'),
             'subtitles': subtitles,
         }
         return info_dict

From 2c302cf66b235aed6be5786489f259c0fa993fae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 25 Jan 2017 23:33:46 +0700
Subject: [PATCH 0081/1696] [ChangeLog] Actualize

---
 ChangeLog | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 4bc30cff7..e0af3f671 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,18 @@
+version <unreleased>
+
+Extractors
++ [openload] Fallback video extension to mp4
++ [extractor/generic] Add support for Openload embeds (#11536, #11812)
+* [srgssr] Fix rts video extraction (#11831)
++ [afreecatv:global] Add support for afreeca.tv (#11807)
++ [crackle] Extract vtt subtitles
++ [crackle] Extract multiple resolutions for thumbnails
++ [crackle] Add support for mobile URLs
++ [konserthusetplay] Extract subtitles (#11823)
++ [konserthusetplay] Add support for HLS videos (#11823)
+* [vimeo:review] Fix config URL extraction (#11821)
+
+
 version 2017.01.24
 
 Extractors

From 2417d41535a907a2da05a8b6490198916279d2ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 25 Jan 2017 23:36:03 +0700
Subject: [PATCH 0082/1696] release 2017.01.25

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 3 ++-
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index f771d72c0..4d409f785 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.01.24*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.01.24**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.01.25*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.01.25**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.01.24
+[debug] youtube-dl version 2017.01.25
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index e0af3f671..ff305d7e8 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.01.25
 
 Extractors
 + [openload] Fallback video extension to mp4
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 2d28b3f72..f640cfcaa 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -33,7 +33,8 @@
  - **AdobeTVVideo**
  - **AdultSwim**
  - **aenetworks**: A+E Networks: A&E, Lifetime, History.com, FYI Network
- - **AfreecaTV**: afreecatv.com
+ - **afreecatv**: afreecatv.com
+ - **afreecatv:global**: afreecatv.com
  - **AirMozilla**
  - **AlJazeera**
  - **Allocine**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 8a66c2fb9..c23fe85de 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.01.24'
+__version__ = '2017.01.25'

From 556dbe7fe35667cb061dbf0ee84d3a065ad11055 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 26 Jan 2017 21:43:14 +0700
Subject: [PATCH 0083/1696] [youtube] Add fallback for duration extraction
 (closes #11841)

---
 youtube_dl/extractor/youtube.py | 36 ++++++++++++++++++++++++++-------
 1 file changed, 29 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 5202beb3e..630586796 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -40,6 +40,7 @@ from ..utils import (
     sanitized_Request,
     smuggle_url,
     str_to_int,
+    try_get,
     unescapeHTML,
     unified_strdate,
     unsmuggle_url,
@@ -383,6 +384,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
                 'categories': ['Science & Technology'],
                 'tags': ['youtube-dl'],
+                'duration': 10,
                 'like_count': int,
                 'dislike_count': int,
                 'start_time': 1,
@@ -402,6 +404,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'tags': ['Icona Pop i love it', 'sweden', 'pop music', 'big beat records', 'big beat', 'charli',
                          'xcx', 'charli xcx', 'girls', 'hbo', 'i love it', "i don't care", 'icona', 'pop',
                          'iconic ep', 'iconic', 'love', 'it'],
+                'duration': 180,
                 'uploader': 'Icona Pop',
                 'uploader_id': 'IconaPop',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IconaPop',
@@ -419,6 +422,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': 'Justin Timberlake - Tunnel Vision (Explicit)',
                 'alt_title': 'Tunnel Vision',
                 'description': 'md5:64249768eec3bc4276236606ea996373',
+                'duration': 419,
                 'uploader': 'justintimberlakeVEVO',
                 'uploader_id': 'justintimberlakeVEVO',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/justintimberlakeVEVO',
@@ -458,6 +462,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
                 'categories': ['Science & Technology'],
                 'tags': ['youtube-dl'],
+                'duration': 10,
                 'like_count': int,
                 'dislike_count': int,
             },
@@ -493,6 +498,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'm4a',
                 'title': 'Afrojack, Spree Wilson - The Spark ft. Spree Wilson',
                 'description': 'md5:12e7067fa6735a77bdcbb58cb1187d2d',
+                'duration': 244,
                 'uploader': 'AfrojackVEVO',
                 'uploader_id': 'AfrojackVEVO',
                 'upload_date': '20131011',
@@ -512,6 +518,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': 'Taylor Swift - Shake It Off',
                 'alt_title': 'Shake It Off',
                 'description': 'md5:95f66187cd7c8b2c13eb78e1223b63c3',
+                'duration': 242,
                 'uploader': 'TaylorSwiftVEVO',
                 'uploader_id': 'TaylorSwiftVEVO',
                 'upload_date': '20140818',
@@ -529,6 +536,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'info_dict': {
                 'id': 'T4XJQO3qol8',
                 'ext': 'mp4',
+                'duration': 219,
                 'upload_date': '20100909',
                 'uploader': 'The Amazing Atheist',
                 'uploader_id': 'TheAmazingAtheist',
@@ -546,6 +554,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
+                'duration': 142,
                 'uploader': 'The Witcher',
                 'uploader_id': 'WitcherGame',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
@@ -562,6 +571,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'title': 'Dedication To My Ex (Miss That) (Lyric Video)',
                 'description': 'md5:33765bb339e1b47e7e72b5490139bb41',
+                'duration': 247,
                 'uploader': 'LloydVEVO',
                 'uploader_id': 'LloydVEVO',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/LloydVEVO',
@@ -576,6 +586,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'info_dict': {
                 'id': '__2ABJjxzNo',
                 'ext': 'mp4',
+                'duration': 266,
                 'upload_date': '20100430',
                 'uploader_id': 'deadmau5',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
@@ -596,6 +607,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'info_dict': {
                 'id': 'lqQg6PlCWgI',
                 'ext': 'mp4',
+                'duration': 6085,
                 'upload_date': '20150827',
                 'uploader_id': 'olympic',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
@@ -615,6 +627,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'id': '_b-2C3KPAM0',
                 'ext': 'mp4',
                 'stretched_ratio': 16 / 9.,
+                'duration': 85,
                 'upload_date': '20110310',
                 'uploader_id': 'AllenMeow',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
@@ -649,6 +662,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
+                'duration': 220,
                 'upload_date': '20150625',
                 'uploader_id': 'dorappi2000',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
@@ -691,6 +705,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'ext': 'mp4',
                     'title': 'teamPGP: Rocket League Noob Stream (Main Camera)',
                     'description': 'md5:dc7872fb300e143831327f1bae3af010',
+                    'duration': 7335,
                     'upload_date': '20150721',
                     'uploader': 'Beer Games Beer',
                     'uploader_id': 'beergamesbeer',
@@ -703,6 +718,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'ext': 'mp4',
                     'title': 'teamPGP: Rocket League Noob Stream (kreestuh)',
                     'description': 'md5:dc7872fb300e143831327f1bae3af010',
+                    'duration': 7337,
                     'upload_date': '20150721',
                     'uploader': 'Beer Games Beer',
                     'uploader_id': 'beergamesbeer',
@@ -715,6 +731,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'ext': 'mp4',
                     'title': 'teamPGP: Rocket League Noob Stream (grizzle)',
                     'description': 'md5:dc7872fb300e143831327f1bae3af010',
+                    'duration': 7337,
                     'upload_date': '20150721',
                     'uploader': 'Beer Games Beer',
                     'uploader_id': 'beergamesbeer',
@@ -727,6 +744,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'ext': 'mp4',
                     'title': 'teamPGP: Rocket League Noob Stream (zim)',
                     'description': 'md5:dc7872fb300e143831327f1bae3af010',
+                    'duration': 7334,
                     'upload_date': '20150721',
                     'uploader': 'Beer Games Beer',
                     'uploader_id': 'beergamesbeer',
@@ -768,6 +786,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
                 'alt_title': 'Dark Walk',
                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
+                'duration': 133,
                 'upload_date': '20151119',
                 'uploader_id': 'IronSoulElf',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
@@ -809,10 +828,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
+                'duration': 721,
                 'upload_date': '20150127',
                 'uploader_id': 'BerkmanCenter',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
-                'uploader': 'BerkmanCenter',
+                'uploader': 'The Berkman Klein Center for Internet & Society',
                 'license': 'Creative Commons Attribution license (reuse allowed)',
             },
             'params': {
@@ -827,6 +847,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
                 'description': 'md5:dda0d780d5a6e120758d1711d062a867',
+                'duration': 4060,
                 'upload_date': '20151119',
                 'uploader': 'Bernie 2016',
                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
@@ -871,7 +892,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'id': 'iqKdEhx-dD4',
                 'ext': 'mp4',
                 'title': 'Isolation - Mind Field (Ep 1)',
-                'description': 'md5:3a72f23c086a1496c9e2c54a25fa0822',
+                'description': 'md5:8013b7ddea787342608f63a13ddc9492',
+                'duration': 2085,
                 'upload_date': '20170118',
                 'uploader': 'Vsauce',
                 'uploader_id': 'Vsauce',
@@ -1516,11 +1538,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         video_subtitles = self.extract_subtitles(video_id, video_webpage)
         automatic_captions = self.extract_automatic_captions(video_id, video_webpage)
 
-        if 'length_seconds' not in video_info:
-            self._downloader.report_warning('unable to extract video duration')
-            video_duration = None
-        else:
-            video_duration = int(compat_urllib_parse_unquote_plus(video_info['length_seconds'][0]))
+        video_duration = try_get(
+            video_info, lambda x: int_or_none(x['length_seconds'][0]))
+        if not video_duration:
+            video_duration = parse_duration(self._html_search_meta(
+                'duration', video_webpage, 'video duration'))
 
         # annotations
         video_annotations = None

From cf0cabbe5011228c78a3d88c1a1b179b10333d6c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 26 Jan 2017 21:49:34 +0700
Subject: [PATCH 0084/1696] [cmt,mtv,southpark] Add support for episode URLs
 (closes #11837)

---
 youtube_dl/extractor/cmt.py       | 2 +-
 youtube_dl/extractor/mtv.py       | 5 ++++-
 youtube_dl/extractor/southpark.py | 4 ++--
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/cmt.py b/youtube_dl/extractor/cmt.py
index f6b794fb3..e701fbeab 100644
--- a/youtube_dl/extractor/cmt.py
+++ b/youtube_dl/extractor/cmt.py
@@ -5,7 +5,7 @@ from .mtv import MTVIE
 
 class CMTIE(MTVIE):
     IE_NAME = 'cmt.com'
-    _VALID_URL = r'https?://(?:www\.)?cmt\.com/(?:videos|shows|full-episodes|video-clips)/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?cmt\.com/(?:videos|shows|(?:full-)?episodes|video-clips)/(?P<id>[^/]+)'
 
     _TESTS = [{
         'url': 'http://www.cmt.com/videos/garth-brooks/989124/the-call-featuring-trisha-yearwood.jhtml#artist=30061',
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index e48ea2481..855c3996f 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -304,7 +304,7 @@ class MTVServicesEmbeddedIE(MTVServicesInfoExtractor):
 
 class MTVIE(MTVServicesInfoExtractor):
     IE_NAME = 'mtv'
-    _VALID_URL = r'https?://(?:www\.)?mtv\.com/(?:video-clips|full-episodes)/(?P<id>[^/?#.]+)'
+    _VALID_URL = r'https?://(?:www\.)?mtv\.com/(?:video-clips|(?:full-)?episodes)/(?P<id>[^/?#.]+)'
     _FEED_URL = 'http://www.mtv.com/feeds/mrss/'
 
     _TESTS = [{
@@ -321,6 +321,9 @@ class MTVIE(MTVServicesInfoExtractor):
     }, {
         'url': 'http://www.mtv.com/full-episodes/94tujl/unlocking-the-truth-gates-of-hell-season-1-ep-101',
         'only_matching': True,
+    }, {
+        'url': 'http://www.mtv.com/episodes/g8xu7q/teen-mom-2-breaking-the-wall-season-7-ep-713',
+        'only_matching': True,
     }]
 
 
diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index 08f8c5744..d8ce416fc 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -6,7 +6,7 @@ from .mtv import MTVServicesInfoExtractor
 
 class SouthParkIE(MTVServicesInfoExtractor):
     IE_NAME = 'southpark.cc.com'
-    _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark\.cc\.com/(?:clips|full-episodes)/(?P<id>.+?)(\?|#|$))'
+    _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark\.cc\.com/(?:clips|(?:full-)?episodes)/(?P<id>.+?)(\?|#|$))'
 
     _FEED_URL = 'http://www.southparkstudios.com/feeds/video-player/mrss'
 
@@ -75,7 +75,7 @@ class SouthParkDeIE(SouthParkIE):
 
 class SouthParkNlIE(SouthParkIE):
     IE_NAME = 'southpark.nl'
-    _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark\.nl/(?:clips|full-episodes)/(?P<id>.+?)(\?|#|$))'
+    _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark\.nl/(?:clips|(?:full-)?episodes)/(?P<id>.+?)(\?|#|$))'
     _FEED_URL = 'http://www.southpark.nl/feeds/video-player/mrss/'
 
     _TESTS = [{

From 9bccdc7004f48963da9a51b6fe24a398d59da725 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 26 Jan 2017 16:06:01 +0100
Subject: [PATCH 0085/1696] [vevo] remove request to old api and catch apiv2
 errors

---
 youtube_dl/extractor/vevo.py | 267 +++++++++++------------------------
 1 file changed, 79 insertions(+), 188 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index f0a8075fb..c4e37f694 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -4,9 +4,9 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_etree_fromstring,
     compat_str,
     compat_urlparse,
+    compat_HTTPError,
 )
 from ..utils import (
     ExtractorError,
@@ -140,21 +140,6 @@ class VevoIE(VevoBaseIE):
         'url': 'http://www.vevo.com/watch/INS171400764',
         'only_matching': True,
     }]
-    _SMIL_BASE_URL = 'http://smil.lvl3.vevo.com'
-    _SOURCE_TYPES = {
-        0: 'youtube',
-        1: 'brightcove',
-        2: 'http',
-        3: 'hls_ios',
-        4: 'hls',
-        5: 'smil',  # http
-        7: 'f4m_cc',
-        8: 'f4m_ak',
-        9: 'f4m_l3',
-        10: 'ism',
-        13: 'smil',  # rtmp
-        18: 'dash',
-    }
     _VERSIONS = {
         0: 'youtube',  # only in AuthenticateVideo videoVersions
         1: 'level3',
@@ -163,41 +148,6 @@ class VevoIE(VevoBaseIE):
         4: 'amazon',
     }
 
-    def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
-        formats = []
-        els = smil.findall('.//{http://www.w3.org/2001/SMIL20/Language}video')
-        for el in els:
-            src = el.attrib['src']
-            m = re.match(r'''(?xi)
-                (?P<ext>[a-z0-9]+):
-                (?P<path>
-                    [/a-z0-9]+     # The directory and main part of the URL
-                    _(?P<tbr>[0-9]+)k
-                    _(?P<width>[0-9]+)x(?P<height>[0-9]+)
-                    _(?P<vcodec>[a-z0-9]+)
-                    _(?P<vbr>[0-9]+)
-                    _(?P<acodec>[a-z0-9]+)
-                    _(?P<abr>[0-9]+)
-                    \.[a-z0-9]+  # File extension
-                )''', src)
-            if not m:
-                continue
-
-            format_url = self._SMIL_BASE_URL + m.group('path')
-            formats.append({
-                'url': format_url,
-                'format_id': 'smil_' + m.group('tbr'),
-                'vcodec': m.group('vcodec'),
-                'acodec': m.group('acodec'),
-                'tbr': int(m.group('tbr')),
-                'vbr': int(m.group('vbr')),
-                'abr': int(m.group('abr')),
-                'ext': m.group('ext'),
-                'width': int(m.group('width')),
-                'height': int(m.group('height')),
-            })
-        return formats
-
     def _initialize_api(self, video_id):
         req = sanitized_Request(
             'http://www.vevo.com/auth', data=b'')
@@ -214,148 +164,91 @@ class VevoIE(VevoBaseIE):
         self._api_url_template = self.http_scheme() + '//apiv2.vevo.com/%s?token=' + auth_info['access_token']
 
     def _call_api(self, path, *args, **kwargs):
-        return self._download_json(self._api_url_template % path, *args, **kwargs)
+        try:
+            data = self._download_json(self._api_url_template % path, *args, **kwargs)
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError):
+                errors = self._parse_json(e.cause.read().decode(), None)['errors']
+                error_message = ', '.join([error['message'] for error in errors])
+                raise ExtractorError('%s said: %s' % (self.IE_NAME, error_message), expected=True)
+            raise
+        return data
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        json_url = 'http://api.vevo.com/VideoService/AuthenticateVideo?isrc=%s' % video_id
-        response = self._download_json(
-            json_url, video_id, 'Downloading video info',
-            'Unable to download info', fatal=False) or {}
-        video_info = response.get('video') or {}
+        self._initialize_api(video_id)
+
+        video_info = self._call_api(
+            'video/%s' % video_id, video_id, 'Downloading api video info',
+            'Failed to download video info')
+
+        video_versions = self._call_api(
+            'video/%s/streams' % video_id, video_id,
+            'Downloading video versions info',
+            'Failed to download video versions info',
+            fatal=False)
+
+        # Some videos are only available via webpage (e.g.
+        # https://github.com/rg3/youtube-dl/issues/9366)
+        if not video_versions:
+            webpage = self._download_webpage(url, video_id)
+            video_versions = self._extract_json(webpage, video_id, 'streams')[video_id][0]
+
+        uploader = None
         artist = None
         featured_artist = None
-        uploader = None
-        view_count = None
+        artists = video_info.get('artists')
+        for curr_artist in artists:
+            if curr_artist.get('role') == 'Featured':
+                featured_artist = curr_artist['name']
+            else:
+                artist = uploader = curr_artist['name']
+
         formats = []
+        for video_version in video_versions:
+            version = self._VERSIONS.get(video_version['version'])
+            version_url = video_version.get('url')
+            if not version_url:
+                continue
 
-        if not video_info:
-            try:
-                self._initialize_api(video_id)
-            except ExtractorError:
-                ytid = response.get('errorInfo', {}).get('ytid')
-                if ytid:
-                    self.report_warning(
-                        'Video is geoblocked, trying with the YouTube video %s' % ytid)
-                    return self.url_result(ytid, 'Youtube', ytid)
-
-                raise
-
-            video_info = self._call_api(
-                'video/%s' % video_id, video_id, 'Downloading api video info',
-                'Failed to download video info')
-
-            video_versions = self._call_api(
-                'video/%s/streams' % video_id, video_id,
-                'Downloading video versions info',
-                'Failed to download video versions info',
-                fatal=False)
-
-            # Some videos are only available via webpage (e.g.
-            # https://github.com/rg3/youtube-dl/issues/9366)
-            if not video_versions:
-                webpage = self._download_webpage(url, video_id)
-                video_versions = self._extract_json(webpage, video_id, 'streams')[video_id][0]
-
-            timestamp = parse_iso8601(video_info.get('releaseDate'))
-            artists = video_info.get('artists')
-            for curr_artist in artists:
-                if curr_artist.get('role') == 'Featured':
-                    featured_artist = curr_artist['name']
-                else:
-                    artist = uploader = curr_artist['name']
-            view_count = int_or_none(video_info.get('views', {}).get('total'))
-
-            for video_version in video_versions:
-                version = self._VERSIONS.get(video_version['version'])
-                version_url = video_version.get('url')
-                if not version_url:
+            if '.ism' in version_url:
+                continue
+            elif '.mpd' in version_url:
+                formats.extend(self._extract_mpd_formats(
+                    version_url, video_id, mpd_id='dash-%s' % version,
+                    note='Downloading %s MPD information' % version,
+                    errnote='Failed to download %s MPD information' % version,
+                    fatal=False))
+            elif '.m3u8' in version_url:
+                formats.extend(self._extract_m3u8_formats(
+                    version_url, video_id, 'mp4', 'm3u8_native',
+                    m3u8_id='hls-%s' % version,
+                    note='Downloading %s m3u8 information' % version,
+                    errnote='Failed to download %s m3u8 information' % version,
+                    fatal=False))
+            else:
+                m = re.search(r'''(?xi)
+                    _(?P<width>[0-9]+)x(?P<height>[0-9]+)
+                    _(?P<vcodec>[a-z0-9]+)
+                    _(?P<vbr>[0-9]+)
+                    _(?P<acodec>[a-z0-9]+)
+                    _(?P<abr>[0-9]+)
+                    \.(?P<ext>[a-z0-9]+)''', version_url)
+                if not m:
                     continue
 
-                if '.ism' in version_url:
-                    continue
-                elif '.mpd' in version_url:
-                    formats.extend(self._extract_mpd_formats(
-                        version_url, video_id, mpd_id='dash-%s' % version,
-                        note='Downloading %s MPD information' % version,
-                        errnote='Failed to download %s MPD information' % version,
-                        fatal=False))
-                elif '.m3u8' in version_url:
-                    formats.extend(self._extract_m3u8_formats(
-                        version_url, video_id, 'mp4', 'm3u8_native',
-                        m3u8_id='hls-%s' % version,
-                        note='Downloading %s m3u8 information' % version,
-                        errnote='Failed to download %s m3u8 information' % version,
-                        fatal=False))
-                else:
-                    m = re.search(r'''(?xi)
-                        _(?P<width>[0-9]+)x(?P<height>[0-9]+)
-                        _(?P<vcodec>[a-z0-9]+)
-                        _(?P<vbr>[0-9]+)
-                        _(?P<acodec>[a-z0-9]+)
-                        _(?P<abr>[0-9]+)
-                        \.(?P<ext>[a-z0-9]+)''', version_url)
-                    if not m:
-                        continue
-
-                    formats.append({
-                        'url': version_url,
-                        'format_id': 'http-%s-%s' % (version, video_version['quality']),
-                        'vcodec': m.group('vcodec'),
-                        'acodec': m.group('acodec'),
-                        'vbr': int(m.group('vbr')),
-                        'abr': int(m.group('abr')),
-                        'ext': m.group('ext'),
-                        'width': int(m.group('width')),
-                        'height': int(m.group('height')),
-                    })
-        else:
-            timestamp = int_or_none(self._search_regex(
-                r'/Date\((\d+)\)/',
-                video_info['releaseDate'], 'release date', fatal=False),
-                scale=1000)
-            artists = video_info.get('mainArtists')
-            if artists:
-                artist = uploader = artists[0]['artistName']
-
-            featured_artists = video_info.get('featuredArtists')
-            if featured_artists:
-                featured_artist = featured_artists[0]['artistName']
-
-            smil_parsed = False
-            for video_version in video_info['videoVersions']:
-                version = self._VERSIONS.get(video_version['version'])
-                if version == 'youtube':
-                    continue
-                else:
-                    source_type = self._SOURCE_TYPES.get(video_version['sourceType'])
-                    renditions = compat_etree_fromstring(video_version['data'])
-                    if source_type == 'http':
-                        for rend in renditions.findall('rendition'):
-                            attr = rend.attrib
-                            formats.append({
-                                'url': attr['url'],
-                                'format_id': 'http-%s-%s' % (version, attr['name']),
-                                'height': int_or_none(attr.get('frameheight')),
-                                'width': int_or_none(attr.get('frameWidth')),
-                                'tbr': int_or_none(attr.get('totalBitrate')),
-                                'vbr': int_or_none(attr.get('videoBitrate')),
-                                'abr': int_or_none(attr.get('audioBitrate')),
-                                'vcodec': attr.get('videoCodec'),
-                                'acodec': attr.get('audioCodec'),
-                            })
-                    elif source_type == 'hls':
-                        formats.extend(self._extract_m3u8_formats(
-                            renditions.find('rendition').attrib['url'], video_id,
-                            'mp4', 'm3u8_native', m3u8_id='hls-%s' % version,
-                            note='Downloading %s m3u8 information' % version,
-                            errnote='Failed to download %s m3u8 information' % version,
-                            fatal=False))
-                    elif source_type == 'smil' and version == 'level3' and not smil_parsed:
-                        formats.extend(self._extract_smil_formats(
-                            renditions.find('rendition').attrib['url'], video_id, False))
-                        smil_parsed = True
+                formats.append({
+                    'url': version_url,
+                    'format_id': 'http-%s-%s' % (version, video_version['quality']),
+                    'vcodec': m.group('vcodec'),
+                    'acodec': m.group('acodec'),
+                    'vbr': int(m.group('vbr')),
+                    'abr': int(m.group('abr')),
+                    'ext': m.group('ext'),
+                    'width': int(m.group('width')),
+                    'height': int(m.group('height')),
+                })
         self._sort_formats(formats)
 
         track = video_info['title']
@@ -376,17 +269,15 @@ class VevoIE(VevoBaseIE):
         else:
             age_limit = None
 
-        duration = video_info.get('duration')
-
         return {
             'id': video_id,
             'title': title,
             'formats': formats,
             'thumbnail': video_info.get('imageUrl') or video_info.get('thumbnailUrl'),
-            'timestamp': timestamp,
+            'timestamp': parse_iso8601(video_info.get('releaseDate')),
             'uploader': uploader,
-            'duration': duration,
-            'view_count': view_count,
+            'duration': int_or_none(video_info.get('duration')),
+            'view_count': int_or_none(video_info.get('views', {}).get('total')),
             'age_limit': age_limit,
             'track': track,
             'artist': uploader,

From b3277115a192b88df34692e42f62f39bd4a65bac Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 26 Jan 2017 16:14:42 +0100
Subject: [PATCH 0086/1696] [disney] Add new extractor(closes #7409)(closes
 #11801)(#4975)(#11000)

---
 youtube_dl/extractor/disney.py     | 115 +++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |   1 +
 2 files changed, 116 insertions(+)
 create mode 100644 youtube_dl/extractor/disney.py

diff --git a/youtube_dl/extractor/disney.py b/youtube_dl/extractor/disney.py
new file mode 100644
index 000000000..396873c6d
--- /dev/null
+++ b/youtube_dl/extractor/disney.py
@@ -0,0 +1,115 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    unified_strdate,
+    compat_str,
+    determine_ext,
+)
+
+
+class DisneyIE(InfoExtractor):
+    _VALID_URL = r'''(?x)
+        https?://(?P<domain>(?:[^/]+\.)?(?:disney\.[a-z]{2,3}(?:\.[a-z]{2})?|disney(?:(?:me|latino)\.com|turkiye\.com\.tr)|starwars\.com))/(?:embed/|(?:[^/]+/)+[\w-]+-)(?P<id>[a-z0-9]{24})'''
+    _TESTS = [{
+        'url': 'http://video.disney.com/watch/moana-trailer-545ed1857afee5a0ec239977',
+        'info_dict': {
+            'id': '545ed1857afee5a0ec239977',
+            'ext': 'mp4',
+            'title': 'Moana - Trailer',
+            'description': 'A fun adventure for the entire Family!  Bring home Moana on Digital HD Feb 21 & Blu-ray March 7',
+            'upload_date': '20170112',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
+    }, {
+        'url': 'http://videos.disneylatino.com/ver/spider-man-de-regreso-a-casa-primer-adelanto-543a33a1850bdcfcca13bae2',
+        'only_matching': True,
+    }, {
+        'url': 'http://video.en.disneyme.com/watch/future-worm/robo-carp-2001-544b66002aa7353cdd3f5114',
+        'only_matching': True,
+    }, {
+        'url': 'http://video.disneyturkiye.com.tr/izle/7c-7-cuceler/kimin-sesi-zaten-5456f3d015f6b36c8afdd0e2',
+        'only_matching': True,
+    }, {
+        'url': 'http://disneyjunior.disney.com/embed/546a4798ddba3d1612e4005d',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.starwars.com/embed/54690d1e6c42e5f09a0fb097',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        domain, video_id = re.match(self._VALID_URL, url).groups()
+        webpage = self._download_webpage(
+            'http://%s/embed/%s' % (domain, video_id), video_id)
+        video_data = self._parse_json(self._search_regex(
+            r'Disney\.EmbedVideo=({.+});', webpage, 'embed data'), video_id)['video']
+
+        for external in video_data.get('externals', []):
+            if external.get('source') == 'vevo':
+                return self.url_result('vevo:' + external['data_id'], 'Vevo')
+
+        title = video_data['title']
+
+        formats = []
+        for flavor in video_data.get('flavors', []):
+            flavor_format = flavor.get('format')
+            flavor_url = flavor.get('url')
+            if not flavor_url or not re.match(r'https?://', flavor_url):
+                continue
+            tbr = int_or_none(flavor.get('bitrate'))
+            if tbr == 99999:
+                formats.extend(self._extract_m3u8_formats(
+                    flavor_url, video_id, 'mp4', m3u8_id=flavor_format, fatal=False))
+                continue
+            format_id = []
+            if flavor_format:
+                format_id.append(flavor_format)
+            if tbr:
+                format_id.append(compat_str(tbr))
+            ext = determine_ext(flavor_url)
+            if flavor_format == 'applehttp' or ext == 'm3u8':
+                ext = 'mp4'
+            width = int_or_none(flavor.get('width'))
+            height = int_or_none(flavor.get('height'))
+            formats.append({
+                'format_id': '-'.join(format_id),
+                'url': flavor_url,
+                'width': width,
+                'height': height,
+                'tbr': tbr,
+                'ext': ext,
+                'vcodec': 'none' if (width == 0 and height == 0) else None,
+            })
+        self._sort_formats(formats)
+
+        subtitles = {}
+        for caption in video_data.get('captions', []):
+            caption_url = caption.get('url')
+            caption_format = caption.get('format')
+            if not caption_url or caption_format.startswith('unknown'):
+                continue
+            subtitles.setdefault(caption.get('language', 'en'), []).append({
+                'url': caption_url,
+                'ext': {
+                    'webvtt': 'vtt',
+                }.get(caption_format, caption_format),
+            })
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': video_data.get('description') or video_data.get('short_desc'),
+            'thumbnail': video_data.get('thumb') or video_data.get('thumb_secure'),
+            'duration': int_or_none(video_data.get('duration_sec')),
+            'upload_date': unified_strdate(video_data.get('publish_date')),
+            'formats': formats,
+            'subtitles': subtitles,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index f09b4cf2c..0c3e081ad 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -251,6 +251,7 @@ from .dumpert import DumpertIE
 from .defense import DefenseGouvFrIE
 from .discovery import DiscoveryIE
 from .discoverygo import DiscoveryGoIE
+from .disney import DisneyIE
 from .dispeak import DigitallySpeakingIE
 from .dropbox import DropboxIE
 from .dw import (

From c19ef77c3138ecf1ce5c988de2d94031f58b4f69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stefan=20P=C3=B6schel?= <github@basicmaster.de>
Date: Wed, 25 Jan 2017 20:44:03 +0100
Subject: [PATCH 0087/1696] [jamendo] Extract full title

---
 youtube_dl/extractor/jamendo.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/jamendo.py b/youtube_dl/extractor/jamendo.py
index 51d19e67d..3db07e79f 100644
--- a/youtube_dl/extractor/jamendo.py
+++ b/youtube_dl/extractor/jamendo.py
@@ -16,7 +16,7 @@ class JamendoIE(InfoExtractor):
             'id': '196219',
             'display_id': 'stories-from-emona-i',
             'ext': 'flac',
-            'title': 'Stories from Emona I',
+            'title': 'Maya Filipič - Stories from Emona I',
             'thumbnail': r're:^https?://.*\.jpg'
         }
     }
@@ -28,7 +28,7 @@ class JamendoIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
-        title = self._html_search_meta('name', webpage, 'title')
+        title = self._search_regex(r'<title>(.*?)\ \|\ Jamendo\ Music\ .*</title>', webpage, 'title')
 
         formats = [{
             'url': 'https://%s.jamendo.com/?trackid=%s&format=%s&from=app-97dab294'
@@ -62,21 +62,21 @@ class JamendoAlbumIE(InfoExtractor):
         'url': 'https://www.jamendo.com/album/121486/duck-on-cover',
         'info_dict': {
             'id': '121486',
-            'title': 'Duck On Cover'
+            'title': 'Shearer - Duck On Cover'
         },
         'playlist': [{
             'md5': 'e1a2fcb42bda30dfac990212924149a8',
             'info_dict': {
                 'id': '1032333',
                 'ext': 'flac',
-                'title': 'Warmachine'
+                'title': 'Shearer - Warmachine'
             }
         }, {
             'md5': '1f358d7b2f98edfe90fd55dac0799d50',
             'info_dict': {
                 'id': '1032330',
                 'ext': 'flac',
-                'title': 'Without Your Ghost'
+                'title': 'Shearer - Without Your Ghost'
             }
         }],
         'params': {
@@ -90,7 +90,7 @@ class JamendoAlbumIE(InfoExtractor):
 
         webpage = self._download_webpage(url, mobj.group('display_id'))
 
-        title = self._html_search_meta('name', webpage, 'title')
+        title = self._search_regex(r'<title>(.*?)\ \|\ Jamendo\ Music\ .*</title>', webpage, 'title')
 
         entries = [
             self.url_result(

From 15846398ca0af9154b88a69f594557568c6a4782 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 26 Jan 2017 23:23:08 +0700
Subject: [PATCH 0088/1696] [utils] Improve parse_duration

---
 test/test_utils.py  | 1 +
 youtube_dl/utils.py | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index e99bf794e..a74d59f34 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -510,6 +510,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_duration('1 hour 3 minutes'), 3780)
         self.assertEqual(parse_duration('87 Min.'), 5220)
         self.assertEqual(parse_duration('PT1H0.040S'), 3600.04)
+        self.assertEqual(parse_duration('PT00H03M30SZ'), 210)
 
     def test_fix_xml_ampersands(self):
         self.assertEqual(
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 98acc2b45..cf46711b9 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1773,7 +1773,7 @@ def parse_duration(s):
     s = s.strip()
 
     days, hours, mins, secs, ms = [None] * 5
-    m = re.match(r'(?:(?:(?:(?P<days>[0-9]+):)?(?P<hours>[0-9]+):)?(?P<mins>[0-9]+):)?(?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?$', s)
+    m = re.match(r'(?:(?:(?:(?P<days>[0-9]+):)?(?P<hours>[0-9]+):)?(?P<mins>[0-9]+):)?(?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?Z?$', s)
     if m:
         days, hours, mins, secs, ms = m.groups()
     else:
@@ -1790,11 +1790,11 @@ def parse_duration(s):
                 )?
                 (?:
                     (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*s(?:ec(?:ond)?s?)?\s*
-                )?$''', s)
+                )?Z?$''', s)
         if m:
             days, hours, mins, secs, ms = m.groups()
         else:
-            m = re.match(r'(?i)(?:(?P<hours>[0-9.]+)\s*(?:hours?)|(?P<mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*)$', s)
+            m = re.match(r'(?i)(?:(?P<hours>[0-9.]+)\s*(?:hours?)|(?P<mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*)Z?$', s)
             if m:
                 hours, mins = m.groups()
             else:

From 3cbecdd11121b9c7ff0284e481992f7230806399 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 26 Jan 2017 23:25:40 +0700
Subject: [PATCH 0089/1696] [jamendo] Improve and extract more metadata (closes
 #11836)

---
 youtube_dl/extractor/jamendo.py | 65 ++++++++++++++++++++++++---------
 1 file changed, 48 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/jamendo.py b/youtube_dl/extractor/jamendo.py
index 3db07e79f..595d7a5b7 100644
--- a/youtube_dl/extractor/jamendo.py
+++ b/youtube_dl/extractor/jamendo.py
@@ -5,9 +5,27 @@ import re
 
 from ..compat import compat_urlparse
 from .common import InfoExtractor
+from ..utils import parse_duration
 
 
-class JamendoIE(InfoExtractor):
+class JamendoBaseIE(InfoExtractor):
+    def _extract_meta(self, webpage, fatal=True):
+        title = self._og_search_title(
+            webpage, default=None) or self._search_regex(
+            r'<title>([^<]+)', webpage,
+            'title', default=None)
+        if title:
+            title = self._search_regex(
+                r'(.+?)\s*\|\s*Jamendo Music', title, 'title', default=None)
+        if not title:
+            title = self._html_search_meta(
+                'name', webpage, 'title', fatal=fatal)
+        mobj = re.search(r'(.+) - (.+)', title or '')
+        artist, second = mobj.groups() if mobj else [None] * 2
+        return title, artist, second
+
+
+class JamendoIE(JamendoBaseIE):
     _VALID_URL = r'https?://(?:www\.)?jamendo\.com/track/(?P<id>[0-9]+)/(?P<display_id>[^/?#&]+)'
     _TEST = {
         'url': 'https://www.jamendo.com/track/196219/stories-from-emona-i',
@@ -17,6 +35,9 @@ class JamendoIE(InfoExtractor):
             'display_id': 'stories-from-emona-i',
             'ext': 'flac',
             'title': 'Maya Filipič - Stories from Emona I',
+            'artist': 'Maya Filipič',
+            'track': 'Stories from Emona I',
+            'duration': 210,
             'thumbnail': r're:^https?://.*\.jpg'
         }
     }
@@ -28,7 +49,7 @@ class JamendoIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
-        title = self._search_regex(r'<title>(.*?)\ \|\ Jamendo\ Music\ .*</title>', webpage, 'title')
+        title, artist, track = self._extract_meta(webpage)
 
         formats = [{
             'url': 'https://%s.jamendo.com/?trackid=%s&format=%s&from=app-97dab294'
@@ -46,17 +67,23 @@ class JamendoIE(InfoExtractor):
 
         thumbnail = self._html_search_meta(
             'image', webpage, 'thumbnail', fatal=False)
+        duration = parse_duration(self._search_regex(
+            r'<span[^>]+itemprop=["\']duration["\'][^>]+content=["\'](.+?)["\']',
+            webpage, 'duration', fatal=False))
 
         return {
             'id': track_id,
             'display_id': display_id,
             'thumbnail': thumbnail,
             'title': title,
+            'duration': duration,
+            'artist': artist,
+            'track': track,
             'formats': formats
         }
 
 
-class JamendoAlbumIE(InfoExtractor):
+class JamendoAlbumIE(JamendoBaseIE):
     _VALID_URL = r'https?://(?:www\.)?jamendo\.com/album/(?P<id>[0-9]+)/(?P<display_id>[\w-]+)'
     _TEST = {
         'url': 'https://www.jamendo.com/album/121486/duck-on-cover',
@@ -69,14 +96,18 @@ class JamendoAlbumIE(InfoExtractor):
             'info_dict': {
                 'id': '1032333',
                 'ext': 'flac',
-                'title': 'Shearer - Warmachine'
+                'title': 'Shearer - Warmachine',
+                'artist': 'Shearer',
+                'track': 'Warmachine',
             }
         }, {
             'md5': '1f358d7b2f98edfe90fd55dac0799d50',
             'info_dict': {
                 'id': '1032330',
                 'ext': 'flac',
-                'title': 'Shearer - Without Your Ghost'
+                'title': 'Shearer - Without Your Ghost',
+                'artist': 'Shearer',
+                'track': 'Without Your Ghost',
             }
         }],
         'params': {
@@ -90,18 +121,18 @@ class JamendoAlbumIE(InfoExtractor):
 
         webpage = self._download_webpage(url, mobj.group('display_id'))
 
-        title = self._search_regex(r'<title>(.*?)\ \|\ Jamendo\ Music\ .*</title>', webpage, 'title')
+        title, artist, album = self._extract_meta(webpage, fatal=False)
 
-        entries = [
-            self.url_result(
-                compat_urlparse.urljoin(url, m.group('path')),
-                ie=JamendoIE.ie_key(),
-                video_id=self._search_regex(
-                    r'/track/(\d+)', m.group('path'),
-                    'track id', default=None))
-            for m in re.finditer(
-                r'<a[^>]+href=(["\'])(?P<path>(?:(?!\1).)+)\1[^>]+class=["\'][^>]*js-trackrow-albumpage-link',
-                webpage)
-        ]
+        entries = [{
+            '_type': 'url_transparent',
+            'url': compat_urlparse.urljoin(url, m.group('path')),
+            'ie_key': JamendoIE.ie_key(),
+            'id': self._search_regex(
+                r'/track/(\d+)', m.group('path'), 'track id', default=None),
+            'artist': artist,
+            'album': album,
+        } for m in re.finditer(
+            r'<a[^>]+href=(["\'])(?P<path>(?:(?!\1).)+)\1[^>]+class=["\'][^>]*js-trackrow-albumpage-link',
+            webpage)]
 
         return self.playlist_result(entries, album_id, title)

From 9463637887ba784e3499410ab0945dcd68002bc1 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 26 Jan 2017 18:36:28 +0100
Subject: [PATCH 0090/1696] [tva] Add new extractor(closes #11842)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/tva.py        | 54 ++++++++++++++++++++++++++++++
 2 files changed, 55 insertions(+)
 create mode 100644 youtube_dl/extractor/tva.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 0c3e081ad..81366f933 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -985,6 +985,7 @@ from .tv2 import (
 )
 from .tv3 import TV3IE
 from .tv4 import TV4IE
+from .tva import TVAIE
 from .tvanouvelles import (
     TVANouvellesIE,
     TVANouvellesArticleIE,
diff --git a/youtube_dl/extractor/tva.py b/youtube_dl/extractor/tva.py
new file mode 100644
index 000000000..3ced098f9
--- /dev/null
+++ b/youtube_dl/extractor/tva.py
@@ -0,0 +1,54 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+    smuggle_url,
+)
+
+
+class TVAIE(InfoExtractor):
+    _VALID_URL = r'https?://videos\.tva\.ca/episode/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://videos.tva.ca/episode/85538',
+        'info_dict': {
+            'id': '85538',
+            'ext': 'mp4',
+            'title': 'Épisode du 25 janvier 2017',
+            'description': 'md5:e9e7fb5532ab37984d2dc87229cadf98',
+            'upload_date': '20170126',
+            'timestamp': 1485442329,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_data = self._download_json(
+            "https://d18jmrhziuoi7p.cloudfront.net/isl/api/v1/dataservice/Items('%s')" % video_id,
+            video_id, query={
+                '$expand': 'Metadata,CustomId',
+                '$select': 'Metadata,Id,Title,ShortDescription,LongDescription,CreatedDate,CustomId,AverageUserRating,Categories,ShowName',
+                '$format': 'json',
+            })
+        metadata = video_data.get('Metadata', {})
+
+        return {
+            '_type': 'url_transparent',
+            'id': video_id,
+            'title': video_data['Title'],
+            'url': smuggle_url('ooyala:' + video_data['CustomId'], {'supportedformats': 'm3u8,hds'}),
+            'description': video_data.get('LongDescription') or video_data.get('ShortDescription'),
+            'series': video_data.get('ShowName'),
+            'episode': metadata.get('EpisodeTitle'),
+            'episode_number': int_or_none(metadata.get('EpisodeNumber')),
+            'categories': video_data.get('Categories'),
+            'average_rating': video_data.get('AverageUserRating'),
+            'timestamp': parse_iso8601(video_data.get('CreatedDate')),
+            'ie_key': 'Ooyala',
+        }

From b51a4ebed45a3944c02bb3c36778630fd9306de7 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 26 Jan 2017 19:15:43 +0100
Subject: [PATCH 0091/1696] [aenetworks] fix season episodes extraction(fixes
 #11669)

---
 youtube_dl/extractor/aenetworks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index c5e079a40..c97317400 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -87,7 +87,7 @@ class AENetworksIE(AENetworksBaseIE):
                     self._html_search_meta('aetn:SeriesTitle', webpage))
             elif url_parts_len == 2:
                 entries = []
-                for episode_item in re.findall(r'(?s)<div[^>]+class="[^"]*episode-item[^"]*"[^>]*>', webpage):
+                for episode_item in re.findall(r'(?s)<[^>]+class="[^"]*(?:episode|program)-item[^"]*"[^>]*>', webpage):
                     episode_attributes = extract_attributes(episode_item)
                     episode_url = compat_urlparse.urljoin(
                         url, episode_attributes['data-canonical'])

From 0b23c222ba099d73c287d024f45f90714c15f289 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 27 Jan 2017 21:31:26 +0700
Subject: [PATCH 0092/1696] [twitch:vod] Expand _VALID_URL (closes #11846)

---
 youtube_dl/extractor/twitch.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 6d67bda86..1ca159a4d 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -209,7 +209,7 @@ class TwitchVodIE(TwitchItemBaseIE):
     _VALID_URL = r'''(?x)
                     https?://
                         (?:
-                            (?:www\.)?twitch\.tv/[^/]+/v/|
+                            (?:www\.)?twitch\.tv/(?:[^/]+/v|videos)/|
                             player\.twitch\.tv/\?.*?\bvideo=v
                         )
                         (?P<id>\d+)
@@ -259,6 +259,9 @@ class TwitchVodIE(TwitchItemBaseIE):
     }, {
         'url': 'http://player.twitch.tv/?t=5m10s&video=v6528877',
         'only_matching': True,
+    }, {
+        'url': 'https://www.twitch.tv/videos/6528877',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 489ffc118232056537e86bd0281488e217fce7d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 27 Jan 2017 22:55:42 +0700
Subject: [PATCH 0093/1696] [soundcloud] Fix track URL extraction (closes
 #11852)

---
 youtube_dl/extractor/soundcloud.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 5a201eaa8..96bebeec5 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -173,11 +173,12 @@ class SoundcloudIE(InfoExtractor):
             })
 
         # We have to retrieve the url
-        streams_url = ('http://api.soundcloud.com/i1/tracks/{0}/streams?'
-                       'client_id={1}&secret_token={2}'.format(track_id, self._IPHONE_CLIENT_ID, secret_token))
         format_dict = self._download_json(
-            streams_url,
-            track_id, 'Downloading track url')
+            'http://api.soundcloud.com/i1/tracks/%s/streams' % track_id,
+            track_id, 'Downloading track url', query={
+                'client_id': self._CLIENT_ID,
+                'secret_token': secret_token,
+            })
 
         for key, stream_url in format_dict.items():
             if key.startswith('http'):

From 9b73471801d24cec678226c82cce9e9ece92732e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 27 Jan 2017 23:08:32 +0700
Subject: [PATCH 0094/1696] [soundcloud] Extract hls formats

---
 youtube_dl/extractor/soundcloud.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 96bebeec5..55c80e1cc 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -198,6 +198,13 @@ class SoundcloudIE(InfoExtractor):
                     'ext': 'flv',
                     'vcodec': 'none',
                 })
+            elif key.startswith('hls'):
+                m3u8_formats = self._extract_m3u8_formats(
+                    stream_url, track_id, 'mp3', entry_protocol='m3u8_native',
+                    m3u8_id=key, fatal=False)
+                for f in m3u8_formats:
+                    f['vcodec'] = 'none'
+                formats.extend(m3u8_formats)
 
             if not formats:
                 # We fallback to the stream_url in the original info, this

From 3a194cb4ecfa8c2590f22236dffc84e1b1565196 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 27 Jan 2017 23:16:30 +0700
Subject: [PATCH 0095/1696] [soundcloud] Improve formats extraction and extract
 audio bitrate

---
 youtube_dl/extractor/soundcloud.py | 48 +++++++++++++++---------------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 55c80e1cc..b3aa4ce26 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -181,46 +181,46 @@ class SoundcloudIE(InfoExtractor):
             })
 
         for key, stream_url in format_dict.items():
+            abr = int_or_none(self._search_regex(
+                r'_(\d+)_url', key, 'audio bitrate', default=None))
             if key.startswith('http'):
-                formats.append({
+                stream_formats = [{
                     'format_id': key,
                     'ext': ext,
                     'url': stream_url,
-                    'vcodec': 'none',
-                })
+                }]
             elif key.startswith('rtmp'):
                 # The url doesn't have an rtmp app, we have to extract the playpath
                 url, path = stream_url.split('mp3:', 1)
-                formats.append({
+                stream_formats = [{
                     'format_id': key,
                     'url': url,
                     'play_path': 'mp3:' + path,
                     'ext': 'flv',
-                    'vcodec': 'none',
-                })
+                }]
             elif key.startswith('hls'):
-                m3u8_formats = self._extract_m3u8_formats(
+                stream_formats = self._extract_m3u8_formats(
                     stream_url, track_id, 'mp3', entry_protocol='m3u8_native',
                     m3u8_id=key, fatal=False)
-                for f in m3u8_formats:
-                    f['vcodec'] = 'none'
-                formats.extend(m3u8_formats)
+            else:
+                continue
 
-            if not formats:
-                # We fallback to the stream_url in the original info, this
-                # cannot be always used, sometimes it can give an HTTP 404 error
-                formats.append({
-                    'format_id': 'fallback',
-                    'url': info['stream_url'] + '?client_id=' + self._CLIENT_ID,
-                    'ext': ext,
-                    'vcodec': 'none',
-                })
+            for f in stream_formats:
+                f['abr'] = abr
 
-            for f in formats:
-                if f['format_id'].startswith('http'):
-                    f['protocol'] = 'http'
-                if f['format_id'].startswith('rtmp'):
-                    f['protocol'] = 'rtmp'
+            formats.extend(stream_formats)
+
+        if not formats:
+            # We fallback to the stream_url in the original info, this
+            # cannot be always used, sometimes it can give an HTTP 404 error
+            formats.append({
+                'format_id': 'fallback',
+                'url': info['stream_url'] + '?client_id=' + self._CLIENT_ID,
+                'ext': ext,
+            })
+
+        for f in formats:
+            f['vcodec'] = 'none'
 
         self._check_formats(formats, track_id)
         self._sort_formats(formats)

From e0b6e50ccd124c6f618bf25bc94361d83cbc8b86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 27 Jan 2017 23:55:55 +0700
Subject: [PATCH 0096/1696] [crunchyroll] Improve series and season metadata
 extraction (closes #11832)

---
 youtube_dl/extractor/crunchyroll.py | 38 ++++++++++++++++++++++++++---
 1 file changed, 35 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 559044352..f811c7f33 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -166,6 +166,25 @@ class CrunchyrollIE(CrunchyrollBaseIE):
             # m3u8 download
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.crunchyroll.com/konosuba-gods-blessing-on-this-wonderful-world/episode-1-give-me-deliverance-from-this-judicial-injustice-727589',
+        'info_dict': {
+            'id': '727589',
+            'ext': 'mp4',
+            'title': "KONOSUBA -God's blessing on this wonderful world! 2 Episode 1 – Give Me Deliverance from this Judicial Injustice!",
+            'description': 'md5:cbcf05e528124b0f3a0a419fc805ea7d',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'uploader': 'Kadokawa Pictures Inc.',
+            'upload_date': '20170118',
+            'series': "KONOSUBA -God's blessing on this wonderful world!",
+            'season_number': 2,
+            'episode': 'Give Me Deliverance from this Judicial Injustice!',
+            'episode_number': 1,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
         'url': 'http://www.crunchyroll.fr/girl-friend-beta/episode-11-goodbye-la-mode-661697',
         'only_matching': True,
@@ -439,6 +458,18 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
 
         subtitles = self.extract_subtitles(video_id, webpage)
 
+        # webpage provide more accurate data than series_title from XML
+        series = self._html_search_regex(
+            r'id=["\']showmedia_about_episode_num[^>]+>\s*<a[^>]+>([^<]+)',
+            webpage, 'series', default=xpath_text(metadata, 'series_title'))
+
+        episode = xpath_text(metadata, 'episode_title')
+        episode_number = int_or_none(xpath_text(metadata, 'episode_number'))
+
+        season_number = int_or_none(self._search_regex(
+            r'(?s)<h4[^>]+id=["\']showmedia_about_episode_num[^>]+>.+?</h4>\s*<h4>\s*Season (\d+)',
+            webpage, 'season number', default=None))
+
         return {
             'id': video_id,
             'title': video_title,
@@ -446,9 +477,10 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             'thumbnail': xpath_text(metadata, 'episode_image_url'),
             'uploader': video_uploader,
             'upload_date': video_upload_date,
-            'series': xpath_text(metadata, 'series_title'),
-            'episode': xpath_text(metadata, 'episode_title'),
-            'episode_number': int_or_none(xpath_text(metadata, 'episode_number')),
+            'series': series,
+            'season_number': season_number,
+            'episode': episode,
+            'episode_number': episode_number,
             'subtitles': subtitles,
             'formats': formats,
         }

From 815d2a36d81c4cc6181d0536ce811b0e2e4a5021 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Jan 2017 00:03:21 +0700
Subject: [PATCH 0097/1696] [ChangeLog] Actualize

---
 ChangeLog | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index ff305d7e8..2c670c62e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,23 @@
+version <unreleased>
+
+Core
+* [utils] Improve parse_duration
+
+Extractors
+* [crunchyroll] Improve series and season metadata extraction (#11832)
+* [soundcloud] Improve formats extraction and extract audio bitrate
++ [soundcloud] Extract HLS formats
+* [soundcloud] Fix track URL extraction (#11852)
++ [twitch:vod] Expand URL regular expressions (#11846)
+* [aenetworks] Fix season episodes extraction (#11669)
++ [tva] Add support for videos.tva.ca (#11842)
+* [jamendo] Improve and extract more metadata (#11836)
++ [disney] Add support for Disney sites (#7409, #11801, #4975, #11000)
+* [vevo] Remove request to old API and catch API v2 errors
++ [cmt,mtv,southpark] Add support for episode URLs (#11837)
++ [youtube] Add fallback for duration extraction (#11841)
+
+
 version 2017.01.25
 
 Extractors

From d41ed6d243c2079db123963a7f65e91f24b390f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Jan 2017 00:33:55 +0700
Subject: [PATCH 0098/1696] release 2017.01.28

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 ++
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 4d409f785..693f3b745 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.01.25*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.01.25**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.01.28*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.01.28**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.01.25
+[debug] youtube-dl version 2017.01.28
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 2c670c62e..8e5a04b42 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.01.28
 
 Core
 * [utils] Improve parse_duration
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index f640cfcaa..6318a862f 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -202,6 +202,7 @@
  - **Digiteka**
  - **Discovery**
  - **DiscoveryGo**
+ - **Disney**
  - **Dotsub**
  - **DouyuTV**: 斗鱼
  - **DPlay**
@@ -785,6 +786,7 @@
  - **TV2Article**
  - **TV3**
  - **TV4**: tv4.se and tv4play.se
+ - **TVA**
  - **TVANouvelles**
  - **TVANouvellesArticle**
  - **TVC**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index c23fe85de..c22c410a8 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.01.25'
+__version__ = '2017.01.28'

From 99a0baf370c7652f6103cff71f878872229b4129 Mon Sep 17 00:00:00 2001
From: Alex Seiler <seileralex@gmail.com>
Date: Tue, 24 Jan 2017 17:42:00 +0100
Subject: [PATCH 0099/1696] [konserthusetplay] Add support for rspoplay.se

---
 youtube_dl/extractor/konserthusetplay.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/konserthusetplay.py b/youtube_dl/extractor/konserthusetplay.py
index 3ae2aa317..c11cbcf47 100644
--- a/youtube_dl/extractor/konserthusetplay.py
+++ b/youtube_dl/extractor/konserthusetplay.py
@@ -11,22 +11,22 @@ from ..utils import (
 
 
 class KonserthusetPlayIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?konserthusetplay\.se/\?.*\bm=(?P<id>[^&]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?(?:konserthusetplay|rspoplay)\.se/\?.*\bm=(?P<id>[^&]+)'
+    _TESTS = [{
         'url': 'http://www.konserthusetplay.se/?m=CKDDnlCY-dhWAAqiMERd-A',
+        'md5': 'e3fd47bf44e864bd23c08e487abe1967',
         'info_dict': {
             'id': 'CKDDnlCY-dhWAAqiMERd-A',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Orkesterns instrument: Valthornen',
             'description': 'md5:f10e1f0030202020396a4d712d2fa827',
             'thumbnail': 're:^https?://.*$',
-            'duration': 398.8,
+            'duration': 398.76,
         },
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        },
-    }
+    }, {
+        'url': 'http://rspoplay.se/?m=elWuEH34SMKvaO4wO_cHBw',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 26e40542dd730b1a18f9d7eebe241972b77810cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Jan 2017 17:50:56 +0700
Subject: [PATCH 0100/1696] [kaltura] Improve uploader_id extraction

---
 youtube_dl/extractor/kaltura.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index c0ddad6f9..a57d913af 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -319,6 +319,6 @@ class KalturaIE(InfoExtractor):
             'thumbnail': info.get('thumbnailUrl'),
             'duration': info.get('duration'),
             'timestamp': info.get('createdAt'),
-            'uploader_id': info.get('userId'),
+            'uploader_id': info.get('userId') if info.get('userId') != 'None' else None,
             'view_count': info.get('plays'),
         }

From ab6f6aee78fc4757fcb65bd8f4699aaf9feac3a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Jan 2017 18:27:42 +0700
Subject: [PATCH 0101/1696] [kaltura] Add fallback for fileExt

---
 youtube_dl/extractor/kaltura.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index a57d913af..5ef382f9f 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -266,9 +266,12 @@ class KalturaIE(InfoExtractor):
             # skip for now.
             if f.get('fileExt') == 'chun':
                 continue
-            if not f.get('fileExt') and f.get('containerFormat') == 'qt':
+            if not f.get('fileExt'):
                 # QT indicates QuickTime; some videos have broken fileExt
-                f['fileExt'] = 'mov'
+                if f.get('containerFormat') == 'qt':
+                    f['fileExt'] = 'mov'
+                else:
+                    f['fileExt'] = 'mp4'
             video_url = sign_url(
                 '%s/flavorId/%s' % (data_url, f['id']))
             # audio-only has no videoCodecId (e.g. kaltura:1926081:0_c03e1b5g

From b92d3c5343536eb0a865afa79e3787fc384ec0ec Mon Sep 17 00:00:00 2001
From: ping <liping.ong@gmail.com>
Date: Tue, 24 Jan 2017 13:52:17 +0800
Subject: [PATCH 0102/1696] [vlive] Add support for channels

---
 youtube_dl/extractor/extractors.py |  5 ++-
 youtube_dl/extractor/vlive.py      | 68 ++++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 81366f933..c781c9b87 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1129,7 +1129,10 @@ from .vk import (
     VKUserVideosIE,
     VKWallPostIE,
 )
-from .vlive import VLiveIE
+from .vlive import (
+    VLiveIE,
+    VLiveChannelIE
+)
 from .vodlocker import VodlockerIE
 from .vodplatform import VODPlatformIE
 from .voicerepublic import VoiceRepublicIE
diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index 540246c79..70bab1f04 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -2,6 +2,8 @@
 from __future__ import unicode_literals
 
 import re
+import time
+import itertools
 
 from .common import InfoExtractor
 from ..utils import (
@@ -169,3 +171,69 @@ class VLiveIE(InfoExtractor):
             'subtitles': subtitles,
         })
         return info
+
+
+class VLiveChannelIE(InfoExtractor):
+    IE_NAME = 'vlive:channel'
+    _VALID_URL = r'https?://channels\.vlive\.tv/(?P<id>[0-9A-Z]+)/video'
+    _TEST = {
+        'url': 'http://channels.vlive.tv/FCD4B/video',
+        'info_dict': {
+            'id': 'FCD4B',
+            'title': 'MAMAMOO',
+        },
+        'playlist_mincount': 110
+    }
+    _APP_ID = '8c6cc7b45d2568fb668be6e05b6e5a3b'
+
+    def _real_extract(self, url):
+        channel_code = self._match_id(url)
+
+        webpage = self._download_webpage(
+            'http://channels.vlive.tv/%s/video' % channel_code, channel_code)
+        app_js_url = self._search_regex(
+            r'(http[^\'\"\s]+app\.js)', webpage, 'app js', default='')
+
+        if app_js_url:
+            app_js = self._download_webpage(app_js_url, channel_code, 'app js')
+            app_id = self._search_regex(
+                r'Global\.VFAN_APP_ID\s*=\s*[\'"]([^\'"]+)[\'"]',
+                app_js, 'app id', default=self._APP_ID)
+        else:
+            app_id = self._APP_ID
+
+        channel_info = self._download_json(
+            'http://api.vfan.vlive.tv/vproxy/channelplus/decodeChannelCode',
+            channel_code, note='decode channel code',
+            query={'app_id': app_id, 'channelCode': channel_code, '_': int(time.time())})
+
+        channel_seq = channel_info['result']['channelSeq']
+        channel_name = None
+        entries = []
+
+        for page_num in itertools.count(1):
+            video_list = self._download_json(
+                'http://api.vfan.vlive.tv/vproxy/channelplus/getChannelVideoList',
+                channel_code, note='channel list %d' % page_num,
+                query={
+                    'app_id': app_id,
+                    'channelSeq': channel_seq,
+                    'maxNumOfRows': 1000,
+                    '_': int(time.time()),
+                    'pageNo': page_num
+                }
+            )
+            if not channel_name:
+                channel_name = video_list['result']['channelInfo']['channelName']
+
+            if not video_list['result'].get('videoList'):
+                break
+
+            for video in video_list['result']['videoList']:
+                video_id = str(video['videoSeq'])
+                entries.append(
+                    self.url_result(
+                        'http://www.vlive.tv/video/%s' % video_id, 'Vlive', video_id))
+
+        return self.playlist_result(
+            entries, channel_code, channel_name)

From 661cc229d2e885dd303d26535477c8905805ddf1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Jan 2017 19:08:01 +0700
Subject: [PATCH 0103/1696] [vlive:channel] Improve

---
 youtube_dl/extractor/vlive.py | 62 ++++++++++++++++++++++++-----------
 1 file changed, 43 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index 70bab1f04..b9718901b 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -6,15 +6,19 @@ import time
 import itertools
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_parse_urlencode,
+    compat_str,
+)
 from ..utils import (
     dict_get,
     ExtractorError,
     float_or_none,
     int_or_none,
     remove_start,
+    try_get,
     urlencode_postdata,
 )
-from ..compat import compat_urllib_parse_urlencode
 
 
 class VLiveIE(InfoExtractor):
@@ -175,9 +179,9 @@ class VLiveIE(InfoExtractor):
 
 class VLiveChannelIE(InfoExtractor):
     IE_NAME = 'vlive:channel'
-    _VALID_URL = r'https?://channels\.vlive\.tv/(?P<id>[0-9A-Z]+)/video'
+    _VALID_URL = r'https?://channels\.vlive\.tv/(?P<id>[0-9A-Z]+)'
     _TEST = {
-        'url': 'http://channels.vlive.tv/FCD4B/video',
+        'url': 'http://channels.vlive.tv/FCD4B',
         'info_dict': {
             'id': 'FCD4B',
             'title': 'MAMAMOO',
@@ -191,21 +195,31 @@ class VLiveChannelIE(InfoExtractor):
 
         webpage = self._download_webpage(
             'http://channels.vlive.tv/%s/video' % channel_code, channel_code)
+
+        app_id = None
+
         app_js_url = self._search_regex(
-            r'(http[^\'\"\s]+app\.js)', webpage, 'app js', default='')
+            r'<script[^>]+src=(["\'])(?P<url>http.+?/app\.js.*?)\1',
+            webpage, 'app js', default=None, group='url')
 
         if app_js_url:
-            app_js = self._download_webpage(app_js_url, channel_code, 'app js')
-            app_id = self._search_regex(
-                r'Global\.VFAN_APP_ID\s*=\s*[\'"]([^\'"]+)[\'"]',
-                app_js, 'app id', default=self._APP_ID)
-        else:
-            app_id = self._APP_ID
+            app_js = self._download_webpage(
+                app_js_url, channel_code, 'Downloading app JS', fatal=False)
+            if app_js:
+                app_id = self._search_regex(
+                    r'Global\.VFAN_APP_ID\s*=\s*[\'"]([^\'"]+)[\'"]',
+                    app_js, 'app id', default=None)
+
+        app_id = app_id or self._APP_ID
 
         channel_info = self._download_json(
             'http://api.vfan.vlive.tv/vproxy/channelplus/decodeChannelCode',
-            channel_code, note='decode channel code',
-            query={'app_id': app_id, 'channelCode': channel_code, '_': int(time.time())})
+            channel_code, note='Downloading decode channel code',
+            query={
+                'app_id': app_id,
+                'channelCode': channel_code,
+                '_': int(time.time())
+            })
 
         channel_seq = channel_info['result']['channelSeq']
         channel_name = None
@@ -214,7 +228,7 @@ class VLiveChannelIE(InfoExtractor):
         for page_num in itertools.count(1):
             video_list = self._download_json(
                 'http://api.vfan.vlive.tv/vproxy/channelplus/getChannelVideoList',
-                channel_code, note='channel list %d' % page_num,
+                channel_code, note='Downloading channel list page #%d' % page_num,
                 query={
                     'app_id': app_id,
                     'channelSeq': channel_seq,
@@ -223,17 +237,27 @@ class VLiveChannelIE(InfoExtractor):
                     'pageNo': page_num
                 }
             )
-            if not channel_name:
-                channel_name = video_list['result']['channelInfo']['channelName']
 
-            if not video_list['result'].get('videoList'):
+            if not channel_name:
+                channel_name = try_get(
+                    video_list,
+                    lambda x: x['result']['channelInfo']['channelName'],
+                    compat_str)
+
+            videos = try_get(
+                video_list, lambda x: x['result']['videoList'], list)
+            if not videos:
                 break
 
-            for video in video_list['result']['videoList']:
-                video_id = str(video['videoSeq'])
+            for video in videos:
+                video_id = video.get('videoSeq')
+                if not video_id:
+                    continue
+                video_id = compat_str(video_id)
                 entries.append(
                     self.url_result(
-                        'http://www.vlive.tv/video/%s' % video_id, 'Vlive', video_id))
+                        'http://www.vlive.tv/video/%s' % video_id,
+                        ie=VLiveIE.ie_key(), video_id=video_id))
 
         return self.playlist_result(
             entries, channel_code, channel_name)

From 008f247077027f10c947060d8f3bb886c9af6aa7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Jan 2017 20:29:22 +0700
Subject: [PATCH 0104/1696] [mtv81] Add extractor (closes #7619)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/mtv.py        | 29 +++++++++++++++++++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index c781c9b87..915291f74 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -553,6 +553,7 @@ from .mtv import (
     MTVVideoIE,
     MTVServicesEmbeddedIE,
     MTVDEIE,
+    MTV81IE,
 )
 from .muenchentv import MuenchenTVIE
 from .musicplayon import MusicPlayOnIE
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 855c3996f..8acea1461 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -327,6 +327,35 @@ class MTVIE(MTVServicesInfoExtractor):
     }]
 
 
+class MTV81IE(InfoExtractor):
+    IE_NAME = 'mtv81'
+    _VALID_URL = r'https?://(?:www\.)?mtv81\.com/videos/(?P<id>[^/?#.]+)'
+
+    _TEST = {
+        'url': 'http://www.mtv81.com/videos/artist-to-watch/the-godfather-of-japanese-hip-hop-segment-1/',
+        'md5': '1edbcdf1e7628e414a8c5dcebca3d32b',
+        'info_dict': {
+            'id': '5e14040d-18a4-47c4-a582-43ff602de88e',
+            'ext': 'mp4',
+            'title': 'Unlocking The Truth|July 18, 2016|1|101|Trailer',
+            'description': '"Unlocking the Truth" premieres August 17th at 11/10c.',
+            'timestamp': 1468846800,
+            'upload_date': '20160718',
+        },
+    }
+
+    def _extract_mgid(self, webpage):
+        return self._search_regex(
+            r'getTheVideo\((["\'])(?P<id>mgid:.+?)\1', webpage,
+            'mgid', group='id')
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        mgid = self._extract_mgid(webpage)
+        return self.url_result('http://media.mtvnservices.com/embed/%s' % mgid)
+
+
 class MTVVideoIE(MTVServicesInfoExtractor):
     IE_NAME = 'mtv:video'
     _VALID_URL = r'''(?x)^https?://

From 732fb3f8be6cca47c60b3befee83ee9b5002984d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Jan 2017 21:06:22 +0700
Subject: [PATCH 0105/1696] [options] Move --abort-on-unavailable-fragment to
 download section

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 0d2ce8d15..5e2936555 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -450,7 +450,7 @@ def parseOpts(overrideArguments=None):
         '--skip-unavailable-fragments',
         action='store_true', dest='skip_unavailable_fragments', default=True,
         help='Skip unavailable fragments (DASH and hlsnative only)')
-    general.add_option(
+    downloader.add_option(
         '--abort-on-unavailable-fragment',
         action='store_false', dest='skip_unavailable_fragments',
         help='Abort downloading when some fragment is not available')

From a71b8d3b3bb399acb82f3ccfbd8a19d411848db4 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 28 Jan 2017 15:51:52 +0100
Subject: [PATCH 0106/1696] [itv] Add new extractor(closes #9240)

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/itv.py        | 181 +++++++++++++++++++++++++++++
 2 files changed, 182 insertions(+)
 create mode 100644 youtube_dl/extractor/itv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 915291f74..086a2296d 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -415,6 +415,7 @@ from .internetvideoarchive import InternetVideoArchiveIE
 from .iprima import IPrimaIE
 from .iqiyi import IqiyiIE
 from .ir90tv import Ir90TvIE
+from .itv import ITVIE
 from .ivi import (
     IviIE,
     IviCompilationIE
diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py
new file mode 100644
index 000000000..d029609c3
--- /dev/null
+++ b/youtube_dl/extractor/itv.py
@@ -0,0 +1,181 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import uuid
+import xml.etree.ElementTree as etree
+import json
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    extract_attributes,
+    xpath_with_ns,
+    xpath_element,
+    xpath_text,
+    int_or_none,
+    parse_duration,
+    ExtractorError,
+    determine_ext,
+)
+
+
+class ITVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?itv\.com/hub/[^/]+/(?P<id>[0-9a-z]+)'
+    _TEST = {
+        'url': 'http://www.itv.com/hub/mr-bean-animated-series/2a2936a0053',
+        'info_dict': {
+            'id': '2a2936a0053',
+            'ext': 'flv',
+            'title': 'Home Movie',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        params = extract_attributes(self._search_regex(
+            r'(?s)(<[^>]+id="video"[^>]*>)', webpage, 'params'))
+
+        ns_map = {
+            'soapenv': 'http://schemas.xmlsoap.org/soap/envelope/',
+            'tem': 'http://tempuri.org/',
+            'itv': 'http://schemas.datacontract.org/2004/07/Itv.BB.Mercury.Common.Types',
+            'com': 'http://schemas.itv.com/2009/05/Common',
+        }
+        for ns, full_ns in ns_map.items():
+            etree.register_namespace(ns, full_ns)
+
+        def _add_ns(name):
+            return xpath_with_ns(name, ns_map)
+
+        def _add_sub_element(element, name):
+            return etree.SubElement(element, _add_ns(name))
+
+        req_env = etree.Element(_add_ns('soapenv:Envelope'))
+        _add_sub_element(req_env, 'soapenv:Header')
+        body = _add_sub_element(req_env, 'soapenv:Body')
+        get_playlist = _add_sub_element(body, ('tem:GetPlaylist'))
+        request = _add_sub_element(get_playlist, 'tem:request')
+        _add_sub_element(request, 'itv:ProductionId').text = params['data-video-id']
+        _add_sub_element(request, 'itv:RequestGuid').text = compat_str(uuid.uuid4()).upper()
+        vodcrid = _add_sub_element(request, 'itv:Vodcrid')
+        _add_sub_element(vodcrid, 'com:Id')
+        _add_sub_element(request, 'itv:Partition')
+        user_info = _add_sub_element(get_playlist, 'tem:userInfo')
+        _add_sub_element(user_info, 'itv:Broadcaster').text = 'Itv'
+        _add_sub_element(user_info, 'itv:DM')
+        _add_sub_element(user_info, 'itv:RevenueScienceValue')
+        _add_sub_element(user_info, 'itv:SessionId')
+        _add_sub_element(user_info, 'itv:SsoToken')
+        _add_sub_element(user_info, 'itv:UserToken')
+        site_info = _add_sub_element(get_playlist, 'tem:siteInfo')
+        _add_sub_element(site_info, 'itv:AdvertisingRestriction').text = 'None'
+        _add_sub_element(site_info, 'itv:AdvertisingSite').text = 'ITV'
+        _add_sub_element(site_info, 'itv:AdvertisingType').text = 'Any'
+        _add_sub_element(site_info, 'itv:Area').text = 'ITVPLAYER.VIDEO'
+        _add_sub_element(site_info, 'itv:Category')
+        _add_sub_element(site_info, 'itv:Platform').text = 'DotCom'
+        _add_sub_element(site_info, 'itv:Site').text = 'ItvCom'
+        device_info = _add_sub_element(get_playlist, 'tem:deviceInfo')
+        _add_sub_element(device_info, 'itv:ScreenSize').text = 'Big'
+        player_info = _add_sub_element(get_playlist, 'tem:playerInfo')
+        _add_sub_element(player_info, 'itv:Version').text = '2'
+
+        headers = self.geo_verification_headers()
+        headers.update({
+            'Content-Type': 'text/xml; charset=utf-8',
+            'SOAPAction': 'http://tempuri.org/PlaylistService/GetPlaylist',
+        })
+        resp_env = self._download_xml(
+            params['data-playlist-url'], video_id,
+            headers=headers, data=etree.tostring(req_env))
+        playlist = xpath_element(resp_env, './/Playlist')
+        if playlist is None:
+            fault_string = xpath_text(resp_env, './/faultstring')
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, fault_string))
+        title = xpath_text(playlist, 'EpisodeTitle', fatal=True)
+        media_files = xpath_element(playlist, 'VideoEntries/Video/MediaFiles', fatal=True)
+        rtmp_url = media_files.attrib['base']
+
+        formats = []
+        for media_file in media_files.findall('MediaFile'):
+            play_path = xpath_text(media_file, 'URL')
+            if not play_path:
+                continue
+            tbr = int_or_none(media_file.get('bitrate'), 1000)
+            formats.append({
+                'format_id': 'rtmp' + ('-%d' % tbr if tbr else ''),
+                'url': rtmp_url,
+                'play_path': play_path,
+                'tbr': tbr,
+                'ext': 'flv',
+            })
+
+        ios_playlist_url = params.get('data-video-playlist')
+        hmac = params.get('data-video-hmac')
+        if ios_playlist_url and hmac:
+            headers = self.geo_verification_headers()
+            headers.update({
+                'Accept': 'application/vnd.itv.vod.playlist.v2+json',
+                'Content-Type': 'application/json',
+                'hmac': hmac.upper(),
+            })
+            ios_playlist = self._download_json(
+                ios_playlist_url, video_id, data=json.dumps({
+                    'user': {
+                        'itvUserId': '',
+                        'entitlements': [],
+                        'token': ''
+                    },
+                    'device': {
+                        'manufacturer': 'Apple',
+                        'model': 'iPad',
+                        'os': {
+                            'name': 'iPhone OS',
+                            'version': '9.3',
+                            'type': 'ios'
+                        }
+                    },
+                    'client': {
+                        'version': '4.1',
+                        'id': 'browser'
+                    },
+                    'variantAvailability': {
+                        'featureset': {
+                            'min': ['hls', 'aes'],
+                            'max': ['hls', 'aes']
+                        },
+                        'platformTag': 'mobile'
+                    }
+                }).encode(), headers=headers, fatal=False)
+            if ios_playlist:
+                video_data = ios_playlist.get('Playlist', {}).get('Video', {})
+                ios_base_url = video_data.get('Base')
+                for media_file in video_data.get('MediaFiles', []):
+                    href = media_file.get('Href')
+                    if not href:
+                        continue
+                    if ios_base_url:
+                        href = ios_base_url + href
+                    ext = determine_ext(href)
+                    if ext == 'm3u8':
+                        formats.extend(self._extract_m3u8_formats(href, video_id, 'mp4', m3u8_id='hls', fatal=False))
+                    else:
+                        formats.append({
+                            'url': href,
+                        })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'episode_title': title,
+            'episode_number': int_or_none(xpath_text(playlist, 'EpisodeNumber')),
+            'series': xpath_text(playlist, 'ProgrammeTitle'),
+            'duartion': parse_duration(xpath_text(playlist, 'Duration')),
+        }

From 24ee6b9721770b7066f10f6a6773f1ce15f82ed0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Jan 2017 22:40:07 +0700
Subject: [PATCH 0107/1696] [options] Remove experimental mark from some
 options

---
 youtube_dl/options.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 5e2936555..09c9387ca 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -216,23 +216,23 @@ def parseOpts(overrideArguments=None):
     network.add_option(
         '--source-address',
         metavar='IP', dest='source_address', default=None,
-        help='Client-side IP address to bind to (experimental)',
+        help='Client-side IP address to bind to',
     )
     network.add_option(
         '-4', '--force-ipv4',
         action='store_const', const='0.0.0.0', dest='source_address',
-        help='Make all connections via IPv4 (experimental)',
+        help='Make all connections via IPv4',
     )
     network.add_option(
         '-6', '--force-ipv6',
         action='store_const', const='::', dest='source_address',
-        help='Make all connections via IPv6 (experimental)',
+        help='Make all connections via IPv6',
     )
     network.add_option(
         '--geo-verification-proxy',
         dest='geo_verification_proxy', default=None, metavar='URL',
         help='Use this proxy to verify the IP address for some geo-restricted sites. '
-        'The default proxy specified by --proxy (or none, if the options is not present) is used for the actual downloading. (experimental)'
+        'The default proxy specified by --proxy (or none, if the options is not present) is used for the actual downloading.'
     )
     network.add_option(
         '--cn-verification-proxy',
@@ -297,7 +297,7 @@ def parseOpts(overrideArguments=None):
         '--match-filter',
         metavar='FILTER', dest='match_filter', default=None,
         help=(
-            'Generic video filter (experimental). '
+            'Generic video filter. '
             'Specify any key (see help for -o for a list of available keys) to'
             ' match if the key is present, '
             '!key to check if the key is not present,'

From f592ff98683794e0f79c96cbec67b737ae8da00c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 28 Jan 2017 17:25:15 +0100
Subject: [PATCH 0108/1696] [itv] extract subtitles

---
 youtube_dl/extractor/itv.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py
index d029609c3..d65cdc6af 100644
--- a/youtube_dl/extractor/itv.py
+++ b/youtube_dl/extractor/itv.py
@@ -20,7 +20,7 @@ from ..utils import (
 
 
 class ITVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?itv\.com/hub/[^/]+/(?P<id>[0-9a-z]+)'
+    _VALID_URL = r'https?://(?:www\.)?itv\.com/hub/[^/]+/(?P<id>[0-9a-zA-Z]+)'
     _TEST = {
         'url': 'http://www.itv.com/hub/mr-bean-animated-series/2a2936a0053',
         'info_dict': {
@@ -98,7 +98,8 @@ class ITVIE(InfoExtractor):
             fault_string = xpath_text(resp_env, './/faultstring')
             raise ExtractorError('%s said: %s' % (self.IE_NAME, fault_string))
         title = xpath_text(playlist, 'EpisodeTitle', fatal=True)
-        media_files = xpath_element(playlist, 'VideoEntries/Video/MediaFiles', fatal=True)
+        video_element = xpath_element(playlist, 'VideoEntries/Video', fatal=True)
+        media_files = xpath_element(video_element, 'MediaFiles', fatal=True)
         rtmp_url = media_files.attrib['base']
 
         formats = []
@@ -170,10 +171,21 @@ class ITVIE(InfoExtractor):
                         })
         self._sort_formats(formats)
 
+        subtitles = {}
+        for caption_url in video_element.findall('ClosedCaptioningURIs/URL'):
+            if not caption_url.text:
+                continue
+            ext = determine_ext(caption_url.text, 'ttml')
+            subtitles.setdefault('en', []).append({
+                'url': caption_url,
+                'ext': 'ttml' if ext == 'xml' else ext,
+            })
+
         return {
             'id': video_id,
             'title': title,
             'formats': formats,
+            'subtitles': subtitles,
             'episode_title': title,
             'episode_number': int_or_none(xpath_text(playlist, 'EpisodeNumber')),
             'series': xpath_text(playlist, 'ProgrammeTitle'),

From 4edeac5bfae76966fd14f636bd68850ea0403ece Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 28 Jan 2017 17:28:18 +0100
Subject: [PATCH 0109/1696] [itv] fix subtitle extraction

---
 youtube_dl/extractor/itv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py
index d65cdc6af..0328c7093 100644
--- a/youtube_dl/extractor/itv.py
+++ b/youtube_dl/extractor/itv.py
@@ -177,7 +177,7 @@ class ITVIE(InfoExtractor):
                 continue
             ext = determine_ext(caption_url.text, 'ttml')
             subtitles.setdefault('en', []).append({
-                'url': caption_url,
+                'url': caption_url.text,
                 'ext': 'ttml' if ext == 'xml' else ext,
             })
 

From acbb2374bce27eda16764b80832f88cf833a51e5 Mon Sep 17 00:00:00 2001
From: Costy Petrisor <costy.petrisor@gmail.com>
Date: Sun, 1 May 2016 12:34:11 +0000
Subject: [PATCH 0110/1696] added --autonumber-start NUMBER as a command line
 option to be able to offset the index at which autonumber formats filenames

---
 youtube_dl/YoutubeDL.py | 2 +-
 youtube_dl/__init__.py  | 1 +
 youtube_dl/options.py   | 4 ++++
 3 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 41d9a63ee..c71e94518 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -584,7 +584,7 @@ class YoutubeDL(object):
             if autonumber_size is None:
                 autonumber_size = 5
             autonumber_templ = '%0' + str(autonumber_size) + 'd'
-            template_dict['autonumber'] = autonumber_templ % self._num_downloads
+            template_dict['autonumber'] = autonumber_templ % (self.params.get('autonumber_start', 1) - 1 + self._num_downloads)
             if template_dict.get('playlist_index') is not None:
                 template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index'])
             if template_dict.get('resolution') is None:
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index dfa4ae839..577bc880f 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -321,6 +321,7 @@ def _real_main(argv=None):
         'listformats': opts.listformats,
         'outtmpl': outtmpl,
         'autonumber_size': opts.autonumber_size,
+        'autonumber_start': opts.autonumber_start,
         'restrictfilenames': opts.restrictfilenames,
         'ignoreerrors': opts.ignoreerrors,
         'force_generic_extractor': opts.force_generic_extractor,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 09c9387ca..571525434 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -663,6 +663,10 @@ def parseOpts(overrideArguments=None):
         '--autonumber-size',
         dest='autonumber_size', metavar='NUMBER',
         help='Specify the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
+    filesystem.add_option(
+        '--autonumber-start',
+        dest='autonumber_start', metavar='NUMBER', type="int", default=1,
+        help='Specify the start value for the %(autonumber)s counter. Defaults to 1.')
     filesystem.add_option(
         '--restrict-filenames',
         action='store_true', dest='restrictfilenames', default=False,

From 1a241a2d02e2507219e81d7b18c18f10937ae6e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Jan 2017 23:57:56 +0700
Subject: [PATCH 0111/1696] [options] Refactor autonumber options and add
 validation (closes #727, closes #2702, closes #9362)

---
 youtube_dl/__init__.py | 6 ++++++
 youtube_dl/options.py  | 8 ++++----
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 577bc880f..2b156342a 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -133,6 +133,12 @@ def _real_main(argv=None):
         parser.error('TV Provider account username missing\n')
     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
         parser.error('using output template conflicts with using title, video ID or auto number')
+    if opts.autonumber_size is not None:
+        if opts.autonumber_size <= 0:
+            parser.error('auto number size must be positive')
+    if opts.autonumber_start is not None:
+        if opts.autonumber_start < 0:
+            parser.error('auto number start must be positive or 0')
     if opts.usetitle and opts.useid:
         parser.error('using title conflicts with using video ID')
     if opts.username is not None and opts.password is None:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 571525434..3abf621c0 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -661,12 +661,12 @@ def parseOpts(overrideArguments=None):
         help=('Output filename template, see the "OUTPUT TEMPLATE" for all the info'))
     filesystem.add_option(
         '--autonumber-size',
-        dest='autonumber_size', metavar='NUMBER',
-        help='Specify the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
+        dest='autonumber_size', metavar='NUMBER', default=5, type=int,
+        help='Specify the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given (default is %default)')
     filesystem.add_option(
         '--autonumber-start',
-        dest='autonumber_start', metavar='NUMBER', type="int", default=1,
-        help='Specify the start value for the %(autonumber)s counter. Defaults to 1.')
+        dest='autonumber_start', metavar='NUMBER', default=1, type=int,
+        help='Specify the start value for %(autonumber)s (default is %default)')
     filesystem.add_option(
         '--restrict-filenames',
         action='store_true', dest='restrictfilenames', default=False,

From c0af11abeeaad75f4387ad77adc751715dfc0cf4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Jan 2017 00:52:23 +0700
Subject: [PATCH 0112/1696] Credit @AVerwer for showroomlive (#11458)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 9e092cccc..90ff4d789 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -191,3 +191,4 @@ Rich Leeper
 Zhong Jianxin
 Thor77
 Mattias Wadman
+Arjan Verwer

From ffcfb7e3e01cec5f5468e4639b2e4d44a0c7bfba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Jan 2017 00:54:31 +0700
Subject: [PATCH 0113/1696] Credit @costypetrisor for autonumber start (#9362)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 90ff4d789..6b6e38613 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -192,3 +192,4 @@ Zhong Jianxin
 Thor77
 Mattias Wadman
 Arjan Verwer
+Costy Petrisor

From 34cea6137e6df158c99d83fd1c1af55f94ee4a38 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Jan 2017 00:57:15 +0700
Subject: [PATCH 0114/1696] Credit @einstein95 for pornflip (#11795) and
 chaturbate fix (#11797)

---
 AUTHORS | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/AUTHORS b/AUTHORS
index 6b6e38613..600e2c55b 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -193,3 +193,5 @@ Thor77
 Mattias Wadman
 Arjan Verwer
 Costy Petrisor
+Logan B
+

From 186f4abe938e0f631b63c5dc1aaa4d622513a366 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Jan 2017 00:59:17 +0700
Subject: [PATCH 0115/1696] Credit @goggle for 20min (#11683) and azmedien
 (#11805)

---
 AUTHORS | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/AUTHORS b/AUTHORS
index 600e2c55b..b3193f7da 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -194,4 +194,4 @@ Mattias Wadman
 Arjan Verwer
 Costy Petrisor
 Logan B
-
+Alex Seiler

From f5169501d2749503e5d19f9c51937aedcce357e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Jan 2017 01:00:17 +0700
Subject: [PATCH 0116/1696] Credit @sudovijay for openload fix (#11646)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index b3193f7da..434217abb 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -195,3 +195,4 @@ Arjan Verwer
 Costy Petrisor
 Logan B
 Alex Seiler
+Vijay Singh

From 4d07b748c2e8057fa6417ab5422cb19be313d7b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Jan 2017 01:01:39 +0700
Subject: [PATCH 0117/1696] Credit @bastik for zdf fix (#11063)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 434217abb..49ffc99aa 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -196,3 +196,4 @@ Costy Petrisor
 Logan B
 Alex Seiler
 Vijay Singh
+Paul Hartmann

From 59c307891ac2cca2b2db42a534a3f4de61820450 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Jan 2017 01:02:28 +0700
Subject: [PATCH 0118/1696] Credit @RPing for cntv (#8541)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 49ffc99aa..78a7a5291 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -197,3 +197,4 @@ Logan B
 Alex Seiler
 Vijay Singh
 Paul Hartmann
+Stephen Chen

From 0842b8241d9f8984dd70266b59aa68241259401f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Jan 2017 01:03:59 +0700
Subject: [PATCH 0119/1696] Credit @fast90 for config location (#10648)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 78a7a5291..022a5de84 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -198,3 +198,4 @@ Alex Seiler
 Vijay Singh
 Paul Hartmann
 Stephen Chen
+Fabian Stahl

From 56fc078da84a7f26d8290b2b425cc2da66a5975a Mon Sep 17 00:00:00 2001
From: Andre Walker <andre.walker@booking.com>
Date: Sat, 28 Jan 2017 16:19:38 +0100
Subject: [PATCH 0120/1696] [npo] Update subtitles url

NPO websites changed the domain they used for subtitles, from
e.omroep.nl to tt888.omroep.nl.
---
 youtube_dl/extractor/npo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index c91f58461..962437145 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -241,7 +241,7 @@ class NPOIE(NPOBaseIE):
         if metadata.get('tt888') == 'ja':
             subtitles['nl'] = [{
                 'ext': 'vtt',
-                'url': 'http://e.omroep.nl/tt888/%s' % video_id,
+                'url': 'http://tt888.omroep.nl/tt888/%s' % video_id,
             }]
 
         return {

From 76aaf1faaed613569cb71e4f9aa7bd218f27c54b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Jan 2017 03:43:46 +0700
Subject: [PATCH 0121/1696] Credit @BagiraHun for videa (#11133)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 022a5de84..3ef2800c9 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -199,3 +199,4 @@ Vijay Singh
 Paul Hartmann
 Stephen Chen
 Fabian Stahl
+Bagira

From d04621daf451d601dba80dc0f2baa29e404e4ca6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Jan 2017 05:36:53 +0700
Subject: [PATCH 0122/1696] [extractor/common] Fix duration per dash segment
 (closes #11868)

---
 youtube_dl/extractor/common.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index dce8c7d0d..a3048fb59 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1751,14 +1751,16 @@ class InfoExtractor(object):
                             # Example: https://www.youtube.com/watch?v=iXZV5uAYMJI
                             # or any YouTube dashsegments video
                             fragments = []
-                            s_num = 0
-                            for segment_url in representation_ms_info['segment_urls']:
-                                s = representation_ms_info['s'][s_num]
+                            segment_index = 0
+                            timescale = representation_ms_info['timescale']
+                            for s in representation_ms_info['s']:
+                                duration = float_or_none(s['d'], timescale)
                                 for r in range(s.get('r', 0) + 1):
                                     fragments.append({
-                                        'url': segment_url,
-                                        'duration': float_or_none(s['d'], representation_ms_info['timescale']),
+                                        'url': representation_ms_info['segment_urls'][segment_index],
+                                        'duration': duration,
                                     })
+                                    segment_index += 1
                             representation_ms_info['fragments'] = fragments
                         # NB: MPD manifest may contain direct URLs to unfragmented media.
                         # No fragments key is present in this case.

From c58c2d63cbde07af66885829b7c3dbcdfbc096dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Jan 2017 05:56:43 +0700
Subject: [PATCH 0123/1696] [extractor/common] Document forgotten fragment base
 and path interfaces

---
 youtube_dl/extractor/common.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a3048fb59..fb484b6f2 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -121,9 +121,19 @@ class InfoExtractor(object):
                                  download, lower-case.
                                  "http", "https", "rtsp", "rtmp", "rtmpe",
                                  "m3u8", "m3u8_native" or "http_dash_segments".
-                    * fragments  A list of fragments of the fragmented media,
-                                 with the following entries:
-                                 * "url" (mandatory) - fragment's URL
+                    * fragment_base_url
+                                 Base URL for fragments. Each fragment's path
+                                 value (if present) will be relative to
+                                 this URL.
+                    * fragments  A list of fragments of a fragmented media.
+                                 Each fragment entry must contain either an url
+                                 or a path. If an url is present it should be
+                                 considered by a client. Otherwise both path and
+                                 fragment_base_url must be present. Here is
+                                 the list of all potential fields:
+                                 * "url" - fragment's URL
+                                 * "path" - fragment's path relative to
+                                            fragment_base_url
                                  * "duration" (optional, int or float)
                                  * "filesize" (optional, int)
                     * preference Order number of this format. If this field is

From e228616c6e73561f0c6d32d6b681bbba321c06aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Jan 2017 06:57:39 +0700
Subject: [PATCH 0124/1696] [extractor/common] Fix initialization template
 (closes #11605, closes #11825)

---
 youtube_dl/extractor/common.py | 48 ++++++++++++++++++++++++----------
 1 file changed, 34 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index fb484b6f2..5a15a9536 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1637,12 +1637,12 @@ class InfoExtractor(object):
                 segment_template = element.find(_add_ns('SegmentTemplate'))
                 if segment_template is not None:
                     extract_common(segment_template)
-                    media_template = segment_template.get('media')
-                    if media_template:
-                        ms_info['media_template'] = media_template
+                    media = segment_template.get('media')
+                    if media:
+                        ms_info['media'] = media
                     initialization = segment_template.get('initialization')
                     if initialization:
-                        ms_info['initialization_url'] = initialization
+                        ms_info['initialization'] = initialization
                     else:
                         extract_Initialization(segment_template)
             return ms_info
@@ -1686,6 +1686,7 @@ class InfoExtractor(object):
                         lang = representation_attrib.get('lang')
                         url_el = representation.find(_add_ns('BaseURL'))
                         filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength') if url_el is not None else None)
+                        bandwidth = int_or_none(representation_attrib.get('bandwidth'))
                         f = {
                             'format_id': '%s-%s' % (mpd_id, representation_id) if mpd_id else representation_id,
                             'url': base_url,
@@ -1693,7 +1694,7 @@ class InfoExtractor(object):
                             'ext': mimetype2ext(mime_type),
                             'width': int_or_none(representation_attrib.get('width')),
                             'height': int_or_none(representation_attrib.get('height')),
-                            'tbr': int_or_none(representation_attrib.get('bandwidth'), 1000),
+                            'tbr': int_or_none(bandwidth, 1000),
                             'asr': int_or_none(representation_attrib.get('audioSamplingRate')),
                             'fps': int_or_none(representation_attrib.get('frameRate')),
                             'language': lang if lang not in ('mul', 'und', 'zxx', 'mis') else None,
@@ -1702,13 +1703,32 @@ class InfoExtractor(object):
                         }
                         f.update(parse_codecs(representation_attrib.get('codecs')))
                         representation_ms_info = extract_multisegment_info(representation, adaption_set_ms_info)
-                        if 'segment_urls' not in representation_ms_info and 'media_template' in representation_ms_info:
 
-                            media_template = representation_ms_info['media_template']
-                            media_template = media_template.replace('$RepresentationID$', representation_id)
-                            media_template = re.sub(r'\$(Number|Bandwidth|Time)\$', r'%(\1)d', media_template)
-                            media_template = re.sub(r'\$(Number|Bandwidth|Time)%([^$]+)\$', r'%(\1)\2', media_template)
-                            media_template.replace('$$', '$')
+                        def prepare_template(template_name, identifiers):
+                            t = representation_ms_info[template_name]
+                            t = t.replace('$RepresentationID$', representation_id)
+                            t = re.sub(r'\$(%s)\$' % '|'.join(identifiers), r'%(\1)d', t)
+                            t = re.sub(r'\$(%s)%%([^$]+)\$' % '|'.join(identifiers), r'%(\1)\2', t)
+                            t.replace('$$', '$')
+                            return t
+
+                        # @initialization is a regular template like @media one
+                        # so it should be handled just the same way (see
+                        # https://github.com/rg3/youtube-dl/issues/11605)
+                        if 'initialization' in representation_ms_info:
+                            initialization_template = prepare_template(
+                                'initialization',
+                                # As per [1, 5.3.9.4.2, Table 15, page 54] $Number$ and
+                                # $Time$ shall not be included for @initialization thus
+                                # only $Bandwidth$ remains
+                                ('Bandwidth', ))
+                            representation_ms_info['initialization_url'] = initialization_template % {
+                                'Bandwidth': bandwidth,
+                            }
+
+                        if 'segment_urls' not in representation_ms_info and 'media' in representation_ms_info:
+
+                            media_template = prepare_template('media', ('Number', 'Bandwidth', 'Time'))
 
                             # As per [1, 5.3.9.4.4, Table 16, page 55] $Number$ and $Time$
                             # can't be used at the same time
@@ -1720,7 +1740,7 @@ class InfoExtractor(object):
                                 representation_ms_info['fragments'] = [{
                                     'url': media_template % {
                                         'Number': segment_number,
-                                        'Bandwidth': int_or_none(representation_attrib.get('bandwidth')),
+                                        'Bandwidth': bandwidth,
                                     },
                                     'duration': segment_duration,
                                 } for segment_number in range(
@@ -1738,7 +1758,7 @@ class InfoExtractor(object):
                                 def add_segment_url():
                                     segment_url = media_template % {
                                         'Time': segment_time,
-                                        'Bandwidth': int_or_none(representation_attrib.get('bandwidth')),
+                                        'Bandwidth': bandwidth,
                                         'Number': segment_number,
                                     }
                                     representation_ms_info['fragments'].append({
@@ -1780,7 +1800,7 @@ class InfoExtractor(object):
                                 'protocol': 'http_dash_segments',
                             })
                             if 'initialization_url' in representation_ms_info:
-                                initialization_url = representation_ms_info['initialization_url'].replace('$RepresentationID$', representation_id)
+                                initialization_url = representation_ms_info['initialization_url']
                                 if not f.get('url'):
                                     f['url'] = initialization_url
                                 f['fragments'].append({'url': initialization_url})

From f13da8af289d7d9365e34ef705a53ac62aa3b570 Mon Sep 17 00:00:00 2001
From: Alex Seiler <seileralex@gmail.com>
Date: Sat, 28 Jan 2017 17:52:07 +0100
Subject: [PATCH 0125/1696] [azmedien:playlist] Add support for topic and
 themen playlists

---
 youtube_dl/extractor/azmedien.py   | 39 +++++++++++++++++++++++++-----
 youtube_dl/extractor/extractors.py |  2 +-
 2 files changed, 34 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/azmedien.py b/youtube_dl/extractor/azmedien.py
index a89f71c20..cbc3ed564 100644
--- a/youtube_dl/extractor/azmedien.py
+++ b/youtube_dl/extractor/azmedien.py
@@ -5,8 +5,9 @@ import re
 from .common import InfoExtractor
 from .kaltura import KalturaIE
 from ..utils import (
-    get_element_by_class,
+    get_element_by_id,
     strip_or_none,
+    urljoin,
 )
 
 
@@ -83,8 +84,8 @@ class AZMedienIE(AZMedienBaseIE):
         return self._kaltura_video(partner_id, entry_id)
 
 
-class AZMedienShowIE(AZMedienBaseIE):
-    IE_DESC = 'AZ Medien shows'
+class AZMedienPlaylistIE(AZMedienBaseIE):
+    IE_DESC = 'AZ Medien playlists'
     _VALID_URL = r'''(?x)
                     https?://
                         (?:www\.)?
@@ -93,7 +94,12 @@ class AZMedienShowIE(AZMedienBaseIE):
                             telebaern\.tv|
                             telem1\.ch
                         )/
-                        (?P<id>[0-9]+-show-[^/\#]+
+                        (?P<id>[0-9]+-
+                            (?:
+                                show|
+                                topic|
+                                themen
+                            )-[^/\#]+
                             (?:
                                 /[0-9]+-episode-[^/\#]+
                             )?
@@ -108,6 +114,18 @@ class AZMedienShowIE(AZMedienBaseIE):
             'title': 'News - Donnerstag, 15. Dezember 2016',
         },
         'playlist_count': 9,
+    }, {
+        # URL with 'themen'
+        'url': 'http://www.telem1.ch/258-themen-tele-m1-classics',
+        'info_dict': {
+            'id': '258-themen-tele-m1-classics',
+            'title': 'Tele M1 Classics',
+        },
+        'playlist_mincount': 15,
+    }, {
+        # URL with 'topic', contains nested playlists
+        'url': 'http://www.telezueri.ch/219-topic-aera-trump-hat-offiziell-begonnen',
+        'only_matching': True,
     }, {
         # URL with 'show' only
         'url': 'http://www.telezueri.ch/86-show-talktaeglich',
@@ -136,10 +154,19 @@ class AZMedienShowIE(AZMedienBaseIE):
                 for m in re.finditer(
                     r'<a[^>]+data-real=(["\'])(?P<url>http.+?)\1', webpage)]
 
+        if not entries:
+            entries = [
+                # May contain nested playlists (e.g. [1]) thus no explicit
+                # ie_key
+                # 1. http://www.telezueri.ch/219-topic-aera-trump-hat-offiziell-begonnen)
+                self.url_result(urljoin(url, m.group('url')))
+                for m in re.finditer(
+                    r'<a[^>]+name=[^>]+href=(["\'])(?P<url>/.+?)\1', webpage)]
+
         title = self._search_regex(
             r'episodeShareTitle\s*=\s*(["\'])(?P<title>(?:(?!\1).)+)\1',
             webpage, 'title',
-            default=strip_or_none(get_element_by_class(
-                'title-block-cell', webpage)), group='title')
+            default=strip_or_none(get_element_by_id(
+                'video-title', webpage)), group='title')
 
         return self.playlist_result(entries, show_id, title)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 086a2296d..2590b5e1b 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -82,7 +82,7 @@ from .awaan import (
 )
 from .azmedien import (
     AZMedienIE,
-    AZMedienShowIE,
+    AZMedienPlaylistIE,
 )
 from .azubu import AzubuIE, AzubuLiveIE
 from .baidu import BaiduVideoIE

From fe323a4800d67d0ad2fecebcc3b627a7a22be427 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Jan 2017 21:21:26 +0700
Subject: [PATCH 0126/1696] [ChangeLog] Actualize

---
 ChangeLog | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 8e5a04b42..ab2818f9e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,24 @@
+version <unreleased>
+
+Core
+* [extractor/common] Fix initialization template (#11605, #11825)
++ [extractor/common] Document fragment_base_url and fragment's path fields
+* [extractor/common] Fix duration per DASH segment (#11868)
++ Introduce --autonumber-start option for initial value of %(autonumber)s
+  template (#727, #2702, #9362, #10457, #10529, #11862)
+
+Extractors
++ [azmedien:playlist] Add support for topic and themen playlists (#11817)
+* [npo] Fix subtitles extraction
++ [itv] Extract subtitles
++ [itv] Add support for itv.com (#9240)
++ [mtv81] Add support for mtv81.com (#7619)
++ [vlive] Add support for channels (#11826)
++ [kaltura] Add fallback for fileExt
++ [kaltura] Improve uploader_id extraction
++ [konserthusetplay] Add support for rspoplay.se (#11828)
+
+
 version 2017.01.28
 
 Core

From 4d2fdb07c47e2d9f96d58f5fbf3da8665a1144a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Jan 2017 13:21:42 +0700
Subject: [PATCH 0127/1696] release 2017.01.29

---
 .github/ISSUE_TEMPLATE.md |  6 ++---
 ChangeLog                 |  2 +-
 README.md                 | 47 +++++++++++++++++++--------------------
 docs/supportedsites.md    |  5 ++++-
 youtube_dl/version.py     |  2 +-
 5 files changed, 32 insertions(+), 30 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 693f3b745..10c982fd0 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.01.28*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.01.28**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.01.29*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.01.29**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.01.28
+[debug] youtube-dl version 2017.01.29
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index ab2818f9e..cd7017f6d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.01.29
 
 Core
 * [extractor/common] Fix initialization template (#11605, #11825)
diff --git a/README.md b/README.md
index 4f677d0cc..2ee00f515 100644
--- a/README.md
+++ b/README.md
@@ -88,8 +88,6 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
     --mark-watched                   Mark videos watched (YouTube only)
     --no-mark-watched                Do not mark videos watched (YouTube only)
     --no-color                       Do not emit color codes in output
-    --abort-on-unavailable-fragment  Abort downloading when some fragment is not
-                                     available
 
 ## Network Options:
     --proxy URL                      Use the specified HTTP/HTTPS/SOCKS proxy.
@@ -99,16 +97,13 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
                                      string (--proxy "") for direct connection
     --socket-timeout SECONDS         Time to wait before giving up, in seconds
     --source-address IP              Client-side IP address to bind to
-                                     (experimental)
     -4, --force-ipv4                 Make all connections via IPv4
-                                     (experimental)
     -6, --force-ipv6                 Make all connections via IPv6
-                                     (experimental)
     --geo-verification-proxy URL     Use this proxy to verify the IP address for
                                      some geo-restricted sites. The default
                                      proxy specified by --proxy (or none, if the
                                      options is not present) is used for the
-                                     actual downloading. (experimental)
+                                     actual downloading.
 
 ## Video Selection:
     --playlist-start NUMBER          Playlist video to start at (default is 1)
@@ -139,23 +134,23 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
                                      COUNT views
     --max-views COUNT                Do not download any videos with more than
                                      COUNT views
-    --match-filter FILTER            Generic video filter (experimental).
-                                     Specify any key (see help for -o for a list
-                                     of available keys) to match if the key is
-                                     present, !key to check if the key is not
-                                     present,key > NUMBER (like "comment_count >
-                                     12", also works with >=, <, <=, !=, =) to
-                                     compare against a number, and & to require
-                                     multiple matches. Values which are not
-                                     known are excluded unless you put a
-                                     question mark (?) after the operator.For
-                                     example, to only match videos that have
-                                     been liked more than 100 times and disliked
-                                     less than 50 times (or the dislike
-                                     functionality is not available at the given
-                                     service), but who also have a description,
-                                     use --match-filter "like_count > 100 &
-                                     dislike_count <? 50 & description" .
+    --match-filter FILTER            Generic video filter. Specify any key (see
+                                     help for -o for a list of available keys)
+                                     to match if the key is present, !key to
+                                     check if the key is not present,key >
+                                     NUMBER (like "comment_count > 12", also
+                                     works with >=, <, <=, !=, =) to compare
+                                     against a number, and & to require multiple
+                                     matches. Values which are not known are
+                                     excluded unless you put a question mark (?)
+                                     after the operator.For example, to only
+                                     match videos that have been liked more than
+                                     100 times and disliked less than 50 times
+                                     (or the dislike functionality is not
+                                     available at the given service), but who
+                                     also have a description, use --match-filter
+                                     "like_count > 100 & dislike_count <? 50 &
+                                     description" .
     --no-playlist                    Download only the video, if the URL refers
                                      to a video and a playlist.
     --yes-playlist                   Download the playlist, if the URL refers to
@@ -178,6 +173,8 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
                                      only)
     --skip-unavailable-fragments     Skip unavailable fragments (DASH and
                                      hlsnative only)
+    --abort-on-unavailable-fragment  Abort downloading when some fragment is not
+                                     available
     --buffer-size SIZE               Size of download buffer (e.g. 1024 or 16K)
                                      (default is 1024)
     --no-resize-buffer               Do not automatically adjust the buffer
@@ -210,7 +207,9 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
     --autonumber-size NUMBER         Specify the number of digits in
                                      %(autonumber)s when it is present in output
                                      filename template or --auto-number option
-                                     is given
+                                     is given (default is 5)
+    --autonumber-start NUMBER        Specify the start value for %(autonumber)s
+                                     (default is 1)
     --restrict-filenames             Restrict filenames to only ASCII
                                      characters, and avoid "&" and spaces in
                                      filenames
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 6318a862f..d4231577b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -76,7 +76,7 @@
  - **awaan:season**
  - **awaan:video**
  - **AZMedien**: AZ Medien videos
- - **AZMedienShow**: AZ Medien shows
+ - **AZMedienPlaylist**: AZ Medien playlists
  - **Azubu**
  - **AzubuLive**
  - **BaiduVideo**: 百度视频
@@ -337,6 +337,7 @@
  - **IPrima**
  - **iqiyi**: 爱奇艺
  - **Ir90Tv**
+ - **ITV**
  - **ivi**: ivi.ru
  - **ivi:compilation**: ivi.ru compilations
  - **ivideon**: Ivideon TV
@@ -445,6 +446,7 @@
  - **mtg**: MTG services
  - **mtv**
  - **mtv.de**
+ - **mtv81**
  - **mtv:video**
  - **mtvservices:embedded**
  - **MuenchenTV**: münchen.tv
@@ -887,6 +889,7 @@
  - **vk:uservideos**: VK - User's Videos
  - **vk:wallpost**
  - **vlive**
+ - **vlive:channel**
  - **Vodlocker**
  - **VODPlatform**
  - **VoiceRepublic**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index c22c410a8..a37a65db9 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.01.28'
+__version__ = '2017.01.29'

From c2d9c25f818da2e0e622b475ffc714f35df0887c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 29 Jan 2017 16:03:39 +0100
Subject: [PATCH 0128/1696] [compat] add compat_etree_register_namespace

---
 youtube_dl/compat.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 02abf8c1e..49e3c90e2 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2529,6 +2529,24 @@ else:
                 el.text = el.text.decode('utf-8')
         return doc
 
+if hasattr(etree, 'register_namespace'):
+    compat_etree_register_namespace = etree.register_namespace
+else:
+    def compat_etree_register_namespace(prefix, uri):
+        """Register a namespace prefix.
+        The registry is global, and any existing mapping for either the
+        given prefix or the namespace URI will be removed.
+        *prefix* is the namespace prefix, *uri* is a namespace uri. Tags and
+        attributes in this namespace will be serialized with prefix if possible.
+        ValueError is raised if prefix is reserved or is invalid.
+        """
+        if re.match(r"ns\d+$", prefix):
+            raise ValueError("Prefix format reserved for internal use")
+        for k, v in list(etree._namespace_map.items()):
+            if k == uri or v == prefix:
+                del etree._namespace_map[k]
+        etree._namespace_map[uri] = prefix
+
 if sys.version_info < (2, 7):
     # Here comes the crazy part: In 2.6, if the xpath is a unicode,
     # .//node does not match if a node is a direct child of . !

From 4719419951ced20e42cddb26b437908ba636debb Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 29 Jan 2017 16:04:15 +0100
Subject: [PATCH 0129/1696] [itv] fix extraction in python 2.6

---
 youtube_dl/extractor/itv.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py
index 0328c7093..b0d860452 100644
--- a/youtube_dl/extractor/itv.py
+++ b/youtube_dl/extractor/itv.py
@@ -6,7 +6,10 @@ import xml.etree.ElementTree as etree
 import json
 
 from .common import InfoExtractor
-from ..compat import compat_str
+from ..compat import (
+    compat_str,
+    compat_etree_register_namespace,
+)
 from ..utils import (
     extract_attributes,
     xpath_with_ns,
@@ -47,7 +50,7 @@ class ITVIE(InfoExtractor):
             'com': 'http://schemas.itv.com/2009/05/Common',
         }
         for ns, full_ns in ns_map.items():
-            etree.register_namespace(ns, full_ns)
+            compat_etree_register_namespace(ns, full_ns)
 
         def _add_ns(name):
             return xpath_with_ns(name, ns_map)

From dadb836139f070da9364439bf3b148eec8bc0b11 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 30 Jan 2017 09:32:31 +0100
Subject: [PATCH 0130/1696] [ruutu] extract dash formats

---
 youtube_dl/extractor/ruutu.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
index f12bc5614..20d01754a 100644
--- a/youtube_dl/extractor/ruutu.py
+++ b/youtube_dl/extractor/ruutu.py
@@ -81,6 +81,9 @@ class RuutuIE(InfoExtractor):
                     elif ext == 'f4m':
                         formats.extend(self._extract_f4m_formats(
                             video_url, video_id, f4m_id='hds', fatal=False))
+                    elif ext == 'mpd':
+                        formats.extend(self._extract_mpd_formats(
+                            video_url, video_id, mpd_id='dash', fatal=False))
                     else:
                         proto = compat_urllib_parse_urlparse(video_url).scheme
                         if not child.tag.startswith('HTTP') and proto != 'rtmp':

From 75822ca7909d7f7e15694f73b05b2bf0f1fa61f3 Mon Sep 17 00:00:00 2001
From: Thomas Christlieb <thomaschristlieb@hotmail.com>
Date: Tue, 31 Jan 2017 10:03:31 +0100
Subject: [PATCH 0131/1696] New parameter --playlist-random to randomize
 playlist download order. Fixes #11889

---
 youtube_dl/YoutubeDL.py | 5 +++++
 youtube_dl/__init__.py  | 1 +
 youtube_dl/options.py   | 4 ++++
 3 files changed, 10 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index c71e94518..a7bf5a1b0 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -24,6 +24,7 @@ import sys
 import time
 import tokenize
 import traceback
+import random
 
 from .compat import (
     compat_basestring,
@@ -159,6 +160,7 @@ class YoutubeDL(object):
     playlistend:       Playlist item to end at.
     playlist_items:    Specific indices of playlist to download.
     playlistreverse:   Download playlist items in reverse order.
+    playlistrandom:    Download playlist items in random order.
     matchtitle:        Download only matching titles.
     rejecttitle:       Reject downloads for matching titles.
     logger:            Log messages to a logging.Logger instance.
@@ -842,6 +844,9 @@ class YoutubeDL(object):
             if self.params.get('playlistreverse', False):
                 entries = entries[::-1]
 
+            if self.params.get('playlistrandom', False):
+                random.shuffle(entries)
+
             for i, entry in enumerate(entries, 1):
                 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
                 extra = {
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 2b156342a..5c5b8094b 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -344,6 +344,7 @@ def _real_main(argv=None):
         'playliststart': opts.playliststart,
         'playlistend': opts.playlistend,
         'playlistreverse': opts.playlist_reverse,
+        'playlistrandom': opts.playlist_random,
         'noplaylist': opts.noplaylist,
         'logtostderr': opts.outtmpl == '-',
         'consoletitle': opts.consoletitle,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 3abf621c0..349f44778 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -470,6 +470,10 @@ def parseOpts(overrideArguments=None):
         '--playlist-reverse',
         action='store_true',
         help='Download playlist videos in reverse order')
+    downloader.add_option(
+        '--playlist-random',
+        action='store_true',
+        help='Download playlist videos in random order')
     downloader.add_option(
         '--xattr-set-filesize',
         dest='xattr_set_filesize', action='store_true',

From ae9a173b6421a3fdf70dd50d2dc0386f8861fe71 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 31 Jan 2017 14:47:56 +0100
Subject: [PATCH 0132/1696] [vimeo] extract both mixed and separated dash
 formats

---
 youtube_dl/extractor/vimeo.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index c12eeadd4..8b6a5cc3c 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -142,10 +142,19 @@ class VimeoBaseInfoExtractor(InfoExtractor):
                         note='Downloading %s m3u8 information' % cdn_name,
                         fatal=False))
                 elif files_type == 'dash':
-                    formats.extend(self._extract_mpd_formats(
-                        manifest_url.replace('/master.json', '/master.mpd'), video_id, format_id,
-                        'Downloading %s MPD information' % cdn_name,
-                        fatal=False))
+                    mpd_pattern = r'/%s/(?:sep/)?video/' % video_id
+                    mpd_manifest_urls = []
+                    if re.search(mpd_pattern, manifest_url):
+                        for suffix, repl in (('', 'video'), ('_sep', 'sep/video')):
+                            mpd_manifest_urls.append((format_id + suffix, re.sub(
+                                mpd_pattern, '/%s/%s/' % (video_id, repl), manifest_url)))
+                    else:
+                        mpd_manifest_urls = [(format_id, manifest_url)]
+                    for f_id, m_url in mpd_manifest_urls:
+                        formats.extend(self._extract_mpd_formats(
+                            m_url.replace('/master.json', '/master.mpd'), video_id, f_id,
+                            'Downloading %s MPD information' % cdn_name,
+                            fatal=False))
 
         subtitles = {}
         text_tracks = config['request'].get('text_tracks')

From 3c90cc8b6fc069930264b41f5505dc34c1077442 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 31 Jan 2017 22:19:29 +0700
Subject: [PATCH 0133/1696] [youtube] Fix extraction for domainless player URLs

Closes #11890
Closes #11891
Closes #11892
Closes #11894
Closes #11895
Closes #11897
Closes #11900
Closes #11903
Closes #11904
Closes #11906
Closes #11907
Closes #11909
Closes #11913
Closes #11914
Closes #11915
Closes #11916
Closes #11917
Closes #11918
Closes #11919
---
 youtube_dl/extractor/youtube.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 630586796..ea398bcc8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1028,8 +1028,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
     def _parse_sig_js(self, jscode):
         funcname = self._search_regex(
-            r'\.sig\|\|([a-zA-Z0-9$]+)\(', jscode,
-            'Initial JS player signature function name')
+            (r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
+             r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\('),
+            jscode, 'Initial JS player signature function name', group='sig')
 
         jsi = JSInterpreter(jscode)
         initial_function = jsi.extract_function(funcname)
@@ -1050,6 +1051,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         if player_url.startswith('//'):
             player_url = 'https:' + player_url
+        elif not re.match(r'https?://', player_url):
+            player_url = compat_urlparse.urljoin(
+                'https://www.youtube.com', player_url)
         try:
             player_id = (player_url, self._signature_cache_id(s))
             if player_id not in self._player_cache:

From 3a528ffd8944417c99b139da18d0dff907ade517 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 31 Jan 2017 22:21:54 +0700
Subject: [PATCH 0134/1696] [ChangeLog] Actualize

---
 ChangeLog | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index cd7017f6d..e331acacc 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,17 @@
+version <unreleased>
+
+Core
++ [compat] Add compat_etree_register_namespace
+
+Extractors
+* [youtube] Fix extraction for domainless player URLs (#11890, #11891, #11892,
+  #11894, #11895, #11897, #11900, #11903, #11904, #11906, #11907, #11909,
+  #11913, #11914, #11915, #11916, #11917, #11918, #11919)
++ [vimeo] Extract both mixed and separated DASH formats
++ [ruutu] Extract DASH formats
+* [itv] Fix extraction for python 2.6
+
+
 version 2017.01.29
 
 Core

From d7e215b42dcaf71298a7e1dc953cf93523b3da81 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 31 Jan 2017 22:24:45 +0700
Subject: [PATCH 0135/1696] release 2017.01.31

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 10c982fd0..180013f72 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.01.29*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.01.29**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.01.31*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.01.31**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.01.29
+[debug] youtube-dl version 2017.01.31
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index e331acacc..d5ab0e0a7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.01.31
 
 Core
 + [compat] Add compat_etree_register_namespace
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a37a65db9..fee0ac7c5 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.01.29'
+__version__ = '2017.01.31'

From 8fd65faece98139def3a6538e98053bebd400263 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Odd=20Str=C3=A5b=C3=B8?= <oddstr13@openshell.no>
Date: Sat, 14 Jan 2017 02:36:04 +0100
Subject: [PATCH 0136/1696] [NRKTV] Added NRKTVSeriesIE

[NRKTV] Added season and episode number to metadata.

[NRKTV] Added category to metadata.

[NRKTV] Added tests to NRKTVSeries.

[NRKTV] Fixed whitespace issues (flake8).
---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/nrk.py        | 49 ++++++++++++++++++++++++++++++
 2 files changed, 50 insertions(+)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 2590b5e1b..06e6d4620 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -668,6 +668,7 @@ from .nrk import (
     NRKTVIE,
     NRKTVDirekteIE,
     NRKTVEpisodesIE,
+    NRKTVSeriesIE,
 )
 from .ntvde import NTVDeIE
 from .ntvru import NTVRuIE
diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index ea7be005a..26604f84f 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -128,6 +128,18 @@ class NRKBaseIE(InfoExtractor):
         series = conviva.get('seriesName') or data.get('seriesTitle')
         episode = conviva.get('episodeName') or data.get('episodeNumberOrDate')
 
+        season_number = None
+        episode_number = None
+        if data.get('mediaElementType') == 'Episode':
+            _season_episode = data.get('scoresStatistics', {}).get('springStreamStream') or \
+                data.get('relativeOriginUrl', '')
+            EPISODENUM_RE = [
+                r'/s(?P<season>\d+)e(?P<episode>\d+)\.',
+                r'/sesong-(?P<season>\d+)/episode-(?P<episode>\d+)',
+            ]
+            season_number = int_or_none(self._search_regex(EPISODENUM_RE, _season_episode, "S##E##", fatal=False, group='season'))
+            episode_number = int_or_none(self._search_regex(EPISODENUM_RE, _season_episode, "S##E##", fatal=False, group='episode'))
+
         thumbnails = None
         images = data.get('images')
         if images and isinstance(images, dict):
@@ -140,11 +152,15 @@ class NRKBaseIE(InfoExtractor):
                 } for image in web_images if image.get('imageUrl')]
 
         description = data.get('description')
+        category = data.get('mediaAnalytics', {}).get('category')
 
         common_info = {
             'description': description,
             'series': series,
             'episode': episode,
+            'season_number': season_number,
+            'episode_number': episode_number,
+            'categories': [category] if category else None,
             'age_limit': parse_age_limit(data.get('legalAge')),
             'thumbnails': thumbnails,
         }
@@ -360,6 +376,39 @@ class NRKTVEpisodesIE(NRKPlaylistBaseIE):
             r'<h1>([^<]+)</h1>', webpage, 'title', fatal=False)
 
 
+class NRKTVSeriesIE(InfoExtractor):
+    _VALID_URL = r'https?://tv\.nrk\.no/serie/(?P<id>[^/]+)/?'
+    _ITEM_RE = r'data-season=["\'](?P<id>\d+)["\']'
+    _TESTS = [{
+        'url': 'https://tv.nrk.no/serie/broedrene-dal-og-spektralsteinene',
+        'playlist_count': 1,
+    }, {
+        'url': 'https://tv.nrk.no/serie/saving-the-human-race',
+        'playlist_count': 1,
+    }, {
+        'url': 'https://tv.nrk.no/serie/postmann-pat',
+        'playlist_count': 3,
+    }, {
+        'url': 'https://tv.nrk.no/serie/groenn-glede',
+        'playlist_count': 9,
+    }]
+
+    def _real_extract(self, url):
+        series_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, series_id)
+
+        entries = [
+            self.url_result('https://tv.nrk.no/program/Episodes/{series}/{season}'.format(
+                series=series_id,
+                season=season_id
+            ))
+            for season_id in re.findall(self._ITEM_RE, webpage)
+        ]
+
+        return self.playlist_result(entries)
+
+
 class NRKSkoleIE(InfoExtractor):
     IE_DESC = 'NRK Skole'
     _VALID_URL = r'https?://(?:www\.)?nrk\.no/skole/?\?.*\bmediaId=(?P<id>\d+)'

From 7c5329e6f4152b48c5476b1b9b8ab931caa10331 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 1 Feb 2017 00:29:29 +0700
Subject: [PATCH 0137/1696] [nrk] Improve extraction and update tests (closes
 #11571)

---
 youtube_dl/extractor/nrk.py | 145 +++++++++++++++++++++++++++---------
 1 file changed, 111 insertions(+), 34 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 26604f84f..fc3c0cd3c 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -134,11 +134,15 @@ class NRKBaseIE(InfoExtractor):
             _season_episode = data.get('scoresStatistics', {}).get('springStreamStream') or \
                 data.get('relativeOriginUrl', '')
             EPISODENUM_RE = [
-                r'/s(?P<season>\d+)e(?P<episode>\d+)\.',
-                r'/sesong-(?P<season>\d+)/episode-(?P<episode>\d+)',
+                r'/s(?P<season>\d{,2})e(?P<episode>\d{,2})\.',
+                r'/sesong-(?P<season>\d{,2})/episode-(?P<episode>\d{,2})',
             ]
-            season_number = int_or_none(self._search_regex(EPISODENUM_RE, _season_episode, "S##E##", fatal=False, group='season'))
-            episode_number = int_or_none(self._search_regex(EPISODENUM_RE, _season_episode, "S##E##", fatal=False, group='episode'))
+            season_number = int_or_none(self._search_regex(
+                EPISODENUM_RE, _season_episode, 'season number',
+                default=None, group='season'))
+            episode_number = int_or_none(self._search_regex(
+                EPISODENUM_RE, _season_episode, 'episode number',
+                default=None, group='episode'))
 
         thumbnails = None
         images = data.get('images')
@@ -243,54 +247,102 @@ class NRKTVIE(NRKBaseIE):
             'title': '20 spørsmål 23.05.2014',
             'description': 'md5:bdea103bc35494c143c6a9acdd84887a',
             'duration': 1741,
+            'series': '20 spørsmål - TV',
+            'episode': '23.05.2014',
         },
     }, {
         'url': 'https://tv.nrk.no/program/mdfp15000514',
-        'md5': '43d0be26663d380603a9cf0c24366531',
         'info_dict': {
             'id': 'MDFP15000514CA',
             'ext': 'mp4',
             'title': 'Grunnlovsjubiléet - Stor ståhei for ingenting 24.05.2014',
             'description': 'md5:89290c5ccde1b3a24bb8050ab67fe1db',
             'duration': 4605,
+            'series': 'Kunnskapskanalen',
+            'episode': '24.05.2014',
+        },
+        'params': {
+            'skip_download': True,
         },
     }, {
         # single playlist video
         'url': 'https://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015#del=2',
-        'md5': 'adbd1dbd813edaf532b0a253780719c2',
         'info_dict': {
             'id': 'MSPO40010515-part2',
             'ext': 'flv',
             'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn 06.01.2015 (del 2:2)',
             'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
         },
-        'skip': 'Only works from Norway',
+        'params': {
+            'skip_download': True,
+        },
+        'expected_warnings': ['Video is geo restricted'],
+        'skip': 'particular part is not supported currently',
     }, {
         'url': 'https://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015',
         'playlist': [{
-            'md5': '9480285eff92d64f06e02a5367970a7a',
             'info_dict': {
-                'id': 'MSPO40010515-part1',
-                'ext': 'flv',
-                'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn 06.01.2015 (del 1:2)',
-                'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
+                'id': 'MSPO40010515AH',
+                'ext': 'mp4',
+                'title': 'Sprint fri teknikk, kvinner og menn 06.01.2015 (Part 1)',
+                'description': 'md5:c03aba1e917561eface5214020551b7a',
+                'duration': 772,
+                'series': 'Tour de Ski',
+                'episode': '06.01.2015',
+            },
+            'params': {
+                'skip_download': True,
             },
         }, {
-            'md5': 'adbd1dbd813edaf532b0a253780719c2',
             'info_dict': {
-                'id': 'MSPO40010515-part2',
-                'ext': 'flv',
-                'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn 06.01.2015 (del 2:2)',
-                'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
+                'id': 'MSPO40010515BH',
+                'ext': 'mp4',
+                'title': 'Sprint fri teknikk, kvinner og menn 06.01.2015 (Part 2)',
+                'description': 'md5:c03aba1e917561eface5214020551b7a',
+                'duration': 6175,
+                'series': 'Tour de Ski',
+                'episode': '06.01.2015',
+            },
+            'params': {
+                'skip_download': True,
             },
         }],
         'info_dict': {
             'id': 'MSPO40010515',
-            'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn',
-            'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
-            'duration': 6947.52,
+            'title': 'Sprint fri teknikk, kvinner og menn 06.01.2015',
+            'description': 'md5:c03aba1e917561eface5214020551b7a',
+        },
+        'expected_warnings': ['Video is geo restricted'],
+    }, {
+        'url': 'https://tv.nrk.no/serie/anno/KMTE50001317/sesong-3/episode-13',
+        'info_dict': {
+            'id': 'KMTE50001317AA',
+            'ext': 'mp4',
+            'title': 'Anno 13:30',
+            'description': 'md5:11d9613661a8dbe6f9bef54e3a4cbbfa',
+            'duration': 2340,
+            'series': 'Anno',
+            'episode': '13:30',
+            'season_number': 3,
+            'episode_number': 13,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://tv.nrk.no/serie/nytt-paa-nytt/MUHH46000317/27-01-2017',
+        'info_dict': {
+            'id': 'MUHH46000317AA',
+            'ext': 'mp4',
+            'title': 'Nytt på Nytt 27.01.2017',
+            'description': 'md5:5358d6388fba0ea6f0b6d11c48b9eb4b',
+            'duration': 1796,
+            'series': 'Nytt på nytt',
+            'episode': '27.01.2017',
+        },
+        'params': {
+            'skip_download': True,
         },
-        'skip': 'Only works from Norway',
     }, {
         'url': 'https://radio.nrk.no/serie/dagsnytt/NPUB21019315/12-07-2015#',
         'only_matching': True,
@@ -377,36 +429,61 @@ class NRKTVEpisodesIE(NRKPlaylistBaseIE):
 
 
 class NRKTVSeriesIE(InfoExtractor):
-    _VALID_URL = r'https?://tv\.nrk\.no/serie/(?P<id>[^/]+)/?'
-    _ITEM_RE = r'data-season=["\'](?P<id>\d+)["\']'
+    _VALID_URL = r'https?://(?:tv|radio)\.nrk(?:super)?\.no/serie/(?P<id>[^/]+)'
+    _ITEM_RE = r'(?:data-season=["\']|id=["\']season-)(?P<id>\d+)'
     _TESTS = [{
+        'url': 'https://tv.nrk.no/serie/groenn-glede',
+        'info_dict': {
+            'id': 'groenn-glede',
+            'title': 'Grønn glede',
+            'description': 'md5:7576e92ae7f65da6993cf90ee29e4608',
+        },
+        'playlist_mincount': 9,
+    }, {
+        'url': 'http://tv.nrksuper.no/serie/labyrint',
+        'info_dict': {
+            'id': 'labyrint',
+            'title': 'Labyrint',
+            'description': 'md5:58afd450974c89e27d5a19212eee7115',
+        },
+        'playlist_mincount': 3,
+    }, {
         'url': 'https://tv.nrk.no/serie/broedrene-dal-og-spektralsteinene',
-        'playlist_count': 1,
+        'only_matching': True,
     }, {
         'url': 'https://tv.nrk.no/serie/saving-the-human-race',
-        'playlist_count': 1,
+        'only_matching': True,
     }, {
         'url': 'https://tv.nrk.no/serie/postmann-pat',
-        'playlist_count': 3,
-    }, {
-        'url': 'https://tv.nrk.no/serie/groenn-glede',
-        'playlist_count': 9,
+        'only_matching': True,
     }]
 
+    @classmethod
+    def suitable(cls, url):
+        return False if NRKTVIE.suitable(url) else super(NRKTVSeriesIE, cls).suitable(url)
+
     def _real_extract(self, url):
         series_id = self._match_id(url)
 
         webpage = self._download_webpage(url, series_id)
 
         entries = [
-            self.url_result('https://tv.nrk.no/program/Episodes/{series}/{season}'.format(
-                series=series_id,
-                season=season_id
-            ))
+            self.url_result(
+                'https://tv.nrk.no/program/Episodes/{series}/{season}'.format(
+                    series=series_id, season=season_id))
             for season_id in re.findall(self._ITEM_RE, webpage)
         ]
 
-        return self.playlist_result(entries)
+        title = self._html_search_meta(
+            'seriestitle', webpage,
+            'title', default=None) or self._og_search_title(
+            webpage, fatal=False)
+
+        description = self._html_search_meta(
+            'series_description', webpage,
+            'description', default=None) or self._og_search_description(webpage)
+
+        return self.playlist_result(entries, series_id, title, description)
 
 
 class NRKSkoleIE(InfoExtractor):

From 363245ad94dfdf0c34b4c2c801e7cf6cea74f39c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 1 Feb 2017 00:30:19 +0700
Subject: [PATCH 0138/1696] Credit @oddstr13 for nrk:series (#11571)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 3ef2800c9..f2875d504 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -200,3 +200,4 @@ Paul Hartmann
 Stephen Chen
 Fabian Stahl
 Bagira
+Odd Stråbø

From c38a67bcd5df639b9d7e7faa8685e76446803527 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 1 Feb 2017 00:49:28 +0700
Subject: [PATCH 0139/1696] [vimeo] Extract license (closes #11880)

---
 youtube_dl/extractor/vimeo.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 8b6a5cc3c..32179e915 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -218,6 +218,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'uploader_id': 'user7108434',
                 'uploader': 'Filippo Valsorda',
                 'duration': 10,
+                'license': 'by-sa',
             },
         },
         {
@@ -486,6 +487,8 @@ class VimeoIE(VimeoBaseInfoExtractor):
                     '%s said: %s' % (self.IE_NAME, seed_status['title']),
                     expected=True)
 
+        cc_license = None
+
         # Extract the config JSON
         try:
             try:
@@ -499,8 +502,9 @@ class VimeoIE(VimeoBaseInfoExtractor):
                     vimeo_clip_page_config = self._search_regex(
                         r'vimeo\.clip_page_config\s*=\s*({.+?});', webpage,
                         'vimeo clip page config')
-                    config_url = self._parse_json(
-                        vimeo_clip_page_config, video_id)['player']['config_url']
+                    page_config = self._parse_json(vimeo_clip_page_config, video_id)
+                    config_url = page_config['player']['config_url']
+                    cc_license = page_config.get('cc_license')
                 config_json = self._download_webpage(config_url, video_id)
                 config = json.loads(config_json)
             except RegexNotFoundError:
@@ -609,6 +613,12 @@ class VimeoIE(VimeoBaseInfoExtractor):
         info_dict = self._parse_config(config, video_id)
         formats.extend(info_dict['formats'])
         self._vimeo_sort_formats(formats)
+
+        if not cc_license:
+            cc_license = self._search_regex(
+                r'<link[^>]+rel=["\']license["\'][^>]+href=(["\'])(?P<license>(?:(?!\1).)+)\1',
+                webpage, 'license', default=None, group='license')
+
         info_dict.update({
             'id': video_id,
             'formats': formats,
@@ -618,6 +628,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             'view_count': view_count,
             'like_count': like_count,
             'comment_count': comment_count,
+            'license': cc_license,
         })
 
         return info_dict

From c15cd296404e164b72fd7f2666d5875f35057d93 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 1 Feb 2017 00:58:02 +0700
Subject: [PATCH 0140/1696] [vimeo] Extract upload timestamp

---
 youtube_dl/extractor/vimeo.py | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 32179e915..8ba222224 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -21,7 +21,9 @@ from ..utils import (
     sanitized_Request,
     smuggle_url,
     std_headers,
+    try_get,
     unified_strdate,
+    unified_timestamp,
     unsmuggle_url,
     urlencode_postdata,
     unescapeHTML,
@@ -213,6 +215,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'ext': 'mp4',
                 'title': "youtube-dl test video - \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550",
                 'description': 'md5:2d3305bad981a06ff79f027f19865021',
+                'timestamp': 1355990239,
                 'upload_date': '20121220',
                 'uploader_url': r're:https?://(?:www\.)?vimeo\.com/user7108434',
                 'uploader_id': 'user7108434',
@@ -259,6 +262,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'id': '68375962',
                 'ext': 'mp4',
                 'title': 'youtube-dl password protected test video',
+                'timestamp': 1371200155,
                 'upload_date': '20130614',
                 'uploader_url': r're:https?://(?:www\.)?vimeo\.com/user18948128',
                 'uploader_id': 'user18948128',
@@ -281,7 +285,8 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'uploader_url': r're:https?://(?:www\.)?vimeo\.com/atencio',
                 'uploader_id': 'atencio',
                 'uploader': 'Peter Atencio',
-                'upload_date': '20130927',
+                'timestamp': 1380339469,
+                'upload_date': '20130928',
                 'duration': 187,
             },
         },
@@ -293,6 +298,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'ext': 'mp4',
                 'title': 'The New Vimeo Player (You Know, For Videos)',
                 'description': 'md5:2ec900bf97c3f389378a96aee11260ea',
+                'timestamp': 1381846109,
                 'upload_date': '20131015',
                 'uploader_url': r're:https?://(?:www\.)?vimeo\.com/staff',
                 'uploader_id': 'staff',
@@ -324,6 +330,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'uploader': 'The DMCI',
                 'uploader_url': r're:https?://(?:www\.)?vimeo\.com/dmci',
                 'uploader_id': 'dmci',
+                'timestamp': 1324343742,
                 'upload_date': '20111220',
                 'description': 'md5:ae23671e82d05415868f7ad1aec21147',
             },
@@ -339,6 +346,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'uploader': 'Casey Donahue',
                 'uploader_url': r're:https?://(?:www\.)?vimeo\.com/caseydonahue',
                 'uploader_id': 'caseydonahue',
+                'timestamp': 1250886430,
                 'upload_date': '20090821',
                 'description': 'md5:bdbf314014e58713e6e5b66eb252f4a6',
             },
@@ -488,6 +496,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                     expected=True)
 
         cc_license = None
+        timestamp = None
 
         # Extract the config JSON
         try:
@@ -505,6 +514,9 @@ class VimeoIE(VimeoBaseInfoExtractor):
                     page_config = self._parse_json(vimeo_clip_page_config, video_id)
                     config_url = page_config['player']['config_url']
                     cc_license = page_config.get('cc_license')
+                    timestamp = try_get(
+                        page_config, lambda x: x['clip']['uploaded_on'],
+                        compat_str)
                 config_json = self._download_webpage(config_url, video_id)
                 config = json.loads(config_json)
             except RegexNotFoundError:
@@ -573,10 +585,10 @@ class VimeoIE(VimeoBaseInfoExtractor):
             self._downloader.report_warning('Cannot find video description')
 
         # Extract upload date
-        video_upload_date = None
-        mobj = re.search(r'<time[^>]+datetime="([^"]+)"', webpage)
-        if mobj is not None:
-            video_upload_date = unified_strdate(mobj.group(1))
+        if not timestamp:
+            timestamp = self._search_regex(
+                r'<time[^>]+datetime="([^"]+)"', webpage,
+                'timestamp', default=None)
 
         try:
             view_count = int(self._search_regex(r'UserPlays:(\d+)', webpage, 'view count'))
@@ -622,7 +634,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         info_dict.update({
             'id': video_id,
             'formats': formats,
-            'upload_date': video_upload_date,
+            'timestamp': unified_timestamp(timestamp),
             'description': video_description,
             'webpage_url': url,
             'view_count': view_count,

From 26c0f09935d51cc8837230ad48db08acd3744dd9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 1 Feb 2017 02:15:52 +0700
Subject: [PATCH 0141/1696] [vimeo] PEP 8

---
 youtube_dl/extractor/vimeo.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 8ba222224..61cc469bf 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -22,13 +22,11 @@ from ..utils import (
     smuggle_url,
     std_headers,
     try_get,
-    unified_strdate,
     unified_timestamp,
     unsmuggle_url,
     urlencode_postdata,
     unescapeHTML,
     parse_filesize,
-    try_get,
 )
 
 

From 2b2d5d319b563a12e26c55966a047fa5bb039cd0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 1 Feb 2017 16:39:32 +0800
Subject: [PATCH 0142/1696] [crunchyroll] Remove ScaledBorderAndShadow settings

See https://github.com/rg3/youtube-dl/pull/9028, especially @lachs0r's
comments for the reason behind this change
---
 youtube_dl/extractor/crunchyroll.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index f811c7f33..109d1c5a8 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -255,8 +255,7 @@ class CrunchyrollIE(CrunchyrollBaseIE):
         output += 'WrapStyle: %s\n' % sub_root.attrib['wrap_style']
         output += 'PlayResX: %s\n' % sub_root.attrib['play_res_x']
         output += 'PlayResY: %s\n' % sub_root.attrib['play_res_y']
-        output += """ScaledBorderAndShadow: no
-
+        output += """
 [V4+ Styles]
 Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
 """

From 7882f1115e8eca2d2c958e2dbb6be45450e4027c Mon Sep 17 00:00:00 2001
From: Thomas Christlieb <thomaschristlieb@hotmail.com>
Date: Wed, 1 Feb 2017 16:00:41 +0100
Subject: [PATCH 0143/1696] Added new Regex for prosiebensat1 Extractor
 Description. Fixes #11810 (#11929)

* Added new Regex for prosiebensat1 Extractor Description. Fixes #11810

* Using _og_search_description() as a Fallback for Description-Regex

* Using _og_search_description() as a Fallback for Description-Regex - Second try

* Also added fallback regex

* Using _og_search_description() as a Fallback for Description-Regex - Third try

* removed fatal=False from search for description regex. default=None should be preferred only

* Using fatal=false for _og_search_description

* Revert "Using fatal=false for _og_search_description"

This reverts commit 2b7e123f9d0f2bd6ada54fa8e4e6035fece5dbf4.

* Deleted default=None Parameter for _og_search_property
---
 youtube_dl/extractor/prosiebensat1.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 03e1b1f7f..6856bacaf 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -375,7 +375,9 @@ class ProSiebenSat1IE(ProSiebenSat1BaseIE):
         title = self._html_search_regex(self._TITLE_REGEXES, webpage, 'title')
         info = self._extract_video_info(url, clip_id)
         description = self._html_search_regex(
-            self._DESCRIPTION_REGEXES, webpage, 'description', fatal=False)
+            self._DESCRIPTION_REGEXES, webpage, 'description', default=None)
+        if description is None: 
+            description = self._og_search_description(webpage)
         thumbnail = self._og_search_thumbnail(webpage)
         upload_date = unified_strdate(self._html_search_regex(
             self._UPLOAD_DATE_REGEXES, webpage, 'upload date', default=None))

From fe5aa197b58be1bbf88a152be0e84f24f1711bd7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 1 Feb 2017 23:13:45 +0800
Subject: [PATCH 0144/1696] [prosiebensat1] PEP8 and update _TESTS

---
 youtube_dl/extractor/prosiebensat1.py | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 6856bacaf..5091d8456 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -147,16 +147,12 @@ class ProSiebenSat1IE(ProSiebenSat1BaseIE):
             'url': 'http://www.prosieben.de/tv/circus-halligalli/videos/218-staffel-2-episode-18-jahresrueckblick-ganze-folge',
             'info_dict': {
                 'id': '2104602',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Episode 18 - Staffel 2',
                 'description': 'md5:8733c81b702ea472e069bc48bb658fc1',
                 'upload_date': '20131231',
                 'duration': 5845.04,
             },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
         },
         {
             'url': 'http://www.prosieben.de/videokatalog/Gesellschaft/Leben/Trends/video-Lady-Umstyling-f%C3%BCr-Audrina-Rebekka-Audrina-Fergen-billig-aussehen-Battal-Modica-700544.html',
@@ -258,7 +254,7 @@ class ProSiebenSat1IE(ProSiebenSat1BaseIE):
             'url': 'http://www.the-voice-of-germany.de/video/31-andreas-kuemmert-rocket-man-clip',
             'info_dict': {
                 'id': '2572814',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Andreas Kümmert: Rocket Man',
                 'description': 'md5:6ddb02b0781c6adf778afea606652e38',
                 'upload_date': '20131017',
@@ -272,7 +268,7 @@ class ProSiebenSat1IE(ProSiebenSat1BaseIE):
             'url': 'http://www.fem.com/wellness/videos/wellness-video-clip-kurztripps-zum-valentinstag.html',
             'info_dict': {
                 'id': '2156342',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Kurztrips zum Valentinstag',
                 'description': 'Romantischer Kurztrip zum Valentinstag? Nina Heinemann verrät, was sich hier wirklich lohnt.',
                 'duration': 307.24,
@@ -289,12 +285,13 @@ class ProSiebenSat1IE(ProSiebenSat1BaseIE):
                 'description': 'md5:63b8963e71f481782aeea877658dec84',
             },
             'playlist_count': 2,
+            'skip': 'This video is unavailable',
         },
         {
             'url': 'http://www.7tv.de/circus-halligalli/615-best-of-circus-halligalli-ganze-folge',
             'info_dict': {
                 'id': '4187506',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Best of Circus HalliGalli',
                 'description': 'md5:8849752efd90b9772c9db6fdf87fb9e9',
                 'upload_date': '20151229',
@@ -376,7 +373,7 @@ class ProSiebenSat1IE(ProSiebenSat1BaseIE):
         info = self._extract_video_info(url, clip_id)
         description = self._html_search_regex(
             self._DESCRIPTION_REGEXES, webpage, 'description', default=None)
-        if description is None: 
+        if description is None:
             description = self._og_search_description(webpage)
         thumbnail = self._og_search_thumbnail(webpage)
         upload_date = unified_strdate(self._html_search_regex(

From 000f207944e277e63dbec5a60007c30e3187d3fd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 1 Feb 2017 23:16:35 +0800
Subject: [PATCH 0145/1696] [prosiebensat1] Update ChangeLog

---
 ChangeLog | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index d5ab0e0a7..da5b75b47 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,8 @@
+version <unreleased>
+
+Extractors
+* [prosiebensat1] Fix extraction of descriptions (#11810, #11929)
+
 version 2017.01.31
 
 Core

From b83ef507b457e6ea8c52265ea42b6c5d2c500a7e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 1 Feb 2017 23:15:38 +0700
Subject: [PATCH 0146/1696] [facebook] Fix extraction (closes #11926)

---
 youtube_dl/extractor/facebook.py | 36 ++++++++++++++++++++++++--------
 1 file changed, 27 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index c0a7fc7d8..47bcc0dbc 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -12,14 +12,16 @@ from ..compat import (
     compat_urllib_parse_unquote_plus,
 )
 from ..utils import (
+    clean_html,
     error_to_compat_str,
     ExtractorError,
+    get_element_by_id,
     int_or_none,
+    js_to_json,
     limit_length,
     sanitized_Request,
+    try_get,
     urlencode_postdata,
-    get_element_by_id,
-    clean_html,
 )
 
 
@@ -243,14 +245,30 @@ class FacebookIE(InfoExtractor):
 
         video_data = None
 
+        def extract_video_data(instances):
+            for item in instances:
+                if item[1][0] == 'VideoConfig':
+                    video_item = item[2][0]
+                    if video_item.get('video_id') == video_id:
+                        return video_item['videoData']
+
         server_js_data = self._parse_json(self._search_regex(
-            r'handleServerJS\(({.+})(?:\);|,")', webpage, 'server js data', default='{}'), video_id)
-        for item in server_js_data.get('instances', []):
-            if item[1][0] == 'VideoConfig':
-                video_item = item[2][0]
-                if video_item.get('video_id') == video_id:
-                    video_data = video_item['videoData']
-                    break
+            r'handleServerJS\(({.+})(?:\);|,")', webpage,
+            'server js data', default='{}'), video_id, fatal=False)
+
+        if server_js_data:
+            video_data = extract_video_data(server_js_data.get('instances', []))
+
+        if not video_data:
+            server_js_data = self._parse_json(
+                self._search_regex(
+                    r'bigPipe\.onPageletArrive\(({.+?})\)\s*;\s*}\s*\)\s*,\s*["\']onPageletArrive\s+stream_pagelet',
+                    webpage, 'js data', default='{}'),
+                video_id, transform_source=js_to_json, fatal=False)
+            if server_js_data:
+                video_data = extract_video_data(try_get(
+                    server_js_data, lambda x: x['jsmods']['instances'],
+                    list) or [])
 
         if not video_data:
             if not fatal_if_no_video:

From b996b8809285c2c8526dfe96f5ea9835ea799fe9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 1 Feb 2017 23:29:59 +0700
Subject: [PATCH 0147/1696] [ChangeLog] Actualize

---
 ChangeLog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index da5b75b47..d24169af8 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,13 @@
 version <unreleased>
 
 Extractors
++ [facebook] Add another fallback extraction scenario (#11926)
 * [prosiebensat1] Fix extraction of descriptions (#11810, #11929)
+- [crunchyroll] Remove ScaledBorderAndShadow settings (#9028)
++ [vimeo] Extract upload timestamp
++ [vimeo] Extract license (#8726, #11880)
++ [nrk:series] Add support for series (#11571, #11711)
+
 
 version 2017.01.31
 

From 50695949937bf399b611ef7957f44aac9fbee9dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 1 Feb 2017 03:20:09 +0700
Subject: [PATCH 0148/1696] release 2017.02.01

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 1 +
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 180013f72..8914569b6 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.01.31*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.01.31**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.01*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.01**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.01.31
+[debug] youtube-dl version 2017.02.01
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index d24169af8..c1e8f643a 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.02.01
 
 Extractors
 + [facebook] Add another fallback extraction scenario (#11926)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index d4231577b..d900f5e12 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -528,6 +528,7 @@
  - **NRKTV**: NRK TV and NRK Radio
  - **NRKTVDirekte**: NRK TV Direkte and NRK Radio Direkte
  - **NRKTVEpisodes**
+ - **NRKTVSeries**
  - **ntv.ru**
  - **Nuvid**
  - **NYTimes**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index fee0ac7c5..0f9b6b703 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.01.31'
+__version__ = '2017.02.01'

From da162c1135febbb653a302b598dba2d24ac4e24e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 1 Feb 2017 20:15:25 +0100
Subject: [PATCH 0149/1696] [compat] add compat_etree_register_namespace to
 __all__ list

---
 youtube_dl/compat.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 49e3c90e2..718902019 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2883,6 +2883,7 @@ __all__ = [
     'compat_cookiejar',
     'compat_cookies',
     'compat_etree_fromstring',
+    'compat_etree_register_namespace',
     'compat_expanduser',
     'compat_get_terminal_size',
     'compat_getenv',

From 020c5df52d61af0630be8c982282e110a83fc8df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 1 Feb 2017 23:48:34 +0100
Subject: [PATCH 0150/1696] [elpais] Fix extraction for some URLs (closes
 #11765)

---
 ChangeLog                      |  1 +
 youtube_dl/extractor/elpais.py | 23 ++++++++++++++++++++---
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index c1e8f643a..8e3a04d7d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -7,6 +7,7 @@ Extractors
 + [vimeo] Extract upload timestamp
 + [vimeo] Extract license (#8726, #11880)
 + [nrk:series] Add support for series (#11571, #11711)
++ [elpais] Fix extraction for some URLs (#11765)
 
 
 version 2017.01.31
diff --git a/youtube_dl/extractor/elpais.py b/youtube_dl/extractor/elpais.py
index 8c725a4e6..99e00cf3c 100644
--- a/youtube_dl/extractor/elpais.py
+++ b/youtube_dl/extractor/elpais.py
@@ -2,7 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import unified_strdate
+from ..utils import strip_jsonp, unified_strdate
 
 
 class ElPaisIE(InfoExtractor):
@@ -29,6 +29,16 @@ class ElPaisIE(InfoExtractor):
             'description': 'Que sí, que las cápsulas son cómodas. Pero si le pides algo más a la vida, quizá deberías aprender a usar bien la cafetera italiana. No tienes más que ver este vídeo y seguir sus siete normas básicas.',
             'upload_date': '20160303',
         }
+    }, {
+        'url': 'http://elpais.com/elpais/2017/01/26/ciencia/1485456786_417876.html',
+        'md5': '9c79923a118a067e1a45789e1e0b0f9c',
+        'info_dict': {
+            'id': '1485456786_417876',
+            'ext': 'mp4',
+            'title': 'Hallado un barco de la antigua Roma que naufragó en Baleares hace 1.800 años',
+            'description': 'La nave portaba cientos de ánforas y se hundió cerca de la isla de Cabrera por razones desconocidas',
+            'upload_date': '20170127',
+        },
     }]
 
     def _real_extract(self, url):
@@ -37,8 +47,15 @@ class ElPaisIE(InfoExtractor):
 
         prefix = self._html_search_regex(
             r'var\s+url_cache\s*=\s*"([^"]+)";', webpage, 'URL prefix')
-        video_suffix = self._search_regex(
-            r"(?:URLMediaFile|urlVideo_\d+)\s*=\s*url_cache\s*\+\s*'([^']+)'", webpage, 'video URL')
+        id_multimedia = self._search_regex(
+            r"id_multimedia\s*=\s*'([^']+)'", webpage, 'ID multimedia', default=None)
+        if id_multimedia:
+            url_info = self._download_json(
+                'http://elpais.com/vdpep/1/?pepid=' + id_multimedia, video_id, transform_source=strip_jsonp)
+            video_suffix = url_info['mp4']
+        else:
+            video_suffix = self._search_regex(
+                r"(?:URLMediaFile|urlVideo_\d+)\s*=\s*url_cache\s*\+\s*'([^']+)'", webpage, 'video URL')
         video_url = prefix + video_suffix
         thumbnail_suffix = self._search_regex(
             r"(?:URLMediaStill|urlFotogramaFijo_\d+)\s*=\s*url_cache\s*\+\s*'([^']+)'",

From 8bdc149441a86e01c56946090087c005a525260e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 2 Feb 2017 08:05:16 +0100
Subject: [PATCH 0151/1696] [downloader/external:ffmpeg] minimize the use of
 aac_adtstoasc filter

---
 youtube_dl/downloader/external.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 5d3e5d8d3..138f353ef 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -17,6 +17,7 @@ from ..utils import (
     encodeArgument,
     handle_youtubedl_headers,
     check_executable,
+    is_outdated_version,
 )
 
 
@@ -264,7 +265,9 @@ class FFmpegFD(ExternalFD):
             if self.params.get('hls_use_mpegts', False) or tmpfilename == '-':
                 args += ['-f', 'mpegts']
             else:
-                args += ['-f', 'mp4', '-bsf:a', 'aac_adtstoasc']
+                args += ['-f', 'mp4']
+                if (ffpp.basename == 'ffmpeg' and is_outdated_version(ffpp._versions['ffmpeg'], '3.2')) and (not info_dict.get('acodec') or info_dict['acodec'].split('.')[0] in ('aac', 'mp4a')):
+                    args += ['-bsf:a', 'aac_adtstoasc']
         elif protocol == 'rtmp':
             args += ['-f', 'flv']
         else:

From 81aeafeb44a16b341e47c3bb85d288252a095eda Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 2 Feb 2017 08:07:06 +0100
Subject: [PATCH 0152/1696] [cbc:watch] extract audio codec for audion only
 formats(fixes #11893)

---
 youtube_dl/extractor/cbc.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/cbc.py b/youtube_dl/extractor/cbc.py
index a291685bf..cf678e7f8 100644
--- a/youtube_dl/extractor/cbc.py
+++ b/youtube_dl/extractor/cbc.py
@@ -296,6 +296,12 @@ class CBCWatchVideoIE(CBCWatchBaseIE):
         formats = self._extract_m3u8_formats(re.sub(r'/([^/]+)/[^/?]+\.m3u8', r'/\1/\1.m3u8', m3u8_url), video_id, 'mp4', fatal=False)
         if len(formats) < 2:
             formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4')
+        for f in formats:
+            format_id = f.get('format_id')
+            if format_id.startswith('AAC'):
+                f['acodec'] = 'aac'
+            elif format_id.startswith('AC3'):
+                f['acodec'] = 'ac-3'
         self._sort_formats(formats)
 
         info = {

From bd8f48c78b952ebe3bf335185c819e265f63cb50 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 2 Feb 2017 21:51:31 +0800
Subject: [PATCH 0153/1696] [bilibili] Support new Bangumi URLs (closes #11845)

To reduce complexity, I don't support old Bangumi URLs directly via
_VALID_URL. Instead, I choose to let it go to generic redirection. An
example can be found in #10190:

http://bangumi.bilibili.com/anime/v/40062
---
 ChangeLog                          |   5 ++
 youtube_dl/extractor/bilibili.py   | 135 ++++++++++++++++++++++++++---
 youtube_dl/extractor/extractors.py |   5 +-
 3 files changed, 134 insertions(+), 11 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 8e3a04d7d..c27907f51 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,8 @@
+version <unreleased>
+
+Extractors
++ [bilibili] Support new Bangumi URLs (#11845)
+
 version 2017.02.01
 
 Extractors
diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 85ea5e6ee..80dd8382e 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -5,19 +5,27 @@ import hashlib
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_parse_qs
+from ..compat import (
+    compat_parse_qs,
+    compat_urlparse,
+)
 from ..utils import (
+    ExtractorError,
     int_or_none,
     float_or_none,
+    parse_iso8601,
+    smuggle_url,
+    strip_jsonp,
     unified_timestamp,
+    unsmuggle_url,
     urlencode_postdata,
 )
 
 
 class BiliBiliIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.|bangumi\.|)bilibili\.(?:tv|com)/(?:video/av|anime/v/)(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.|bangumi\.|)bilibili\.(?:tv|com)/(?:video/av|anime/(?P<anime_id>\d+)/play#)(?P<id>\d+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.bilibili.tv/video/av1074402/',
         'md5': '9fa226fe2b8a9a4d5a69b4c6a183417e',
         'info_dict': {
@@ -32,25 +40,61 @@ class BiliBiliIE(InfoExtractor):
             'uploader': '菊子桑',
             'uploader_id': '156160',
         },
-    }
+    }, {
+        # Tested in BiliBiliBangumiIE
+        'url': 'http://bangumi.bilibili.com/anime/1869/play#40062',
+        'only_matching': True,
+    }, {
+        'url': 'http://bangumi.bilibili.com/anime/5802/play#100643',
+        'md5': '3f721ad1e75030cc06faf73587cfec57',
+        'info_dict': {
+            'id': '100643',
+            'ext': 'mp4',
+            'title': 'CHAOS;CHILD',
+            'description': '如果你是神明，并且能够让妄想成为现实。那你会进行怎么样的妄想？是淫靡的世界？独裁社会？毁灭性的制裁？还是……2015年，涩谷。从6年前发生的大灾害“涩谷地震”之后复兴了的这个街区里新设立的私立高中...',
+        },
+        'skip': 'Geo-restricted to China',
+    }]
 
     _APP_KEY = '84956560bc028eb7'
     _BILIBILI_KEY = '94aba54af9065f71de72f5508f1cd42e'
 
+    def _report_error(self, result):
+        if 'message' in result:
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, result['message']), expected=True)
+        elif 'code' in result:
+            raise ExtractorError('%s returns error %d' % (self.IE_NAME, result['code']), expected=True)
+        else:
+            raise ExtractorError('Can\'t extract Bangumi episode ID')
+
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        url, smuggled_data = unsmuggle_url(url, {})
+
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        anime_id = mobj.group('anime_id')
         webpage = self._download_webpage(url, video_id)
 
-        if 'anime/v' not in url:
+        if 'anime/' not in url:
             cid = compat_parse_qs(self._search_regex(
                 [r'EmbedPlayer\([^)]+,\s*"([^"]+)"\)',
                  r'<iframe[^>]+src="https://secure\.bilibili\.com/secure,([^"]+)"'],
                 webpage, 'player parameters'))['cid'][0]
         else:
+            if 'no_bangumi_tip' not in smuggled_data:
+                self.to_screen('Downloading episode %s. To download all videos in anime %s, re-run youtube-dl with %s' % (
+                    video_id, anime_id, compat_urlparse.urljoin(url, '//bangumi.bilibili.com/anime/%s' % anime_id)))
+            headers = {
+                'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
+            }
+            headers.update(self.geo_verification_headers())
+
             js = self._download_json(
                 'http://bangumi.bilibili.com/web_api/get_source', video_id,
                 data=urlencode_postdata({'episode_id': video_id}),
-                headers={'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8'})
+                headers=headers)
+            if 'result' not in js:
+                self._report_error(js)
             cid = js['result']['cid']
 
         payload = 'appkey=%s&cid=%s&otype=json&quality=2&type=mp4' % (self._APP_KEY, cid)
@@ -58,7 +102,11 @@ class BiliBiliIE(InfoExtractor):
 
         video_info = self._download_json(
             'http://interface.bilibili.com/playurl?%s&sign=%s' % (payload, sign),
-            video_id, note='Downloading video info page')
+            video_id, note='Downloading video info page',
+            headers=self.geo_verification_headers())
+
+        if 'durl' not in video_info:
+            self._report_error(video_info)
 
         entries = []
 
@@ -85,7 +133,7 @@ class BiliBiliIE(InfoExtractor):
         title = self._html_search_regex('<h1[^>]+title="([^"]+)">', webpage, 'title')
         description = self._html_search_meta('description', webpage)
         timestamp = unified_timestamp(self._html_search_regex(
-            r'<time[^>]+datetime="([^"]+)"', webpage, 'upload time', fatal=False))
+            r'<time[^>]+datetime="([^"]+)"', webpage, 'upload time', default=None))
         thumbnail = self._html_search_meta(['og:image', 'thumbnailUrl'], webpage)
 
         # TODO 'view_count' requires deobfuscating Javascript
@@ -99,7 +147,7 @@ class BiliBiliIE(InfoExtractor):
         }
 
         uploader_mobj = re.search(
-            r'<a[^>]+href="https?://space\.bilibili\.com/(?P<id>\d+)"[^>]+title="(?P<name>[^"]+)"',
+            r'<a[^>]+href="(?:https?:)?//space\.bilibili\.com/(?P<id>\d+)"[^>]+title="(?P<name>[^"]+)"',
             webpage)
         if uploader_mobj:
             info.update({
@@ -123,3 +171,70 @@ class BiliBiliIE(InfoExtractor):
                 'description': description,
                 'entries': entries,
             }
+
+
+class BiliBiliBangumiIE(InfoExtractor):
+    _VALID_URL = r'https?://bangumi\.bilibili\.com/anime/(?P<id>\d+)'
+
+    IE_NAME = 'bangumi.bilibili.com'
+    IE_DESC = 'BiliBili番剧'
+
+    _TESTS = [{
+        'url': 'http://bangumi.bilibili.com/anime/1869',
+        'info_dict': {
+            'id': '1869',
+            'title': '混沌武士',
+            'description': 'md5:6a9622b911565794c11f25f81d6a97d2',
+        },
+        'playlist_count': 26,
+    }, {
+        'url': 'http://bangumi.bilibili.com/anime/1869',
+        'info_dict': {
+            'id': '1869',
+            'title': '混沌武士',
+            'description': 'md5:6a9622b911565794c11f25f81d6a97d2',
+        },
+        'playlist': [{
+            'md5': '91da8621454dd58316851c27c68b0c13',
+            'info_dict': {
+                'id': '40062',
+                'ext': 'mp4',
+                'title': '混沌武士',
+                'description': '故事发生在日本的江户时代。风是一个小酒馆的打工女。一日，酒馆里来了一群恶霸，虽然他们的举动令风十分不满，但是毕竟风只是一届女流，无法对他们采取什么行动，只能在心里嘟哝。这时，酒家里又进来了个“不良份子...',
+                'timestamp': 1414538739,
+                'upload_date': '20141028',
+                'episode': '疾风怒涛 Tempestuous Temperaments',
+                'episode_number': 1,
+            },
+        }],
+        'params': {
+            'playlist_items': '1',
+        },
+    }]
+
+    @classmethod
+    def suitable(cls, url):
+        return False if BiliBiliIE.suitable(url) else super(BiliBiliBangumiIE, cls).suitable(url)
+
+    def _real_extract(self, url):
+        bangumi_id = self._match_id(url)
+
+        # Sometimes this API returns a JSONP response
+        season_info = self._download_json(
+            'http://bangumi.bilibili.com/jsonp/seasoninfo/%s.ver' % bangumi_id,
+            bangumi_id, transform_source=strip_jsonp)['result']
+
+        entries = [{
+            '_type': 'url_transparent',
+            'url': smuggle_url(episode['webplay_url'], {'no_bangumi_tip': 1}),
+            'ie_key': BiliBiliIE.ie_key(),
+            'timestamp': parse_iso8601(episode.get('update_time'), delimiter=' '),
+            'episode': episode.get('index_title'),
+            'episode_number': int_or_none(episode.get('index')),
+        } for episode in season_info['episodes']]
+
+        entries = sorted(entries, key=lambda entry: entry.get('episode_number'))
+
+        return self.playlist_result(
+            entries, bangumi_id,
+            season_info.get('bangumi_title'), season_info.get('evaluate'))
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 06e6d4620..1d1c05d42 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -103,7 +103,10 @@ from .beatport import BeatportIE
 from .bet import BetIE
 from .bigflix import BigflixIE
 from .bild import BildIE
-from .bilibili import BiliBiliIE
+from .bilibili import (
+    BiliBiliIE,
+    BiliBiliBangumiIE,
+)
 from .biobiochiletv import BioBioChileTVIE
 from .biqle import BIQLEIE
 from .bleacherreport import (

From a685751051f277b8ce99ee0949420bca4ea28c28 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 2 Feb 2017 22:01:11 +0700
Subject: [PATCH 0154/1696] [youtube:playlist] Recognize TL playlists (closes
 #11945)

---
 youtube_dl/extractor/youtube.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index ea398bcc8..0e67fdd12 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1857,13 +1857,13 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
                             youtu\.be/[0-9A-Za-z_-]{11}\?.*?\blist=
                         )
                         (
-                            (?:PL|LL|EC|UU|FL|RD|UL)?[0-9A-Za-z-_]{10,}
+                            (?:PL|LL|EC|UU|FL|RD|UL|TL)?[0-9A-Za-z-_]{10,}
                             # Top tracks, they can also include dots
                             |(?:MC)[\w\.]*
                         )
                         .*
                      |
-                        ((?:PL|LL|EC|UU|FL|RD|UL)[0-9A-Za-z-_]{10,})
+                        ((?:PL|LL|EC|UU|FL|RD|UL|TL)[0-9A-Za-z-_]{10,})
                      )"""
     _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s&disable_polymer=true'
     _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})&amp;[^"]*?index=(?P<index>\d+)(?:[^>]+>(?P<title>[^<]+))?'
@@ -1985,6 +1985,9 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
     }, {
         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
         'only_matching': True,
+    }, {
+        'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
+        'only_matching': True,
     }]
 
     def _real_initialize(self):

From 5a116e13020813f9f1d952504455043986c28b9b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 2 Feb 2017 22:45:18 +0700
Subject: [PATCH 0155/1696] [facebook] Fix title extraction (closes #11941)

---
 youtube_dl/extractor/facebook.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 47bcc0dbc..b325c8200 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -73,7 +73,7 @@ class FacebookIE(InfoExtractor):
         'info_dict': {
             'id': '274175099429670',
             'ext': 'mp4',
-            'title': 'Facebook video #274175099429670',
+            'title': 'Asif Nawab Butt posted a video to his Timeline.',
             'uploader': 'Asif Nawab Butt',
             'upload_date': '20140506',
             'timestamp': 1399398998,
@@ -318,10 +318,16 @@ class FacebookIE(InfoExtractor):
             video_title = self._html_search_regex(
                 r'(?s)<span class="fbPhotosPhotoCaption".*?id="fbPhotoPageCaption"><span class="hasCaption">(.*?)</span>',
                 webpage, 'alternative title', default=None)
-            video_title = limit_length(video_title, 80)
         if not video_title:
+            video_title = self._html_search_meta(
+                'description', webpage, 'title')
+        if video_title:
+            video_title = limit_length(video_title, 80)
+        else:
             video_title = 'Facebook video #%s' % video_id
-        uploader = clean_html(get_element_by_id('fbPhotoPageAuthorName', webpage))
+        uploader = clean_html(get_element_by_id(
+            'fbPhotoPageAuthorName', webpage)) or self._search_regex(
+            r'ownerName\s*:\s*"([^"]+)"', webpage, 'uploader', fatal=False)
         timestamp = int_or_none(self._search_regex(
             r'<abbr[^>]+data-utime=["\'](\d+)', webpage,
             'timestamp', default=None))

From c54c01f82dba6d3e982c73c81ad71c49f31d8af1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 2 Feb 2017 23:03:38 +0700
Subject: [PATCH 0156/1696] [go] Relax video id regex (closes #11937)

---
 youtube_dl/extractor/go.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/go.py b/youtube_dl/extractor/go.py
index c7776b186..a34779b16 100644
--- a/youtube_dl/extractor/go.py
+++ b/youtube_dl/extractor/go.py
@@ -43,7 +43,10 @@ class GoIE(InfoExtractor):
         sub_domain, video_id, display_id = re.match(self._VALID_URL, url).groups()
         if not video_id:
             webpage = self._download_webpage(url, display_id)
-            video_id = self._search_regex(r'data-video-id=["\']VDKA(\w+)', webpage, 'video id')
+            video_id = self._search_regex(
+                # There may be inner quotes, e.g. data-video-id="'VDKA3609139'"
+                # from http://freeform.go.com/shows/shadowhunters/episodes/season-2/1-this-guilty-blood
+                r'data-video-id=["\']*VDKA(\w+)', webpage, 'video id')
         brand = self._BRANDS[sub_domain]
         video_data = self._download_json(
             'http://api.contents.watchabc.go.com/vp2/ws/contents/3000/videos/%s/001/-1/-1/-1/%s/-1/-1.json' % (brand, video_id),

From a22b2fd19bd8c08d50f884d1903486d4f00f76ec Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 3 Feb 2017 01:28:24 +0800
Subject: [PATCH 0157/1696] [youtube] Fix ytsearch* when cookies are provided

Closes #11924

The API with `page` is no longer used in browsers, and YouTube always
returns {'reload': 'now'} when cookies are provided.

See http://youtube.github.io/spfjs/documentation/start/ for how SPF
works. Basically appending static link with a `spf` parameter yields the
corresponding dynamic link.
---
 ChangeLog                       |  1 +
 youtube_dl/extractor/youtube.py | 22 ++++++++++++++--------
 2 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index c27907f51..c80126cfb 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [youtube] Fix ytsearch when cookies are provided (#11924)
 + [bilibili] Support new Bangumi URLs (#11845)
 
 version 2017.02.01
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 0e67fdd12..f2f751104 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2348,18 +2348,18 @@ class YoutubeSearchIE(SearchInfoExtractor, YoutubePlaylistIE):
         videos = []
         limit = n
 
+        url_query = {
+            'search_query': query.encode('utf-8'),
+        }
+        url_query.update(self._EXTRA_QUERY_ARGS)
+        result_url = 'https://www.youtube.com/results?' + compat_urllib_parse_urlencode(url_query)
+
         for pagenum in itertools.count(1):
-            url_query = {
-                'search_query': query.encode('utf-8'),
-                'page': pagenum,
-                'spf': 'navigate',
-            }
-            url_query.update(self._EXTRA_QUERY_ARGS)
-            result_url = 'https://www.youtube.com/results?' + compat_urllib_parse_urlencode(url_query)
             data = self._download_json(
                 result_url, video_id='query "%s"' % query,
                 note='Downloading page %s' % pagenum,
-                errnote='Unable to download API page')
+                errnote='Unable to download API page',
+                query={'spf': 'navigate'})
             html_content = data[1]['body']['content']
 
             if 'class="search-message' in html_content:
@@ -2371,6 +2371,12 @@ class YoutubeSearchIE(SearchInfoExtractor, YoutubePlaylistIE):
             videos += new_videos
             if not new_videos or len(videos) > limit:
                 break
+            next_link = self._html_search_regex(
+                r'href="(/results\?[^"]*\bsp=[^"]+)"[^>]*>\s*<span[^>]+class="[^"]*\byt-uix-button-content\b[^"]*"[^>]*>Next',
+                html_content, 'next link', default=None)
+            if next_link is None:
+                break
+            result_url = compat_urlparse.urljoin('https://www.youtube.com/', next_link)
 
         if len(videos) > n:
             videos = videos[:n]

From b3ee552e4b918fb720111b23147e24fa5475a74b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Michal=20=C4=8Ciha=C5=99?= <michal@cihar.com>
Date: Tue, 31 Jan 2017 07:54:53 +0100
Subject: [PATCH 0158/1696] [utils] Handle single-line comments in js_to_json

---
 test/test_utils.py  | 3 +++
 youtube_dl/utils.py | 4 ++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index a74d59f34..954bb7d8b 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -791,6 +791,9 @@ class TestUtil(unittest.TestCase):
         on = js_to_json('{ 0: /* " \n */ ",]" , }')
         self.assertEqual(json.loads(on), {'0': ',]'})
 
+        on = js_to_json('{ 0: // comment\n1 }')
+        self.assertEqual(json.loads(on), {'0': 1})
+
         on = js_to_json(r'["<p>x<\/p>"]')
         self.assertEqual(json.loads(on), ['<p>x</p>'])
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index cf46711b9..6c462625b 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2107,7 +2107,7 @@ def js_to_json(code):
         v = m.group(0)
         if v in ('true', 'false', 'null'):
             return v
-        elif v.startswith('/*') or v == ',':
+        elif v.startswith('/*') or v.startswith('//') or v == ',':
             return ""
 
         if v[0] in ("'", '"'):
@@ -2134,7 +2134,7 @@ def js_to_json(code):
     return re.sub(r'''(?sx)
         "(?:[^"\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^"\\]*"|
         '(?:[^'\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^'\\]*'|
-        /\*.*?\*/|,(?=\s*[\]}])|
+        /\*.*?\*/|//[^\n]*|,(?=\s*[\]}])|
         [a-zA-Z_][.a-zA-Z_0-9]*|
         \b(?:0[xX][0-9a-fA-F]+|0+[0-7]+)(?:\s*:)?|
         [0-9]+(?=\s*:)

From 0bbcc8a10a4bd339540bf149dd263419fd8b6e66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Michal=20=C4=8Ciha=C5=99?= <michal@cihar.com>
Date: Tue, 31 Jan 2017 07:59:55 +0100
Subject: [PATCH 0159/1696] [iprima] Fix extraction (closes #11920, closes
 #11896)

---
 youtube_dl/extractor/iprima.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index da2cdc656..0fe576883 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -65,7 +65,7 @@ class IPrimaIE(InfoExtractor):
 
         options = self._parse_json(
             self._search_regex(
-                r'(?s)var\s+playerOptions\s*=\s*({.+?});',
+                r'(?s)(?:TDIPlayerOptions|playerOptions)\s*=\s*({.+?});\s*\]\]',
                 playerpage, 'player options', default='{}'),
             video_id, transform_source=js_to_json, fatal=False)
         if options:

From 4195096ea8da8237a63e1ba3876dc8856b8605c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 3 Feb 2017 02:55:06 +0700
Subject: [PATCH 0160/1696] [utils] Improve comments processing in js_to_json
 (closes #11947)

---
 test/test_utils.py  | 24 ++++++++++++++++++++++++
 youtube_dl/utils.py | 20 +++++++++++---------
 2 files changed, 35 insertions(+), 9 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 954bb7d8b..edc712f07 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -785,12 +785,24 @@ class TestUtil(unittest.TestCase):
         on = js_to_json('["abc", "def",]')
         self.assertEqual(json.loads(on), ['abc', 'def'])
 
+        on = js_to_json('[/*comment\n*/"abc"/*comment\n*/,/*comment\n*/"def",/*comment\n*/]')
+        self.assertEqual(json.loads(on), ['abc', 'def'])
+
+        on = js_to_json('[//comment\n"abc" //comment\n,//comment\n"def",//comment\n]')
+        self.assertEqual(json.loads(on), ['abc', 'def'])
+
         on = js_to_json('{"abc": "def",}')
         self.assertEqual(json.loads(on), {'abc': 'def'})
 
+        on = js_to_json('{/*comment\n*/"abc"/*comment\n*/:/*comment\n*/"def"/*comment\n*/,/*comment\n*/}')
+        self.assertEqual(json.loads(on), {'abc': 'def'})
+
         on = js_to_json('{ 0: /* " \n */ ",]" , }')
         self.assertEqual(json.loads(on), {'0': ',]'})
 
+        on = js_to_json('{ /*comment\n*/0/*comment\n*/: /* " \n */ ",]" , }')
+        self.assertEqual(json.loads(on), {'0': ',]'})
+
         on = js_to_json('{ 0: // comment\n1 }')
         self.assertEqual(json.loads(on), {'0': 1})
 
@@ -803,15 +815,27 @@ class TestUtil(unittest.TestCase):
         on = js_to_json("['a\\\nb']")
         self.assertEqual(json.loads(on), ['ab'])
 
+        on = js_to_json("/*comment\n*/[/*comment\n*/'a\\\nb'/*comment\n*/]/*comment\n*/")
+        self.assertEqual(json.loads(on), ['ab'])
+
         on = js_to_json('{0xff:0xff}')
         self.assertEqual(json.loads(on), {'255': 255})
 
+        on = js_to_json('{/*comment\n*/0xff/*comment\n*/:/*comment\n*/0xff/*comment\n*/}')
+        self.assertEqual(json.loads(on), {'255': 255})
+
         on = js_to_json('{077:077}')
         self.assertEqual(json.loads(on), {'63': 63})
 
+        on = js_to_json('{/*comment\n*/077/*comment\n*/:/*comment\n*/077/*comment\n*/}')
+        self.assertEqual(json.loads(on), {'63': 63})
+
         on = js_to_json('{42:42}')
         self.assertEqual(json.loads(on), {'42': 42})
 
+        on = js_to_json('{/*comment\n*/42/*comment\n*/:/*comment\n*/42/*comment\n*/}')
+        self.assertEqual(json.loads(on), {'42': 42})
+
     def test_extract_attributes(self):
         self.assertEqual(extract_attributes('<e x="y">'), {'x': 'y'})
         self.assertEqual(extract_attributes("<e x='y'>"), {'x': 'y'})
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 6c462625b..67a847eba 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2103,6 +2103,13 @@ def strip_jsonp(code):
 
 
 def js_to_json(code):
+    COMMENT_RE = r'/\*(?:(?!\*/).)*?\*/|//[^\n]*'
+    SKIP_RE = r'\s*(?:{comment})?\s*'.format(comment=COMMENT_RE)
+    INTEGER_TABLE = (
+        (r'(?s)^(0[xX][0-9a-fA-F]+){skip}:?$'.format(skip=SKIP_RE), 16),
+        (r'(?s)^(0+[0-7]+){skip}:?$'.format(skip=SKIP_RE), 8),
+    )
+
     def fix_kv(m):
         v = m.group(0)
         if v in ('true', 'false', 'null'):
@@ -2118,11 +2125,6 @@ def js_to_json(code):
                 '\\x': '\\u00',
             }.get(m.group(0), m.group(0)), v[1:-1])
 
-        INTEGER_TABLE = (
-            (r'^(0[xX][0-9a-fA-F]+)\s*:?$', 16),
-            (r'^(0+[0-7]+)\s*:?$', 8),
-        )
-
         for regex, base in INTEGER_TABLE:
             im = re.match(regex, v)
             if im:
@@ -2134,11 +2136,11 @@ def js_to_json(code):
     return re.sub(r'''(?sx)
         "(?:[^"\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^"\\]*"|
         '(?:[^'\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^'\\]*'|
-        /\*.*?\*/|//[^\n]*|,(?=\s*[\]}])|
+        {comment}|,(?={skip}[\]}}])|
         [a-zA-Z_][.a-zA-Z_0-9]*|
-        \b(?:0[xX][0-9a-fA-F]+|0+[0-7]+)(?:\s*:)?|
-        [0-9]+(?=\s*:)
-        ''', fix_kv, code)
+        \b(?:0[xX][0-9a-fA-F]+|0+[0-7]+)(?:{skip}:)?|
+        [0-9]+(?={skip}:)
+        '''.format(comment=COMMENT_RE, skip=SKIP_RE), fix_kv, code)
 
 
 def qualities(quality_ids):

From 33da98f4933ddc54c944bae985cfcc7b53563208 Mon Sep 17 00:00:00 2001
From: Justsoos <justso@gmail.com>
Date: Wed, 1 Feb 2017 21:30:01 +0800
Subject: [PATCH 0161/1696] [douyutv] Improve room id regex

http://www.douyu.com/t/lpl  source get extra '\' with "room_id\" (from js coding)
---
 youtube_dl/extractor/douyutv.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/douyutv.py b/youtube_dl/extractor/douyutv.py
index 2f3c5113e..911594413 100644
--- a/youtube_dl/extractor/douyutv.py
+++ b/youtube_dl/extractor/douyutv.py
@@ -18,7 +18,7 @@ from ..utils import (
 
 class DouyuTVIE(InfoExtractor):
     IE_DESC = '斗鱼'
-    _VALID_URL = r'https?://(?:www\.)?douyu(?:tv)?\.com/(?P<id>[A-Za-z0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?douyu(?:tv)?\.com/(?:[^/]+/)*(?P<id>[A-Za-z0-9]+)'
     _TESTS = [{
         'url': 'http://www.douyutv.com/iseven',
         'info_dict': {
@@ -68,6 +68,10 @@ class DouyuTVIE(InfoExtractor):
     }, {
         'url': 'http://www.douyu.com/xiaocang',
         'only_matching': True,
+    }, {
+        # \"room_id\"
+        'url': 'http://www.douyu.com/t/lpl',
+        'only_matching': True,
     }]
 
     # Decompile core.swf in webpage by ffdec "Search SWFs in memory". core.swf
@@ -82,7 +86,7 @@ class DouyuTVIE(InfoExtractor):
         else:
             page = self._download_webpage(url, video_id)
             room_id = self._html_search_regex(
-                r'"room_id"\s*:\s*(\d+),', page, 'room id')
+                r'"room_id\\?"\s*:\s*(\d+),', page, 'room id')
 
         room = self._download_json(
             'http://m.douyu.com/html5/live?roomId=%s' % room_id, video_id,

From 45024183aea169dc898902388f782485de02cbac Mon Sep 17 00:00:00 2001
From: Mattias Wadman <mattias.wadman@gmail.com>
Date: Fri, 3 Feb 2017 05:10:13 +0100
Subject: [PATCH 0162/1696] [infoq] Add audio only format if available (#11565)

* [infoq] Add audio only format if available

Refactor cookie code into a function.
Renamed formats to http_video, http_audio, rtmp_video
Renamed extract functions to video instead of videos as they return
one or no video.

* [infoq] Rename to _extract_cookies as it more than one

* [infoq] Remove redundant determine_ext

* [infoq] Add comment about hardcoded URL

* [infoq] Use _hidden_inputs instead of messy regex

* [infoq] Probe if audio URL is valid

Make it possible to pass headers to _is_valid_url

* [infoq] Add audio only test
---
 youtube_dl/extractor/common.py |  4 +--
 youtube_dl/extractor/infoq.py  | 63 ++++++++++++++++++++++++++++------
 2 files changed, 55 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5a15a9536..2c8ec1417 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1025,13 +1025,13 @@ class InfoExtractor(object):
                 unique_formats.append(f)
         formats[:] = unique_formats
 
-    def _is_valid_url(self, url, video_id, item='video'):
+    def _is_valid_url(self, url, video_id, item='video', headers={}):
         url = self._proto_relative_url(url, scheme='http:')
         # For now assume non HTTP(S) URLs always valid
         if not (url.startswith('http://') or url.startswith('https://')):
             return True
         try:
-            self._request_webpage(url, video_id, 'Checking %s URL' % item)
+            self._request_webpage(url, video_id, 'Checking %s URL' % item, headers=headers)
             return True
         except ExtractorError as e:
             if isinstance(e.cause, compat_urllib_error.URLError):
diff --git a/youtube_dl/extractor/infoq.py b/youtube_dl/extractor/infoq.py
index cca0b8a93..9fb71e8ef 100644
--- a/youtube_dl/extractor/infoq.py
+++ b/youtube_dl/extractor/infoq.py
@@ -4,7 +4,10 @@ from __future__ import unicode_literals
 
 import base64
 
-from ..compat import compat_urllib_parse_unquote
+from ..compat import (
+    compat_urllib_parse_unquote,
+    compat_urlparse,
+)
 from ..utils import determine_ext
 from .bokecc import BokeCCBaseIE
 
@@ -33,9 +36,21 @@ class InfoQIE(BokeCCBaseIE):
             'ext': 'flv',
             'description': 'md5:308d981fb28fa42f49f9568322c683ff',
         },
+    }, {
+        'url': 'https://www.infoq.com/presentations/Simple-Made-Easy',
+        'md5': '0e34642d4d9ef44bf86f66f6399672db',
+        'info_dict': {
+            'id': 'Simple-Made-Easy',
+            'title': 'Simple Made Easy',
+            'ext': 'mp3',
+            'description': 'md5:3e0e213a8bbd074796ef89ea35ada25b',
+        },
+        'params': {
+            'format': 'bestaudio',
+        },
     }]
 
-    def _extract_rtmp_videos(self, webpage):
+    def _extract_rtmp_video(self, webpage):
         # The server URL is hardcoded
         video_url = 'rtmpe://video.infoq.com/cfx/st/'
 
@@ -47,28 +62,53 @@ class InfoQIE(BokeCCBaseIE):
         playpath = 'mp4:' + real_id
 
         return [{
-            'format_id': 'rtmp',
+            'format_id': 'rtmp_video',
             'url': video_url,
             'ext': determine_ext(playpath),
             'play_path': playpath,
         }]
 
-    def _extract_http_videos(self, webpage):
-        http_video_url = self._search_regex(r'P\.s\s*=\s*\'([^\']+)\'', webpage, 'video URL')
-
+    def _extract_cookies(self, webpage):
         policy = self._search_regex(r'InfoQConstants.scp\s*=\s*\'([^\']+)\'', webpage, 'policy')
         signature = self._search_regex(r'InfoQConstants.scs\s*=\s*\'([^\']+)\'', webpage, 'signature')
         key_pair_id = self._search_regex(r'InfoQConstants.sck\s*=\s*\'([^\']+)\'', webpage, 'key-pair-id')
+        return 'CloudFront-Policy=%s; CloudFront-Signature=%s; CloudFront-Key-Pair-Id=%s' % (
+            policy, signature, key_pair_id)
 
+    def _extract_http_video(self, webpage):
+        http_video_url = self._search_regex(r'P\.s\s*=\s*\'([^\']+)\'', webpage, 'video URL')
         return [{
-            'format_id': 'http',
+            'format_id': 'http_video',
             'url': http_video_url,
             'http_headers': {
-                'Cookie': 'CloudFront-Policy=%s; CloudFront-Signature=%s; CloudFront-Key-Pair-Id=%s' % (
-                    policy, signature, key_pair_id),
+                'Cookie': self._extract_cookies(webpage)
             },
         }]
 
+    def _extract_http_audio(self, webpage, video_id):
+        fields = self._hidden_inputs(webpage)
+        http_audio_url = fields['filename']
+        if http_audio_url is None:
+            return []
+
+        cookies_header = {'Cookie': self._extract_cookies(webpage)}
+
+        # base URL is found in the Location header in the response returned by
+        # GET https://www.infoq.com/mp3download.action?filename=... when logged in.
+        http_audio_url = compat_urlparse.urljoin('http://res.infoq.com/downloads/mp3downloads/', http_audio_url)
+
+        # audio file seem to be missing some times even if there is a download link
+        # so probe URL to make sure
+        if not self._is_valid_url(http_audio_url, video_id, headers=cookies_header):
+            return []
+
+        return [{
+            'format_id': 'http_audio',
+            'url': http_audio_url,
+            'vcodec': 'none',
+            'http_headers': cookies_header,
+        }]
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
@@ -80,7 +120,10 @@ class InfoQIE(BokeCCBaseIE):
             # for China videos, HTTP video URL exists but always fails with 403
             formats = self._extract_bokecc_formats(webpage, video_id)
         else:
-            formats = self._extract_rtmp_videos(webpage) + self._extract_http_videos(webpage)
+            formats = (
+                self._extract_rtmp_video(webpage) +
+                self._extract_http_video(webpage) +
+                self._extract_http_audio(webpage, video_id))
 
         self._sort_formats(formats)
 

From d7f9242e301fa7c08542932c9348140cf2e07172 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 3 Feb 2017 12:13:24 +0800
Subject: [PATCH 0163/1696] [ChangeLog] Update after #11565

---
 ChangeLog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChangeLog b/ChangeLog
index c80126cfb..487ed3f0f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
++ [infoq] Add audio only formats (#11565)
 * [youtube] Fix ytsearch when cookies are provided (#11924)
 + [bilibili] Support new Bangumi URLs (#11845)
 

From 4ce3407d089ae8c34341e6d68267910683d4b500 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 3 Feb 2017 10:15:03 +0100
Subject: [PATCH 0164/1696] [filmon] improve extraction

---
 youtube_dl/extractor/extractors.py |   5 +-
 youtube_dl/extractor/filmon.py     | 236 +++++++++++++++++------------
 2 files changed, 139 insertions(+), 102 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index c9b9ebd23..e4ee43ee3 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -287,7 +287,10 @@ from .fc2 import (
     FC2EmbedIE,
 )
 from .fczenit import FczenitIE
-from .filmon import FilmOnIE, FilmOnVODIE
+from .filmon import (
+    FilmOnIE,
+    FilmOnChannelIE,
+)
 from .firstpost import FirstpostIE
 from .firsttv import FirstTVIE
 from .fivemin import FiveMinIE
diff --git a/youtube_dl/extractor/filmon.py b/youtube_dl/extractor/filmon.py
index 987792fec..f775fe0ba 100644
--- a/youtube_dl/extractor/filmon.py
+++ b/youtube_dl/extractor/filmon.py
@@ -2,74 +2,21 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import qualities
-from ..compat import compat_urllib_request
-
-
-_QUALITY = qualities(('low', 'high'))
+from ..compat import (
+    compat_str,
+    compat_HTTPError,
+)
+from ..utils import (
+    qualities,
+    strip_or_none,
+    int_or_none,
+    ExtractorError,
+)
 
 
 class FilmOnIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?filmon\.com/(?:tv|channel)/(?P<id>[a-z0-9-]+)'
-    _TESTS = [{
-        'url': 'https://www.filmon.com/channel/filmon-sports',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.filmon.com/tv/2894',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        channel_id = self._match_id(url)
-
-        request = compat_urllib_request.Request('https://www.filmon.com/channel/%s' % (channel_id))
-        request.add_header('X-Requested-With', 'XMLHttpRequest')
-        channel_info = self._download_json(request, channel_id)
-        now_playing = channel_info['now_playing']
-
-        thumbnails = []
-        for thumb in now_playing.get('images', ()):
-            if thumb['type'] != '2':
-                continue
-            thumbnails.append({
-                'url': thumb['url'],
-                'width': int(thumb['width']),
-                'height': int(thumb['height']),
-            })
-
-        formats = []
-
-        for stream in channel_info['streams']:
-            formats.append({
-                'format_id': str(stream['id']),
-                # this is an m3u8 stream, but we are deliberately not using _extract_m3u8_formats
-                # because 0) it doesn't have bitrate variants anyway, and 1) the ids generated
-                # by that method are highly unstable (because the bitrate is variable)
-                'url': stream['url'],
-                'resolution': stream['name'],
-                'format_note': 'expires after %u seconds' % int(stream['watch-timeout']),
-                'ext': 'mp4',
-                'quality': _QUALITY(stream['quality']),
-                'preference': int(stream['watch-timeout']),
-            })
-        self._sort_formats(formats)
-
-        return {
-            'id': str(channel_info['id']),
-            'display_id': channel_info['alias'],
-            'formats': formats,
-            # XXX: use the channel description (channel_info['description'])?
-            'uploader_id': channel_info['alias'],
-            'uploader': channel_info['title'], # XXX: kinda stretching it...
-            'title': now_playing.get('programme_name') or channel_info['title'],
-            'description': now_playing.get('programme_description'),
-            'thumbnails': thumbnails,
-            'is_live': True,
-        }
-
-
-class FilmOnVODIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?filmon\.com/vod/view/(?P<id>\d+)'
+    IE_NAME = 'filmon'
+    _VALID_URL = r'(?:https?://(?:www\.)?filmon\.com/vod/view/|filmon:)(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://www.filmon.com/vod/view/24869-0-plan-9-from-outer-space',
         'info_dict': {
@@ -83,62 +30,149 @@ class FilmOnVODIE(InfoExtractor):
         'info_dict': {
             'id': '2825',
             'title': 'Popeye Series 1',
+            'description': 'The original series of Popeye.',
         },
-        'playlist_count': 8,
+        'playlist_mincount': 8,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        result = self._download_json('https://www.filmon.com/api/vod/movie?id=%s' % (video_id), video_id)
-        if result['code'] != 200:
-            raise ExtractorError('FilmOn said: %s' % (result['reason']), expected=True)
+        try:
+            response = self._download_json(
+                'https://www.filmon.com/api/vod/movie?id=%s' % video_id,
+                video_id)['response']
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError):
+                errmsg = self._parse_json(e.cause.read().decode(), video_id)['reason']
+                raise ExtractorError('%s said: %s' % (self.IE_NAME, errmsg), expected=True)
+            raise
 
-        response = result['response']
+        title = response['title']
+        description = strip_or_none(response.get('description'))
 
-        if response.get('episodes'):
-            return {
-                '_type': 'playlist',
-                'id': video_id,
-                'title': response['title'],
-                'entries': [{
-                    '_type': 'url',
-                    'url': 'https://www.filmon.com/vod/view/%s' % (ep),
-                } for ep in response['episodes']]
-            }
+        if response.get('type_id') == 1:
+            entries = [self.url_result('filmon:' + episode_id) for episode_id in response.get('episodes', [])]
+            return self.playlist_result(entries, video_id, title, description)
 
+        QUALITY = qualities(('low', 'high'))
         formats = []
-        for (id, stream) in response['streams'].items():
+        for format_id, stream in response.get('streams', {}).items():
+            stream_url = stream.get('url')
+            if not stream_url:
+                continue
             formats.append({
-                'format_id': id,
-                'url': stream['url'],
-                'resolution': stream['name'],
-                'format_note': 'expires after %u seconds' % int(stream['watch-timeout']),
+                'format_id': format_id,
+                'url': stream_url,
                 'ext': 'mp4',
-                'quality': _QUALITY(stream['quality']),
-                'preference': int(stream['watch-timeout']),
+                'quality': QUALITY(stream.get('quality')),
+                'protocol': 'm3u8_native',
             })
         self._sort_formats(formats)
 
-        poster = response['poster']
-        thumbnails = [{
-            'id': 'poster',
-            'url': poster['url'],
-            'width': poster['width'],
-            'height': poster['height'],
-        }]
-        for (id, thumb) in poster['thumbs'].items():
+        thumbnails = []
+        poster = response.get('poster', {})
+        thumbs = poster.get('thumbs', {})
+        thumbs['poster'] = poster
+        for thumb_id, thumb in thumbs.items():
+            thumb_url = thumb.get('url')
+            if not thumb_url:
+                continue
             thumbnails.append({
-                'id': id,
-                'url': thumb['url'],
-                'width': thumb['width'],
-                'height': thumb['height'],
+                'id': thumb_id,
+                'url': thumb_url,
+                'width': int_or_none(thumb.get('width')),
+                'height': int_or_none(thumb.get('height')),
             })
 
         return {
             'id': video_id,
-            'title': response['title'],
+            'title': title,
             'formats': formats,
-            'description': response['description'],
+            'description': description,
             'thumbnails': thumbnails,
         }
+
+
+class FilmOnChannelIE(InfoExtractor):
+    IE_NAME = 'filmon:channel'
+    _VALID_URL = r'https?://(?:www\.)?filmon\.com/(?:tv|channel)/(?P<id>[a-z0-9-]+)'
+    _TESTS = [{
+        # VOD
+        'url': 'http://www.filmon.com/tv/sports-haters',
+        'info_dict': {
+            'id': '4190',
+            'ext': 'mp4',
+            'title': 'Sports Haters',
+            'description': 'md5:dabcb4c1d9cfc77085612f1a85f8275d',
+        },
+    }, {
+        # LIVE
+        'url': 'https://www.filmon.com/channel/filmon-sports',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.filmon.com/tv/2894',
+        'only_matching': True,
+    }]
+
+    _THUMBNAIL_RES = [
+        ('logo', 56, 28),
+        ('big_logo', 106, 106),
+        ('extra_big_logo', 300, 300),
+    ]
+
+    def _real_extract(self, url):
+        channel_id = self._match_id(url)
+
+        try:
+            channel_data = self._download_json(
+                'http://www.filmon.com/api-v2/channel/' + channel_id, channel_id)['data']
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError):
+                errmsg = self._parse_json(e.cause.read().decode(), channel_id)['message']
+                raise ExtractorError('%s said: %s' % (self.IE_NAME, errmsg), expected=True)
+            raise
+
+        channel_id = compat_str(channel_data['id'])
+        is_live = not channel_data.get('is_vod') and not channel_data.get('is_vox')
+        title = channel_data['title']
+
+        QUALITY = qualities(('low', 'high'))
+        formats = []
+        for stream in channel_data.get('streams', []):
+            stream_url = stream.get('url')
+            if not stream_url:
+                continue
+            if not is_live:
+                formats.extend(self._extract_wowza_formats(
+                    stream_url, channel_id, skip_protocols=['dash', 'rtmp', 'rtsp']))
+                continue
+            quality = stream.get('quality')
+            formats.append({
+                'format_id': quality,
+                # this is an m3u8 stream, but we are deliberately not using _extract_m3u8_formats
+                # because it doesn't have bitrate variants anyway
+                'url': stream_url,
+                'ext': 'mp4',
+                'quality': QUALITY(quality),
+            })
+        self._sort_formats(formats)
+
+        thumbnails = []
+        for name, width, height in self._THUMBNAIL_RES:
+            thumbnails.append({
+                'id': name,
+                'url': 'http://static.filmon.com/assets/channels/%s/%s.png' % (channel_id, name),
+                'width': width,
+                'height': height,
+            })
+
+        return {
+            'id': channel_id,
+            'display_id': channel_data.get('alias'),
+            'title': self._live_title(title) if is_live else title,
+            'description': channel_data.get('description'),
+            'thumbnails': thumbnails,
+            'formats': formats,
+            'is_live': is_live,
+        }

From daac118bf4e8bf3dc1ec202fe8b21b9319d15dbf Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 3 Feb 2017 18:56:40 +0800
Subject: [PATCH 0165/1696] [ChangeLog] Update after #11901

---
 ChangeLog | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 487ed3f0f..947590b94 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,10 +1,14 @@
 version <unreleased>
 
+Core
++ Add --playlist-random to shuffle playlists (#11889, #11901)
+
 Extractors
 + [infoq] Add audio only formats (#11565)
 * [youtube] Fix ytsearch when cookies are provided (#11924)
 + [bilibili] Support new Bangumi URLs (#11845)
 
+
 version 2017.02.01
 
 Extractors

From f7a10d8cd6d1378d5f8e67b4b3572fa474b47cde Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 3 Feb 2017 21:25:44 +0700
Subject: [PATCH 0166/1696] [sportbox] Remove extractor (closes #11954) Covered
 by generic extractor

---
 youtube_dl/extractor/sportbox.py | 54 --------------------------------
 1 file changed, 54 deletions(-)

diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index b512cd20f..05a0b5a80 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -11,60 +11,6 @@ from ..utils import (
 )
 
 
-class SportBoxIE(InfoExtractor):
-    _VALID_URL = r'https?://news\.sportbox\.ru/(?:[^/]+/)+spbvideo_NI\d+_(?P<display_id>.+)'
-    _TESTS = [{
-        'url': 'http://news.sportbox.ru/Vidy_sporta/Avtosport/Rossijskij/spbvideo_NI483529_Gonka-2-zaezd-Obyedinenniy-2000-klassi-Turing-i-S',
-        'md5': 'ff56a598c2cf411a9a38a69709e97079',
-        'info_dict': {
-            'id': '80822',
-            'ext': 'mp4',
-            'title': 'Гонка 2  заезд ««Объединенный 2000»: классы Туринг и Супер-продакшн',
-            'description': 'md5:3d72dc4a006ab6805d82f037fdc637ad',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'upload_date': '20140928',
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }, {
-        'url': 'http://news.sportbox.ru/Vidy_sporta/billiard/spbvideo_NI486287_CHempionat-mira-po-dinamichnoy-piramide-4',
-        'only_matching': True,
-    }, {
-        'url': 'http://news.sportbox.ru/video/no_ads/spbvideo_NI536574_V_Novorossijske_proshel_detskij_turnir_Pole_slavy_bojevoj?ci=211355',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('display_id')
-
-        webpage = self._download_webpage(url, display_id)
-
-        player = self._search_regex(
-            r'src="/?(vdl/player/[^"]+)"', webpage, 'player')
-
-        title = self._html_search_regex(
-            [r'"nodetitle"\s*:\s*"([^"]+)"', r'class="node-header_{1,2}title">([^<]+)'],
-            webpage, 'title')
-        description = self._og_search_description(webpage) or self._html_search_meta(
-            'description', webpage, 'description')
-        thumbnail = self._og_search_thumbnail(webpage)
-        upload_date = unified_strdate(self._html_search_meta(
-            'dateCreated', webpage, 'upload date'))
-
-        return {
-            '_type': 'url_transparent',
-            'url': compat_urlparse.urljoin(url, '/%s' % player),
-            'display_id': display_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'upload_date': upload_date,
-        }
-
-
 class SportBoxEmbedIE(InfoExtractor):
     _VALID_URL = r'https?://news\.sportbox\.ru/vdl/player(?:/[^/]+/|\?.*?\bn?id=)(?P<id>\d+)'
     _TESTS = [{

From b7cc5f078eca4d90b3e3d31d1247452953dba1fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 3 Feb 2017 21:56:10 +0700
Subject: [PATCH 0167/1696] [extractors] Remove remnants of sportbox extractor
 (#11954)

---
 youtube_dl/extractor/extractors.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index aa235bec1..eaf3676df 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -888,10 +888,7 @@ from .spiegeltv import SpiegeltvIE
 from .spike import SpikeIE
 from .stitcher import StitcherIE
 from .sport5 import Sport5IE
-from .sportbox import (
-    SportBoxIE,
-    SportBoxEmbedIE,
-)
+from .sportbox import SportBoxEmbedIE
 from .sportdeutschland import SportDeutschlandIE
 from .sportschau import SportschauIE
 from .srgssr import (

From f962790ee53c634758021d9fc752ae476c6a142b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 3 Feb 2017 21:56:48 +0700
Subject: [PATCH 0168/1696] [vine] Fix extraction (closes #11955)

---
 youtube_dl/extractor/vine.py | 103 +++++++++++++++--------------------
 1 file changed, 44 insertions(+), 59 deletions(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index 0183f052a..4957a07f7 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -6,8 +6,9 @@ import itertools
 
 from .common import InfoExtractor
 from ..utils import (
+    determine_ext,
     int_or_none,
-    unified_strdate,
+    unified_timestamp,
 )
 
 
@@ -20,50 +21,16 @@ class VineIE(InfoExtractor):
             'id': 'b9KOOWX7HUx',
             'ext': 'mp4',
             'title': 'Chicken.',
-            'alt_title': 'Vine by Jack Dorsey',
+            'alt_title': 'Vine by Jack',
+            'timestamp': 1368997951,
             'upload_date': '20130519',
-            'uploader': 'Jack Dorsey',
+            'uploader': 'Jack',
             'uploader_id': '76',
             'view_count': int,
             'like_count': int,
             'comment_count': int,
             'repost_count': int,
         },
-    }, {
-        'url': 'https://vine.co/v/MYxVapFvz2z',
-        'md5': '7b9a7cbc76734424ff942eb52c8f1065',
-        'info_dict': {
-            'id': 'MYxVapFvz2z',
-            'ext': 'mp4',
-            'title': 'Fuck Da Police #Mikebrown #justice #ferguson #prayforferguson #protesting #NMOS14',
-            'alt_title': 'Vine by Mars Ruiz',
-            'upload_date': '20140815',
-            'uploader': 'Mars Ruiz',
-            'uploader_id': '1102363502380728320',
-            'view_count': int,
-            'like_count': int,
-            'comment_count': int,
-            'repost_count': int,
-        },
-    }, {
-        'url': 'https://vine.co/v/bxVjBbZlPUH',
-        'md5': 'ea27decea3fa670625aac92771a96b73',
-        'info_dict': {
-            'id': 'bxVjBbZlPUH',
-            'ext': 'mp4',
-            'title': '#mw3 #ac130 #killcam #angelofdeath',
-            'alt_title': 'Vine by Z3k3',
-            'upload_date': '20130430',
-            'uploader': 'Z3k3',
-            'uploader_id': '936470460173008896',
-            'view_count': int,
-            'like_count': int,
-            'comment_count': int,
-            'repost_count': int,
-        },
-    }, {
-        'url': 'https://vine.co/oembed/MYxVapFvz2z.json',
-        'only_matching': True,
     }, {
         'url': 'https://vine.co/v/e192BnZnZ9V',
         'info_dict': {
@@ -71,6 +38,7 @@ class VineIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'ยิ้ม~ เขิน~ อาย~ น่าร้ากอ้ะ >//< @n_whitewo @orlameena #lovesicktheseries  #lovesickseason2',
             'alt_title': 'Vine by Pimry_zaa',
+            'timestamp': 1436057405,
             'upload_date': '20150705',
             'uploader': 'Pimry_zaa',
             'uploader_id': '1135760698325307392',
@@ -82,43 +50,60 @@ class VineIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        'url': 'https://vine.co/v/MYxVapFvz2z',
+        'only_matching': True,
+    }, {
+        'url': 'https://vine.co/v/bxVjBbZlPUH',
+        'only_matching': True,
+    }, {
+        'url': 'https://vine.co/oembed/MYxVapFvz2z.json',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage('https://vine.co/v/' + video_id, video_id)
 
-        data = self._parse_json(
-            self._search_regex(
-                r'window\.POST_DATA\s*=\s*({.+?});\s*</script>',
-                webpage, 'vine data'),
-            video_id)
+        data = self._download_json(
+            'https://archive.vine.co/posts/%s.json' % video_id, video_id)
 
-        data = data[list(data.keys())[0]]
-
-        formats = [{
-            'format_id': '%(format)s-%(rate)s' % f,
-            'vcodec': f.get('format'),
-            'quality': f.get('rate'),
-            'url': f['videoUrl'],
-        } for f in data['videoUrls'] if f.get('videoUrl')]
+        def video_url(kind):
+            for url_suffix in ('Url', 'URL'):
+                format_url = data.get('video%s%s' % (kind, url_suffix))
+                if format_url:
+                    return format_url
 
+        formats = []
+        for quality, format_id in enumerate(('low', '', 'dash')):
+            format_url = video_url(format_id.capitalize())
+            if not format_url:
+                continue
+            # DASH link returns plain mp4
+            if format_id == 'dash' and determine_ext(format_url) == 'mpd':
+                formats.extend(self._extract_mpd_formats(
+                    format_url, video_id, mpd_id='dash', fatal=False))
+            else:
+                formats.append({
+                    'url': format_url,
+                    'format_id': format_id or 'standard',
+                    'quality': quality,
+                })
         self._sort_formats(formats)
 
         username = data.get('username')
 
         return {
             'id': video_id,
-            'title': data.get('description') or self._og_search_title(webpage),
-            'alt_title': 'Vine by %s' % username if username else self._og_search_description(webpage, default=None),
+            'title': data.get('description'),
+            'alt_title': 'Vine by %s' % username if username else None,
             'thumbnail': data.get('thumbnailUrl'),
-            'upload_date': unified_strdate(data.get('created')),
+            'timestamp': unified_timestamp(data.get('created')),
             'uploader': username,
             'uploader_id': data.get('userIdStr'),
-            'view_count': int_or_none(data.get('loops', {}).get('count')),
-            'like_count': int_or_none(data.get('likes', {}).get('count')),
-            'comment_count': int_or_none(data.get('comments', {}).get('count')),
-            'repost_count': int_or_none(data.get('reposts', {}).get('count')),
+            'view_count': int_or_none(data.get('loops')),
+            'like_count': int_or_none(data.get('likes')),
+            'comment_count': int_or_none(data.get('comments')),
+            'repost_count': int_or_none(data.get('reposts')),
             'formats': formats,
         }
 

From 605fd6392fedd2599115e1f1e12df2a6212df1ae Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 3 Feb 2017 17:59:48 +0100
Subject: [PATCH 0169/1696] [youtube] add format info for itag 325 and 328

---
 youtube_dl/extractor/youtube.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index f2f751104..76710931a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -329,6 +329,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'preference': -50, 'container': 'm4a_dash'},
         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'preference': -50, 'container': 'm4a_dash'},
         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'preference': -50, 'container': 'm4a_dash'},
+        '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'preference': -50, 'container': 'm4a_dash'},
+        '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'preference': -50, 'container': 'm4a_dash'},
 
         # Dash webm
         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},

From f65dba7cdb98bb5444ad5656c9626a15d210f6f6 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 3 Feb 2017 22:25:19 +0100
Subject: [PATCH 0170/1696] [myspace] fix extraction and extract hls and http
 formats

---
 youtube_dl/extractor/myspace.py | 108 +++++++++++++++++---------------
 1 file changed, 58 insertions(+), 50 deletions(-)

diff --git a/youtube_dl/extractor/myspace.py b/youtube_dl/extractor/myspace.py
index ab32e632e..f281238c9 100644
--- a/youtube_dl/extractor/myspace.py
+++ b/youtube_dl/extractor/myspace.py
@@ -17,9 +17,10 @@ class MySpaceIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'https://myspace.com/fiveminutestothestage/video/little-big-town/109594919',
+            'md5': '9c1483c106f4a695c47d2911feed50a7',
             'info_dict': {
                 'id': '109594919',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Little Big Town',
                 'description': 'This country quartet was all smiles while playing a sold out show at the Pacific Amphitheatre in Orange County, California.',
                 'uploader': 'Five Minutes to the Stage',
@@ -27,37 +28,30 @@ class MySpaceIE(InfoExtractor):
                 'timestamp': 1414108751,
                 'upload_date': '20141023',
             },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
         },
         # songs
         {
             'url': 'https://myspace.com/killsorrow/music/song/of-weakened-soul...-93388656-103880681',
+            'md5': '1d7ee4604a3da226dd69a123f748b262',
             'info_dict': {
                 'id': '93388656',
-                'ext': 'flv',
+                'ext': 'm4a',
                 'title': 'Of weakened soul...',
                 'uploader': 'Killsorrow',
                 'uploader_id': 'killsorrow',
             },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
         }, {
-            'add_ie': ['Vevo'],
+            'add_ie': ['Youtube'],
             'url': 'https://myspace.com/threedaysgrace/music/song/animal-i-have-become-28400208-28218041',
             'info_dict': {
-                'id': 'USZM20600099',
-                'ext': 'mp4',
-                'title': 'Animal I Have Become',
-                'uploader': 'Three Days Grace',
-                'timestamp': int,
-                'upload_date': '20060502',
+                'id': 'xqds0B_meys',
+                'ext': 'webm',
+                'title': 'Three Days Grace - Animal I Have Become',
+                'description': 'md5:8bd86b3693e72a077cf863a8530c54bb',
+                'uploader': 'ThreeDaysGraceVEVO',
+                'uploader_id': 'ThreeDaysGraceVEVO',
+                'upload_date': '20091002',
             },
-            'skip': 'VEVO is only available in some countries',
         }, {
             'add_ie': ['Youtube'],
             'url': 'https://myspace.com/starset2/music/song/first-light-95799905-106964426',
@@ -76,24 +70,46 @@ class MySpaceIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
+        is_song = mobj.group('mediatype').startswith('music/song')
         webpage = self._download_webpage(url, video_id)
         player_url = self._search_regex(
-            r'playerSwf":"([^"?]*)', webpage, 'player URL')
+            r'videoSwf":"([^"?]*)', webpage, 'player URL', fatal=False)
 
-        def rtmp_format_from_stream_url(stream_url, width=None, height=None):
-            rtmp_url, play_path = stream_url.split(';', 1)
-            return {
-                'format_id': 'rtmp',
-                'url': rtmp_url,
-                'play_path': play_path,
-                'player_url': player_url,
-                'protocol': 'rtmp',
-                'ext': 'flv',
-                'width': width,
-                'height': height,
-            }
+        def formats_from_stream_urls(stream_url, hls_stream_url, http_stream_url, width=None, height=None):
+            formats = []
+            vcodec = 'none' if is_song else None
+            if hls_stream_url:
+                formats.append({
+                    'format_id': 'hls',
+                    'url': hls_stream_url,
+                    'protocol': 'm3u8_native',
+                    'ext': 'm4a' if is_song else 'mp4',
+                    'vcodec': vcodec,
+                })
+            if stream_url and player_url:
+                rtmp_url, play_path = stream_url.split(';', 1)
+                formats.append({
+                    'format_id': 'rtmp',
+                    'url': rtmp_url,
+                    'play_path': play_path,
+                    'player_url': player_url,
+                    'protocol': 'rtmp',
+                    'ext': 'flv',
+                    'width': width,
+                    'height': height,
+                    'vcodec': vcodec,
+                })
+            if http_stream_url:
+                formats.append({
+                    'format_id': 'http',
+                    'url': http_stream_url,
+                    'width': width,
+                    'height': height,
+                    'vcodec': vcodec,
+                })
+            return formats
 
-        if mobj.group('mediatype').startswith('music/song'):
+        if is_song:
             # songs don't store any useful info in the 'context' variable
             song_data = self._search_regex(
                 r'''<button.*data-song-id=(["\'])%s\1.*''' % video_id,
@@ -108,8 +124,10 @@ class MySpaceIE(InfoExtractor):
                 return self._search_regex(
                     r'''data-%s=([\'"])(?P<data>.*?)\1''' % name,
                     song_data, name, default='', group='data')
-            stream_url = search_data('stream-url')
-            if not stream_url:
+            formats = formats_from_stream_urls(
+                search_data('stream-url'), search_data('hls-stream-url'),
+                search_data('http-stream-url'))
+            if not formats:
                 vevo_id = search_data('vevo-id')
                 youtube_id = search_data('youtube-id')
                 if vevo_id:
@@ -121,6 +139,7 @@ class MySpaceIE(InfoExtractor):
                 else:
                     raise ExtractorError(
                         'Found song but don\'t know how to download it')
+            self._sort_formats(formats)
             return {
                 'id': video_id,
                 'title': self._og_search_title(webpage),
@@ -128,27 +147,16 @@ class MySpaceIE(InfoExtractor):
                 'uploader_id': search_data('artist-username'),
                 'thumbnail': self._og_search_thumbnail(webpage),
                 'duration': int_or_none(search_data('duration')),
-                'formats': [rtmp_format_from_stream_url(stream_url)]
+                'formats': formats,
             }
         else:
             video = self._parse_json(self._search_regex(
                 r'context = ({.*?});', webpage, 'context'),
                 video_id)['video']
-            formats = []
-            hls_stream_url = video.get('hlsStreamUrl')
-            if hls_stream_url:
-                formats.append({
-                    'format_id': 'hls',
-                    'url': hls_stream_url,
-                    'protocol': 'm3u8_native',
-                    'ext': 'mp4',
-                })
-            stream_url = video.get('streamUrl')
-            if stream_url:
-                formats.append(rtmp_format_from_stream_url(
-                    stream_url,
-                    int_or_none(video.get('width')),
-                    int_or_none(video.get('height'))))
+            formats = formats_from_stream_urls(
+                video.get('streamUrl'), video.get('hlsStreamUrl'),
+                video.get('mp4StreamUrl'), int_or_none(video.get('width')),
+                int_or_none(video.get('height')))
             self._sort_formats(formats)
             return {
                 'id': video_id,

From 2c15db829c1bd8311ed82e2884661271f0cf73ed Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 4 Feb 2017 08:38:28 +0100
Subject: [PATCH 0171/1696] [drtv] add support for live and radio
 sections(closes #1827)(closes #3427)

---
 youtube_dl/extractor/drtv.py       | 74 +++++++++++++++++++++++++++---
 youtube_dl/extractor/extractors.py |  5 +-
 2 files changed, 72 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index 88d096b30..e966d7483 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -9,12 +9,13 @@ from ..utils import (
     mimetype2ext,
     parse_iso8601,
     remove_end,
+    update_url_query,
 )
 
 
 class DRTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?dr\.dk/(?:tv/se|nyheder)/(?:[^/]+/)*(?P<id>[\da-z-]+)(?:[/#?]|$)'
-
+    _VALID_URL = r'https?://(?:www\.)?dr\.dk/(?:tv/se|nyheder|radio/ondemand)/(?:[^/]+/)*(?P<id>[\da-z-]+)(?:[/#?]|$)'
+    IE_NAME = 'drtv'
     _TESTS = [{
         'url': 'https://www.dr.dk/tv/se/boern/ultra/klassen-ultra/klassen-darlig-taber-10',
         'md5': '25e659cccc9a2ed956110a299fdf5983',
@@ -79,9 +80,10 @@ class DRTVIE(InfoExtractor):
         subtitles = {}
 
         for asset in data['Assets']:
-            if asset.get('Kind') == 'Image':
+            kind = asset.get('Kind')
+            if kind == 'Image':
                 thumbnail = asset.get('Uri')
-            elif asset.get('Kind') == 'VideoResource':
+            elif kind in ('VideoResource', 'AudioResource'):
                 duration = float_or_none(asset.get('DurationInMilliseconds'), 1000)
                 restricted_to_denmark = asset.get('RestrictedToDenmark')
                 spoken_subtitles = asset.get('Target') == 'SpokenSubtitles'
@@ -96,9 +98,13 @@ class DRTVIE(InfoExtractor):
                         preference = -1
                         format_id += '-spoken-subtitles'
                     if target == 'HDS':
-                        formats.extend(self._extract_f4m_formats(
+                        f4m_formats = self._extract_f4m_formats(
                             uri + '?hdcore=3.3.0&plugin=aasp-3.3.0.99.43',
-                            video_id, preference, f4m_id=format_id))
+                            video_id, preference, f4m_id=format_id)
+                        if kind == 'AudioResource':
+                            for f in f4m_formats:
+                                f['vcodec'] = 'none'
+                        formats.extend(f4m_formats)
                     elif target == 'HLS':
                         formats.extend(self._extract_m3u8_formats(
                             uri, video_id, 'mp4', entry_protocol='m3u8_native',
@@ -112,6 +118,7 @@ class DRTVIE(InfoExtractor):
                             'format_id': format_id,
                             'tbr': int_or_none(bitrate),
                             'ext': link.get('FileFormat'),
+                            'vcodec': 'none' if kind == 'AudioResource' else None,
                         })
                 subtitles_list = asset.get('SubtitlesList')
                 if isinstance(subtitles_list, list):
@@ -144,3 +151,58 @@ class DRTVIE(InfoExtractor):
             'formats': formats,
             'subtitles': subtitles,
         }
+
+
+class DRTVLiveIE(InfoExtractor):
+    IE_NAME = 'drtv:live'
+    _VALID_URL = r'https?://(?:www\.)?dr\.dk/(?:tv|TV)/live/(?P<id>[\da-z-]+)'
+    _TEST = {
+        'url': 'https://www.dr.dk/tv/live/dr1',
+        'info_dict': {
+            'id': 'dr1',
+            'ext': 'mp4',
+            'title': 're:^DR1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        channel_id = self._match_id(url)
+        channel_data = self._download_json(
+            'https://www.dr.dk/mu-online/api/1.0/channel/' + channel_id,
+            channel_id)
+        title = self._live_title(channel_data['Title'])
+
+        formats = []
+        for streaming_server in channel_data.get('StreamingServers', []):
+            server = streaming_server.get('Server')
+            if not server:
+                continue
+            link_type = streaming_server.get('LinkType')
+            for quality in streaming_server.get('Qualities', []):
+                for stream in quality.get('Streams', []):
+                    stream_path = stream.get('Stream')
+                    if not stream_path:
+                        continue
+                    stream_url = update_url_query(
+                        '%s/%s' % (server, stream_path), {'b': ''})
+                    if link_type == 'HLS':
+                        formats.extend(self._extract_m3u8_formats(
+                            stream_url, channel_id, 'mp4',
+                            m3u8_id=link_type, fatal=False, live=True))
+                    elif link_type == 'HDS':
+                        formats.extend(self._extract_f4m_formats(update_url_query(
+                            '%s/%s' % (server, stream_path), {'hdcore': '3.7.0'}),
+                            channel_id, f4m_id=link_type, fatal=False))
+        self._sort_formats(formats)
+
+        return {
+            'id': channel_id,
+            'title': title,
+            'thumbnail': channel_data.get('PrimaryImageUri'),
+            'formats': formats,
+            'is_live': True,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index eaf3676df..32420937c 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -248,7 +248,10 @@ from .dramafever import (
 from .dreisat import DreiSatIE
 from .drbonanza import DRBonanzaIE
 from .drtuber import DrTuberIE
-from .drtv import DRTVIE
+from .drtv import (
+    DRTVIE,
+    DRTVLiveIE,
+)
 from .dvtv import DVTVIE
 from .dumpert import DumpertIE
 from .defense import DefenseGouvFrIE

From 36fce54816eb1f1d792ac7ed4d07e292d44d62f5 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 4 Feb 2017 15:23:46 +0100
Subject: [PATCH 0172/1696] [turner] fix downloading of secure hls formats
 using ffmpeg(closes #11358)(closes #11373)(closes #11800)

---
 youtube_dl/downloader/external.py | 9 +++++++++
 youtube_dl/extractor/turner.py    | 8 ++++++--
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 138f353ef..41e37261d 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -199,6 +199,15 @@ class FFmpegFD(ExternalFD):
 
         args = [ffpp.executable, '-y']
 
+        seekable = info_dict.get('_seekable')
+        if seekable is not None:
+            # setting -seekable prevents ffmpeg from guessing if the server
+            # supports seeking(by adding the header `Range: bytes=0-`), which
+            # can cause problems in some cases
+            # https://github.com/rg3/youtube-dl/issues/11800#issuecomment-275037127
+            # http://trac.ffmpeg.org/ticket/6125#comment:10
+            args += ['-seekable', '1' if seekable else '0']
+
         args += self._configuration_args()
 
         # start_time = info_dict.get('start_time') or 0
diff --git a/youtube_dl/extractor/turner.py b/youtube_dl/extractor/turner.py
index 57ffedb87..1c0be9fc6 100644
--- a/youtube_dl/extractor/turner.py
+++ b/youtube_dl/extractor/turner.py
@@ -100,9 +100,13 @@ class TurnerBaseIE(AdobePassIE):
                 formats.extend(self._extract_smil_formats(
                     video_url, video_id, fatal=False))
             elif ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
+                m3u8_formats = self._extract_m3u8_formats(
                     video_url, video_id, 'mp4',
-                    m3u8_id=format_id or 'hls', fatal=False))
+                    m3u8_id=format_id or 'hls', fatal=False)
+                if '/secure/' in video_url and '?hdnea=' in video_url:
+                    for f in m3u8_formats:
+                        f['_seekable'] = False
+                formats.extend(m3u8_formats)
             elif ext == 'f4m':
                 formats.extend(self._extract_f4m_formats(
                     update_url_query(video_url, {'hdcore': '3.7.0'}),

From 643dc0fcfed5e5eb152000190d0c7ba9dd577ef8 Mon Sep 17 00:00:00 2001
From: A Connecticut Princess <bugchecker@dibaby.org>
Date: Sat, 4 Feb 2017 13:23:35 +0500
Subject: [PATCH 0173/1696] [vk] Catch author blocked error message

Example link (video in blocked group):
https://vk.com/search?c%5Bq%5D=%D0%9F%D1%80%D1%8B%D0%B6%D0%BE%D0%BA%20c%20%D0%BA%D1%80%D0%B0%D0%BD%D0%B0%20%D0%B2%20%D1%81%D1%82%D0%B8%D0%BB%D0%B5%20%D0%A7%D0%B5%D0%BB%D0%BE%D0%B2%D0%B5%D0%BA%D0%B0-%D0%BF%D0%B0%D1%83%D0%BA%D0%B0&c%5Bsection%5D=video&c%5Bsort%5D=2&z=video-10639516_456240611
---
 youtube_dl/extractor/vk.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 6e6c3a0e1..7c42a4f54 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -281,6 +281,11 @@ class VKIE(VKBaseIE):
         {
             'url': 'http://new.vk.com/video205387401_165548505',
             'only_matching': True,
+        },
+        {
+            # This video is no longer available, because its author has been blocked.
+            'url': 'https://vk.com/video-10639516_456240611',
+            'only_matching': True,
         }
     ]
 
@@ -328,6 +333,12 @@ class VKIE(VKBaseIE):
 
             r'<!>Access denied':
             'Access denied to video %s.',
+
+            r'<!>Видеозапись недоступна, так как её автор был заблокирован.':
+            'Video %s is no longer available, because its author has been blocked.',
+
+            r'<!>This video is no longer available, because its author has been blocked.':
+            'Video %s is no longer available, because its author has been blocked.',
         }
 
         for error_re, error_msg in ERRORS.items():

From c2521c1ac6bbd24cd5d01ba764f2d084b16c506f Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Sat, 4 Feb 2017 10:23:14 -0500
Subject: [PATCH 0174/1696] [Piksel] Add another app token regex

---
 youtube_dl/extractor/piksel.py | 43 ++++++++++++++++++++++++----------
 1 file changed, 30 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/piksel.py b/youtube_dl/extractor/piksel.py
index d44edcdfb..c0c276a50 100644
--- a/youtube_dl/extractor/piksel.py
+++ b/youtube_dl/extractor/piksel.py
@@ -16,18 +16,33 @@ from ..utils import (
 
 class PikselIE(InfoExtractor):
     _VALID_URL = r'https?://player\.piksel\.com/v/(?P<id>[a-z0-9]+)'
-    _TEST = {
-        'url': 'http://player.piksel.com/v/nv60p12f',
-        'md5': 'd9c17bbe9c3386344f9cfd32fad8d235',
-        'info_dict': {
-            'id': 'nv60p12f',
-            'ext': 'mp4',
-            'title': 'فن الحياة  - الحلقة 1',
-            'description': 'احدث برامج الداعية الاسلامي " مصطفي حسني " فى رمضان 2016علي النهار نور',
-            'timestamp': 1465231790,
-            'upload_date': '20160606',
+    _TESTS = [
+        {
+            'url': 'http://player.piksel.com/v/nv60p12f',
+            'md5': 'd9c17bbe9c3386344f9cfd32fad8d235',
+            'info_dict': {
+                'id': 'nv60p12f',
+                'ext': 'mp4',
+                'title': 'فن الحياة  - الحلقة 1',
+                'description': 'احدث برامج الداعية الاسلامي " مصطفي حسني " فى رمضان 2016علي النهار نور',
+                'timestamp': 1465231790,
+                'upload_date': '20160606',
+            }
+        },
+        {
+            # Original source: http://www.uscourts.gov/cameras-courts/state-washington-vs-donald-j-trump-et-al
+            'url': 'https://player.piksel.com/v/v80kqp41',
+            'md5': '753ddcd8cc8e4fa2dda4b7be0e77744d',
+            'info_dict': {
+                'id': 'v80kqp41',
+                'ext': 'mp4',
+                'title': 'WAW- State of Washington vs. Donald J. Trump, et al',
+                'description': 'State of Washington vs. Donald J. Trump, et al, Case Number 17-CV-00141-JLR, TRO Hearing, Civil Rights Case, 02/3/2017, 1:00 PM (PST), Seattle Federal Courthouse, Seattle, WA, Judge James L. Robart presiding.',
+                'timestamp': 1486171129,
+                'upload_date': '20170204',
+            }
         }
-    }
+    ]
 
     @staticmethod
     def _extract_url(webpage):
@@ -40,8 +55,10 @@ class PikselIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        app_token = self._search_regex(
-            r'clientAPI\s*:\s*"([^"]+)"', webpage, 'app token')
+        app_token = self._search_regex([
+            r'clientAPI\s*:\s*"([^"]+)"',
+            r'data-de-api-key\s*=\s*"([^"]+)"'
+        ], webpage, 'app token')
         response = self._download_json(
             'http://player.piksel.com/ws/ws_program/api/%s/mode/json/apiv/5' % app_token,
             video_id, query={

From 31487eb9746123b7c4e28be7e48908773beab40c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Feb 2017 22:57:48 +0700
Subject: [PATCH 0175/1696] release 2017.02.04

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 README.md                 | 1 +
 docs/supportedsites.md    | 7 +++++--
 youtube_dl/version.py     | 2 +-
 5 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 8914569b6..11fd56038 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.01*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.01**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.04*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.04**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.02.01
+[debug] youtube-dl version 2017.02.04
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 947590b94..5323769d8 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.02.04
 
 Core
 + Add --playlist-random to shuffle playlists (#11889, #11901)
diff --git a/README.md b/README.md
index 2ee00f515..89876bd7a 100644
--- a/README.md
+++ b/README.md
@@ -182,6 +182,7 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
                                      automatically resized from an initial value
                                      of SIZE.
     --playlist-reverse               Download playlist videos in reverse order
+    --playlist-random                Download playlist videos in random order
     --xattr-set-filesize             Set file xattribute ytdl.filesize with
                                      expected file size (experimental)
     --hls-prefer-native              Use the native HLS downloader instead of
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index d900f5e12..50a339bc4 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -84,6 +84,7 @@
  - **bambuser:channel**
  - **Bandcamp**
  - **Bandcamp:album**
+ - **bangumi.bilibili.com**: BiliBili番剧
  - **bbc**: BBC
  - **bbc.co.uk**: BBC iPlayer
  - **bbc.co.uk:article**: BBC articles
@@ -211,7 +212,8 @@
  - **DRBonanza**
  - **Dropbox**
  - **DrTuber**
- - **DRTV**
+ - **drtv**
+ - **drtv:live**
  - **Dumpert**
  - **dvtv**: http://video.aktualne.cz/
  - **dw**
@@ -247,6 +249,8 @@
  - **fc2:embed**
  - **Fczenit**
  - **fernsehkritik.tv**
+ - **filmon**
+ - **filmon:channel**
  - **Firstpost**
  - **FiveTV**
  - **Flickr**
@@ -703,7 +707,6 @@
  - **Spiegeltv**
  - **Spike**
  - **Sport5**
- - **SportBox**
  - **SportBoxEmbed**
  - **SportDeutschland**
  - **Sportschau**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0f9b6b703..376b31397 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.02.01'
+__version__ = '2017.02.04'

From 8e4041cf3f8e769ee2188f3db4747b7133ab5c2d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 4 Feb 2017 17:02:12 +0100
Subject: [PATCH 0176/1696] [radiocanada] fix extraction for toutv rtmp formats

---
 youtube_dl/extractor/radiocanada.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/radiocanada.py b/youtube_dl/extractor/radiocanada.py
index 321917ad0..3b40002a8 100644
--- a/youtube_dl/extractor/radiocanada.py
+++ b/youtube_dl/extractor/radiocanada.py
@@ -54,9 +54,8 @@ class RadioCanadaIE(InfoExtractor):
             raise ExtractorError('This video is DRM protected.', expected=True)
 
         device_types = ['ipad']
-        if app_code != 'toutv':
-            device_types.append('flash')
         if not smuggled_data:
+            device_types.append('flash')
             device_types.append('android')
 
         formats = []
@@ -103,7 +102,7 @@ class RadioCanadaIE(InfoExtractor):
                         continue
                     f_url = re.sub(r'\d+\.%s' % ext, '%d.%s' % (tbr, ext), v_url)
                     protocol = determine_protocol({'url': f_url})
-                    formats.append({
+                    f = {
                         'format_id': '%s-%d' % (protocol, tbr),
                         'url': f_url,
                         'ext': 'flv' if protocol == 'rtmp' else ext,
@@ -111,7 +110,14 @@ class RadioCanadaIE(InfoExtractor):
                         'width': int_or_none(url_e.get('width')),
                         'height': int_or_none(url_e.get('height')),
                         'tbr': tbr,
-                    })
+                    }
+                    mobj = re.match(r'(?P<url>rtmp://[^/]+/[^/]+)/(?P<playpath>[^?]+)(?P<auth>\?.+)', f_url)
+                    if mobj:
+                        f.update({
+                            'url': mobj.group('url') + mobj.group('auth'),
+                            'play_path': mobj.group('playpath'),
+                        })
+                    formats.append(f)
                     if protocol == 'rtsp':
                         base_url = self._search_regex(
                             r'rtsp://([^?]+)', f_url, 'base url', default=None)

From 9db8f6c54021a9c809c8ae65a37544ad566ed159 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Feb 2017 23:21:07 +0700
Subject: [PATCH 0177/1696] [twitch:stream] Improve _VALID_URL (closes #11971)

---
 youtube_dl/extractor/twitch.py | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 1ca159a4d..bbba394b0 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -447,7 +447,14 @@ class TwitchHighlightsIE(TwitchVideosBaseIE):
 
 class TwitchStreamIE(TwitchBaseIE):
     IE_NAME = 'twitch:stream'
-    _VALID_URL = r'%s/(?P<id>[^/#?]+)/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:
+                            (?:www\.)?twitch\.tv/|
+                            player\.twitch\.tv/\?.*?\bchannel=
+                        )
+                        (?P<id>[^/#?]+)
+                    '''
 
     _TESTS = [{
         'url': 'http://www.twitch.tv/shroomztv',
@@ -471,8 +478,25 @@ class TwitchStreamIE(TwitchBaseIE):
     }, {
         'url': 'http://www.twitch.tv/miracle_doto#profile-0',
         'only_matching': True,
+    }, {
+        'url': 'https://player.twitch.tv/?channel=lotsofs',
+        'only_matching': True,
     }]
 
+    @classmethod
+    def suitable(cls, url):
+        return (False
+                if any(ie.suitable(url) for ie in (
+                    TwitchVideoIE,
+                    TwitchChapterIE,
+                    TwitchVodIE,
+                    TwitchProfileIE,
+                    TwitchAllVideosIE,
+                    TwitchUploadsIE,
+                    TwitchPastBroadcastsIE,
+                    TwitchHighlightsIE))
+                else super(TwitchStreamIE, cls).suitable(url))
+
     def _real_extract(self, url):
         channel_id = self._match_id(url)
 

From 3144eccf551afc4c5e66e06de541c033e6f90681 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Feb 2017 23:22:28 +0700
Subject: [PATCH 0178/1696] [ChangeLog] Actualize

---
 ChangeLog | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 5323769d8..fe9cd3440 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,12 +1,38 @@
+version <unreleased>
+
+Extractors
++ [twitch:stream] Add support for player.twitch.tv (#11971)
+
+
 version 2017.02.04
 
 Core
 + Add --playlist-random to shuffle playlists (#11889, #11901)
+* [utils] Improve comments processing in js_to_json (#11947)
+* [utils] Handle single-line comments in js_to_json
+* [downloader/external:ffmpeg] Minimize the use of aac_adtstoasc filter
 
 Extractors
++ [piksel] Add another app token pattern (#11969)
++ [vk] Capture and output author blocked error message (#11965)
++ [turner] Fix secure HLS formats downloading with ffmpeg (#11358, #11373,
+  #11800)
++ [drtv] Add support for live and radio sections (#1827, #3427)
+* [myspace] Fix extraction and extract HLS and HTTP formats
++ [youtube] Add format info for itag 325 and 328
+* [vine] Fix extraction (#11955)
+- [sportbox] Remove extractor (#11954)
++ [filmon] Add support for filmon.com (#11187)
 + [infoq] Add audio only formats (#11565)
+* [douyutv] Improve room id regular expression (#11931)
+* [iprima] Fix extraction (#11920, #11896)
 * [youtube] Fix ytsearch when cookies are provided (#11924)
+* [go] Relax video id regular expression (#11937)
+* [facebook] Fix title extraction (#11941)
++ [youtube:playlist] Recognize TL playlists (#11945)
 + [bilibili] Support new Bangumi URLs (#11845)
++ [cbc:watch] Extract audio codec for audio only formats (#11893)
++ [elpais] Fix extraction for some URLs (#11765)
 
 
 version 2017.02.01
@@ -18,7 +44,6 @@ Extractors
 + [vimeo] Extract upload timestamp
 + [vimeo] Extract license (#8726, #11880)
 + [nrk:series] Add support for series (#11571, #11711)
-+ [elpais] Fix extraction for some URLs (#11765)
 
 
 version 2017.01.31

From 7bccd5fc8ac35b1a3952522c0aa176c982f20206 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Feb 2017 23:23:38 +0700
Subject: [PATCH 0179/1696] [ChangeLog] Actualize

---
 ChangeLog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChangeLog b/ChangeLog
index fe9cd3440..76be8dbd9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -2,6 +2,7 @@ version <unreleased>
 
 Extractors
 + [twitch:stream] Add support for player.twitch.tv (#11971)
+* [radiocanada] Fix extraction for toutv rtmp formats
 
 
 version 2017.02.04

From a713a86755ba864a7b765fd2ce9a5ac8a8f4cc63 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Feb 2017 23:26:39 +0700
Subject: [PATCH 0180/1696] release 2017.02.04.1

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 11fd56038..15e7d4944 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.04*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.04**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.04.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.04.1**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.02.04
+[debug] youtube-dl version 2017.02.04.1
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 76be8dbd9..23a729559 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.02.04.1
 
 Extractors
 + [twitch:stream] Add support for player.twitch.tv (#11971)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 376b31397..5dde47a26 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.02.04'
+__version__ = '2017.02.04.1'

From 3d2c2752c5cd70fc7f9cebe8c4683a1de626017d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 4 Feb 2017 18:18:03 +0100
Subject: [PATCH 0181/1696] [afreecatv] extract rtmp formats

---
 youtube_dl/extractor/afreecatv.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py
index 4f6cdb8a2..e0a0f7c57 100644
--- a/youtube_dl/extractor/afreecatv.py
+++ b/youtube_dl/extractor/afreecatv.py
@@ -221,10 +221,23 @@ class AfreecaTVGlobalIE(AfreecaTVIE):
                 s_url = s.get('purl')
                 if not s_url:
                     continue
-                # TODO: extract rtmp formats
-                if s.get('stype') == 'HLS':
+                stype = s.get('stype')
+                if stype == 'HLS':
                     formats.extend(self._extract_m3u8_formats(
-                        s_url, channel_id, 'mp4', fatal=False))
+                        s_url, channel_id, 'mp4', m3u8_id=stype, fatal=False))
+                elif stype == 'RTMP':
+                    format_id = [stype]
+                    label = s.get('label')
+                    if label:
+                        format_id.append(label)
+                    formats.append({
+                        'format_id': '-'.join(format_id),
+                        'url': s_url,
+                        'tbr': int_or_none(s.get('bps')),
+                        'height': int_or_none(s.get('brt')),
+                        'ext': 'flv',
+                        'rtmp_live': True,
+                    })
             self._sort_formats(formats)
 
             info.update({

From 49bd8d5e2e5c4de8c1c409adffc557cb198f7eee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Feb 2017 02:41:22 +0700
Subject: [PATCH 0182/1696] [travis] Add python 3.6

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index c74c9cc12..4833c76e9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,6 +6,7 @@ python:
   - "3.3"
   - "3.4"
   - "3.5"
+  - "3.6"
 sudo: false
 script: nosetests test --verbose
 notifications:

From 6fd138bed892ac8ae1714d64f4a53d8ea7a1d5bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Feb 2017 13:36:52 +0700
Subject: [PATCH 0183/1696] [sportbox] PEP 8

---
 youtube_dl/extractor/sportbox.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index 05a0b5a80..e7bd5bf91 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -4,11 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
-from ..utils import (
-    js_to_json,
-    unified_strdate,
-)
+from ..utils import js_to_json
 
 
 class SportBoxEmbedIE(InfoExtractor):

From 6ef3e65a7b244d5e432e764772177c7d48cab237 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Feb 2017 13:37:27 +0700
Subject: [PATCH 0184/1696] [videopress] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/generic.py    | 22 +++++++
 youtube_dl/extractor/videopress.py | 99 ++++++++++++++++++++++++++++++
 3 files changed, 122 insertions(+)
 create mode 100644 youtube_dl/extractor/videopress.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 32420937c..cf608faee 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1095,6 +1095,7 @@ from .videomore import (
     VideomoreSeasonIE,
 )
 from .videopremium import VideoPremiumIE
+from .videopress import VideoPressIE
 from .vidio import VidioIE
 from .vidme import (
     VidmeIE,
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a23486620..4156cf27d 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -81,6 +81,7 @@ from .videa import VideaIE
 from .twentymin import TwentyMinutenIE
 from .ustream import UstreamIE
 from .openload import OpenloadIE
+from .videopress import VideoPressIE
 
 
 class GenericIE(InfoExtractor):
@@ -1473,6 +1474,21 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
             'add_ie': [TwentyMinutenIE.ie_key()],
+        },
+        {
+            # VideoPress embed
+            'url': 'https://en.support.wordpress.com/videopress/',
+            'info_dict': {
+                'id': 'OcobLTqC',
+                'ext': 'm4v',
+                'title': 'IMG_5786',
+                'timestamp': 1435711927,
+                'upload_date': '20150701',
+            },
+            'params': {
+                'skip_download': True,
+            },
+            'add_ie': [VideoPressIE.ie_key()],
         }
         # {
         #     # TODO: find another test
@@ -2438,6 +2454,12 @@ class GenericIE(InfoExtractor):
             return _playlist_from_matches(
                 openload_urls, ie=OpenloadIE.ie_key())
 
+        # Look for VideoPress embeds
+        videopress_urls = VideoPressIE._extract_urls(webpage)
+        if videopress_urls:
+            return _playlist_from_matches(
+                videopress_urls, ie=VideoPressIE.ie_key())
+
         # Looking for http://schema.org/VideoObject
         json_ld = self._search_json_ld(
             webpage, video_id, default={}, expected_type='VideoObject')
diff --git a/youtube_dl/extractor/videopress.py b/youtube_dl/extractor/videopress.py
new file mode 100644
index 000000000..049db25a5
--- /dev/null
+++ b/youtube_dl/extractor/videopress.py
@@ -0,0 +1,99 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import random
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    determine_ext,
+    float_or_none,
+    parse_age_limit,
+    qualities,
+    try_get,
+    unified_timestamp,
+    urljoin,
+)
+
+
+class VideoPressIE(InfoExtractor):
+    _VALID_URL = r'https?://videopress\.com/embed/(?P<id>[\da-zA-Z]+)'
+    _TESTS = [{
+        'url': 'https://videopress.com/embed/kUJmAcSf',
+        'md5': '706956a6c875873d51010921310e4bc6',
+        'info_dict': {
+            'id': 'kUJmAcSf',
+            'ext': 'mp4',
+            'title': 'VideoPress Demo',
+            'thumbnail': r're:^https?://.*\.jpg',
+            'duration': 634.6,
+            'timestamp': 1434983935,
+            'upload_date': '20150622',
+            'age_limit': 0,
+        },
+    }, {
+        # 17+, requires birth_* params
+        'url': 'https://videopress.com/embed/iH3gstfZ',
+        'only_matching': True,
+    }]
+
+    @staticmethod
+    def _extract_urls(webpage):
+        return re.findall(
+            r'<iframe[^>]+src=["\']((?:https?://)?videopress\.com/embed/[\da-zA-Z]+)',
+            webpage)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        video = self._download_json(
+            'https://public-api.wordpress.com/rest/v1.1/videos/%s' % video_id,
+            video_id, query={
+                'birth_month': random.randint(1, 12),
+                'birth_day': random.randint(1, 31),
+                'birth_year': random.randint(1950, 1995),
+            })
+
+        title = video['title']
+
+        def base_url(scheme):
+            return try_get(
+                video, lambda x: x['file_url_base'][scheme], compat_str)
+
+        base_url = base_url('https') or base_url('http')
+
+        QUALITIES = ('std', 'dvd', 'hd')
+        quality = qualities(QUALITIES)
+
+        formats = []
+        for format_id, f in video['files'].items():
+            if not isinstance(f, dict):
+                continue
+            for ext, path in f.items():
+                if ext in ('mp4', 'ogg'):
+                    formats.append({
+                        'url': urljoin(base_url, path),
+                        'format_id': '%s-%s' % (format_id, ext),
+                        'ext': determine_ext(path, ext),
+                        'quality': quality(format_id),
+                    })
+        original_url = try_get(video, lambda x: x['original'], compat_str)
+        if original_url:
+            formats.append({
+                'url': original_url,
+                'format_id': 'original',
+                'quality': len(QUALITIES),
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': video.get('description'),
+            'thumbnail': video.get('poster'),
+            'duration': float_or_none(video.get('duration'), 1000),
+            'timestamp': unified_timestamp(video.get('upload_date')),
+            'age_limit': parse_age_limit(video.get('rating')),
+            'formats': formats,
+        }

From e4e50f60b1040a4b6aa8ecb9139f7d5de195f407 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 5 Feb 2017 21:41:08 +0800
Subject: [PATCH 0185/1696] [googledrive] Fix extraction on Python 3.6

Since Python 3.6, invalid escape sequences are deprecated. It's likely
that there are invalid escape sequences somewhere on the webpage, so
instead of unescaping the whole webpage, just unescape the URL.

See https://bugs.python.org/issue27364. That change was designed for
string literals, while it affects the 'unicode_escape' encoding as well.
The code path is:

str.decode('unicode_escape')
    codecs.unicode_escape_decode()
        PyUnicode_DecodeUnicodeEscape()
---
 ChangeLog                           | 6 ++++++
 youtube_dl/extractor/googledrive.py | 9 +++++----
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 23a729559..a0025ab91 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [googledrive] Fix extraction on Python 3.6
+
+
 version 2017.02.04.1
 
 Extractors
diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
index 766fc26d0..fec36cbbb 100644
--- a/youtube_dl/extractor/googledrive.py
+++ b/youtube_dl/extractor/googledrive.py
@@ -6,6 +6,7 @@ from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     int_or_none,
+    lowercase_escape,
 )
 
 
@@ -13,12 +14,12 @@ class GoogleDriveIE(InfoExtractor):
     _VALID_URL = r'https?://(?:(?:docs|drive)\.google\.com/(?:uc\?.*?id=|file/d/)|video\.google\.com/get_player\?.*?docid=)(?P<id>[a-zA-Z0-9_-]{28,})'
     _TESTS = [{
         'url': 'https://drive.google.com/file/d/0ByeS4oOUV-49Zzh4R1J6R09zazQ/edit?pli=1',
-        'md5': '881f7700aec4f538571fa1e0eed4a7b6',
+        'md5': 'd109872761f7e7ecf353fa108c0dbe1e',
         'info_dict': {
             'id': '0ByeS4oOUV-49Zzh4R1J6R09zazQ',
             'ext': 'mp4',
             'title': 'Big Buck Bunny.mp4',
-            'duration': 46,
+            'duration': 45,
         }
     }, {
         # video id is longer than 28 characters
@@ -55,7 +56,7 @@ class GoogleDriveIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(
-            'http://docs.google.com/file/d/%s' % video_id, video_id, encoding='unicode_escape')
+            'http://docs.google.com/file/d/%s' % video_id, video_id)
 
         reason = self._search_regex(r'"reason"\s*,\s*"([^"]+)', webpage, 'reason', default=None)
         if reason:
@@ -74,7 +75,7 @@ class GoogleDriveIE(InfoExtractor):
             resolution = fmt.split('/')[1]
             width, height = resolution.split('x')
             formats.append({
-                'url': fmt_url,
+                'url': lowercase_escape(fmt_url),
                 'format_id': fmt_id,
                 'resolution': resolution,
                 'width': int_or_none(width),

From caf0f5f8b7d0854caaf6778fe3a646ee0d7668fe Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 5 Feb 2017 21:48:13 +0800
Subject: [PATCH 0186/1696] [iwara] Fix extraction (closes #11781)

---
 ChangeLog                     |  1 +
 youtube_dl/extractor/iwara.py | 41 +++++++++++++++++++++++++----------
 2 files changed, 30 insertions(+), 12 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index a0025ab91..77286dbef 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [iwara] Fix extraction (#11781)
 * [googledrive] Fix extraction on Python 3.6
 
 
diff --git a/youtube_dl/extractor/iwara.py b/youtube_dl/extractor/iwara.py
index 8d7e7f472..011274b02 100644
--- a/youtube_dl/extractor/iwara.py
+++ b/youtube_dl/extractor/iwara.py
@@ -3,14 +3,18 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse_urlparse
-from ..utils import remove_end
+from ..utils import (
+    int_or_none,
+    mimetype2ext,
+    remove_end,
+)
 
 
 class IwaraIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.|ecchi\.)?iwara\.tv/videos/(?P<id>[a-zA-Z0-9]+)'
     _TESTS = [{
         'url': 'http://iwara.tv/videos/amVwUl1EHpAD9RD',
-        'md5': '1d53866b2c514b23ed69e4352fdc9839',
+        # md5 is unstable
         'info_dict': {
             'id': 'amVwUl1EHpAD9RD',
             'ext': 'mp4',
@@ -23,17 +27,17 @@ class IwaraIE(InfoExtractor):
         'info_dict': {
             'id': '0B1LvuHnL-sRFNXB1WHNqbGw4SXc',
             'ext': 'mp4',
-            'title': '[3D Hentai] Kyonyu Ã\x97 Genkai Ã\x97 Emaki Shinobi Girls.mp4',
+            'title': '[3D Hentai] Kyonyu × Genkai × Emaki Shinobi Girls.mp4',
             'age_limit': 18,
         },
         'add_ie': ['GoogleDrive'],
     }, {
         'url': 'http://www.iwara.tv/videos/nawkaumd6ilezzgq',
-        'md5': '1d85f1e5217d2791626cff5ec83bb189',
+        # md5 is unstable
         'info_dict': {
             'id': '6liAP9s2Ojc',
             'ext': 'mp4',
-            'age_limit': 0,
+            'age_limit': 18,
             'title': '[MMD] Do It Again Ver.2 [1080p 60FPS] (Motion,Camera,Wav+DL)',
             'description': 'md5:590c12c0df1443d833fbebe05da8c47a',
             'upload_date': '20160910',
@@ -52,9 +56,9 @@ class IwaraIE(InfoExtractor):
         # ecchi is 'sexy' in Japanese
         age_limit = 18 if hostname.split('.')[0] == 'ecchi' else 0
 
-        entries = self._parse_html5_media_entries(url, webpage, video_id)
+        video_data = self._download_json('http://www.iwara.tv/api/video/%s' % video_id, video_id)
 
-        if not entries:
+        if not video_data:
             iframe_url = self._html_search_regex(
                 r'<iframe[^>]+src=([\'"])(?P<url>[^\'"]+)\1',
                 webpage, 'iframe URL', group='url')
@@ -67,11 +71,24 @@ class IwaraIE(InfoExtractor):
         title = remove_end(self._html_search_regex(
             r'<title>([^<]+)</title>', webpage, 'title'), ' | Iwara')
 
-        info_dict = entries[0]
-        info_dict.update({
+        formats = []
+        for a_format in video_data:
+            format_id = a_format.get('resolution')
+            height = int_or_none(self._search_regex(
+                r'(\d+)p', format_id, 'height', default=None))
+            formats.append({
+                'url': a_format['uri'],
+                'format_id': format_id,
+                'ext': mimetype2ext(a_format.get('mime')) or 'mp4',
+                'height': height,
+                'quality': 1 if format_id == 'Source' else 0,
+            })
+
+        self._sort_formats(formats)
+
+        return {
             'id': video_id,
             'title': title,
             'age_limit': age_limit,
-        })
-
-        return info_dict
+            'formats': formats,
+        }

From 2ab2c0d1f53f66614eda4fefb042e851e78097f0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 5 Feb 2017 22:30:13 +0800
Subject: [PATCH 0187/1696] [iwara] Add width (closes #11724)

The heuristic is from #11724
---
 youtube_dl/extractor/iwara.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/iwara.py b/youtube_dl/extractor/iwara.py
index 011274b02..a7514fc80 100644
--- a/youtube_dl/extractor/iwara.py
+++ b/youtube_dl/extractor/iwara.py
@@ -81,6 +81,7 @@ class IwaraIE(InfoExtractor):
                 'format_id': format_id,
                 'ext': mimetype2ext(a_format.get('mime')) or 'mp4',
                 'height': height,
+                'width': int_or_none(height / 9.0 * 16.0 if height else None),
                 'quality': 1 if format_id == 'Source' else 0,
             })
 

From 019f4c03717bfd2b887309e5a4c96ea82cbedf34 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 5 Feb 2017 22:47:04 +0800
Subject: [PATCH 0188/1696] [bandcamp] Fix extraction for incomplete albums

Closes #11727
---
 ChangeLog                        |  1 +
 youtube_dl/extractor/bandcamp.py | 19 ++++++++++++++++---
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 77286dbef..984191925 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [bandcamp] Fix extraction for incomplete albums (#11727)
 * [iwara] Fix extraction (#11781)
 * [googledrive] Fix extraction on Python 3.6
 
diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index 88c590e98..056e06376 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -209,6 +209,15 @@ class BandcampAlbumIE(InfoExtractor):
             'id': 'entropy-ep',
         },
         'playlist_mincount': 3,
+    }, {
+        # not all tracks have songs
+        'url': 'https://insulters.bandcamp.com/album/we-are-the-plague',
+        'info_dict': {
+            'id': 'we-are-the-plague',
+            'title': 'WE ARE THE PLAGUE',
+            'uploader_id': 'insulters',
+        },
+        'playlist_count': 2,
     }]
 
     def _real_extract(self, url):
@@ -217,12 +226,16 @@ class BandcampAlbumIE(InfoExtractor):
         album_id = mobj.group('album_id')
         playlist_id = album_id or uploader_id
         webpage = self._download_webpage(url, playlist_id)
-        tracks_paths = re.findall(r'<a href="(.*?)" itemprop="url">', webpage)
-        if not tracks_paths:
+        track_elements = re.findall(
+            r'(?s)<div[^>]*>(.*?<a[^>]+href="([^"]+?)"[^>]+itemprop="url"[^>]*>.*?)</div>', webpage)
+        if not track_elements:
             raise ExtractorError('The page doesn\'t contain any tracks')
+        # Only tracks with duration info have songs
         entries = [
             self.url_result(compat_urlparse.urljoin(url, t_path), ie=BandcampIE.ie_key())
-            for t_path in tracks_paths]
+            for elem_content, t_path in track_elements
+            if self._html_search_meta('duration', elem_content, default=None)]
+
         title = self._html_search_regex(
             r'album_title\s*:\s*"((?:\\.|[^"\\])+?)"',
             webpage, 'title', fatal=False)

From e705755739e32779ef819370124d78981b04414c Mon Sep 17 00:00:00 2001
From: motophil <phil@rrr.de>
Date: Sun, 5 Feb 2017 17:19:37 +0100
Subject: [PATCH 0189/1696] [gaskrank] Add new extractor (#11685)

* [gaskrank] Add new extractor

* [gaskrank] Add new extractor - fixes as requested

* [gaskrank] Add new extractor - style fix

* [Gaskrank] Add new extractor - requested fixes

* [Gaskrank] Add new extractor - fix md5 checksum

* [gaskrank] Add new extractor - more requested fixes

* [Gaskrank] Add new extractor - fixed all but one quantified code issues

* [Gaskrank] add new extractor - more fields extracted, added second test

* [Gaskrank] Add new extractor - requested fixes.

* [Gaskrank] Add new extractor - requested changes.

* [Gaskrank] Add new extractor - final(?) fixes.
---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/gaskrank.py   | 121 +++++++++++++++++++++++++++++
 2 files changed, 122 insertions(+)
 create mode 100644 youtube_dl/extractor/gaskrank.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index cf608faee..12cda36cc 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -349,6 +349,7 @@ from .gameone import (
 from .gamersyde import GamersydeIE
 from .gamespot import GameSpotIE
 from .gamestar import GameStarIE
+from .gaskrank import GaskrankIE
 from .gazeta import GazetaIE
 from .gdcvault import GDCVaultIE
 from .generic import GenericIE
diff --git a/youtube_dl/extractor/gaskrank.py b/youtube_dl/extractor/gaskrank.py
new file mode 100644
index 000000000..972b47bf2
--- /dev/null
+++ b/youtube_dl/extractor/gaskrank.py
@@ -0,0 +1,121 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+from .common import InfoExtractor
+from ..utils import float_or_none
+from ..utils import int_or_none
+from ..utils import js_to_json
+from ..utils import unified_strdate
+
+
+class GaskrankIE(InfoExtractor):
+    """InfoExtractor for gaskrank.tv"""
+    _VALID_URL = r'https?://(?:www\.)?gaskrank\.tv/tv/(?P<categories>[^/]+)/(?P<id>[^/]+)\.html?'
+    _TESTS = [
+        {
+            'url': 'http://www.gaskrank.tv/tv/motorrad-fun/strike-einparken-durch-anfaenger-crash-mit-groesserem-flurschaden.htm',
+            'md5': '1ae88dbac97887d85ebd1157a95fc4f9',
+            'info_dict': {
+                'id': '201601/26955',
+                'ext': 'mp4',
+                'title': 'Strike! Einparken können nur Männer - Flurschaden hält sich in Grenzen *lol*',
+                'thumbnail': r're:^https?://.*\.jpg$',
+                'categories': ['motorrad-fun'],
+                'display_id': 'strike-einparken-durch-anfaenger-crash-mit-groesserem-flurschaden',
+                'uploader_id': 'Bikefun',
+                'upload_date': '20170110',
+                'uploader_url': None,
+            }
+        },
+        {
+            'url': 'http://www.gaskrank.tv/tv/racing/isle-of-man-tt-2011-michael-du-15920.htm',
+            'md5': 'c33ee32c711bc6c8224bfcbe62b23095',
+            'info_dict': {
+                'id': '201106/15920',
+                'ext': 'mp4',
+                'title': 'Isle of Man - Michael Dunlop vs Guy Martin - schwindelig kucken',
+                'thumbnail': r're:^https?://.*\.jpg$',
+                'categories': ['racing'],
+                'display_id': 'isle-of-man-tt-2011-michael-du-15920',
+                'uploader_id': 'IOM',
+                'upload_date': '20160506',
+                'uploader_url': 'www.iomtt.com',
+            }
+        }
+    ]
+
+    def _real_extract(self, url):
+        """extract information from gaskrank.tv"""
+        def fix_json(code):
+            """Removes trailing comma in json: {{},} --> {{}}"""
+            return re.sub(r',\s*}', r'}', js_to_json(code))
+
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        categories = [re.match(self._VALID_URL, url).group('categories')]
+        title = self._search_regex(
+            r'movieName\s*:\s*\'([^\']*)\'',
+            webpage, 'title')
+        thumbnail = self._search_regex(
+            r'poster\s*:\s*\'([^\']*)\'',
+            webpage, 'thumbnail', default=None)
+
+        mobj = re.search(
+            r'Video von:\s*(?P<uploader_id>[^|]*?)\s*\|\s*vom:\s*(?P<upload_date>[0-9][0-9]\.[0-9][0-9]\.[0-9][0-9][0-9][0-9])',
+            webpage)
+        if mobj is not None:
+            uploader_id = mobj.groupdict().get('uploader_id')
+            upload_date = unified_strdate(mobj.groupdict().get('upload_date'))
+
+        uploader_url = self._search_regex(
+            r'Homepage:\s*<[^>]*>(?P<uploader_url>[^<]*)',
+            webpage, 'uploader_url', default=None)
+        tags = re.findall(
+            r'/tv/tags/[^/]+/"\s*>(?P<tag>[^<]*?)<',
+            webpage)
+
+        view_count = self._search_regex(
+            r'class\s*=\s*"gkRight"(?:[^>]*>\s*<[^>]*)*icon-eye-open(?:[^>]*>\s*<[^>]*)*>\s*(?P<view_count>[0-9\.]*)',
+            webpage, 'view_count', default=None)
+        if view_count:
+            view_count = int_or_none(view_count.replace('.', ''))
+
+        average_rating = self._search_regex(
+            r'itemprop\s*=\s*"ratingValue"[^>]*>\s*(?P<average_rating>[0-9,]+)',
+            webpage, 'average_rating')
+        if average_rating:
+            average_rating = float_or_none(average_rating.replace(',', '.'))
+
+        playlist = self._parse_json(
+            self._search_regex(
+                r'playlist\s*:\s*\[([^\]]*)\]',
+                webpage, 'playlist', default='{}'),
+            display_id, transform_source=fix_json, fatal=False)
+
+        video_id = self._search_regex(
+            r'https?://movies\.gaskrank\.tv/([^-]*?)(-[^\.]*)?\.mp4',
+            playlist.get('0').get('src'), 'video id')
+
+        formats = []
+        for key in playlist:
+            formats.append({
+                'url': playlist[key]['src'],
+                'format_id': key,
+                'quality': playlist[key].get('quality')})
+        self._sort_formats(formats, field_preference=['format_id'])
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': thumbnail,
+            'categories': categories,
+            'display_id': display_id,
+            'uploader_id': uploader_id,
+            'upload_date': upload_date,
+            'uploader_url': uploader_url,
+            'tags': tags,
+            'view_count': view_count,
+            'average_rating': average_rating,
+        }

From 9c14fe96811a54788f5173fd341aa71d49511e52 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 6 Feb 2017 00:24:36 +0800
Subject: [PATCH 0190/1696] [gaskrank] Minor change and update ChangeLog after
 #11685

---
 ChangeLog                        |  1 +
 youtube_dl/extractor/gaskrank.py | 10 ++++++----
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 984191925..62eef4a2e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
++ [gaskrank] Add new extractor (#11685)
 * [bandcamp] Fix extraction for incomplete albums (#11727)
 * [iwara] Fix extraction (#11781)
 * [googledrive] Fix extraction on Python 3.6
diff --git a/youtube_dl/extractor/gaskrank.py b/youtube_dl/extractor/gaskrank.py
index 972b47bf2..36ba7d8cf 100644
--- a/youtube_dl/extractor/gaskrank.py
+++ b/youtube_dl/extractor/gaskrank.py
@@ -3,10 +3,12 @@ from __future__ import unicode_literals
 
 import re
 from .common import InfoExtractor
-from ..utils import float_or_none
-from ..utils import int_or_none
-from ..utils import js_to_json
-from ..utils import unified_strdate
+from ..utils import (
+    float_or_none,
+    int_or_none,
+    js_to_json,
+    unified_strdate,
+)
 
 
 class GaskrankIE(InfoExtractor):

From 815482d4eb03c2b3ea319288cc66b38406dbb9d5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 6 Feb 2017 00:37:56 +0800
Subject: [PATCH 0191/1696] Credit @motophil for gaskrank.py (#11685)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index f2875d504..247c0ea13 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -201,3 +201,4 @@ Stephen Chen
 Fabian Stahl
 Bagira
 Odd Stråbø
+Philip Herzog

From 2aec7256ae3ef877cc57f7d27f0fa171a7f25a98 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 6 Feb 2017 00:20:30 +0700
Subject: [PATCH 0192/1696] [extractor/common] Speed-up media tags regex
 (closes #11979)

---
 youtube_dl/extractor/common.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 2c8ec1417..519188622 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1959,7 +1959,12 @@ class InfoExtractor(object):
         media_tags = [(media_tag, media_type, '')
                       for media_tag, media_type
                       in re.findall(r'(?s)(<(video|audio)[^>]*/>)', webpage)]
-        media_tags.extend(re.findall(r'(?s)(<(?P<tag>video|audio)[^>]*>)(.*?)</(?P=tag)>', webpage))
+        media_tags.extend(re.findall(
+            # We only allow video|audio followed by a whitespace or '>'.
+            # Allowing more characters may end up in significant slow down (see
+            # https://github.com/rg3/youtube-dl/issues/11979, example URL:
+            # http://www.porntrex.com/maps/videositemap.xml).
+            r'(?s)(<(?P<tag>video|audio)(?:\s+[^>]*)?>)(.*?)</(?P=tag)>', webpage))
         for media_tag, media_type, media_content in media_tags:
             media_info = {
                 'formats': [],

From 7097bffba6769302d9aef7e839baf4c6f30de066 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 6 Feb 2017 23:07:59 +0700
Subject: [PATCH 0193/1696] [downloader/fragment] Respect --no-part

---
 youtube_dl/downloader/fragment.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 60df627a6..56f975266 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -61,6 +61,7 @@ class FragmentFD(FileDownloader):
                 'noprogress': True,
                 'ratelimit': self.params.get('ratelimit'),
                 'retries': self.params.get('retries', 0),
+                'nopart': self.params.get('nopart', False),
                 'test': self.params.get('test', False),
             }
         )

From b7a8c1bcfaccb797609ea763a1aa3140148bcddc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 6 Feb 2017 23:23:40 +0700
Subject: [PATCH 0194/1696] [extractor/generic] Improve rtmp support (closes
 #11993)

---
 youtube_dl/extractor/generic.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 4156cf27d..1fb0e0673 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -29,6 +29,7 @@ from ..utils import (
     UnsupportedError,
     xpath_text,
 )
+from .commonprotocols import RtmpIE
 from .brightcove import (
     BrightcoveLegacyIE,
     BrightcoveNewIE,
@@ -2487,6 +2488,8 @@ class GenericIE(InfoExtractor):
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True
+            if RtmpIE.suitable(vurl):
+                return True
             vpath = compat_urlparse.urlparse(vurl).path
             vext = determine_ext(vpath)
             return '.' in vpath and vext not in ('swf', 'png', 'jpg', 'srt', 'sbv', 'sub', 'vtt', 'ttml', 'js')
@@ -2584,6 +2587,10 @@ class GenericIE(InfoExtractor):
                 entries.append(self.url_result(video_url, 'Youtube'))
                 continue
 
+            if RtmpIE.suitable(video_url):
+                entries.append(self.url_result(video_url, RtmpIE.ie_key()))
+                continue
+
             # here's a fun little line of code for you:
             video_id = os.path.splitext(video_id)[0]
 

From 5620f840f68ffb688131aaa3a40462110f9da468 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 6 Feb 2017 23:31:58 +0700
Subject: [PATCH 0195/1696] [extractor/generic] Add test for #11993 and more
 metadata for rtmp

---
 youtube_dl/extractor/generic.py | 26 ++++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 1fb0e0673..1c233f038 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -948,6 +948,19 @@ class GenericIE(InfoExtractor):
                 'title': 'Webinar: Using Discovery, The National Archives’ online catalogue',
             },
         },
+        # jwplayer rtmp
+        {
+            'url': 'http://www.suffolk.edu/sjc/',
+            'info_dict': {
+                'id': 'sjclive',
+                'ext': 'flv',
+                'title': 'Massachusetts Supreme Judicial Court Oral Arguments',
+                'uploader': 'www.suffolk.edu',
+            },
+            'params': {
+                'skip_download': True,
+            }
+        },
         # rtl.nl embed
         {
             'url': 'http://www.rtlnieuws.nl/nieuws/buitenland/aanslagen-kopenhagen',
@@ -2587,10 +2600,6 @@ class GenericIE(InfoExtractor):
                 entries.append(self.url_result(video_url, 'Youtube'))
                 continue
 
-            if RtmpIE.suitable(video_url):
-                entries.append(self.url_result(video_url, RtmpIE.ie_key()))
-                continue
-
             # here's a fun little line of code for you:
             video_id = os.path.splitext(video_id)[0]
 
@@ -2601,6 +2610,15 @@ class GenericIE(InfoExtractor):
                 'age_limit': age_limit,
             }
 
+            if RtmpIE.suitable(video_url):
+                entry_info_dict.update({
+                    '_type': 'url_transparent',
+                    'ie_key': RtmpIE.ie_key(),
+                    'url': video_url,
+                })
+                entries.append(entry_info_dict)
+                continue
+
             ext = determine_ext(video_url)
             if ext == 'smil':
                 entry_info_dict['formats'] = self._extract_smil_formats(video_url, video_id)

From d5d904ff7d712e126885915d9e3e35b5f3115888 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 6 Feb 2017 23:53:42 +0700
Subject: [PATCH 0196/1696] [canalplus] Add support for cstar.fr (#11990)

---
 youtube_dl/extractor/canalplus.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index b3f76a7b1..4b9fa2ddf 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -27,6 +27,7 @@ class CanalplusIE(InfoExtractor):
                                     (?:www\.)?d8\.tv|
                                     (?:www\.)?c8\.fr|
                                     (?:www\.)?d17\.tv|
+                                    (?:(?:football|www)\.)?cstar\.fr|
                                     (?:www\.)?itele\.fr
                                 )/(?:(?:[^/]+/)*(?P<display_id>[^/?#&]+))?(?:\?.*\bvid=(?P<vid>\d+))?|
                                 player\.canalplus\.fr/#/(?P<id>\d+)
@@ -40,6 +41,7 @@ class CanalplusIE(InfoExtractor):
         'd8': 'd8',
         'c8': 'd8',
         'd17': 'd17',
+        'cstar': 'd17',
         'itele': 'itele',
     }
 
@@ -86,6 +88,19 @@ class CanalplusIE(InfoExtractor):
             'description': 'Chaque matin du lundi au vendredi, Michaël Darmon reçoit un invité politique à 8h25.',
             'upload_date': '20161014',
         },
+    }, {
+        'url': 'http://football.cstar.fr/cstar-minisite-foot/pid7566-feminines-videos.html?vid=1416769',
+        'info_dict': {
+            'id': '1416769',
+            'display_id': 'pid7566-feminines-videos',
+            'ext': 'mp4',
+            'title': 'France - Albanie : les temps forts de la soirée - 20/09/2016',
+            'description': 'md5:c3f30f2aaac294c1c969b3294de6904e',
+            'upload_date': '20160921',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'http://m.canalplus.fr/?vid=1398231',
         'only_matching': True,

From 242a14a1f6ea14b8e7de9aa1eddaf938de825e1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 7 Feb 2017 00:22:16 +0700
Subject: [PATCH 0197/1696] [extractor/common] Fix audio only with audio group
 in m3u8 (closes #11995)

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 519188622..0b4e2ac20 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1315,8 +1315,8 @@ class InfoExtractor(object):
                         'abr': abr,
                     })
                 f.update(parse_codecs(last_info.get('CODECS')))
-                if audio_in_video_stream.get(last_info.get('AUDIO')) is False:
-                    # TODO: update acodec for for audio only formats with the same GROUP-ID
+                if audio_in_video_stream.get(last_info.get('AUDIO')) is False and f['vcodec'] != 'none':
+                    # TODO: update acodec for audio only formats with the same GROUP-ID
                     f['acodec'] = 'none'
                 formats.append(f)
                 last_info = {}

From f28aeff26416515fbaf46ce087e8332a15958c23 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 7 Feb 2017 01:52:59 +0700
Subject: [PATCH 0198/1696] [pornhub] Fix extraction (closes #11997)

---
 youtube_dl/extractor/pornhub.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 3eaf56973..017f6c552 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -156,7 +156,12 @@ class PornHubIE(InfoExtractor):
         comment_count = self._extract_count(
             r'All Comments\s*<span>\(([\d,.]+)\)', webpage, 'comment')
 
-        video_urls = list(map(compat_urllib_parse_unquote, re.findall(r"player_quality_[0-9]{3}p\s*=\s*'([^']+)'", webpage)))
+        video_urls = []
+        for quote, video_url in re.findall(
+                r'player_quality_[0-9]{3,4}p\s*=\s*(["\'])(.+?)\1;', webpage):
+            video_urls.append(compat_urllib_parse_unquote(re.sub(
+                r'{0}\s*\+\s*{0}'.format(quote), '', video_url)))
+
         if webpage.find('"encrypted":true') != -1:
             password = compat_urllib_parse_unquote_plus(
                 self._search_regex(r'"video_title":"([^"]+)', webpage, 'password'))

From c87f95f991901d98c3418fef8c34807f16d56aac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 7 Feb 2017 01:58:57 +0700
Subject: [PATCH 0199/1696] [ChangeLog] Actualize

---
 ChangeLog | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 62eef4a2e..ee3c30b7c 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,10 +1,20 @@
 version <unreleased>
 
+Core
+* [extractor/common] Fix audio only with audio group in m3u8 (#11995)
++ [downloader/fragment] Respect --no-part
+* [extractor/common] Speed-up HTML5 media entries extraction (#11979)
+
 Extractors
-+ [gaskrank] Add new extractor (#11685)
+* [pornhub] Fix extraction (#11997)
++ [canalplus] Add support for cstar.fr (#11990)
++ [extractor/generic] Improve RTMP support (#11993)
++ [gaskrank] Add support for gaskrank.tv (#11685)
 * [bandcamp] Fix extraction for incomplete albums (#11727)
 * [iwara] Fix extraction (#11781)
 * [googledrive] Fix extraction on Python 3.6
++ [videopress] Add support for videopress.com
++ [afreecatv] Extract RTMP formats
 
 
 version 2017.02.04.1

From 013877298de439365331887903417deeac9d7ab0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 7 Feb 2017 02:04:50 +0700
Subject: [PATCH 0200/1696] release 2017.02.07

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 ++
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 15e7d4944..20a726fc4 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.04.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.04.1**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.07*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.07**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.02.04.1
+[debug] youtube-dl version 2017.02.07
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index ee3c30b7c..7e2afaacf 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.02.07
 
 Core
 * [extractor/common] Fix audio only with audio group in m3u8 (#11995)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 50a339bc4..2d82cc321 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -282,6 +282,7 @@
  - **Gamersyde**
  - **GameSpot**
  - **GameStar**
+ - **Gaskrank**
  - **Gazeta**
  - **GDCVault**
  - **generic**: Generic downloader that works on some sites
@@ -859,6 +860,7 @@
  - **videomore:season**
  - **videomore:video**
  - **VideoPremium**
+ - **VideoPress**
  - **videoweed**: VideoWeed
  - **Vidio**
  - **vidme**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 5dde47a26..a73e9d89c 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.02.04.1'
+__version__ = '2017.02.07'

From fdf9b959bc600c9377d6567d29081baa4f23f0da Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 8 Feb 2017 16:23:42 +0100
Subject: [PATCH 0201/1696] [nbc] add support adobe pass auth(closes #12006)

---
 youtube_dl/extractor/nbc.py | 69 +++++++++++++++++++++++++++++--------
 1 file changed, 54 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 434a94de4..d2a44d05d 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -4,23 +4,26 @@ import re
 
 from .common import InfoExtractor
 from .theplatform import ThePlatformIE
+from .adobepass import AdobePassIE
+from ..compat import compat_urllib_parse_urlparse
 from ..utils import (
     find_xpath_attr,
     lowercase_escape,
     smuggle_url,
     unescapeHTML,
     update_url_query,
+    int_or_none,
 )
 
 
-class NBCIE(InfoExtractor):
+class NBCIE(AdobePassIE):
     _VALID_URL = r'https?://(?:www\.)?nbc\.com/(?:[^/]+/)+(?P<id>n?\d+)'
 
     _TESTS = [
         {
-            'url': 'http://www.nbc.com/the-tonight-show/segments/112966',
+            'url': 'http://www.nbc.com/the-tonight-show/video/jimmy-fallon-surprises-fans-at-ben-jerrys/2848237',
             'info_dict': {
-                'id': '112966',
+                'id': '2848237',
                 'ext': 'mp4',
                 'title': 'Jimmy Fallon Surprises Fans at Ben & Jerry\'s',
                 'description': 'Jimmy gives out free scoops of his new "Tonight Dough" ice cream flavor by surprising customers at the Ben & Jerry\'s scoop shop.',
@@ -69,7 +72,7 @@ class NBCIE(InfoExtractor):
             # HLS streams requires the 'hdnea3' cookie
             'url': 'http://www.nbc.com/Kings/video/goliath/n1806',
             'info_dict': {
-                'id': 'n1806',
+                'id': '101528f5a9e8127b107e98c5e6ce4638',
                 'ext': 'mp4',
                 'title': 'Goliath',
                 'description': 'When an unknown soldier saves the life of the King\'s son in battle, he\'s thrust into the limelight and politics of the kingdom.',
@@ -87,21 +90,57 @@ class NBCIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        theplatform_url = unescapeHTML(lowercase_escape(self._html_search_regex(
-            [
-                r'(?:class="video-player video-player-full" data-mpx-url|class="player" src)="(.*?)"',
-                r'<iframe[^>]+src="((?:https?:)?//player\.theplatform\.com/[^"]+)"',
-                r'"embedURL"\s*:\s*"([^"]+)"'
-            ],
-            webpage, 'theplatform url').replace('_no_endcard', '').replace('\\/', '/')))
-        if theplatform_url.startswith('//'):
-            theplatform_url = 'http:' + theplatform_url
-        return {
+        info = {
             '_type': 'url_transparent',
             'ie_key': 'ThePlatform',
-            'url': smuggle_url(theplatform_url, {'source_url': url}),
             'id': video_id,
         }
+        video_data = None
+        preload = self._search_regex(
+            r'PRELOAD\s*=\s*({.+})', webpage, 'preload data', default=None)
+        if preload:
+            preload_data = self._parse_json(preload, video_id)
+            path = compat_urllib_parse_urlparse(url).path.rstrip('/')
+            entity_id = preload_data.get('xref', {}).get(path)
+            video_data = preload_data.get('entities', {}).get(entity_id)
+        if video_data:
+            query = {
+                'mbr': 'true',
+                'manifest': 'm3u',
+            }
+            video_id = video_data['guid']
+            title = video_data['title']
+            if video_data.get('entitlement') == 'auth':
+                resource = self._get_mvpd_resource(
+                    'nbcentertainment', title, video_id,
+                    video_data.get('vChipRating'))
+                query['auth'] = self._extract_mvpd_auth(
+                    url, video_id, 'nbcentertainment', resource)
+            theplatform_url = smuggle_url(update_url_query(
+                'http://link.theplatform.com/s/NnzsPC/media/guid/2410887629/' + video_id,
+                query), {'force_smil_url': True})
+            info.update({
+                'id': video_id,
+                'title': title,
+                'url': theplatform_url,
+                'description': video_data.get('description'),
+                'keywords': video_data.get('keywords'),
+                'season_number': int_or_none(video_data.get('seasonNumber')),
+                'episode_number': int_or_none(video_data.get('episodeNumber')),
+                'series': video_data.get('showName'),
+            })
+        else:
+            theplatform_url = unescapeHTML(lowercase_escape(self._html_search_regex(
+                [
+                    r'(?:class="video-player video-player-full" data-mpx-url|class="player" src)="(.*?)"',
+                    r'<iframe[^>]+src="((?:https?:)?//player\.theplatform\.com/[^"]+)"',
+                    r'"embedURL"\s*:\s*"([^"]+)"'
+                ],
+                webpage, 'theplatform url').replace('_no_endcard', '').replace('\\/', '/')))
+            if theplatform_url.startswith('//'):
+                theplatform_url = 'http:' + theplatform_url
+            info['url'] = smuggle_url(theplatform_url, {'source_url': url})
+        return info
 
 
 class NBCSportsVPlayerIE(InfoExtractor):

From b9c9cb5f7958f924949adbe37824df1fc1530519 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 8 Feb 2017 23:15:39 +0700
Subject: [PATCH 0202/1696] [6play] Fix extraction (closes #12011)

---
 youtube_dl/extractor/sixplay.py | 69 +++++++++++++++++++++------------
 1 file changed, 44 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/sixplay.py b/youtube_dl/extractor/sixplay.py
index d3aba58a2..460e18e26 100644
--- a/youtube_dl/extractor/sixplay.py
+++ b/youtube_dl/extractor/sixplay.py
@@ -2,63 +2,82 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
-    qualities,
-    int_or_none,
-    mimetype2ext,
     determine_ext,
+    int_or_none,
+    try_get,
+    qualities,
 )
 
 
 class SixPlayIE(InfoExtractor):
     _VALID_URL = r'(?:6play:|https?://(?:www\.)?6play\.fr/.+?-c_)(?P<id>[0-9]+)'
     _TEST = {
-        'url': 'http://www.6play.fr/jamel-et-ses-amis-au-marrakech-du-rire-p_1316/jamel-et-ses-amis-au-marrakech-du-rire-2015-c_11495320',
+        'url': 'http://www.6play.fr/le-meilleur-patissier-p_1807/le-meilleur-patissier-special-fetes-mercredi-a-21-00-sur-m6-c_11638450',
         'md5': '42310bffe4ba3982db112b9cd3467328',
         'info_dict': {
-            'id': '11495320',
+            'id': '11638450',
             'ext': 'mp4',
-            'title': 'Jamel et ses amis au Marrakech du rire 2015',
-            'description': 'md5:ba2149d5c321d5201b78070ee839d872',
+            'title': 'Le Meilleur Pâtissier, spécial fêtes mercredi à 21:00 sur M6',
+            'description': 'md5:308853f6a5f9e2d55a30fc0654de415f',
+            'duration': 39,
+            'series': 'Le meilleur pâtissier',
+        },
+        'params': {
+            'skip_download': True,
         },
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        clip_data = self._download_json(
-            'https://player.m6web.fr/v2/video/config/6play-auth/FR/%s.json' % video_id,
-            video_id)
-        video_data = clip_data['videoInfo']
+
+        data = self._download_json(
+            'https://pc.middleware.6play.fr/6play/v2/platforms/m6group_web/services/6play/videos/clip_%s' % video_id,
+            video_id, query={
+                'csa': 5,
+                'with': 'clips',
+            })
+
+        clip_data = data['clips'][0]
+        title = clip_data['title']
 
         quality_key = qualities(['lq', 'sd', 'hq', 'hd'])
         formats = []
-        for source in clip_data['sources']:
-            source_type, source_url = source.get('type'), source.get('src')
-            if not source_url or source_type == 'hls/primetime':
+        for asset in clip_data['assets']:
+            asset_url = asset.get('full_physical_path')
+            if not asset_url:
                 continue
-            ext = mimetype2ext(source_type) or determine_ext(source_url)
-            if ext == 'm3u8':
+            container = asset.get('video_container')
+            ext = determine_ext(asset_url)
+            if container == 'm3u8' or ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
-                    source_url, video_id, 'mp4', 'm3u8_native',
+                    asset_url, video_id, 'mp4', 'm3u8_native',
                     m3u8_id='hls', fatal=False))
                 formats.extend(self._extract_f4m_formats(
-                    source_url.replace('.m3u8', '.f4m'),
+                    asset_url.replace('.m3u8', '.f4m'),
                     video_id, f4m_id='hds', fatal=False))
-            elif ext == 'mp4':
-                quality = source.get('quality')
+            elif container == 'mp4' or ext == 'mp4':
+                quality = asset.get('video_quality')
                 formats.append({
-                    'url': source_url,
+                    'url': asset_url,
                     'format_id': quality,
                     'quality': quality_key(quality),
                     'ext': ext,
                 })
         self._sort_formats(formats)
 
+        def get(getter):
+            for src in (data, clip_data):
+                v = try_get(src, getter, compat_str)
+                if v:
+                    return v
+
         return {
             'id': video_id,
-            'title': video_data['title'].strip(),
-            'description': video_data.get('description'),
-            'duration': int_or_none(video_data.get('duration')),
-            'series': video_data.get('titlePgm'),
+            'title': title,
+            'description': get(lambda x: x['description']),
+            'duration': int_or_none(clip_data.get('duration')),
+            'series': get(lambda x: x['program']['title']),
             'formats': formats,
         }

From ae8d5a5c594f480ba047f0a45836813fccd3d2c7 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 8 Feb 2017 18:56:38 +0100
Subject: [PATCH 0203/1696] [go] add support for adobe pass auth(closes
 #11468)(closes #10831)

---
 youtube_dl/extractor/go.py | 60 ++++++++++++++++++++++++++++----------
 1 file changed, 44 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/go.py b/youtube_dl/extractor/go.py
index a34779b16..f28e6fbf5 100644
--- a/youtube_dl/extractor/go.py
+++ b/youtube_dl/extractor/go.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 
 import re
 
-from .common import InfoExtractor
+from .adobepass import AdobePassIE
 from ..utils import (
     int_or_none,
     determine_ext,
@@ -13,15 +13,30 @@ from ..utils import (
 )
 
 
-class GoIE(InfoExtractor):
-    _BRANDS = {
-        'abc': '001',
-        'freeform': '002',
-        'watchdisneychannel': '004',
-        'watchdisneyjunior': '008',
-        'watchdisneyxd': '009',
+class GoIE(AdobePassIE):
+    _SITE_INFO = {
+        'abc': {
+            'brand': '001',
+            'requestor_id': 'ABC',
+        },
+        'freeform': {
+            'brand': '002',
+            'requestor_id': 'ABCFamily',
+        },
+        'watchdisneychannel': {
+            'brand': '004',
+            'requestor_id': 'Disney',
+        },
+        'watchdisneyjunior': {
+            'brand': '008',
+            'requestor_id': 'DisneyJunior',
+        },
+        'watchdisneyxd': {
+            'brand': '009',
+            'requestor_id': 'DisneyXD',
+        }
     }
-    _VALID_URL = r'https?://(?:(?P<sub_domain>%s)\.)?go\.com/(?:[^/]+/)*(?:vdka(?P<id>\w+)|season-\d+/\d+-(?P<display_id>[^/?#]+))' % '|'.join(_BRANDS.keys())
+    _VALID_URL = r'https?://(?:(?P<sub_domain>%s)\.)?go\.com/(?:[^/]+/)*(?:vdka(?P<id>\w+)|season-\d+/\d+-(?P<display_id>[^/?#]+))' % '|'.join(_SITE_INFO.keys())
     _TESTS = [{
         'url': 'http://abc.go.com/shows/castle/video/most-recent/vdka0_g86w5onx',
         'info_dict': {
@@ -47,7 +62,8 @@ class GoIE(InfoExtractor):
                 # There may be inner quotes, e.g. data-video-id="'VDKA3609139'"
                 # from http://freeform.go.com/shows/shadowhunters/episodes/season-2/1-this-guilty-blood
                 r'data-video-id=["\']*VDKA(\w+)', webpage, 'video id')
-        brand = self._BRANDS[sub_domain]
+        site_info = self._SITE_INFO[sub_domain]
+        brand = site_info['brand']
         video_data = self._download_json(
             'http://api.contents.watchabc.go.com/vp2/ws/contents/3000/videos/%s/001/-1/-1/-1/%s/-1/-1.json' % (brand, video_id),
             video_id)['video'][0]
@@ -63,14 +79,26 @@ class GoIE(InfoExtractor):
             if ext == 'm3u8':
                 video_type = video_data.get('type')
                 if video_type == 'lf':
+                    data = {
+                        'video_id': video_data['id'],
+                        'video_type': video_type,
+                        'brand': brand,
+                        'device': '001',
+                    }
+                    if video_data.get('accesslevel') == '1':
+                        requestor_id = site_info['requestor_id']
+                        resource = self._get_mvpd_resource(
+                            requestor_id, title, video_id, None)
+                        auth = self._extract_mvpd_auth(
+                            url, video_id, requestor_id, resource)
+                        data.update({
+                            'token': auth,
+                            'token_type': 'ap',
+                            'adobe_requestor_id': requestor_id,
+                        })
                     entitlement = self._download_json(
                         'https://api.entitlement.watchabc.go.com/vp2/ws-secure/entitlement/2020/authorize.json',
-                        video_id, data=urlencode_postdata({
-                            'video_id': video_data['id'],
-                            'video_type': video_type,
-                            'brand': brand,
-                            'device': '001',
-                        }))
+                        video_id, data=urlencode_postdata(data), headers=self.geo_verification_headers())
                     errors = entitlement.get('errors', {}).get('errors', [])
                     if errors:
                         error_message = ', '.join([error['message'] for error in errors])

From 829801827383324f1375b16276435781992a6619 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 8 Feb 2017 20:44:23 +0100
Subject: [PATCH 0204/1696] [scrippsnetworks:watch] Add new extractor(closes
 #10765)

---
 youtube_dl/extractor/extractors.py      |  1 +
 youtube_dl/extractor/scrippsnetworks.py | 60 +++++++++++++++++++++++++
 2 files changed, 61 insertions(+)
 create mode 100644 youtube_dl/extractor/scrippsnetworks.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 12cda36cc..795843dae 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -838,6 +838,7 @@ from .sbs import SBSIE
 from .scivee import SciVeeIE
 from .screencast import ScreencastIE
 from .screencastomatic import ScreencastOMaticIE
+from .scrippsnetworks import ScrippsNetworksWatchIE
 from .seeker import SeekerIE
 from .senateisvp import SenateISVPIE
 from .sendtonews import SendtoNewsIE
diff --git a/youtube_dl/extractor/scrippsnetworks.py b/youtube_dl/extractor/scrippsnetworks.py
new file mode 100644
index 000000000..597d6f543
--- /dev/null
+++ b/youtube_dl/extractor/scrippsnetworks.py
@@ -0,0 +1,60 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .adobepass import AdobePassIE
+from ..utils import (
+    int_or_none,
+    smuggle_url,
+    update_url_query,
+)
+
+
+class ScrippsNetworksWatchIE(AdobePassIE):
+    IE_NAME = 'scrippsnetworks:watch'
+    _VALID_URL = r'https?://watch\.(?:hgtv|foodnetwork|travelchannel|diynetwork|cookingchanneltv)\.com/player\.[A-Z0-9]+\.html#(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://watch.hgtv.com/player.HNT.html#0256538',
+        'md5': '26545fd676d939954c6808274bdb905a',
+        'info_dict': {
+            'id': '0256538',
+            'ext': 'mp4',
+            'title': 'Seeking a Wow House',
+            'description': 'Buyers retiring in Palm Springs, California, want a modern house with major wow factor. They\'re also looking for a pool and a large, open floorplan with tall windows looking out at the views.',
+            'uploader': 'SCNI',
+            'upload_date': '20170207',
+            'timestamp': 1486450493,
+        },
+        'skip': 'requires TV provider authentication',
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        channel = self._parse_json(self._search_regex(
+            r'"channels"\s*:\s*(\[.+\])',
+            webpage, 'channels'), video_id)[0]
+        video_data = next(v for v in channel['videos'] if v.get('nlvid') == video_id)
+        title = video_data['title']
+        release_url = video_data['releaseUrl']
+        if video_data.get('restricted'):
+            requestor_id = self._search_regex(
+                r'requestorId\s*=\s*"([^"]+)";', webpage, 'requestor id')
+            resource = self._get_mvpd_resource(
+                requestor_id, title, video_id,
+                video_data.get('ratings', [{}])[0].get('rating'))
+            auth = self._extract_mvpd_auth(
+                url, video_id, requestor_id, resource)
+            release_url = update_url_query(release_url, {'auth': auth})
+
+        return {
+            '_type': 'url_transparent',
+            'id': video_id,
+            'title': title,
+            'url': smuggle_url(release_url, {'force_smil_url': True}),
+            'description': video_data.get('description'),
+            'thumbnail': video_data.get('thumbnailUrl'),
+            'series': video_data.get('showTitle'),
+            'season_number': int_or_none(video_data.get('season')),
+            'episode_number': int_or_none(video_data.get('episodeNumber')),
+            'ie_key': 'ThePlatform',
+        }

From c8b8fb0a990421d6e9724fd3ba0060b357fcea8e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 8 Feb 2017 22:56:10 +0100
Subject: [PATCH 0205/1696] [sixplay] improve extraction

- skip drm protected formats
- extract more and better formats
- skip duplicate asset urls
---
 youtube_dl/extractor/sixplay.py | 32 +++++++++++++++++++++++++-------
 1 file changed, 25 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/sixplay.py b/youtube_dl/extractor/sixplay.py
index 460e18e26..8d60cbd73 100644
--- a/youtube_dl/extractor/sixplay.py
+++ b/youtube_dl/extractor/sixplay.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
@@ -12,6 +14,7 @@ from ..utils import (
 
 
 class SixPlayIE(InfoExtractor):
+    IE_NAME = '6play'
     _VALID_URL = r'(?:6play:|https?://(?:www\.)?6play\.fr/.+?-c_)(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.6play.fr/le-meilleur-patissier-p_1807/le-meilleur-patissier-special-fetes-mercredi-a-21-00-sur-m6-c_11638450',
@@ -42,21 +45,36 @@ class SixPlayIE(InfoExtractor):
         clip_data = data['clips'][0]
         title = clip_data['title']
 
+        urls = []
         quality_key = qualities(['lq', 'sd', 'hq', 'hd'])
         formats = []
         for asset in clip_data['assets']:
             asset_url = asset.get('full_physical_path')
-            if not asset_url:
+            protocol = asset.get('protocol')
+            if not asset_url or protocol == 'primetime' or asset_url in urls:
                 continue
+            urls.append(asset_url)
             container = asset.get('video_container')
             ext = determine_ext(asset_url)
             if container == 'm3u8' or ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    asset_url, video_id, 'mp4', 'm3u8_native',
-                    m3u8_id='hls', fatal=False))
-                formats.extend(self._extract_f4m_formats(
-                    asset_url.replace('.m3u8', '.f4m'),
-                    video_id, f4m_id='hds', fatal=False))
+                if protocol == 'usp':
+                    asset_url = re.sub(r'/([^/]+)\.ism/[^/]*\.m3u8', r'/\1.ism/\1.m3u8', asset_url)
+                    formats.extend(self._extract_m3u8_formats(
+                        asset_url, video_id, 'mp4', 'm3u8_native',
+                        m3u8_id='hls', fatal=False))
+                    formats.extend(self._extract_f4m_formats(
+                        asset_url.replace('.m3u8', '.f4m'),
+                        video_id, f4m_id='hds', fatal=False))
+                    formats.extend(self._extract_mpd_formats(
+                        asset_url.replace('.m3u8', '.mpd'),
+                        video_id, mpd_id='dash', fatal=False))
+                    formats.extend(self._extract_ism_formats(
+                        re.sub('/[^/]+\.m3u8', '/Manifest', asset_url),
+                        video_id, ism_id='mss', fatal=False))
+                else:
+                    formats.extend(self._extract_m3u8_formats(
+                        asset_url, video_id, 'mp4', 'm3u8_native',
+                        m3u8_id='hls', fatal=False))
             elif container == 'mp4' or ext == 'mp4':
                 quality = asset.get('video_quality')
                 formats.append({

From b04975733c02f3bc0043c02d170bd3a042c67b89 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 9 Feb 2017 09:13:29 +0100
Subject: [PATCH 0206/1696] [sprout] Add new extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/sprout.py     | 52 ++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+)
 create mode 100644 youtube_dl/extractor/sprout.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 795843dae..3445e7d40 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -896,6 +896,7 @@ from .sport5 import Sport5IE
 from .sportbox import SportBoxEmbedIE
 from .sportdeutschland import SportDeutschlandIE
 from .sportschau import SportschauIE
+from .sprout import SproutIE
 from .srgssr import (
     SRGSSRIE,
     SRGSSRPlayIE,
diff --git a/youtube_dl/extractor/sprout.py b/youtube_dl/extractor/sprout.py
new file mode 100644
index 000000000..8467bf49d
--- /dev/null
+++ b/youtube_dl/extractor/sprout.py
@@ -0,0 +1,52 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .adobepass import AdobePassIE
+from ..utils import (
+    extract_attributes,
+    update_url_query,
+    smuggle_url,
+)
+
+
+class SproutIE(AdobePassIE):
+    _VALID_URL = r'https?://(?:www\.)?sproutonline\.com/watch/(?P<id>[^/?#]+)'
+    _TEST = {
+        'url': 'http://www.sproutonline.com/watch/cowboy-adventure',
+        'md5': '74bf14128578d1e040c3ebc82088f45f',
+        'info_dict': {
+            'id': '9dexnwtmh8_X',
+            'ext': 'mp4',
+            'title': 'A Cowboy Adventure',
+            'description': 'Ruff-Ruff, Tweet and Dave get to be cowboys for the day at Six Cow Corral.',
+            'timestamp': 1437758640,
+            'upload_date': '20150724',
+            'uploader': 'NBCU-SPROUT-NEW',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        video_component = self._search_regex(
+            r'(?s)(<div[^>]+data-component="video"[^>]*?>)',
+            webpage, 'video component', default=None)
+        if video_component:
+            options = self._parse_json(extract_attributes(
+                video_component)['data-options'], video_id)
+            theplatform_url = options['video']
+            query = {
+                'mbr': 'true',
+                'manifest': 'm3u',
+            }
+            if options.get('protected'):
+                query['auth'] = self._extract_mvpd_auth(url, options['pid'], 'sprout', 'sprout')
+            theplatform_url = smuggle_url(update_url_query(
+                theplatform_url, query), {'force_smil_url': True})
+        else:
+            iframe = self._search_regex(
+                r'(<iframe[^>]+id="sproutVideoIframe"[^>]*?>)',
+                webpage, 'iframe')
+            theplatform_url = extract_attributes(iframe)['src']
+
+        return self.url_result(theplatform_url, 'ThePlatform')

From 37084f6641c07dbe6580b366f330eb4126d18bbe Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 9 Feb 2017 16:24:54 +0100
Subject: [PATCH 0207/1696] [kaltura] improve embed partner id extraction(fixes
 #12041)

---
 youtube_dl/extractor/kaltura.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 5ef382f9f..c6483bcf0 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -27,7 +27,7 @@ class KalturaIE(InfoExtractor):
                         (?:
                             (?:
                                 # flash player
-                                index\.php/kwidget|
+                                index\.php/(?:kwidget|extwidget/preview)|
                                 # html5 player
                                 html5/html5lib/[^/]+/mwEmbedFrame\.php
                             )
@@ -94,6 +94,10 @@ class KalturaIE(InfoExtractor):
             'params': {
                 'skip_download': True,
             },
+        },
+        {
+            'url': 'https://www.kaltura.com/index.php/extwidget/preview/partner_id/1770401/uiconf_id/37307382/entry_id/0_58u8kme7/embed/iframe?&flashvars[streamerType]=auto',
+            'only_matching': True,
         }
     ]
 
@@ -112,7 +116,7 @@ class KalturaIE(InfoExtractor):
             re.search(
                 r'''(?xs)
                     (?P<q1>["\'])
-                        (?:https?:)?//cdnapi(?:sec)?\.kaltura\.com/(?:(?!(?P=q1)).)*(?:p|partner_id)/(?P<partner_id>\d+)(?:(?!(?P=q1)).)*
+                        (?:https?:)?//cdnapi(?:sec)?\.kaltura\.com(?:(?!(?P=q1)).)*/(?:p|partner_id)/(?P<partner_id>\d+)(?:(?!(?P=q1)).)*
                     (?P=q1).*?
                     (?:
                         entry_?[Ii]d|
@@ -209,6 +213,8 @@ class KalturaIE(InfoExtractor):
                 partner_id = params['wid'][0][1:]
             elif 'p' in params:
                 partner_id = params['p'][0]
+            elif 'partner_id' in params:
+                partner_id = params['partner_id'][0]
             else:
                 raise ExtractorError('Invalid URL', expected=True)
             if 'entry_id' in params:

From be670b8e8f9c32fe3d37666b28c4889d780d5964 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 9 Feb 2017 17:36:59 +0100
Subject: [PATCH 0208/1696] [external:ffmpeg] do not assume that ffmpeg unknown
 version format is new

---
 youtube_dl/downloader/external.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 41e37261d..bdd3545a2 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -275,7 +275,7 @@ class FFmpegFD(ExternalFD):
                 args += ['-f', 'mpegts']
             else:
                 args += ['-f', 'mp4']
-                if (ffpp.basename == 'ffmpeg' and is_outdated_version(ffpp._versions['ffmpeg'], '3.2')) and (not info_dict.get('acodec') or info_dict['acodec'].split('.')[0] in ('aac', 'mp4a')):
+                if (ffpp.basename == 'ffmpeg' and is_outdated_version(ffpp._versions['ffmpeg'], '3.2', False)) and (not info_dict.get('acodec') or info_dict['acodec'].split('.')[0] in ('aac', 'mp4a')):
                     args += ['-bsf:a', 'aac_adtstoasc']
         elif protocol == 'rtmp':
             args += ['-f', 'flv']

From 78ef214d2d8010f2fc7ab451c9b4ae137c2569dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Feb 2017 23:42:40 +0700
Subject: [PATCH 0209/1696] [facebook] Improve JS data regex (closes #12042)

---
 youtube_dl/extractor/facebook.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index b325c8200..4a3c839f4 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -134,6 +134,20 @@ class FacebookIE(InfoExtractor):
             'upload_date': '20161030',
             'uploader': 'CNN',
         },
+    }, {
+        # bigPipe.onPageletArrive ... onPageletArrive pagelet_group_mall
+        'url': 'https://www.facebook.com/yaroslav.korpan/videos/1417995061575415/',
+        'info_dict': {
+            'id': '1417995061575415',
+            'ext': 'mp4',
+            'title': 'md5:a7b86ca673f51800cd54687b7f4012fe',
+            'timestamp': 1486648217,
+            'upload_date': '20170209',
+            'uploader': 'Yaroslav Korpan',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'https://www.facebook.com/video.php?v=10204634152394104',
         'only_matching': True,
@@ -262,7 +276,7 @@ class FacebookIE(InfoExtractor):
         if not video_data:
             server_js_data = self._parse_json(
                 self._search_regex(
-                    r'bigPipe\.onPageletArrive\(({.+?})\)\s*;\s*}\s*\)\s*,\s*["\']onPageletArrive\s+stream_pagelet',
+                    r'bigPipe\.onPageletArrive\(({.+?})\)\s*;\s*}\s*\)\s*,\s*["\']onPageletArrive\s+(?:stream_pagelet|pagelet_group_mall)',
                     webpage, 'js data', default='{}'),
                 video_id, transform_source=js_to_json, fatal=False)
             if server_js_data:

From e64b0fca147c1512c8d31d02aedefed78411bbd9 Mon Sep 17 00:00:00 2001
From: Thomas Christlieb <thomaschristlieb@hotmail.com>
Date: Wed, 8 Feb 2017 13:53:39 +0100
Subject: [PATCH 0210/1696] [pornhub] Fix extraction (closes #12007)

---
 youtube_dl/extractor/pornhub.py | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 017f6c552..5e930f45e 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -156,11 +156,24 @@ class PornHubIE(InfoExtractor):
         comment_count = self._extract_count(
             r'All Comments\s*<span>\(([\d,.]+)\)', webpage, 'comment')
 
+        video_variables = {}
+        for video_variablename, quote, video_variable in re.findall(
+                r'(player_quality_[0-9]{3,4}p[0-9a-z]+?)=\s*(["\'])(.*?)\2;', webpage):
+            video_variables[video_variablename] = video_variable
+
+        encoded_video_urls = []
+        for encoded_video_url in re.findall(
+                r'player_quality_[0-9]{3,4}p\s*=(.*?);', webpage):
+            encoded_video_urls.append(encoded_video_url)
+
+        # Decode the URLs 
         video_urls = []
-        for quote, video_url in re.findall(
-                r'player_quality_[0-9]{3,4}p\s*=\s*(["\'])(.+?)\1;', webpage):
-            video_urls.append(compat_urllib_parse_unquote(re.sub(
-                r'{0}\s*\+\s*{0}'.format(quote), '', video_url)))
+        for url in encoded_video_urls:
+            for varname, varval in video_variables.items():
+                url = url.replace(varname, varval)
+            url = url.replace('+', '')
+            url = url.replace(' ', '')
+            video_urls.append(url)
 
         if webpage.find('"encrypted":true') != -1:
             password = compat_urllib_parse_unquote_plus(

From b7f9843bec27d04f66c0656da22137e32fa157ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Feb 2017 00:57:44 +0700
Subject: [PATCH 0211/1696] [pornhub] Simplify (closes #12018)

---
 youtube_dl/extractor/pornhub.py | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 5e930f45e..818d99c1f 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -158,22 +158,15 @@ class PornHubIE(InfoExtractor):
 
         video_variables = {}
         for video_variablename, quote, video_variable in re.findall(
-                r'(player_quality_[0-9]{3,4}p[0-9a-z]+?)=\s*(["\'])(.*?)\2;', webpage):
+                r'(player_quality_[0-9]{3,4}p\w+)\s*=\s*(["\'])(.+?)\2;', webpage):
             video_variables[video_variablename] = video_variable
 
-        encoded_video_urls = []
-        for encoded_video_url in re.findall(
-                r'player_quality_[0-9]{3,4}p\s*=(.*?);', webpage):
-            encoded_video_urls.append(encoded_video_url)
-
-        # Decode the URLs 
         video_urls = []
-        for url in encoded_video_urls:
+        for encoded_video_url in re.findall(
+                r'player_quality_[0-9]{3,4}p\s*=(.+?);', webpage):
             for varname, varval in video_variables.items():
-                url = url.replace(varname, varval)
-            url = url.replace('+', '')
-            url = url.replace(' ', '')
-            video_urls.append(url)
+                encoded_video_url = encoded_video_url.replace(varname, varval)
+            video_urls.append(re.sub(r'[\s+]', '', encoded_video_url))
 
         if webpage.find('"encrypted":true') != -1:
             password = compat_urllib_parse_unquote_plus(

From 9150d1eb6936e7f3ad168095f57dc0c8f56d2364 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Feb 2017 01:03:35 +0700
Subject: [PATCH 0212/1696] [xtube] Fix extraction (closes #12023)

---
 youtube_dl/extractor/xtube.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index 83bc1fef2..91bae7ee7 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -60,7 +60,8 @@ class XTubeIE(InfoExtractor):
         webpage = self._download_webpage(req, display_id)
 
         sources = self._parse_json(self._search_regex(
-            r'sources\s*:\s*({.+?}),', webpage, 'sources'), video_id)
+            r'(["\'])sources\1\s*:\s*(?P<sources>{.+?}),',
+            webpage, 'sources', group='sources'), video_id)
 
         formats = []
         for format_id, format_url in sources.items():
@@ -81,10 +82,10 @@ class XTubeIE(InfoExtractor):
              r'<span[^>]+class="nickname"[^>]*>([^<]+)'),
             webpage, 'uploader', fatal=False)
         duration = parse_duration(self._search_regex(
-            r'<dt>Runtime:</dt>\s*<dd>([^<]+)</dd>',
+            r'<dt>Runtime:?</dt>\s*<dd>([^<]+)</dd>',
             webpage, 'duration', fatal=False))
         view_count = str_to_int(self._search_regex(
-            r'<dt>Views:</dt>\s*<dd>([\d,\.]+)</dd>',
+            r'<dt>Views:?</dt>\s*<dd>([\d,\.]+)</dd>',
             webpage, 'view count', fatal=False))
         comment_count = str_to_int(self._html_search_regex(
             r'>Comments? \(([\d,\.]+)\)<',

From fbc6dc525e525565544b377a1d16cd915cb11a7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Feb 2017 01:05:48 +0700
Subject: [PATCH 0213/1696] [xtube] Fix shortcuts

---
 youtube_dl/extractor/xtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index 91bae7ee7..11717fe98 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -53,7 +53,7 @@ class XTubeIE(InfoExtractor):
 
         if not display_id:
             display_id = video_id
-            url = 'http://www.xtube.com/watch.php?v=%s' % video_id
+            url = 'http://www.xtube.com/video-watch/-%s' % video_id
 
         req = sanitized_Request(url)
         req.add_header('Cookie', 'age_verified=1; cookiesAccepted=1')

From ff24261ba0359d51dfa54fe4c84a9db157e3b76c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Feb 2017 01:24:14 +0700
Subject: [PATCH 0214/1696] [kaltura] Add explicit port to regexes They should
 not match e.g. cdnapi.kaltura.computernetworks.com/...

---
 youtube_dl/extractor/kaltura.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index c6483bcf0..54374ea76 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -23,7 +23,7 @@ class KalturaIE(InfoExtractor):
                 (?:
                     kaltura:(?P<partner_id>\d+):(?P<id>[0-9a-z_]+)|
                     https?://
-                        (:?(?:www|cdnapi(?:sec)?)\.)?kaltura\.com/
+                        (:?(?:www|cdnapi(?:sec)?)\.)?kaltura\.com(?::\d+)?/
                         (?:
                             (?:
                                 # flash player
@@ -98,6 +98,10 @@ class KalturaIE(InfoExtractor):
         {
             'url': 'https://www.kaltura.com/index.php/extwidget/preview/partner_id/1770401/uiconf_id/37307382/entry_id/0_58u8kme7/embed/iframe?&flashvars[streamerType]=auto',
             'only_matching': True,
+        },
+        {
+            'url': 'https://www.kaltura.com:443/index.php/extwidget/preview/partner_id/1770401/uiconf_id/37307382/entry_id/0_58u8kme7/embed/iframe?&flashvars[streamerType]=auto',
+            'only_matching': True,
         }
     ]
 
@@ -116,7 +120,7 @@ class KalturaIE(InfoExtractor):
             re.search(
                 r'''(?xs)
                     (?P<q1>["\'])
-                        (?:https?:)?//cdnapi(?:sec)?\.kaltura\.com(?:(?!(?P=q1)).)*/(?:p|partner_id)/(?P<partner_id>\d+)(?:(?!(?P=q1)).)*
+                        (?:https?:)?//cdnapi(?:sec)?\.kaltura\.com(?::\d+)?/(?:(?!(?P=q1)).)*\b(?:p|partner_id)/(?P<partner_id>\d+)(?:(?!(?P=q1)).)*
                     (?P=q1).*?
                     (?:
                         entry_?[Ii]d|

From 61ee556aea69f60c5853f27bc92240d4758d7362 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Feb 2017 01:26:00 +0700
Subject: [PATCH 0215/1696] [ChangeLog] Actualize

---
 ChangeLog | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 7e2afaacf..237be5130 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,18 @@
+version <unreleased>
+
+Extractors
+* [xtube] Fix extraction (#12023)
+* [pornhub] Fix extraction (#12007, #12018)
+* [facebook] Improve JS data regular expression (#12042)
+* [kaltura] Improve embed partner id extraction (#12041)
++ [sprout] Add support for sproutonline.com
+* [6play] Improve extraction
++ [scrippsnetworks:watch] Add support for Scripps Networks sites (#10765)
++ [go] Add support for Adobe Pass authentication (#11468, #10831)
+* [6play] Fix extraction (#12011)
++ [nbc] Add support for Adobe Pass authentication (#12006)
+
+
 version 2017.02.07
 
 Core

From 55d4de2283d94c8846357e9a4fd8c74df9eb2835 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Feb 2017 01:27:33 +0700
Subject: [PATCH 0216/1696] release 2017.02.10

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 4 +++-
 youtube_dl/version.py     | 2 +-
 4 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 20a726fc4..0c1569fd4 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.07*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.07**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.10*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.10**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.02.07
+[debug] youtube-dl version 2017.02.10
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 237be5130..d4c8081f7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.02.10
 
 Extractors
 * [xtube] Fix extraction (#12023)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 2d82cc321..76882f3b5 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -11,6 +11,7 @@
  - **4tube**
  - **56.com**
  - **5min**
+ - **6play**
  - **8tracks**
  - **91porn**
  - **9c9media**
@@ -667,6 +668,7 @@
  - **screen.yahoo:search**: Yahoo screen search
  - **Screencast**
  - **ScreencastOMatic**
+ - **scrippsnetworks:watch**
  - **Seeker**
  - **SenateISVP**
  - **SendtoNews**
@@ -676,7 +678,6 @@
  - **Shared**: shared.sx
  - **ShowRoomLive**
  - **Sina**
- - **SixPlay**
  - **skynewsarabia:article**
  - **skynewsarabia:video**
  - **SkySports**
@@ -711,6 +712,7 @@
  - **SportBoxEmbed**
  - **SportDeutschland**
  - **Sportschau**
+ - **Sprout**
  - **sr:mediathek**: Saarländischer Rundfunk
  - **SRGSSR**
  - **SRGSSRPlay**: srf.ch, rts.ch, rsi.ch, rtr.ch and swissinfo.ch play sites
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a73e9d89c..a8395ce04 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.02.07'
+__version__ = '2017.02.10'

From 4d32b6385160e8cf9117839022ed795ce02b107d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 9 Feb 2017 23:07:43 +0100
Subject: [PATCH 0217/1696] [tvplayer] Add new extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/tvplayer.py   | 75 ++++++++++++++++++++++++++++++
 2 files changed, 76 insertions(+)
 create mode 100644 youtube_dl/extractor/tvplayer.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 3445e7d40..5115e1a0c 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1019,6 +1019,7 @@ from .tvplay import (
     TVPlayIE,
     ViafreeIE,
 )
+from .tvplayer import TVPlayerIE
 from .tweakers import TweakersIE
 from .twentyfourvideo import TwentyFourVideoIE
 from .twentymin import TwentyMinutenIE
diff --git a/youtube_dl/extractor/tvplayer.py b/youtube_dl/extractor/tvplayer.py
new file mode 100644
index 000000000..b6537141a
--- /dev/null
+++ b/youtube_dl/extractor/tvplayer.py
@@ -0,0 +1,75 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_HTTPError
+from ..utils import (
+    extract_attributes,
+    urlencode_postdata,
+    ExtractorError,
+)
+
+
+class TVPlayerIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?tvplayer\.com/watch/(?P<id>[^/?#]+)'
+    _TEST = {
+        'url': 'http://tvplayer.com/watch/bbcone',
+        'info_dict': {
+            'id': '89',
+            'ext': 'mp4',
+            'title': r're:^BBC One [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        current_channel = extract_attributes(self._search_regex(
+            r'(<div[^>]+class="[^"]*current-channel[^"]*"[^>]*>)',
+            webpage, 'channel element'))
+        title = current_channel['data-name']
+
+        resource_id = self._search_regex(
+            r'resourceId\s*=\s*"(\d+)"', webpage, 'resource id')
+        platform = self._search_regex(
+            r'platform\s*=\s*"([^"]+)"', webpage, 'platform')
+        token = self._search_regex(
+            r'token\s*=\s*"([^"]+)"', webpage, 'token', default='null')
+        validate = self._search_regex(
+            r'validate\s*=\s*"([^"]+)"', webpage, 'validate', default='null')
+
+        try:
+            response = self._download_json(
+                'http://api.tvplayer.com/api/v2/stream/live',
+                resource_id, headers={
+                    'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
+                }, data=urlencode_postdata({
+                    'service': 1,
+                    'platform': platform,
+                    'id': resource_id,
+                    'token': token,
+                    'validate': validate,
+                }))['tvplayer']['response']
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError):
+                response = self._parse_json(
+                    e.cause.read().decode(), resource_id)['tvplayer']['response']
+                raise ExtractorError(
+                    '%s said: %s' % (self.IE_NAME, response['error']), expected=True)
+            raise
+
+        formats = self._extract_m3u8_formats(response['stream'], resource_id, 'mp4')
+        self._sort_formats(formats)
+
+        return {
+            'id': resource_id,
+            'display_id': display_id,
+            'title': self._live_title(title),
+            'formats': formats,
+            'is_live': True,
+        }

From e01bfc19c345f189b65f3c6e4064b304f1cd337f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Feb 2017 09:39:24 +0700
Subject: [PATCH 0218/1696] [extractor/commonmistakes] Restrict _VALID_URL
 (closes #12050)

---
 youtube_dl/extractor/commonmistakes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/commonmistakes.py b/youtube_dl/extractor/commonmistakes.py
index 2f86e2381..d3ed4a9a4 100644
--- a/youtube_dl/extractor/commonmistakes.py
+++ b/youtube_dl/extractor/commonmistakes.py
@@ -7,7 +7,7 @@ from ..utils import ExtractorError
 class CommonMistakesIE(InfoExtractor):
     IE_DESC = False  # Do not list
     _VALID_URL = r'''(?x)
-        (?:url|URL)
+        (?:url|URL)$
     '''
 
     _TESTS = [{

From 5abcca9060dfd82e86ec590e0fba6ccd111b54a5 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 10 Feb 2017 09:34:23 +0100
Subject: [PATCH 0219/1696] [sixplay] use raw string for regex

---
 youtube_dl/extractor/sixplay.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/sixplay.py b/youtube_dl/extractor/sixplay.py
index 8d60cbd73..547be8f95 100644
--- a/youtube_dl/extractor/sixplay.py
+++ b/youtube_dl/extractor/sixplay.py
@@ -69,7 +69,7 @@ class SixPlayIE(InfoExtractor):
                         asset_url.replace('.m3u8', '.mpd'),
                         video_id, mpd_id='dash', fatal=False))
                     formats.extend(self._extract_ism_formats(
-                        re.sub('/[^/]+\.m3u8', '/Manifest', asset_url),
+                        re.sub(r'/[^/]+\.m3u8', '/Manifest', asset_url),
                         video_id, ism_id='mss', fatal=False))
                 else:
                     formats.extend(self._extract_m3u8_formats(

From 951070957516f0044c5dc89bc5ff3c74c30bd0db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Feb 2017 22:16:20 +0700
Subject: [PATCH 0220/1696] [bloomberg] Add another video id regex (closes
 #12062)

---
 youtube_dl/extractor/bloomberg.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/bloomberg.py b/youtube_dl/extractor/bloomberg.py
index c5e11e8eb..2fbfad1ba 100644
--- a/youtube_dl/extractor/bloomberg.py
+++ b/youtube_dl/extractor/bloomberg.py
@@ -33,6 +33,10 @@ class BloombergIE(InfoExtractor):
         'params': {
             'format': 'best[format_id^=hds]',
         },
+    }, {
+        # data-bmmrid=
+        'url': 'https://www.bloomberg.com/politics/articles/2017-02-08/le-pen-aide-briefed-french-central-banker-on-plan-to-print-money',
+        'only_matching': True,
     }, {
         'url': 'http://www.bloomberg.com/news/articles/2015-11-12/five-strange-things-that-have-been-happening-in-financial-markets',
         'only_matching': True,
@@ -45,9 +49,10 @@ class BloombergIE(InfoExtractor):
         name = self._match_id(url)
         webpage = self._download_webpage(url, name)
         video_id = self._search_regex(
-            (r'["\']bmmrId["\']\s*:\s*(["\'])(?P<url>(?:(?!\1).)+)\1',
-             r'videoId\s*:\s*(["\'])(?P<url>(?:(?!\1).)+)\1'),
-            webpage, 'id', group='url', default=None)
+            (r'["\']bmmrId["\']\s*:\s*(["\'])(?P<id>(?:(?!\1).)+)\1',
+             r'videoId\s*:\s*(["\'])(?P<id>(?:(?!\1).)+)\1',
+             r'data-bmmrid=(["\'])(?P<id>(?:(?!\1).)+)\1'),
+            webpage, 'id', group='id', default=None)
         if not video_id:
             bplayer_data = self._parse_json(self._search_regex(
                 r'BPlayer\(null,\s*({[^;]+})\);', webpage, 'id'), name)

From 9dd5408c99c6ff101ac8302dea5d5c5bfdfc40c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Feb 2017 22:48:11 +0700
Subject: [PATCH 0221/1696] [pluralsight] Detect blocked account error message
 (#12070)

---
 youtube_dl/extractor/pluralsight.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 5c798e874..00b275eb1 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -92,6 +92,10 @@ class PluralsightIE(PluralsightBaseIE):
             raise ExtractorError('Unable to login: %s' % error, expected=True)
 
         if all(p not in response for p in ('__INITIAL_STATE__', '"currentUser"')):
+            BLOCKED = 'Your account has been blocked due to suspicious activity'
+            if BLOCKED in response:
+                raise ExtractorError(
+                    'Unable to login: %s' % BLOCKED, expected=True)
             raise ExtractorError('Unable to log in')
 
     def _get_subtitles(self, author, clip_id, lang, name, duration, video_id):

From 08a00eef79c8f8b12211513b241404394ef6120c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 10 Feb 2017 16:51:41 +0100
Subject: [PATCH 0222/1696] [extractor/common] skip m3u8 manifests protected
 with Adobe Flash Access

---
 youtube_dl/extractor/common.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 0b4e2ac20..9681453ca 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1208,6 +1208,9 @@ class InfoExtractor(object):
         m3u8_doc, urlh = res
         m3u8_url = urlh.geturl()
 
+        if '#EXT-X-FAXS-CM:' in m3u8_doc:  # Adobe Flash Access
+            return []
+
         formats = [self._m3u8_meta_format(m3u8_url, ext, preference, m3u8_id)]
 
         format_url = lambda u: (

From c7d6f614f35c81cf64941affdf683a478078274b Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 10 Feb 2017 16:59:49 +0100
Subject: [PATCH 0223/1696] [corus] Add new extractor(closes #12060)(#9164)

---
 youtube_dl/extractor/corus.py       | 72 +++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py  |  6 +--
 youtube_dl/extractor/hgtv.py        | 44 ------------------
 youtube_dl/extractor/theplatform.py |  5 +-
 4 files changed, 77 insertions(+), 50 deletions(-)
 create mode 100644 youtube_dl/extractor/corus.py

diff --git a/youtube_dl/extractor/corus.py b/youtube_dl/extractor/corus.py
new file mode 100644
index 000000000..7b2f5008b
--- /dev/null
+++ b/youtube_dl/extractor/corus.py
@@ -0,0 +1,72 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .theplatform import ThePlatformFeedIE
+from ..utils import int_or_none
+
+
+class CorusIE(ThePlatformFeedIE):
+    _VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:globaltv|etcanada)\.com|(?:hgtv|foodnetwork|slice)\.ca)/(?:video/|(?:[^/]+/)+(?:videos/[a-z0-9-]+-|video\.html\?.*?\bv=))(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://www.hgtv.ca/shows/bryan-inc/videos/movie-night-popcorn-with-bryan-870923331648/',
+        'md5': '05dcbca777bf1e58c2acbb57168ad3a6',
+        'info_dict': {
+            'id': '870923331648',
+            'ext': 'mp4',
+            'title': 'Movie Night Popcorn with Bryan',
+            'description': 'Bryan whips up homemade popcorn, the old fashion way for Jojo and Lincoln.',
+            'uploader': 'SHWM-NEW',
+            'upload_date': '20170206',
+            'timestamp': 1486392197,
+        },
+    }, {
+        'url': 'http://www.foodnetwork.ca/shows/chopped/video/episode/chocolate-obsession/video.html?v=872683587753',
+        'only_matching': True,
+    }, {
+        'url': 'http://etcanada.com/video/873675331955/meet-the-survivor-game-changers-castaways-part-2/',
+        'only_matching': True,
+    }]
+
+    _TP_FEEDS = {
+        'globaltv': {
+            'feed_id': 'ChQqrem0lNUp',
+            'account_id': 2269680845,
+        },
+        'etcanada': {
+            'feed_id': 'ChQqrem0lNUp',
+            'account_id': 2269680845,
+        },
+        'hgtv': {
+            'feed_id': 'L0BMHXi2no43',
+            'account_id': 2414428465,
+        },
+        'foodnetwork': {
+            'feed_id': 'ukK8o58zbRmJ',
+            'account_id': 2414429569,
+        },
+        'slice': {
+            'feed_id': '5tUJLgV2YNJ5',
+            'account_id': 2414427935,
+        },
+    }
+
+    def _real_extract(self, url):
+        domain, video_id = re.match(self._VALID_URL, url).groups()
+        feed_info = self._TP_FEEDS[domain.split('.')[0]]
+        return self._extract_feed_info('dtjsEC', feed_info['feed_id'], 'byId=' + video_id, video_id, lambda e: {
+            'episode_number': int_or_none(e.get('pl1$episode')),
+            'season_number': int_or_none(e.get('pl1$season')),
+            'series': e.get('pl1$show'),
+        }, {
+            'HLS': {
+                'manifest': 'm3u',
+            },
+            'DesktopHLS Default': {
+                'manifest': 'm3u',
+            },
+            'MP4 MBR': {
+                'manifest': 'm3u',
+            },
+        }, feed_info['account_id'])
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 5115e1a0c..76ad7c40b 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -202,6 +202,7 @@ from .commonprotocols import (
     RtmpIE,
 )
 from .condenast import CondeNastIE
+from .corus import CorusIE
 from .cracked import CrackedIE
 from .crackle import CrackleIE
 from .criterion import CriterionIE
@@ -381,10 +382,7 @@ from .heise import HeiseIE
 from .hellporno import HellPornoIE
 from .helsinki import HelsinkiIE
 from .hentaistigma import HentaiStigmaIE
-from .hgtv import (
-    HGTVIE,
-    HGTVComShowIE,
-)
+from .hgtv import HGTVComShowIE
 from .historicfilms import HistoricFilmsIE
 from .hitbox import HitboxIE, HitboxLiveIE
 from .hitrecord import HitRecordIE
diff --git a/youtube_dl/extractor/hgtv.py b/youtube_dl/extractor/hgtv.py
index 69543bff2..e854300c7 100644
--- a/youtube_dl/extractor/hgtv.py
+++ b/youtube_dl/extractor/hgtv.py
@@ -2,50 +2,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    js_to_json,
-    smuggle_url,
-)
-
-
-class HGTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?hgtv\.ca/[^/]+/video/(?P<id>[^/]+)/video.html'
-    _TEST = {
-        'url': 'http://www.hgtv.ca/homefree/video/overnight-success/video.html?v=738081859718&p=1&s=da#video',
-        'md5': '',
-        'info_dict': {
-            'id': 'aFH__I_5FBOX',
-            'ext': 'mp4',
-            'title': 'Overnight Success',
-            'description': 'After weeks of hard work, high stakes, breakdowns and pep talks, the final 2 contestants compete to win the ultimate dream.',
-            'uploader': 'SHWM-NEW',
-            'timestamp': 1470320034,
-            'upload_date': '20160804',
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-        embed_vars = self._parse_json(self._search_regex(
-            r'(?s)embed_vars\s*=\s*({.*?});',
-            webpage, 'embed vars'), display_id, js_to_json)
-        return {
-            '_type': 'url_transparent',
-            'url': smuggle_url(
-                'http://link.theplatform.com/s/dtjsEC/%s?mbr=true&manifest=m3u' % embed_vars['pid'], {
-                    'force_smil_url': True
-                }),
-            'series': embed_vars.get('show'),
-            'season_number': int_or_none(embed_vars.get('season')),
-            'episode_number': int_or_none(embed_vars.get('episode')),
-            'ie_key': 'ThePlatform',
-        }
 
 
 class HGTVComShowIE(InfoExtractor):
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 192d8fa29..5c5987c6a 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -306,9 +306,10 @@ class ThePlatformFeedIE(ThePlatformBaseIE):
         },
     }]
 
-    def _extract_feed_info(self, provider_id, feed_id, filter_query, video_id, custom_fields=None, asset_types_query={}):
+    def _extract_feed_info(self, provider_id, feed_id, filter_query, video_id, custom_fields=None, asset_types_query={}, account_id=None):
         real_url = self._URL_TEMPLATE % (self.http_scheme(), provider_id, feed_id, filter_query)
         entry = self._download_json(real_url, video_id)['entries'][0]
+        main_smil_url = 'http://link.theplatform.com/s/%s/media/guid/%d/%s' % (provider_id, account_id, entry['guid']) if account_id else None
 
         formats = []
         subtitles = {}
@@ -333,7 +334,7 @@ class ThePlatformFeedIE(ThePlatformBaseIE):
                 if asset_type in asset_types_query:
                     query.update(asset_types_query[asset_type])
                 cur_formats, cur_subtitles = self._extract_theplatform_smil(update_url_query(
-                    smil_url, query), video_id, 'Downloading SMIL data for %s' % asset_type)
+                    main_smil_url or smil_url, query), video_id, 'Downloading SMIL data for %s' % asset_type)
                 formats.extend(cur_formats)
                 subtitles = self._merge_subtitles(subtitles, cur_subtitles)
 

From 71cdd75628f851975eef551a96b1a24f94066d74 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Feb 2017 01:04:09 +0700
Subject: [PATCH 0224/1696] [facebook] Relax video id matching (closes #11017,
 closes #12055, closes #12056)

---
 youtube_dl/extractor/facebook.py | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 4a3c839f4..78a4ad666 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -148,6 +148,32 @@ class FacebookIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        'url': 'https://www.facebook.com/LaGuiaDelVaron/posts/1072691702860471',
+        'info_dict': {
+            'id': '1072691702860471',
+            'ext': 'mp4',
+            'title': 'md5:ae2d22a93fbb12dad20dc393a869739d',
+            'timestamp': 1477305000,
+            'upload_date': '20161024',
+            'uploader': 'La Guía Del Varón',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://www.facebook.com/groups/1024490957622648/permalink/1396382447100162/',
+        'info_dict': {
+            'id': '1396382447100162',
+            'ext': 'mp4',
+            'title': 'md5:e2d2700afdf84e121f5d0f999bad13a3',
+            'timestamp': 1486035494,
+            'upload_date': '20170202',
+            'uploader': 'Elisabeth Ahtn',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'https://www.facebook.com/video.php?v=10204634152394104',
         'only_matching': True,
@@ -263,7 +289,7 @@ class FacebookIE(InfoExtractor):
             for item in instances:
                 if item[1][0] == 'VideoConfig':
                     video_item = item[2][0]
-                    if video_item.get('video_id') == video_id:
+                    if video_item.get('video_id'):
                         return video_item['videoData']
 
         server_js_data = self._parse_json(self._search_regex(

From 97eb9bd2acffd352558b626f923862d039efbcb7 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 10 Feb 2017 19:45:48 +0100
Subject: [PATCH 0225/1696] [bbc] extract m3u8 formats with 320k audio

---
 youtube_dl/extractor/bbc.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index b17916137..8a2ed0ab6 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -225,6 +225,8 @@ class BBCCoUkIE(InfoExtractor):
         }
     ]
 
+    _USP_RE = r'/([^/]+?)\.ism(?:\.hlsv2\.ism)?/[^/]+\.m3u8'
+
     class MediaSelectionError(Exception):
         def __init__(self, id):
             self.id = id
@@ -336,6 +338,15 @@ class BBCCoUkIE(InfoExtractor):
                         formats.extend(self._extract_m3u8_formats(
                             href, programme_id, ext='mp4', entry_protocol='m3u8_native',
                             m3u8_id=format_id, fatal=False))
+                        if re.search(self._USP_RE, href):
+                            usp_formats = self._extract_m3u8_formats(
+                                re.sub(self._USP_RE, r'/\1.ism/\1.m3u8', href),
+                                programme_id, ext='mp4', entry_protocol='m3u8_native',
+                                m3u8_id=format_id, fatal=False)
+                            for f in usp_formats:
+                                if f.get('height') and f['height'] > 720:
+                                    continue
+                                formats.append(f)
                     elif transfer_format == 'hds':
                         formats.extend(self._extract_f4m_formats(
                             href, programme_id, f4m_id=format_id, fatal=False))

From 2af12ad9d2c3e41d0e91fe6e2f35827469d587bf Mon Sep 17 00:00:00 2001
From: Thomas Christlieb <thomaschristlieb@hotmail.com>
Date: Sat, 11 Feb 2017 10:16:54 +0100
Subject: [PATCH 0226/1696] Introduce get_elements_by_class and
 get_elements_by_attribute utility functions

---
 test/test_utils.py  | 29 +++++++++++++++++++++++++++++
 youtube_dl/utils.py | 32 ++++++++++++++++++++++----------
 2 files changed, 51 insertions(+), 10 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index edc712f07..3cdb21d40 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -34,6 +34,9 @@ from youtube_dl.utils import (
     find_xpath_attr,
     fix_xml_ampersands,
     get_element_by_class,
+    get_element_by_attribute,
+    get_elements_by_class,
+    get_elements_by_attribute,
     InAdvancePagedList,
     intlist_to_bytes,
     is_html,
@@ -1124,6 +1127,32 @@ The first line
         self.assertEqual(get_element_by_class('foo', html), 'nice')
         self.assertEqual(get_element_by_class('no-such-class', html), None)
 
+    def test_get_element_by_attribute(self):
+        html = '''
+            <span class="foo bar">nice</span>
+        '''
+
+        self.assertEqual(get_element_by_attribute('class', 'foo bar', html), 'nice')
+        self.assertEqual(get_element_by_attribute('class', 'foo', html), None)
+        self.assertEqual(get_element_by_attribute('class', 'no-such-foo', html), None)
+
+    def test_get_elements_by_class(self):
+        html = '''
+            <span class="foo bar">nice</span><span class="foo bar">also nice</span>
+        '''
+
+        self.assertEqual(get_elements_by_class('foo', html), ['nice', 'also nice'])
+        self.assertEqual(get_elements_by_class('no-such-class', html), [])
+
+    def test_get_elements_by_attribute(self):
+        html = '''
+            <span class="foo bar">nice</span><span class="foo bar">also nice</span>
+        '''
+
+        self.assertEqual(get_elements_by_attribute('class', 'foo bar', html), ['nice', 'also nice'])
+        self.assertEqual(get_elements_by_attribute('class', 'foo', html), [])
+        self.assertEqual(get_elements_by_attribute('class', 'no-such-foo', html), [])
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 67a847eba..a81fe7d30 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -337,17 +337,30 @@ def get_element_by_id(id, html):
 
 
 def get_element_by_class(class_name, html):
-    return get_element_by_attribute(
+    """Return the content of the first tag with the specified class in the passed HTML document"""
+    retval = get_elements_by_class(class_name, html)
+    return retval[0] if retval else None
+
+
+def get_element_by_attribute(attribute, value, html, escape_value=True):
+    retval = get_elements_by_attribute(attribute, value, html, escape_value)
+    return retval[0] if retval else None
+
+
+def get_elements_by_class(class_name, html):
+    """Return the content of all tags with the specified class in the passed HTML document as a list"""
+    return get_elements_by_attribute(
         'class', r'[^\'"]*\b%s\b[^\'"]*' % re.escape(class_name),
         html, escape_value=False)
 
 
-def get_element_by_attribute(attribute, value, html, escape_value=True):
+def get_elements_by_attribute(attribute, value, html, escape_value=True):
     """Return the content of the tag with the specified attribute in the passed HTML document"""
 
     value = re.escape(value) if escape_value else value
 
-    m = re.search(r'''(?xs)
+    retlist = []
+    for m in re.finditer(r'''(?xs)
         <([a-zA-Z0-9:._-]+)
          (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'))*?
          \s+%s=['"]?%s['"]?
@@ -355,16 +368,15 @@ def get_element_by_attribute(attribute, value, html, escape_value=True):
         \s*>
         (?P<content>.*?)
         </\1>
-    ''' % (re.escape(attribute), value), html)
+    ''' % (re.escape(attribute), value), html):
+        res = m.group('content')
 
-    if not m:
-        return None
-    res = m.group('content')
+        if res.startswith('"') or res.startswith("'"):
+            res = res[1:-1]
 
-    if res.startswith('"') or res.startswith("'"):
-        res = res[1:-1]
+        retlist.append(unescapeHTML(res))
 
-    return unescapeHTML(res)
+    return retlist
 
 
 class HTMLAttributeParser(compat_HTMLParser):

From 624c4b92ffaa5586dd821db6d10106aa1a067a2e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Feb 2017 16:18:45 +0700
Subject: [PATCH 0227/1696] [facebook] Add coding cookie

---
 youtube_dl/extractor/facebook.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 78a4ad666..70b8c95c5 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

From 3d7e3aaa0e32e34fd81f3220d5da74c83c2e9866 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Feb 2017 17:00:52 +0700
Subject: [PATCH 0228/1696] [pluralsight:course] Fix extraction (closes #12075)

---
 youtube_dl/extractor/pluralsight.py | 38 ++++++++++++++++++++++-------
 1 file changed, 29 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 00b275eb1..e0cbd045e 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -18,6 +18,7 @@ from ..utils import (
     parse_duration,
     qualities,
     srt_subtitles_timecode,
+    update_url_query,
     urlencode_postdata,
 )
 
@@ -331,25 +332,44 @@ class PluralsightCourseIE(PluralsightBaseIE):
         # TODO: PSM cookie
 
         course = self._download_json(
-            '%s/data/course/%s' % (self._API_BASE, course_id),
-            course_id, 'Downloading course JSON')
+            '%s/player/functions/rpc' % self._API_BASE, course_id,
+            'Downloading course JSON',
+            data=json.dumps({
+                'fn': 'bootstrapPlayer',
+                'payload': {
+                    'courseId': course_id,
+                }
+            }).encode('utf-8'),
+            headers={
+                'Content-Type': 'application/json;charset=utf-8'
+            })['payload']['course']
 
         title = course['title']
+        course_name = course['name']
+        course_data = course['modules']
         description = course.get('description') or course.get('shortDescription')
 
-        course_data = self._download_json(
-            '%s/data/course/content/%s' % (self._API_BASE, course_id),
-            course_id, 'Downloading course data JSON')
-
         entries = []
         for num, module in enumerate(course_data, 1):
+            author = module.get('author')
+            module_name = module.get('name')
+            if not author or not module_name:
+                continue
             for clip in module.get('clips', []):
-                player_parameters = clip.get('playerParameters')
-                if not player_parameters:
+                clip_index = int_or_none(clip.get('index'))
+                if clip_index is None:
                     continue
+                clip_url = update_url_query(
+                    '%s/player' % self._API_BASE, query={
+                        'mode': 'live',
+                        'course': course_name,
+                        'author': author,
+                        'name': module_name,
+                        'clip': clip_index,
+                    })
                 entries.append({
                     '_type': 'url_transparent',
-                    'url': '%s/training/player?%s' % (self._API_BASE, player_parameters),
+                    'url': clip_url,
                     'ie_key': PluralsightIE.ie_key(),
                     'chapter': module.get('title'),
                     'chapter_number': num,

From 3e2274c8b7abf34738c5579b7bb4aa8e12d68fe7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Feb 2017 17:08:22 +0700
Subject: [PATCH 0229/1696] [ChangeLog] Actualize

---
 ChangeLog | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index d4c8081f7..76310c063 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,21 @@
+version <unreleased>
+
+Core
++ [utils] Introduce get_elements_by_class and get_elements_by_attribute
+  utility functions
++ [extractor/common] Skip m3u8 manifests protected with Adobe Flash Access
+
+Extractor
+* [pluralsight:course] Fix extraction (#12075)
++ [bbc] Extract m3u8 formats with 320k audio
+* [facebook] Relax video id matching (#11017, #12055, #12056)
++ [corus] Add support for Corus Entertainment sites (#12060, #9164)
++ [pluralsight] Detect blocked account error message (#12070)
++ [bloomberg] Add another video id pattern (#12062)
+* [extractor/commonmistakes] Restrict URL regular expression (#12050)
++ [tvplayer] Add support for tvplayer.com
+
+
 version 2017.02.10
 
 Extractors

From 9b92a5917ba32644511d13a8af546d6f41282ffc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Feb 2017 03:24:00 +0700
Subject: [PATCH 0230/1696] release 2017.02.11

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 3 ++-
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 0c1569fd4..7bd301cc8 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.10*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.10**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.11*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.11**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.02.10
+[debug] youtube-dl version 2017.02.11
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 76310c063..a0e8e5680 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.02.11
 
 Core
 + [utils] Introduce get_elements_by_class and get_elements_by_attribute
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 76882f3b5..3e84f1237 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -169,6 +169,7 @@
  - **ComedyCentralShortname**
  - **ComedyCentralTV**
  - **CondeNast**: Condé Nast media group: Allure, Architectural Digest, Ars Technica, Bon Appétit, Brides, Condé Nast, Condé Nast Traveler, Details, Epicurious, GQ, Glamour, Golf Digest, SELF, Teen Vogue, The New Yorker, Vanity Fair, Vogue, W Magazine, WIRED
+ - **Corus**
  - **Coub**
  - **Cracked**
  - **Crackle**
@@ -309,7 +310,6 @@
  - **HellPorno**
  - **Helsinki**: helsinki.fi
  - **HentaiStigma**
- - **HGTV**
  - **hgtv.com:show**
  - **HistoricFilms**
  - **history:topic**: History.com Topic
@@ -806,6 +806,7 @@
  - **tvp**: Telewizja Polska
  - **tvp:embed**: Telewizja Polska
  - **tvp:series**
+ - **TVPlayer**
  - **Tweakers**
  - **twitch:chapter**
  - **twitch:clips**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a8395ce04..1f84acfea 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.02.10'
+__version__ = '2017.02.11'

From 68c22c4c15a608a7067f658f5facc1ad8334c03a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Feb 2017 22:27:45 +0800
Subject: [PATCH 0231/1696] [iqiyi] Update _TESTS

---
 youtube_dl/extractor/iqiyi.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 01c7b3042..2af6a6db4 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -173,11 +173,12 @@ class IqiyiIE(InfoExtractor):
         }
     }, {
         'url': 'http://www.iqiyi.com/v_19rrhnnclk.html',
-        'md5': '667171934041350c5de3f5015f7f1152',
+        'md5': 'b7dc800a4004b1b57749d9abae0472da',
         'info_dict': {
             'id': 'e3f585b550a280af23c98b6cb2be19fb',
             'ext': 'mp4',
-            'title': '名侦探柯南 国语版：第752集 迫近灰原秘密的黑影 下篇',
+            # This can be either Simplified Chinese or Traditional Chinese
+            'title': r're:^(?:名侦探柯南 国语版：第752集 迫近灰原秘密的黑影 下篇|名偵探柯南 國語版：第752集 迫近灰原秘密的黑影 下篇)$',
         },
         'skip': 'Geo-restricted to China',
     }, {

From 2f49bcd69076626db1b789192dd113458b1eb01c Mon Sep 17 00:00:00 2001
From: Ricardo Constantino <wiiaboo@gmail.com>
Date: Fri, 10 Feb 2017 21:05:09 +0000
Subject: [PATCH 0232/1696] utils.py: Workaround TypeError with Python 2.7.13
 in Windows

Fixes #11540

Tested with Windows Python 2.7.12 and 2.7.13.
---
 youtube_dl/utils.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a81fe7d30..1279a9042 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1684,6 +1684,11 @@ def setproctitle(title):
         libc = ctypes.cdll.LoadLibrary('libc.so.6')
     except OSError:
         return
+    except TypeError:
+        # LoadLibrary in Windows Python 2.7.13 only expects
+        # a bytestring, but since unicode_literals turns
+        # every string into a unicode string, it fails.
+        return
     title_bytes = title.encode('utf-8')
     buf = ctypes.create_string_buffer(len(title_bytes))
     buf.value = title_bytes

From 8d38dafbbff2b4f5b414e2555f47ad9352fb83b4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 12 Feb 2017 00:45:37 +0800
Subject: [PATCH 0233/1696] ChangeLog: update after #12085

---
 ChangeLog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index a0e8e5680..cba47a296 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Core
+* TypeError is fixed with Python 2.7.13 on Windows (#11540, #12085)
+
+
 version 2017.02.11
 
 Core

From 6310acf512136a1e37ef5905c4bdce8cf14ba5a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Feb 2017 18:09:53 +0700
Subject: [PATCH 0234/1696] [youtube] Fix parsing codecs (closes #12091)

---
 youtube_dl/extractor/youtube.py | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 76710931a..dec02804b 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -34,6 +34,7 @@ from ..utils import (
     int_or_none,
     mimetype2ext,
     orderedSet,
+    parse_codecs,
     parse_duration,
     remove_quotes,
     remove_start,
@@ -1696,15 +1697,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                                     codecs = mobj.group('val')
                                     break
                             if codecs:
-                                codecs = codecs.split(',')
-                                if len(codecs) == 2:
-                                    acodec, vcodec = codecs[1], codecs[0]
-                                else:
-                                    acodec, vcodec = (codecs[0], 'none') if kind == 'audio' else ('none', codecs[0])
-                                dct.update({
-                                    'acodec': acodec,
-                                    'vcodec': vcodec,
-                                })
+                                dct.update(parse_codecs(codecs))
                 formats.append(dct)
         elif video_info.get('hlsvp'):
             manifest_url = video_info['hlsvp'][0]

From 459818e2808e0dfccc5a3efb7c053afa847cb632 Mon Sep 17 00:00:00 2001
From: Aniruddh-J <anirudha1987@gmail.com>
Date: Sun, 12 Feb 2017 17:48:11 +0530
Subject: [PATCH 0235/1696] [aenetworks] Add support for lifetimemovieclub.com

---
 youtube_dl/extractor/aenetworks.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index c97317400..dd96a47ce 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -23,7 +23,7 @@ class AENetworksBaseIE(ThePlatformIE):
 class AENetworksIE(AENetworksBaseIE):
     IE_NAME = 'aenetworks'
     IE_DESC = 'A+E Networks: A&E, Lifetime, History.com, FYI Network'
-    _VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:history|aetv|mylifetime)\.com|fyi\.tv)/(?:shows/(?P<show_path>[^/]+(?:/[^/]+){0,2})|movies/(?P<movie_display_id>[^/]+)/full-movie)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:history|aetv|mylifetime|lifetimemovieclub)\.com|fyi\.tv)/(?:shows/(?P<show_path>[^/]+(?:/[^/]+){0,2})|movies/(?P<movie_display_id>[^/]+)(?:/full-movie)?)'
     _TESTS = [{
         'url': 'http://www.history.com/shows/mountain-men/season-1/episode-1',
         'md5': 'a97a65f7e823ae10e9244bc5433d5fe6',
@@ -62,11 +62,15 @@ class AENetworksIE(AENetworksBaseIE):
     }, {
         'url': 'http://www.mylifetime.com/movies/center-stage-on-pointe/full-movie',
         'only_matching': True
+    }, {
+        'url': 'https://www.lifetimemovieclub.com/movies/a-killer-among-us',
+        'only_matching': True
     }]
     _DOMAIN_TO_REQUESTOR_ID = {
         'history.com': 'HISTORY',
         'aetv.com': 'AETV',
         'mylifetime.com': 'LIFETIME',
+        'lifetimemovieclub.com': 'LIFETIMEMOVIECLUB',
         'fyi.tv': 'FYI',
     }
 

From f8514630db9ba72a9bddc000c393698f4c116c81 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 12 Feb 2017 20:53:55 +0800
Subject: [PATCH 0236/1696] [einthusan] Fix extraction (closes #11416)

The old test URLs are no longer valid, so I replace them with the one
from #11416
---
 ChangeLog                         |   3 +
 youtube_dl/extractor/einthusan.py | 117 ++++++++++++++++++------------
 2 files changed, 75 insertions(+), 45 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index cba47a296..089449dfb 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -3,6 +3,9 @@ version <unreleased>
 Core
 * TypeError is fixed with Python 2.7.13 on Windows (#11540, #12085)
 
+Extractor
+* [einthusan] Fix extraction (#11416)
+
 
 version 2017.02.11
 
diff --git a/youtube_dl/extractor/einthusan.py b/youtube_dl/extractor/einthusan.py
index 6ca07a13d..8a2a17b63 100644
--- a/youtube_dl/extractor/einthusan.py
+++ b/youtube_dl/extractor/einthusan.py
@@ -1,67 +1,94 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import base64
+import json
+
 from .common import InfoExtractor
-from ..compat import compat_urlparse
+from ..compat import (
+    compat_urlparse,
+    compat_str,
+)
 from ..utils import (
-    remove_start,
-    sanitized_Request,
+    extract_attributes,
+    ExtractorError,
+    get_elements_by_class,
+    urlencode_postdata,
 )
 
 
 class EinthusanIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?einthusan\.com/movies/watch.php\?([^#]*?)id=(?P<id>[0-9]+)'
-    _TESTS = [
-        {
-            'url': 'http://www.einthusan.com/movies/watch.php?id=2447',
-            'md5': 'd71379996ff5b7f217eca034c34e3461',
-            'info_dict': {
-                'id': '2447',
-                'ext': 'mp4',
-                'title': 'Ek Villain',
-                'thumbnail': r're:^https?://.*\.jpg$',
-                'description': 'md5:9d29fc91a7abadd4591fb862fa560d93',
-            }
-        },
-        {
-            'url': 'http://www.einthusan.com/movies/watch.php?id=1671',
-            'md5': 'b16a6fd3c67c06eb7c79c8a8615f4213',
-            'info_dict': {
-                'id': '1671',
-                'ext': 'mp4',
-                'title': 'Soodhu Kavvuum',
-                'thumbnail': r're:^https?://.*\.jpg$',
-                'description': 'md5:b40f2bf7320b4f9414f3780817b2af8c',
-            }
-        },
-    ]
+    _VALID_URL = r'https?://einthusan\.tv/movie/watch/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'https://einthusan.tv/movie/watch/9097/',
+        'md5': 'ff0f7f2065031b8a2cf13a933731c035',
+        'info_dict': {
+            'id': '9097',
+            'ext': 'mp4',
+            'title': 'Ae Dil Hai Mushkil',
+            'description': 'md5:33ef934c82a671a94652a9b4e54d931b',
+            'thumbnail': r're:^https?://.*\.jpg$',
+        }
+    }
+
+    # reversed from jsoncrypto.prototype.decrypt() in einthusan-PGMovieWatcher.js
+    def _decrypt(self, encrypted_data, video_id):
+        return self._parse_json(base64.b64decode((
+            encrypted_data[:10] + encrypted_data[-1] + encrypted_data[12:-1]
+        ).encode('ascii')).decode('utf-8'), video_id)
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        request = sanitized_Request(url)
-        request.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0')
-        webpage = self._download_webpage(request, video_id)
+        webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(
-            r'<h1><a[^>]+class=["\']movie-title["\'][^>]*>(.+?)</a></h1>',
-            webpage, 'title')
+        title = self._html_search_regex(r'<h3>([^<]+)</h3>', webpage, 'title')
 
-        video_id = self._search_regex(
-            r'data-movieid=["\'](\d+)', webpage, 'video id', default=video_id)
+        player_params = extract_attributes(self._search_regex(
+            r'(<section[^>]+id="UIVideoPlayer"[^>]+>)', webpage, 'player parameters'))
 
-        m3u8_url = self._download_webpage(
-            'http://cdn.einthusan.com/geturl/%s/hd/London,Washington,Toronto,Dallas,San,Sydney/'
-            % video_id, video_id, headers={'Referer': url})
-        formats = self._extract_m3u8_formats(
-            m3u8_url, video_id, ext='mp4', entry_protocol='m3u8_native')
+        page_id = self._html_search_regex(
+            '<html[^>]+data-pageid="([^"]+)"', webpage, 'page ID')
+        video_data = self._download_json(
+            'https://einthusan.tv/ajax/movie/watch/%s/' % video_id, video_id,
+            data=urlencode_postdata({
+                'xEvent': 'UIVideoPlayer.PingOutcome',
+                'xJson': json.dumps({
+                    'EJOutcomes': player_params['data-ejpingables'],
+                    'NativeHLS': False
+                }),
+                'arcVersion': 3,
+                'appVersion': 59,
+                'gorilla.csrf.Token': page_id,
+            }))['Data']
 
-        description = self._html_search_meta('description', webpage)
+        if isinstance(video_data, compat_str) and video_data.startswith('/ratelimited/'):
+            raise ExtractorError(
+                'Download rate reached. Please try again later.', expected=True)
+
+        ej_links = self._decrypt(video_data['EJLinks'], video_id)
+
+        formats = []
+
+        m3u8_url = ej_links.get('HLSLink')
+        if m3u8_url:
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, ext='mp4', entry_protocol='m3u8_native'))
+
+        mp4_url = ej_links.get('MP4Link')
+        if mp4_url:
+            formats.append({
+                'url': mp4_url,
+            })
+
+        self._sort_formats(formats)
+
+        description = get_elements_by_class('synopsis', webpage)[0]
         thumbnail = self._html_search_regex(
-            r'''<a class="movie-cover-wrapper".*?><img src=["'](.*?)["'].*?/></a>''',
-            webpage, "thumbnail url", fatal=False)
+            r'''<img[^>]+src=(["'])(?P<url>(?!\1).+?/moviecovers/(?!\1).+?)\1''',
+            webpage, 'thumbnail url', fatal=False, group='url')
         if thumbnail is not None:
-            thumbnail = compat_urlparse.urljoin(url, remove_start(thumbnail, '..'))
+            thumbnail = compat_urlparse.urljoin(url, thumbnail)
 
         return {
             'id': video_id,

From 0dac7cbb092c804f1548c4a60f15ac29a7db06b9 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 12 Feb 2017 17:24:45 +0100
Subject: [PATCH 0237/1696] [hotstar] improve extraction(closes #12096)

- extract all qualities
- detect drm protected videos
- extract more metadata
---
 youtube_dl/extractor/hotstar.py | 46 +++++++++++++++++++++++----------
 1 file changed, 32 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/hotstar.py b/youtube_dl/extractor/hotstar.py
index f05d765d6..3a7a66a34 100644
--- a/youtube_dl/extractor/hotstar.py
+++ b/youtube_dl/extractor/hotstar.py
@@ -34,11 +34,9 @@ class HotStarIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    _GET_CONTENT_TEMPLATE = 'http://account.hotstar.com/AVS/besc?action=GetAggregatedContentDetails&channel=PCTV&contentId=%s'
-    _GET_CDN_TEMPLATE = 'http://getcdn.hotstar.com/AVS/besc?action=GetCDN&asJson=Y&channel=%s&id=%s&type=%s'
-
-    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata', fatal=True):
-        json_data = super(HotStarIE, self)._download_json(url_or_request, video_id, note, fatal=fatal)
+    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata', fatal=True, query=None):
+        json_data = super(HotStarIE, self)._download_json(
+            url_or_request, video_id, note, fatal=fatal, query=query)
         if json_data['resultCode'] != 'OK':
             if fatal:
                 raise ExtractorError(json_data['errorDescription'])
@@ -48,20 +46,37 @@ class HotStarIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         video_data = self._download_json(
-            self._GET_CONTENT_TEMPLATE % video_id,
-            video_id)['contentInfo'][0]
+            'http://account.hotstar.com/AVS/besc', video_id, query={
+                'action': 'GetAggregatedContentDetails',
+                'channel': 'PCTV',
+                'contentId': video_id,
+            })['contentInfo'][0]
+        title = video_data['episodeTitle']
+
+        if video_data.get('encrypted') == 'Y':
+            raise ExtractorError('This video is DRM protected.', expected=True)
 
         formats = []
-        # PCTV for extracting f4m manifest
-        for f in ('TABLET',):
+        for f in ('JIO',):
             format_data = self._download_json(
-                self._GET_CDN_TEMPLATE % (f, video_id, 'VOD'),
-                video_id, 'Downloading %s JSON metadata' % f, fatal=False)
+                'http://getcdn.hotstar.com/AVS/besc',
+                video_id, 'Downloading %s JSON metadata' % f,
+                fatal=False, query={
+                    'action': 'GetCDN',
+                    'asJson': 'Y',
+                    'channel': f,
+                    'id': video_id,
+                    'type': 'VOD',
+                })
             if format_data:
-                format_url = format_data['src']
+                format_url = format_data.get('src')
+                if not format_url:
+                    continue
                 ext = determine_ext(format_url)
                 if ext == 'm3u8':
-                    formats.extend(self._extract_m3u8_formats(format_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
+                    formats.extend(self._extract_m3u8_formats(
+                        format_url, video_id, 'mp4',
+                        m3u8_id='hls', fatal=False))
                 elif ext == 'f4m':
                     # produce broken files
                     continue
@@ -75,9 +90,12 @@ class HotStarIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'title': video_data['episodeTitle'],
+            'title': title,
             'description': video_data.get('description'),
             'duration': int_or_none(video_data.get('duration')),
             'timestamp': int_or_none(video_data.get('broadcastDate')),
             'formats': formats,
+            'episode': title,
+            'episode_number': int_or_none(video_data.get('episodeNumber')),
+            'series': video_data.get('contentTitle'),
         }

From 1e2c3f61fc952620a52a8a3a79bcd1a6f7d8ecae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Feb 2017 03:33:23 +0700
Subject: [PATCH 0238/1696] [travis] Separate builds for core and download

---
 .travis.yml             |  7 ++++++-
 devscripts/run_tests.sh | 19 +++++++++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)
 create mode 100644 devscripts/run_tests.sh

diff --git a/.travis.yml b/.travis.yml
index 4833c76e9..8ba93ec02 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,7 +8,12 @@ python:
   - "3.5"
   - "3.6"
 sudo: false
-script: nosetests test --verbose
+env:
+  - YTDL_TEST_SET=core
+  - YTDL_TEST_SET=download
+before_script:
+  - chmod +x ./devscripts/run_tests.sh
+script: ./devscripts/run_tests.sh
 notifications:
   email:
     - filippo.valsorda@gmail.com
diff --git a/devscripts/run_tests.sh b/devscripts/run_tests.sh
new file mode 100644
index 000000000..7f4c1e083
--- /dev/null
+++ b/devscripts/run_tests.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+DOWNLOAD_TESTS="age_restriction|download|subtitles|write_annotations|iqiyi_sdk_interpreter"
+
+test_set=""
+
+case "$YTDL_TEST_SET" in
+    core)
+        test_set="-I test_($DOWNLOAD_TESTS)\.py"
+    ;;
+    download)
+        test_set="-I test_(?!$DOWNLOAD_TESTS).+\.py"
+    ;;
+    *)
+        break
+    ;;
+esac
+
+nosetests test --verbose $test_set

From 9dad94185367cdfde0de21cd8e595094cbe31acc Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 13 Feb 2017 11:43:20 +0100
Subject: [PATCH 0239/1696] [disney] improve extraction

- add support for more urls
- detect expired videos
- skip Adobe Flash Access protected videos

closes #4975
closes #11000
closes #11882
closes #11936
---
 youtube_dl/extractor/disney.py  | 60 ++++++++++++++++++++++++++++-----
 youtube_dl/extractor/generic.py | 13 -------
 2 files changed, 52 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/disney.py b/youtube_dl/extractor/disney.py
index 396873c6d..939d1338c 100644
--- a/youtube_dl/extractor/disney.py
+++ b/youtube_dl/extractor/disney.py
@@ -9,13 +9,15 @@ from ..utils import (
     unified_strdate,
     compat_str,
     determine_ext,
+    ExtractorError,
 )
 
 
 class DisneyIE(InfoExtractor):
     _VALID_URL = r'''(?x)
-        https?://(?P<domain>(?:[^/]+\.)?(?:disney\.[a-z]{2,3}(?:\.[a-z]{2})?|disney(?:(?:me|latino)\.com|turkiye\.com\.tr)|starwars\.com))/(?:embed/|(?:[^/]+/)+[\w-]+-)(?P<id>[a-z0-9]{24})'''
+        https?://(?P<domain>(?:[^/]+\.)?(?:disney\.[a-z]{2,3}(?:\.[a-z]{2})?|disney(?:(?:me|latino)\.com|turkiye\.com\.tr)|(?:starwars|marvelkids)\.com))/(?:(?:embed/|(?:[^/]+/)+[\w-]+-)(?P<id>[a-z0-9]{24})|(?:[^/]+/)?(?P<display_id>[^/?#]+))'''
     _TESTS = [{
+        # Disney.EmbedVideo
         'url': 'http://video.disney.com/watch/moana-trailer-545ed1857afee5a0ec239977',
         'info_dict': {
             'id': '545ed1857afee5a0ec239977',
@@ -28,6 +30,20 @@ class DisneyIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         }
+    }, {
+        # Grill.burger
+        'url': 'http://www.starwars.com/video/rogue-one-a-star-wars-story-intro-featurette',
+        'info_dict': {
+            'id': '5454e9f4e9804a552e3524c8',
+            'ext': 'mp4',
+            'title': '"Intro" Featurette: Rogue One: A Star Wars Story',
+            'upload_date': '20170104',
+            'description': 'Go behind-the-scenes of Rogue One: A Star Wars Story in this featurette with Director Gareth Edwards and the cast of the film.',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
     }, {
         'url': 'http://videos.disneylatino.com/ver/spider-man-de-regreso-a-casa-primer-adelanto-543a33a1850bdcfcca13bae2',
         'only_matching': True,
@@ -43,31 +59,55 @@ class DisneyIE(InfoExtractor):
     }, {
         'url': 'http://www.starwars.com/embed/54690d1e6c42e5f09a0fb097',
         'only_matching': True,
+    }, {
+        'url': 'http://spiderman.marvelkids.com/embed/522900d2ced3c565e4cc0677',
+        'only_matching': True,
+    }, {
+        'url': 'http://spiderman.marvelkids.com/videos/contest-of-champions-part-four-clip-1',
+        'only_matching': True,
+    }, {
+        'url': 'http://disneyjunior.en.disneyme.com/dj/watch-my-friends-tigger-and-pooh-promo',
+        'only_matching': True,
+    }, {
+        'url': 'http://disneyjunior.disney.com/galactech-the-galactech-grab-galactech-an-admiral-rescue',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        domain, video_id = re.match(self._VALID_URL, url).groups()
-        webpage = self._download_webpage(
-            'http://%s/embed/%s' % (domain, video_id), video_id)
-        video_data = self._parse_json(self._search_regex(
-            r'Disney\.EmbedVideo=({.+});', webpage, 'embed data'), video_id)['video']
+        domain, video_id, display_id = re.match(self._VALID_URL, url).groups()
+        if not video_id:
+            webpage = self._download_webpage(url, display_id)
+            grill = re.sub(r'"\s*\+\s*"', '', self._search_regex(
+                r'Grill\.burger\s*=\s*({.+})\s*:',
+                webpage, 'grill data'))
+            page_data = next(s for s in self._parse_json(grill, display_id)['stack'] if s.get('type') == 'video')
+            video_data = page_data['data'][0]
+        else:
+            webpage = self._download_webpage(
+                'http://%s/embed/%s' % (domain, video_id), video_id)
+            page_data = self._parse_json(self._search_regex(
+                r'Disney\.EmbedVideo\s*=\s*({.+});',
+                webpage, 'embed data'), video_id)
+            video_data = page_data['video']
 
         for external in video_data.get('externals', []):
             if external.get('source') == 'vevo':
                 return self.url_result('vevo:' + external['data_id'], 'Vevo')
 
+        video_id = video_data['id']
         title = video_data['title']
 
         formats = []
         for flavor in video_data.get('flavors', []):
             flavor_format = flavor.get('format')
             flavor_url = flavor.get('url')
-            if not flavor_url or not re.match(r'https?://', flavor_url):
+            if not flavor_url or not re.match(r'https?://', flavor_url) or flavor_format == 'mp4_access':
                 continue
             tbr = int_or_none(flavor.get('bitrate'))
             if tbr == 99999:
                 formats.extend(self._extract_m3u8_formats(
-                    flavor_url, video_id, 'mp4', m3u8_id=flavor_format, fatal=False))
+                    flavor_url, video_id, 'mp4',
+                    m3u8_id=flavor_format, fatal=False))
                 continue
             format_id = []
             if flavor_format:
@@ -88,6 +128,10 @@ class DisneyIE(InfoExtractor):
                 'ext': ext,
                 'vcodec': 'none' if (width == 0 and height == 0) else None,
             })
+        if not formats and video_data.get('expired'):
+            raise ExtractorError(
+                '%s said: %s' % (self.IE_NAME, page_data['translations']['video_expired']),
+                expected=True)
         self._sort_formats(formats)
 
         subtitles = {}
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 1c233f038..494cc3c84 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -991,19 +991,6 @@ class GenericIE(InfoExtractor):
                 'title': 'Os Guinness // Is It Fools Talk? // Unbelievable? Conference 2014',
             },
         },
-        # Kaltura embed protected with referrer
-        {
-            'url': 'http://www.disney.nl/disney-channel/filmpjes/achter-de-schermen#/videoId/violetta-achter-de-schermen-ruggero',
-            'info_dict': {
-                'id': '1_g4fbemnq',
-                'ext': 'mp4',
-                'title': 'Violetta - Achter De Schermen - Ruggero',
-                'description': 'Achter de schermen met Ruggero',
-                'timestamp': 1435133761,
-                'upload_date': '20150624',
-                'uploader_id': 'echojecka',
-            },
-        },
         # Kaltura embed with single quotes
         {
             'url': 'http://fod.infobase.com/p_ViewPlaylist.aspx?AssignmentID=NUN8ZY',

From 1de9f78e71214e130b5882662cdcd716b737e6ca Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Mon, 13 Feb 2017 18:56:05 +0800
Subject: [PATCH 0240/1696] [travis] Separate builds for core and download

---
 .travis.yml             |  7 ++++++-
 devscripts/run_tests.sh | 19 +++++++++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)
 create mode 100644 devscripts/run_tests.sh

diff --git a/.travis.yml b/.travis.yml
index 4833c76e9..8ba93ec02 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,7 +8,12 @@ python:
   - "3.5"
   - "3.6"
 sudo: false
-script: nosetests test --verbose
+env:
+  - YTDL_TEST_SET=core
+  - YTDL_TEST_SET=download
+before_script:
+  - chmod +x ./devscripts/run_tests.sh
+script: ./devscripts/run_tests.sh
 notifications:
   email:
     - filippo.valsorda@gmail.com
diff --git a/devscripts/run_tests.sh b/devscripts/run_tests.sh
new file mode 100644
index 000000000..7f4c1e083
--- /dev/null
+++ b/devscripts/run_tests.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+DOWNLOAD_TESTS="age_restriction|download|subtitles|write_annotations|iqiyi_sdk_interpreter"
+
+test_set=""
+
+case "$YTDL_TEST_SET" in
+    core)
+        test_set="-I test_($DOWNLOAD_TESTS)\.py"
+    ;;
+    download)
+        test_set="-I test_(?!$DOWNLOAD_TESTS).+\.py"
+    ;;
+    *)
+        break
+    ;;
+esac
+
+nosetests test --verbose $test_set

From 454e5cdb17dd4e77f3d387045b083f3d3ed61ae0 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 13 Feb 2017 14:28:30 +0100
Subject: [PATCH 0241/1696] [limelight] add support referer protected videos

---
 youtube_dl/extractor/generic.py   |  9 ++++++---
 youtube_dl/extractor/limelight.py | 23 ++++++++++++++++-------
 2 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 494cc3c84..a2b0298ec 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2337,8 +2337,9 @@ class GenericIE(InfoExtractor):
                 'Channel': 'channel',
                 'ChannelList': 'channel_list',
             }
-            return self.url_result('limelight:%s:%s' % (
-                lm[mobj.group(1)], mobj.group(2)), 'Limelight%s' % mobj.group(1), mobj.group(2))
+            return self.url_result(smuggle_url('limelight:%s:%s' % (
+                lm[mobj.group(1)], mobj.group(2)), {'source_url': url}),
+                'Limelight%s' % mobj.group(1), mobj.group(2))
 
         mobj = re.search(
             r'''(?sx)
@@ -2348,7 +2349,9 @@ class GenericIE(InfoExtractor):
                         value=(["\'])(?:(?!\3).)*mediaId=(?P<id>[a-z0-9]{32})
             ''', webpage)
         if mobj:
-            return self.url_result('limelight:media:%s' % mobj.group('id'))
+            return self.url_result(smuggle_url(
+                'limelight:media:%s' % mobj.group('id'),
+                {'source_url': url}), 'LimelightMedia', mobj.group('id'))
 
         # Look for AdobeTVVideo embeds
         mobj = re.search(
diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index e635f3c4d..a3712665b 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -8,6 +8,7 @@ from ..utils import (
     determine_ext,
     float_or_none,
     int_or_none,
+    unsmuggle_url,
 )
 
 
@@ -15,20 +16,23 @@ class LimelightBaseIE(InfoExtractor):
     _PLAYLIST_SERVICE_URL = 'http://production-ps.lvp.llnw.net/r/PlaylistService/%s/%s/%s'
     _API_URL = 'http://api.video.limelight.com/rest/organizations/%s/%s/%s/%s.json'
 
-    def _call_playlist_service(self, item_id, method, fatal=True):
+    def _call_playlist_service(self, item_id, method, fatal=True, referer=None):
+        headers = {}
+        if referer:
+            headers['Referer'] = referer
         return self._download_json(
             self._PLAYLIST_SERVICE_URL % (self._PLAYLIST_SERVICE_PATH, item_id, method),
-            item_id, 'Downloading PlaylistService %s JSON' % method, fatal=fatal)
+            item_id, 'Downloading PlaylistService %s JSON' % method, fatal=fatal, headers=headers)
 
     def _call_api(self, organization_id, item_id, method):
         return self._download_json(
             self._API_URL % (organization_id, self._API_PATH, item_id, method),
             item_id, 'Downloading API %s JSON' % method)
 
-    def _extract(self, item_id, pc_method, mobile_method, meta_method):
-        pc = self._call_playlist_service(item_id, pc_method)
+    def _extract(self, item_id, pc_method, mobile_method, meta_method, referer=None):
+        pc = self._call_playlist_service(item_id, pc_method, referer=referer)
         metadata = self._call_api(pc['orgId'], item_id, meta_method)
-        mobile = self._call_playlist_service(item_id, mobile_method, fatal=False)
+        mobile = self._call_playlist_service(item_id, mobile_method, fatal=False, referer=referer)
         return pc, mobile, metadata
 
     def _extract_info(self, streams, mobile_urls, properties):
@@ -207,10 +211,13 @@ class LimelightMediaIE(LimelightBaseIE):
     _API_PATH = 'media'
 
     def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
         video_id = self._match_id(url)
 
         pc, mobile, metadata = self._extract(
-            video_id, 'getPlaylistByMediaId', 'getMobilePlaylistByMediaId', 'properties')
+            video_id, 'getPlaylistByMediaId',
+            'getMobilePlaylistByMediaId', 'properties',
+            smuggled_data.get('source_url'))
 
         return self._extract_info(
             pc['playlistItems'][0].get('streams', []),
@@ -247,11 +254,13 @@ class LimelightChannelIE(LimelightBaseIE):
     _API_PATH = 'channels'
 
     def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
         channel_id = self._match_id(url)
 
         pc, mobile, medias = self._extract(
             channel_id, 'getPlaylistByChannelId',
-            'getMobilePlaylistWithNItemsByChannelId?begin=0&count=-1', 'media')
+            'getMobilePlaylistWithNItemsByChannelId?begin=0&count=-1',
+            'media', smuggled_data.get('source_url'))
 
         entries = [
             self._extract_info(

From 89c6691f9d130ec63552a6ece4743caa572fc962 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 13 Feb 2017 15:08:48 +0100
Subject: [PATCH 0242/1696] [bellmedia] accept longer video id(closes #12114)

---
 youtube_dl/extractor/bellmedia.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bellmedia.py b/youtube_dl/extractor/bellmedia.py
index 32326ed9e..1f5b6ed92 100644
--- a/youtube_dl/extractor/bellmedia.py
+++ b/youtube_dl/extractor/bellmedia.py
@@ -24,7 +24,7 @@ class BellMediaIE(InfoExtractor):
                 space
             )\.ca|
             much\.com
-        )/.*?(?:\bvid=|-vid|~|%7E|/(?:episode)?)(?P<id>[0-9]{6})'''
+        )/.*?(?:\bvid=|-vid|~|%7E|/(?:episode)?)(?P<id>[0-9]{6,})'''
     _TESTS = [{
         'url': 'http://www.ctv.ca/video/player?vid=706966',
         'md5': 'ff2ebbeae0aa2dcc32a830c3fd69b7b0',
@@ -55,6 +55,9 @@ class BellMediaIE(InfoExtractor):
     }, {
         'url': 'http://www.much.com/shows/the-almost-impossible-gameshow/928979/episode-6',
         'only_matching': True,
+    }, {
+        'url': 'http://www.ctv.ca/DCs-Legends-of-Tomorrow/Video/S2E11-Turncoat-vid1051430',
+        'only_matching': True,
     }]
     _DOMAINS = {
         'thecomedynetwork': 'comedy',

From 6e5956e6ba32c5e4d186e79fbaff0842818ae56b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 13 Feb 2017 23:17:48 +0700
Subject: [PATCH 0243/1696] [lemonde] Fallback delegate extraction to generic
 extractor (closes #12115, closes #12116)

---
 youtube_dl/extractor/lemonde.py | 32 ++++++++++++++++++++++++++++----
 1 file changed, 28 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/lemonde.py b/youtube_dl/extractor/lemonde.py
index 42568f315..3306892e8 100644
--- a/youtube_dl/extractor/lemonde.py
+++ b/youtube_dl/extractor/lemonde.py
@@ -7,20 +7,40 @@ class LemondeIE(InfoExtractor):
     _VALID_URL = r'https?://(?:.+?\.)?lemonde\.fr/(?:[^/]+/)*(?P<id>[^/]+)\.html'
     _TESTS = [{
         'url': 'http://www.lemonde.fr/police-justice/video/2016/01/19/comprendre-l-affaire-bygmalion-en-cinq-minutes_4849702_1653578.html',
-        'md5': '01fb3c92de4c12c573343d63e163d302',
+        'md5': 'da120c8722d8632eec6ced937536cc98',
         'info_dict': {
             'id': 'lqm3kl',
             'ext': 'mp4',
             'title': "Comprendre l'affaire Bygmalion en 5 minutes",
             'thumbnail': r're:^https?://.*\.jpg',
-            'duration': 320,
+            'duration': 309,
             'upload_date': '20160119',
             'timestamp': 1453194778,
             'uploader_id': '3pmkp',
         },
+    }, {
+        # standard iframe embed
+        'url': 'http://www.lemonde.fr/les-decodeurs/article/2016/10/18/tout-comprendre-du-ceta-le-petit-cousin-du-traite-transatlantique_5015920_4355770.html',
+        'info_dict': {
+            'id': 'uzsxms',
+            'ext': 'mp4',
+            'title': "CETA : quelles suites pour l'accord commercial entre l'Europe et le Canada ?",
+            'thumbnail': r're:^https?://.*\.jpg',
+            'duration': 325,
+            'upload_date': '20161021',
+            'timestamp': 1477044540,
+            'uploader_id': '3pmkp',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'http://redaction.actu.lemonde.fr/societe/video/2016/01/18/calais-debut-des-travaux-de-defrichement-dans-la-jungle_4849233_3224.html',
         'only_matching': True,
+    }, {
+        # YouTube embeds
+        'url': 'http://www.lemonde.fr/pixels/article/2016/12/09/pourquoi-pewdiepie-superstar-de-youtube-a-menace-de-fermer-sa-chaine_5046649_4408996.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -30,5 +50,9 @@ class LemondeIE(InfoExtractor):
 
         digiteka_url = self._proto_relative_url(self._search_regex(
             r'url\s*:\s*(["\'])(?P<url>(?:https?://)?//(?:www\.)?(?:digiteka\.net|ultimedia\.com)/deliver/.+?)\1',
-            webpage, 'digiteka url', group='url'))
-        return self.url_result(digiteka_url, 'Digiteka')
+            webpage, 'digiteka url', group='url', default=None))
+
+        if digiteka_url:
+            return self.url_result(digiteka_url, 'Digiteka')
+
+        return self.url_result(url, 'Generic')

From f6d6ca1db3020e7c7771880d0c4b58fdf732a8d5 Mon Sep 17 00:00:00 2001
From: Vobe <vobe@mailinator.com>
Date: Sat, 11 Feb 2017 21:11:55 +0100
Subject: [PATCH 0244/1696] [xtube] Improve title extraction

---
 youtube_dl/extractor/xtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index 11717fe98..ed3a37649 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -53,7 +53,7 @@ class XTubeIE(InfoExtractor):
 
         if not display_id:
             display_id = video_id
-            url = 'http://www.xtube.com/video-watch/-%s' % video_id
+            url = 'http://www.xtube.com/watch.php?v=%s' % video_id
 
         req = sanitized_Request(url)
         req.add_header('Cookie', 'age_verified=1; cookiesAccepted=1')
@@ -73,7 +73,7 @@ class XTubeIE(InfoExtractor):
         self._sort_formats(formats)
 
         title = self._search_regex(
-            (r'<h1>(?P<title>[^<]+)</h1>', r'videoTitle\s*:\s*(["\'])(?P<title>.+?)\1'),
+            (r'<h1>\s*(?P<title>[^<]+?)\s*</h1>', r'videoTitle\s*:\s*(["\'])(?P<title>.+?)\1'),
             webpage, 'title', group='title')
         description = self._search_regex(
             r'</h1>\s*<p>([^<]+)', webpage, 'description', fatal=False)

From 085f169ffebc17ec8b2bfc63aec8f5df57c7bdcf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 13 Feb 2017 23:44:43 +0700
Subject: [PATCH 0245/1696] [xtube] Fix extraction for both kinds of video id
 (closes #12088)

---
 youtube_dl/extractor/xtube.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index ed3a37649..5584674a0 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -44,6 +44,9 @@ class XTubeIE(InfoExtractor):
     }, {
         'url': 'xtube:625837',
         'only_matching': True,
+    }, {
+        'url': 'xtube:kVTUy_G222_',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -53,11 +56,16 @@ class XTubeIE(InfoExtractor):
 
         if not display_id:
             display_id = video_id
-            url = 'http://www.xtube.com/watch.php?v=%s' % video_id
 
-        req = sanitized_Request(url)
-        req.add_header('Cookie', 'age_verified=1; cookiesAccepted=1')
-        webpage = self._download_webpage(req, display_id)
+        if video_id.isdigit() and len(video_id) < 11:
+            url_pattern = 'http://www.xtube.com/video-watch/-%s'
+        else:
+            url_pattern = 'http://www.xtube.com/watch.php?v=%s'
+
+        webpage = self._download_webpage(
+            url_pattern % video_id, display_id, headers={
+                'Cookie': 'age_verified=1; cookiesAccepted=1',
+            })
 
         sources = self._parse_json(self._search_regex(
             r'(["\'])sources\1\s*:\s*(?P<sources>{.+?}),',

From 50de3dbad39d0b8cc1529113894f146f6f3f24b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Feb 2017 01:00:06 +0700
Subject: [PATCH 0246/1696] [zdf] Fix extraction (closes #12117)

---
 youtube_dl/extractor/zdf.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index a365923fb..523bb5c95 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -20,9 +20,9 @@ from ..utils import (
 
 
 class ZDFBaseIE(InfoExtractor):
-    def _call_api(self, url, player, referrer, video_id):
+    def _call_api(self, url, player, referrer, video_id, item):
         return self._download_json(
-            url, video_id, 'Downloading JSON content',
+            url, video_id, 'Downloading JSON %s' % item,
             headers={
                 'Referer': referrer,
                 'Api-Auth': 'Bearer %s' % player['apiToken'],
@@ -104,7 +104,7 @@ class ZDFIE(ZDFBaseIE):
             })
             formats.append(f)
 
-    def _extract_entry(self, url, content, video_id):
+    def _extract_entry(self, url, player, content, video_id):
         title = content.get('title') or content['teaserHeadline']
 
         t = content['mainVideoContent']['http://zdf.de/rels/target']
@@ -116,7 +116,8 @@ class ZDFIE(ZDFBaseIE):
                 'http://zdf.de/rels/streams/ptmd-template'].replace(
                 '{playerId}', 'portal')
 
-        ptmd = self._download_json(urljoin(url, ptmd_path), video_id)
+        ptmd = self._call_api(
+            urljoin(url, ptmd_path), player, url, video_id, 'metadata')
 
         formats = []
         track_uris = set()
@@ -174,8 +175,9 @@ class ZDFIE(ZDFBaseIE):
         }
 
     def _extract_regular(self, url, player, video_id):
-        content = self._call_api(player['content'], player, url, video_id)
-        return self._extract_entry(player['content'], content, video_id)
+        content = self._call_api(
+            player['content'], player, url, video_id, 'content')
+        return self._extract_entry(player['content'], player, content, video_id)
 
     def _extract_mobile(self, video_id):
         document = self._download_json(

From cedf08ff54d192a0e32ecb3b943f50299cda7ea2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Feb 2017 01:07:35 +0700
Subject: [PATCH 0247/1696] [ChangeLog] Actualize

---
 ChangeLog | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 089449dfb..d651f8880 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -4,7 +4,17 @@ Core
 * TypeError is fixed with Python 2.7.13 on Windows (#11540, #12085)
 
 Extractor
+* [zdf] Fix extraction (#12117)
+* [xtube] Fix extraction for both kinds of video id (#12088)
+* [xtube] Improve title extraction (#12088)
++ [lemonde] Fallback delegate extraction to generic extractor (#12115, #12116)
+* [bellmedia] Allow video id longer than 6 characters (#12114)
++ [limelight] Add support for referer protected videos
+* [disney] Improve extraction (#4975, #11000, #11882, #11936)
+* [hotstar] Improve extraction (#12096)
 * [einthusan] Fix extraction (#11416)
++ [aenetworks] Add support for lifetimemovieclub.com (#12097)
+* [youtube] Fix parsing codecs (#12091)
 
 
 version 2017.02.11

From 58a65ba852443075fe38a3ef74798de05dd57bda Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Feb 2017 01:09:18 +0700
Subject: [PATCH 0248/1696] release 2017.02.14

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 7bd301cc8..32aa55d83 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.11*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.11**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.14*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.14**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.02.11
+[debug] youtube-dl version 2017.02.14
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index d651f8880..9242b3eee 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.02.14
 
 Core
 * TypeError is fixed with Python 2.7.13 on Windows (#11540, #12085)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1f84acfea..3e7e7c0bf 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.02.11'
+__version__ = '2017.02.14'

From fcca0d53a8fa47614a39a433a3da7d1ab1d88ed9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ond=C5=99ej=20Caletka?= <ondrej@caletka.cz>
Date: Tue, 14 Feb 2017 15:57:17 +0100
Subject: [PATCH 0249/1696] [ceskatelevize] Quick fix to revert to using old
 HLS-based playlist

This fixes recent changes in iVysilani. Proper patch should migrate to
MPEG-DASH version, which is now the default.
---
 youtube_dl/extractor/ceskatelevize.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index 4f88c31ad..0f1453b99 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -21,10 +21,10 @@ class CeskaTelevizeIE(InfoExtractor):
     _TESTS = [{
         'url': 'http://www.ceskatelevize.cz/ivysilani/ivysilani/10441294653-hyde-park-civilizace/214411058091220',
         'info_dict': {
-            'id': '61924494876951776',
+            'id': '61924494877246241',
             'ext': 'mp4',
-            'title': 'Hyde Park Civilizace',
-            'description': 'md5:fe93f6eda372d150759d11644ebbfb4a',
+            'title': 'Hyde Park Civilizace: Život v Grónsku',
+            'description': 'md5:3fec8f6bb497be5cdb0c9e8781076626',
             'thumbnail': r're:^https?://.*\.jpg',
             'duration': 3350,
         },
@@ -121,6 +121,7 @@ class CeskaTelevizeIE(InfoExtractor):
         req.add_header('Content-type', 'application/x-www-form-urlencoded')
         req.add_header('x-addr', '127.0.0.1')
         req.add_header('X-Requested-With', 'XMLHttpRequest')
+        req.add_header('User-agent', 'Mozilla/5.0')
         req.add_header('Referer', url)
 
         playlistpage = self._download_json(req, playlist_id)

From 5cb2d36c82abf3b753910afe3013b274e31a247a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Feb 2017 22:56:39 +0700
Subject: [PATCH 0250/1696] [ceskatelevize] Extract DASH formats (closes
 #12119, closes #12133)

---
 youtube_dl/extractor/ceskatelevize.py | 142 +++++++++++++++-----------
 1 file changed, 83 insertions(+), 59 deletions(-)

diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index 0f1453b99..e08bf264c 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -13,6 +13,7 @@ from ..utils import (
     float_or_none,
     sanitized_Request,
     urlencode_postdata,
+    USER_AGENTS,
 )
 
 
@@ -114,71 +115,94 @@ class CeskaTelevizeIE(InfoExtractor):
             'requestSource': 'iVysilani',
         }
 
-        req = sanitized_Request(
-            'http://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist',
-            data=urlencode_postdata(data))
-
-        req.add_header('Content-type', 'application/x-www-form-urlencoded')
-        req.add_header('x-addr', '127.0.0.1')
-        req.add_header('X-Requested-With', 'XMLHttpRequest')
-        req.add_header('User-agent', 'Mozilla/5.0')
-        req.add_header('Referer', url)
-
-        playlistpage = self._download_json(req, playlist_id)
-
-        playlist_url = playlistpage['url']
-        if playlist_url == 'error_region':
-            raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)
-
-        req = sanitized_Request(compat_urllib_parse_unquote(playlist_url))
-        req.add_header('Referer', url)
-
-        playlist_title = self._og_search_title(webpage, default=None)
-        playlist_description = self._og_search_description(webpage, default=None)
-
-        playlist = self._download_json(req, playlist_id)['playlist']
-        playlist_len = len(playlist)
-
         entries = []
-        for item in playlist:
-            is_live = item.get('type') == 'LIVE'
-            formats = []
-            for format_id, stream_url in item['streamUrls'].items():
-                formats.extend(self._extract_m3u8_formats(
-                    stream_url, playlist_id, 'mp4',
-                    entry_protocol='m3u8' if is_live else 'm3u8_native',
-                    fatal=False))
-            self._sort_formats(formats)
 
-            item_id = item.get('id') or item['assetId']
-            title = item['title']
+        for user_agent in (None, USER_AGENTS['Safari']):
+            req = sanitized_Request(
+                'http://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist',
+                data=urlencode_postdata(data))
 
-            duration = float_or_none(item.get('duration'))
-            thumbnail = item.get('previewImageUrl')
+            req.add_header('Content-type', 'application/x-www-form-urlencoded')
+            req.add_header('x-addr', '127.0.0.1')
+            req.add_header('X-Requested-With', 'XMLHttpRequest')
+            if user_agent:
+                req.add_header('User-Agent', user_agent)
+            req.add_header('Referer', url)
 
-            subtitles = {}
-            if item.get('type') == 'VOD':
-                subs = item.get('subtitles')
-                if subs:
-                    subtitles = self.extract_subtitles(episode_id, subs)
+            playlistpage = self._download_json(req, playlist_id, fatal=False)
 
-            if playlist_len == 1:
-                final_title = playlist_title or title
-                if is_live:
-                    final_title = self._live_title(final_title)
-            else:
-                final_title = '%s (%s)' % (playlist_title, title)
+            if not playlistpage:
+                continue
 
-            entries.append({
-                'id': item_id,
-                'title': final_title,
-                'description': playlist_description if playlist_len == 1 else None,
-                'thumbnail': thumbnail,
-                'duration': duration,
-                'formats': formats,
-                'subtitles': subtitles,
-                'is_live': is_live,
-            })
+            playlist_url = playlistpage['url']
+            if playlist_url == 'error_region':
+                raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)
+
+            req = sanitized_Request(compat_urllib_parse_unquote(playlist_url))
+            req.add_header('Referer', url)
+
+            playlist_title = self._og_search_title(webpage, default=None)
+            playlist_description = self._og_search_description(webpage, default=None)
+
+            playlist = self._download_json(req, playlist_id, fatal=False)
+            if not playlist:
+                continue
+
+            playlist = playlist.get('playlist')
+            if not isinstance(playlist, list):
+                continue
+
+            playlist_len = len(playlist)
+
+            for num, item in enumerate(playlist):
+                is_live = item.get('type') == 'LIVE'
+                formats = []
+                for format_id, stream_url in item.get('streamUrls', {}).items():
+                    if 'playerType=flash' in stream_url:
+                        formats.extend(self._extract_m3u8_formats(
+                            stream_url, playlist_id, 'mp4',
+                            entry_protocol='m3u8' if is_live else 'm3u8_native',
+                            fatal=False))
+                    else:
+                        formats.extend(self._extract_mpd_formats(
+                            stream_url, playlist_id, fatal=False))
+
+                if user_agent and len(entries) == playlist_len:
+                    entries[num]['formats'].extend(formats)
+                    continue
+
+                item_id = item.get('id') or item['assetId']
+                title = item['title']
+
+                duration = float_or_none(item.get('duration'))
+                thumbnail = item.get('previewImageUrl')
+
+                subtitles = {}
+                if item.get('type') == 'VOD':
+                    subs = item.get('subtitles')
+                    if subs:
+                        subtitles = self.extract_subtitles(episode_id, subs)
+
+                if playlist_len == 1:
+                    final_title = playlist_title or title
+                    if is_live:
+                        final_title = self._live_title(final_title)
+                else:
+                    final_title = '%s (%s)' % (playlist_title, title)
+
+                entries.append({
+                    'id': item_id,
+                    'title': final_title,
+                    'description': playlist_description if playlist_len == 1 else None,
+                    'thumbnail': thumbnail,
+                    'duration': duration,
+                    'formats': formats,
+                    'subtitles': subtitles,
+                    'is_live': is_live,
+                })
+
+        for e in entries:
+            self._sort_formats(e['formats'])
 
         return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
 

From 9a372f14b422de15acf91e25a90375688b2ba3fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Feb 2017 23:52:41 +0700
Subject: [PATCH 0251/1696] [pornhub] Extract video URL from tv platform site
 (#12007, #12129)

---
 youtube_dl/extractor/pornhub.py | 44 ++++++++++++++++++++++-----------
 1 file changed, 30 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 818d99c1f..7a2737032 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -2,27 +2,27 @@
 from __future__ import unicode_literals
 
 import itertools
-import os
+# import os
 import re
 
 from .common import InfoExtractor
 from ..compat import (
     compat_HTTPError,
-    compat_urllib_parse_unquote,
-    compat_urllib_parse_unquote_plus,
-    compat_urllib_parse_urlparse,
+    # compat_urllib_parse_unquote,
+    # compat_urllib_parse_unquote_plus,
+    # compat_urllib_parse_urlparse,
 )
 from ..utils import (
     ExtractorError,
     int_or_none,
     js_to_json,
     orderedSet,
-    sanitized_Request,
+    # sanitized_Request,
     str_to_int,
 )
-from ..aes import (
-    aes_decrypt_text
-)
+# from ..aes import (
+#     aes_decrypt_text
+# )
 
 
 class PornHubIE(InfoExtractor):
@@ -109,10 +109,14 @@ class PornHubIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        req = sanitized_Request(
-            'http://www.pornhub.com/view_video.php?viewkey=%s' % video_id)
-        req.add_header('Cookie', 'age_verified=1')
-        webpage = self._download_webpage(req, video_id)
+        def dl_webpage(platform):
+            return self._download_webpage(
+                'http://www.pornhub.com/view_video.php?viewkey=%s' % video_id,
+                video_id, headers={
+                    'Cookie': 'age_verified=1; platform=%s' % platform,
+                })
+
+        webpage = dl_webpage('pc')
 
         error_msg = self._html_search_regex(
             r'(?s)<div[^>]+class=(["\'])(?:(?!\1).)*\b(?:removed|userMessageSection)\b(?:(?!\1).)*\1[^>]*>(?P<error>.+?)</div>',
@@ -123,10 +127,19 @@ class PornHubIE(InfoExtractor):
                 'PornHub said: %s' % error_msg,
                 expected=True, video_id=video_id)
 
+        tv_webpage = dl_webpage('tv')
+
+        video_url = self._search_regex(
+            r'<video[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//.+?)\1', tv_webpage,
+            'video url', group='url')
+
+        title = self._search_regex(
+            r'<h1>([^>]+)</h1>', tv_webpage, 'title', default=None)
+
         # video_title from flashvars contains whitespace instead of non-ASCII (see
         # http://www.pornhub.com/view_video.php?viewkey=1331683002), not relying
         # on that anymore.
-        title = self._html_search_meta(
+        title = title or self._html_search_meta(
             'twitter:title', webpage, default=None) or self._search_regex(
             (r'<h1[^>]+class=["\']title["\'][^>]*>(?P<title>[^<]+)',
              r'<div[^>]+data-video-title=(["\'])(?P<title>.+?)\1',
@@ -156,6 +169,7 @@ class PornHubIE(InfoExtractor):
         comment_count = self._extract_count(
             r'All Comments\s*<span>\(([\d,.]+)\)', webpage, 'comment')
 
+        """
         video_variables = {}
         for video_variablename, quote, video_variable in re.findall(
                 r'(player_quality_[0-9]{3,4}p\w+)\s*=\s*(["\'])(.+?)\2;', webpage):
@@ -197,6 +211,7 @@ class PornHubIE(InfoExtractor):
                 'height': height,
             })
         self._sort_formats(formats)
+        """
 
         page_params = self._parse_json(self._search_regex(
             r'page_params\.zoneDetails\[([\'"])[^\'"]+\1\]\s*=\s*(?P<data>{[^}]+})',
@@ -209,6 +224,7 @@ class PornHubIE(InfoExtractor):
 
         return {
             'id': video_id,
+            'url': video_url,
             'uploader': video_uploader,
             'title': title,
             'thumbnail': thumbnail,
@@ -217,7 +233,7 @@ class PornHubIE(InfoExtractor):
             'like_count': like_count,
             'dislike_count': dislike_count,
             'comment_count': comment_count,
-            'formats': formats,
+            # 'formats': formats,
             'age_limit': 18,
             'tags': tags,
             'categories': categories,

From 22ce9ad2bdad2bf79b22f82cfff7f58156c9d349 Mon Sep 17 00:00:00 2001
From: Marek Rusinowski <marekrusinowski@gmail.com>
Date: Mon, 13 Feb 2017 21:42:26 +0100
Subject: [PATCH 0252/1696] [vod.pl] Add new extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/vodpl.py      | 36 ++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+)
 create mode 100644 youtube_dl/extractor/vodpl.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 76ad7c40b..657e45e6f 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1147,6 +1147,7 @@ from .vlive import (
     VLiveChannelIE
 )
 from .vodlocker import VodlockerIE
+from .vodpl import VODPlIE
 from .vodplatform import VODPlatformIE
 from .voicerepublic import VoiceRepublicIE
 from .voxmedia import VoxMediaIE
diff --git a/youtube_dl/extractor/vodpl.py b/youtube_dl/extractor/vodpl.py
new file mode 100644
index 000000000..f612347ce
--- /dev/null
+++ b/youtube_dl/extractor/vodpl.py
@@ -0,0 +1,36 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .onet import OnetBaseIE
+from ..utils import clean_html
+
+
+class VODPlIE(OnetBaseIE):
+    _VALID_URL = r'https?://vod\.pl/(?:.*/)?(?P<id>[0-9a-zA-Z]+)'
+
+    _TEST = {
+        'url': 'https://vod.pl/filmy/chlopaki-nie-placza/3ep3jns',
+        'md5': 'a7dc3b2f7faa2421aefb0ecaabf7ec74',
+        'info_dict': {
+            'id': '3ep3jns',
+            'ext': 'mp4',
+            'title': 'Chłopaki nie płaczą',
+            'description': 'Kuba Brenner aby pomóc swojemu nieśmiałemu przyjacielowi Oskarowi wynajmuje w agencji towarzyskiej dwie panie. Po upojnej nocy okazuje się, że chłopcy nie byli przygotowani finansowo. "Opiekun artystyczny" dziewczyn zabiera w ramach rekompensaty drogocenną rzeźbę należącą do wujka Oskara. Kłopoty chłopców zaczynają się, gdy Kuba udaje się do agencji aby wykupić figurkę i trafia w sam środek mafijnej transakcji... Idiotyczny przypadek sprawia, że w klubie dochodzi do strzelaniny podczas której Grucha i Bolec zostają ranni, ginie również walizka z pieniędzmi... Podejrzenie pada na Kubę.',
+            'timestamp': 1463415154,
+            'duration': 5765,
+            'upload_date': '20160516',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        mvp_id = self._search_mvp_id(webpage)
+
+        info_dict = self._extract_from_id(mvp_id, webpage)
+        info_dict.update({
+            'id': video_id,
+            'description': clean_html(info_dict['description']).strip().replace('\r', '\n')
+        })
+
+        return info_dict

From 6092ccd05844976ea946ba5277f2b00ccb5c7920 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 15 Feb 2017 00:52:31 +0700
Subject: [PATCH 0253/1696] [vodpl] Make more robust and add another test
 (closes #12122)

---
 youtube_dl/extractor/vodpl.py | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/vodpl.py b/youtube_dl/extractor/vodpl.py
index f612347ce..9e919708e 100644
--- a/youtube_dl/extractor/vodpl.py
+++ b/youtube_dl/extractor/vodpl.py
@@ -2,35 +2,31 @@
 from __future__ import unicode_literals
 
 from .onet import OnetBaseIE
-from ..utils import clean_html
 
 
 class VODPlIE(OnetBaseIE):
-    _VALID_URL = r'https?://vod\.pl/(?:.*/)?(?P<id>[0-9a-zA-Z]+)'
+    _VALID_URL = r'https?://vod\.pl/(?:[^/]+/)+(?P<id>[0-9a-zA-Z]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'https://vod.pl/filmy/chlopaki-nie-placza/3ep3jns',
         'md5': 'a7dc3b2f7faa2421aefb0ecaabf7ec74',
         'info_dict': {
             'id': '3ep3jns',
             'ext': 'mp4',
             'title': 'Chłopaki nie płaczą',
-            'description': 'Kuba Brenner aby pomóc swojemu nieśmiałemu przyjacielowi Oskarowi wynajmuje w agencji towarzyskiej dwie panie. Po upojnej nocy okazuje się, że chłopcy nie byli przygotowani finansowo. "Opiekun artystyczny" dziewczyn zabiera w ramach rekompensaty drogocenną rzeźbę należącą do wujka Oskara. Kłopoty chłopców zaczynają się, gdy Kuba udaje się do agencji aby wykupić figurkę i trafia w sam środek mafijnej transakcji... Idiotyczny przypadek sprawia, że w klubie dochodzi do strzelaniny podczas której Grucha i Bolec zostają ranni, ginie również walizka z pieniędzmi... Podejrzenie pada na Kubę.',
+            'description': 'md5:f5f03b84712e55f5ac9f0a3f94445224',
             'timestamp': 1463415154,
             'duration': 5765,
             'upload_date': '20160516',
         },
-    }
+    }, {
+        'url': 'https://vod.pl/seriale/belfer-na-planie-praca-kamery-online/2c10heh',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        mvp_id = self._search_mvp_id(webpage)
-
-        info_dict = self._extract_from_id(mvp_id, webpage)
-        info_dict.update({
-            'id': video_id,
-            'description': clean_html(info_dict['description']).strip().replace('\r', '\n')
-        })
-
+        info_dict = self._extract_from_id(self._search_mvp_id(webpage), webpage)
+        info_dict['id'] = video_id
         return info_dict

From d31aa74fdb3f69071ba869feba03525f67e974f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 15 Feb 2017 00:58:18 +0700
Subject: [PATCH 0254/1696] [onetmvp] Add shortcut extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/onet.py       | 20 +++++++++++++++++---
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 657e45e6f..b2ee0c1b0 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -694,6 +694,7 @@ from .ondemandkorea import OnDemandKoreaIE
 from .onet import (
     OnetIE,
     OnetChannelIE,
+    OnetMVPIE,
 )
 from .onionstudios import OnionStudiosIE
 from .ooyala import (
diff --git a/youtube_dl/extractor/onet.py b/youtube_dl/extractor/onet.py
index 0a501b3e5..46bad492a 100644
--- a/youtube_dl/extractor/onet.py
+++ b/youtube_dl/extractor/onet.py
@@ -23,7 +23,7 @@ class OnetBaseIE(InfoExtractor):
         return self._search_regex(
             r'id=(["\'])mvp:(?P<id>.+?)\1', webpage, 'mvp id', group='id')
 
-    def _extract_from_id(self, video_id, webpage):
+    def _extract_from_id(self, video_id, webpage=None):
         response = self._download_json(
             'http://qi.ckm.onetapi.pl/', video_id,
             query={
@@ -74,8 +74,10 @@ class OnetBaseIE(InfoExtractor):
 
         meta = video.get('meta', {})
 
-        title = self._og_search_title(webpage, default=None) or meta['title']
-        description = self._og_search_description(webpage, default=None) or meta.get('description')
+        title = (self._og_search_title(
+            webpage, default=None) if webpage else None) or meta['title']
+        description = (self._og_search_description(
+            webpage, default=None) if webpage else None) or meta.get('description')
         duration = meta.get('length') or meta.get('lenght')
         timestamp = parse_iso8601(meta.get('addDate'), ' ')
 
@@ -89,6 +91,18 @@ class OnetBaseIE(InfoExtractor):
         }
 
 
+class OnetMVPIE(OnetBaseIE):
+    _VALID_URL = r'onetmvp:(?P<id>\d+\.\d+)'
+
+    _TEST = {
+        'url': 'onetmvp:381027.1509591944',
+        'only_matching': True,
+    }
+
+    def _real_extract(self, url):
+        return self._extract_from_id(self._match_id(url))
+
+
 class OnetIE(OnetBaseIE):
     _VALID_URL = r'https?://(?:www\.)?onet\.tv/[a-z]/[a-z]+/(?P<display_id>[0-9a-z-]+)/(?P<id>[0-9a-z]+)'
     IE_NAME = 'onet.tv'

From 43a3d9edfcdad8eb33758c4a7f4f912322001b8a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 15 Feb 2017 01:14:06 +0700
Subject: [PATCH 0255/1696] [onetpl] Add support for onet.pl (closes #10507)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/onet.py       | 32 ++++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index b2ee0c1b0..be3688d5a 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -695,6 +695,7 @@ from .onet import (
     OnetIE,
     OnetChannelIE,
     OnetMVPIE,
+    OnetPlIE,
 )
 from .onionstudios import OnionStudiosIE
 from .ooyala import (
diff --git a/youtube_dl/extractor/onet.py b/youtube_dl/extractor/onet.py
index 46bad492a..801aadbff 100644
--- a/youtube_dl/extractor/onet.py
+++ b/youtube_dl/extractor/onet.py
@@ -181,3 +181,35 @@ class OnetChannelIE(OnetBaseIE):
         channel_title = strip_or_none(get_element_by_class('o_channelName', webpage))
         channel_description = strip_or_none(get_element_by_class('o_channelDesc', webpage))
         return self.playlist_result(entries, channel_id, channel_title, channel_description)
+
+
+class OnetPlIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:[^/]+\.)?onet\.pl/(?:[^/]+/)+(?P<id>[0-9a-z]+)'
+    IE_NAME = 'onet.pl'
+
+    _TESTS = [{
+        'url': 'http://eurosport.onet.pl/zimowe/skoki-narciarskie/ziobro-wygral-kwalifikacje-w-pjongczangu/9ckrly',
+        'md5': 'b94021eb56214c3969380388b6e73cb0',
+        'info_dict': {
+            'id': '1561707.1685479',
+            'ext': 'mp4',
+            'title': 'Ziobro wygrał kwalifikacje w Pjongczangu',
+            'description': 'md5:61fb0740084d2d702ea96512a03585b4',
+            'upload_date': '20170214',
+            'timestamp': 1487078046,
+        },
+    }, {
+        'url': 'http://film.onet.pl/zwiastuny/ghost-in-the-shell-drugi-zwiastun-pl/5q6yl3',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        mvp_id = self._search_regex(
+            r'data-params-mvp=["\'](\d+\.\d+)', webpage, 'mvp id')
+
+        return self.url_result(
+            'onetmvp:%s' % mvp_id, OnetMVPIE.ie_key(), video_id=mvp_id)

From 04a741232f8e03cc91a3539066c66aed802076b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 15 Feb 2017 01:23:55 +0700
Subject: [PATCH 0256/1696] [onetpl] Add support for businessinsider.com.pl and
 plejada.pl

---
 youtube_dl/extractor/onet.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/onet.py b/youtube_dl/extractor/onet.py
index 801aadbff..94f57990b 100644
--- a/youtube_dl/extractor/onet.py
+++ b/youtube_dl/extractor/onet.py
@@ -184,7 +184,7 @@ class OnetChannelIE(OnetBaseIE):
 
 
 class OnetPlIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:[^/]+\.)?onet\.pl/(?:[^/]+/)+(?P<id>[0-9a-z]+)'
+    _VALID_URL = r'https?://(?:[^/]+\.)?(?:onet|businessinsider\.com|plejada)\.pl/(?:[^/]+/)+(?P<id>[0-9a-z]+)'
     IE_NAME = 'onet.pl'
 
     _TESTS = [{
@@ -201,6 +201,15 @@ class OnetPlIE(InfoExtractor):
     }, {
         'url': 'http://film.onet.pl/zwiastuny/ghost-in-the-shell-drugi-zwiastun-pl/5q6yl3',
         'only_matching': True,
+    }, {
+        'url': 'http://moto.onet.pl/jak-wybierane-sa-miejsca-na-fotoradary/6rs04e',
+        'only_matching': True,
+    }, {
+        'url': 'http://businessinsider.com.pl/wideo/scenariusz-na-koniec-swiata-wedlug-nasa/dwnqptk',
+        'only_matching': True,
+    }, {
+        'url': 'http://plejada.pl/weronika-rosati-o-swoim-domniemanym-slubie/n2bq89',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 3021cf83b7cd45283fd1a72859e46f44e67ce7bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 15 Feb 2017 02:08:32 +0700
Subject: [PATCH 0257/1696] [pinkbike] Fix uploader extraction (closes #12054)

---
 youtube_dl/extractor/pinkbike.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pinkbike.py b/youtube_dl/extractor/pinkbike.py
index 6a4580d54..9f3501f77 100644
--- a/youtube_dl/extractor/pinkbike.py
+++ b/youtube_dl/extractor/pinkbike.py
@@ -64,7 +64,8 @@ class PinkbikeIE(InfoExtractor):
             'video:duration', webpage, 'duration'))
 
         uploader = self._search_regex(
-            r'un:\s*"([^"]+)"', webpage, 'uploader', fatal=False)
+            r'<a[^>]+\brel=["\']author[^>]+>([^<]+)', webpage,
+            'uploader', fatal=False)
         upload_date = unified_strdate(self._search_regex(
             r'class="fullTime"[^>]+title="([^"]+)"',
             webpage, 'upload date', fatal=False))

From 1bd05345ea4b91598ec04b8e0d33fd14f9e2eddc Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 15 Feb 2017 14:18:50 +0100
Subject: [PATCH 0258/1696] [amcnetworks] fix extraction(closes #12127)

---
 youtube_dl/extractor/amcnetworks.py | 30 ++++++++++++++++++++---------
 1 file changed, 21 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/amcnetworks.py b/youtube_dl/extractor/amcnetworks.py
index 87c803e94..b71d1a093 100644
--- a/youtube_dl/extractor/amcnetworks.py
+++ b/youtube_dl/extractor/amcnetworks.py
@@ -53,20 +53,30 @@ class AMCNetworksIE(ThePlatformIE):
             'mbr': 'true',
             'manifest': 'm3u',
         }
-        media_url = self._search_regex(r'window\.platformLinkURL\s*=\s*[\'"]([^\'"]+)', webpage, 'media url')
+        media_url = self._search_regex(
+            r'window\.platformLinkURL\s*=\s*[\'"]([^\'"]+)',
+            webpage, 'media url')
         theplatform_metadata = self._download_theplatform_metadata(self._search_regex(
-            r'https?://link.theplatform.com/s/([^?]+)', media_url, 'theplatform_path'), display_id)
+            r'link\.theplatform\.com/s/([^?]+)',
+            media_url, 'theplatform_path'), display_id)
         info = self._parse_theplatform_metadata(theplatform_metadata)
         video_id = theplatform_metadata['pid']
         title = theplatform_metadata['title']
         rating = theplatform_metadata['ratings'][0]['rating']
-        auth_required = self._search_regex(r'window\.authRequired\s*=\s*(true|false);', webpage, 'auth required')
+        auth_required = self._search_regex(
+            r'window\.authRequired\s*=\s*(true|false);',
+            webpage, 'auth required')
         if auth_required == 'true':
-            requestor_id = self._search_regex(r'window\.requestor_id\s*=\s*[\'"]([^\'"]+)', webpage, 'requestor id')
-            resource = self._get_mvpd_resource(requestor_id, title, video_id, rating)
-            query['auth'] = self._extract_mvpd_auth(url, video_id, requestor_id, resource)
+            requestor_id = self._search_regex(
+                r'window\.requestor_id\s*=\s*[\'"]([^\'"]+)',
+                webpage, 'requestor id')
+            resource = self._get_mvpd_resource(
+                requestor_id, title, video_id, rating)
+            query['auth'] = self._extract_mvpd_auth(
+                url, video_id, requestor_id, resource)
         media_url = update_url_query(media_url, query)
-        formats, subtitles = self._extract_theplatform_smil(media_url, video_id)
+        formats, subtitles = self._extract_theplatform_smil(
+            media_url, video_id)
         self._sort_formats(formats)
         info.update({
             'id': video_id,
@@ -78,9 +88,11 @@ class AMCNetworksIE(ThePlatformIE):
         if ns_keys:
             ns = list(ns_keys)[0]
             series = theplatform_metadata.get(ns + '$show')
-            season_number = int_or_none(theplatform_metadata.get(ns + '$season'))
+            season_number = int_or_none(
+                theplatform_metadata.get(ns + '$season'))
             episode = theplatform_metadata.get(ns + '$episodeTitle')
-            episode_number = int_or_none(theplatform_metadata.get(ns + '$episode'))
+            episode_number = int_or_none(
+                theplatform_metadata.get(ns + '$episode'))
             if season_number:
                 title = 'Season %d - %s' % (season_number, title)
             if series:

From db13c16ef8968613680e2bbc85f373c3e74faf98 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 15 Feb 2017 23:12:10 +0700
Subject: [PATCH 0259/1696] [utils] Add support for quoted string literals in
 --match-filter (closes #8050, closes #12142, closes #12144)

---
 test/test_YoutubeDL.py | 24 ++++++++++++++++++++++++
 youtube_dl/utils.py    |  9 +++++++--
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 8bf00bea9..d07c35be8 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# coding: utf-8
 
 from __future__ import unicode_literals
 
@@ -606,6 +607,8 @@ class TestYoutubeDL(unittest.TestCase):
             'duration': 30,
             'filesize': 10 * 1024,
             'playlist_id': '42',
+            'uploader': "變態妍字幕版 太妍 тест",
+            'creator': "тест ' 123 ' тест--",
         }
         second = {
             'id': '2',
@@ -616,6 +619,7 @@ class TestYoutubeDL(unittest.TestCase):
             'description': 'foo',
             'filesize': 5 * 1024,
             'playlist_id': '43',
+            'uploader': "тест 123",
         }
         videos = [first, second]
 
@@ -656,6 +660,26 @@ class TestYoutubeDL(unittest.TestCase):
         res = get_videos(f)
         self.assertEqual(res, ['1'])
 
+        f = match_filter_func('uploader = "變態妍字幕版 太妍 тест"')
+        res = get_videos(f)
+        self.assertEqual(res, ['1'])
+
+        f = match_filter_func('uploader != "變態妍字幕版 太妍 тест"')
+        res = get_videos(f)
+        self.assertEqual(res, ['2'])
+
+        f = match_filter_func('creator = "тест \' 123 \' тест--"')
+        res = get_videos(f)
+        self.assertEqual(res, ['1'])
+
+        f = match_filter_func("creator = 'тест \\' 123 \\' тест--'")
+        res = get_videos(f)
+        self.assertEqual(res, ['1'])
+
+        f = match_filter_func(r"creator = 'тест \' 123 \' тест--' & duration > 30")
+        res = get_videos(f)
+        self.assertEqual(res, [])
+
     def test_playlist_items_selection(self):
         entries = [{
             'id': compat_str(i),
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 1279a9042..07c07be6f 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2383,6 +2383,7 @@ def _match_one(filter_part, dct):
         \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
         (?:
             (?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
+            (?P<quote>["\'])(?P<quotedstrval>(?:\\.|(?!(?P=quote)|\\).)+?)(?P=quote)|
             (?P<strval>(?![0-9.])[a-z0-9A-Z]*)
         )
         \s*$
@@ -2391,7 +2392,8 @@ def _match_one(filter_part, dct):
     if m:
         op = COMPARISON_OPERATORS[m.group('op')]
         actual_value = dct.get(m.group('key'))
-        if (m.group('strval') is not None or
+        if (m.group('quotedstrval') is not None or
+            m.group('strval') is not None or
             # If the original field is a string and matching comparisonvalue is
             # a number we should respect the origin of the original field
             # and process comparison value as a string (see
@@ -2401,7 +2403,10 @@ def _match_one(filter_part, dct):
             if m.group('op') not in ('=', '!='):
                 raise ValueError(
                     'Operator %s does not support string values!' % m.group('op'))
-            comparison_value = m.group('strval') or m.group('intval')
+            comparison_value = m.group('quotedstrval') or m.group('strval') or m.group('intval')
+            quote = m.group('quote')
+            if quote is not None:
+                comparison_value = comparison_value.replace(r'\%s' % quote, quote)
         else:
             try:
                 comparison_value = int(m.group('intval'))

From 398dea321001b99ac4ad28d3d60a5317c4a439d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 15 Feb 2017 23:20:46 +0700
Subject: [PATCH 0260/1696] [test_YoutubeDL] Fix invalid escape sequences

---
 test/test_YoutubeDL.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index d07c35be8..2cfcf743a 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -541,10 +541,10 @@ class TestYoutubeDL(unittest.TestCase):
         self.assertEqual(ydl._format_note({}), '')
         assertRegexpMatches(self, ydl._format_note({
             'vbr': 10,
-        }), '^\s*10k$')
+        }), r'^\s*10k$')
         assertRegexpMatches(self, ydl._format_note({
             'fps': 30,
-        }), '^30fps$')
+        }), r'^30fps$')
 
     def test_postprocessors(self):
         filename = 'post-processor-testfile.mp4'

From 099cfdb770f458de7cfdf3e814fbb9f43db217ea Mon Sep 17 00:00:00 2001
From: Anisse Astier <anisse@astier.eu>
Date: Wed, 15 Feb 2017 17:28:31 +0100
Subject: [PATCH 0261/1696] [devscripts/run_tests.sh] Change permission for
 script to 755

---
 .travis.yml             | 2 --
 devscripts/run_tests.sh | 0
 2 files changed, 2 deletions(-)
 mode change 100644 => 100755 devscripts/run_tests.sh

diff --git a/.travis.yml b/.travis.yml
index 8ba93ec02..f41e11137 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,8 +11,6 @@ sudo: false
 env:
   - YTDL_TEST_SET=core
   - YTDL_TEST_SET=download
-before_script:
-  - chmod +x ./devscripts/run_tests.sh
 script: ./devscripts/run_tests.sh
 notifications:
   email:
diff --git a/devscripts/run_tests.sh b/devscripts/run_tests.sh
old mode 100644
new mode 100755

From de4d378c0cd9035d4ab93dc6826a17c76f388641 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 15 Feb 2017 23:38:00 +0700
Subject: [PATCH 0262/1696] [ceskatelevize] Prefix format ids

---
 youtube_dl/extractor/ceskatelevize.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index e08bf264c..1b16e5aaa 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -162,10 +162,10 @@ class CeskaTelevizeIE(InfoExtractor):
                         formats.extend(self._extract_m3u8_formats(
                             stream_url, playlist_id, 'mp4',
                             entry_protocol='m3u8' if is_live else 'm3u8_native',
-                            fatal=False))
+                            m3u8_id='hls', fatal=False))
                     else:
                         formats.extend(self._extract_mpd_formats(
-                            stream_url, playlist_id, fatal=False))
+                            stream_url, playlist_id, mpd_id='dash', fatal=False))
 
                 if user_agent and len(entries) == playlist_len:
                     entries[num]['formats'].extend(formats)

From eafaeb226a277008fb8df72bf0326f2b369ff6a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Feb 2017 00:04:15 +0700
Subject: [PATCH 0263/1696] [ceskatelevize] Lower priority for audio
 description sources (#12119)

---
 youtube_dl/extractor/ceskatelevize.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index 1b16e5aaa..b1dfacf80 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -159,13 +159,19 @@ class CeskaTelevizeIE(InfoExtractor):
                 formats = []
                 for format_id, stream_url in item.get('streamUrls', {}).items():
                     if 'playerType=flash' in stream_url:
-                        formats.extend(self._extract_m3u8_formats(
+                        stream_formats = self._extract_m3u8_formats(
                             stream_url, playlist_id, 'mp4',
                             entry_protocol='m3u8' if is_live else 'm3u8_native',
-                            m3u8_id='hls', fatal=False))
+                            m3u8_id='hls-%s' % format_id, fatal=False)
                     else:
-                        formats.extend(self._extract_mpd_formats(
-                            stream_url, playlist_id, mpd_id='dash', fatal=False))
+                        stream_formats = self._extract_mpd_formats(
+                            stream_url, playlist_id,
+                            mpd_id='dash-%s' % format_id, fatal=False)
+                    # See https://github.com/rg3/youtube-dl/issues/12119#issuecomment-280037031
+                    if format_id == 'audioDescription':
+                        for f in stream_formats:
+                            f['source_preference'] = -10
+                    formats.extend(stream_formats)
 
                 if user_agent and len(entries) == playlist_len:
                     entries[num]['formats'].extend(formats)

From 3aa25395aa02b7a33e0fbf6d38e39fffee268255 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Feb 2017 00:08:56 +0700
Subject: [PATCH 0264/1696] [ChangeLog] Actualize

---
 ChangeLog | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 9242b3eee..912e1bbdc 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,21 @@
+version <unreleased>
+
+Core
++ [utils] Add support for quoted string literals in --match-filter (#8050,
+  #12142, #12144)
+
+Extractors
+* [ceskatelevize] Lower priority for audio description sources (#12119)
+* [amcnetworks] Fix extraction (#12127)
+* [pinkbike] Fix uploader extraction (#12054)
++ [onetpl] Add support for businessinsider.com.pl and plejada.pl
++ [onetpl] Add support for onet.pl (#10507)
++ [onetmvp] Add shortcut extractor
++ [vodpl] Add support for vod.pl (#12122)
++ [pornhub] Extract video URL from tv platform site (#12007, #12129)
++ [ceskatelevize] Extract DASH formats (#12119, #12133)
+
+
 version 2017.02.14
 
 Core

From 2480b056c137e514662b70053ec2df1391b6c2ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Feb 2017 00:10:04 +0700
Subject: [PATCH 0265/1696] release 2017.02.16

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 3 +++
 youtube_dl/version.py     | 2 +-
 4 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 32aa55d83..06711f73b 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.14*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.14**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.16*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.16**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.02.14
+[debug] youtube-dl version 2017.02.16
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 912e1bbdc..8ef8a8307 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.02.16
 
 Core
 + [utils] Add support for quoted string literals in --match-filter (#8050,
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 3e84f1237..5a436e8f7 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -546,8 +546,10 @@
  - **OktoberfestTV**
  - **on.aol.com**
  - **OnDemandKorea**
+ - **onet.pl**
  - **onet.tv**
  - **onet.tv:channel**
+ - **OnetMVP**
  - **OnionStudios**
  - **Ooyala**
  - **OoyalaExternal**
@@ -900,6 +902,7 @@
  - **vlive**
  - **vlive:channel**
  - **Vodlocker**
+ - **VODPl**
  - **VODPlatform**
  - **VoiceRepublic**
  - **VoxMedia**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 3e7e7c0bf..323e80954 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.02.14'
+__version__ = '2017.02.16'

From b898f0a173fa040ddf95dbd97650cec07a8f19f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Feb 2017 04:57:42 +0700
Subject: [PATCH 0266/1696] [elpais] Fix typo and improve extraction (closes
 #12139)

---
 youtube_dl/extractor/elpais.py | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/elpais.py b/youtube_dl/extractor/elpais.py
index 99e00cf3c..b89f6db62 100644
--- a/youtube_dl/extractor/elpais.py
+++ b/youtube_dl/extractor/elpais.py
@@ -39,6 +39,18 @@ class ElPaisIE(InfoExtractor):
             'description': 'La nave portaba cientos de ánforas y se hundió cerca de la isla de Cabrera por razones desconocidas',
             'upload_date': '20170127',
         },
+    }, {
+        'url': 'http://epv.elpais.com/epv/2017/02/14/programa_la_voz_de_inaki/1487062137_075943.html',
+        'info_dict': {
+            'id': '1487062137_075943',
+            'ext': 'mp4',
+            'title': 'Disyuntivas',
+            'description': 'md5:a0fb1485c4a6a8a917e6f93878e66218',
+            'upload_date': '20170214',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
@@ -59,14 +71,15 @@ class ElPaisIE(InfoExtractor):
         video_url = prefix + video_suffix
         thumbnail_suffix = self._search_regex(
             r"(?:URLMediaStill|urlFotogramaFijo_\d+)\s*=\s*url_cache\s*\+\s*'([^']+)'",
-            webpage, 'thumbnail URL', fatal=False)
+            webpage, 'thumbnail URL', default=None)
         thumbnail = (
             None if thumbnail_suffix is None
-            else prefix + thumbnail_suffix)
+            else prefix + thumbnail_suffix) or self._og_search_thumbnail(webpage)
         title = self._html_search_regex(
-            (r"tituloVideo\s*=\s*'([^']+)'", webpage, 'title',
-             r'<h2 class="entry-header entry-title.*?>(.*?)</h2>'),
-            webpage, 'title')
+            (r"tituloVideo\s*=\s*'([^']+)'",
+             r'<h2 class="entry-header entry-title.*?>(.*?)</h2>',
+             r'<h1[^>]+class="titulo"[^>]*>([^<]+)'),
+            webpage, 'title', default=None) or self._og_search_title(webpage)
         upload_date = unified_strdate(self._search_regex(
             r'<p class="date-header date-int updated"\s+title="([^"]+)">',
             webpage, 'upload date', default=None) or self._html_search_meta(

From a4a554a79354981fcab55de8eaab7b95a40bbb48 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 16 Feb 2017 23:42:36 +0800
Subject: [PATCH 0267/1696] [generic] Try parsing JWPlayer embedded videos
 (closes #12030)

---
 ChangeLog                                |   6 ++
 youtube_dl/extractor/archiveorg.py       |   4 +-
 youtube_dl/extractor/common.py           | 118 ++++++++++++++++++++
 youtube_dl/extractor/generic.py          |  20 ++++
 youtube_dl/extractor/jwplatform.py       | 132 +----------------------
 youtube_dl/extractor/ondemandkorea.py    |   4 +-
 youtube_dl/extractor/pornhub.py          |  44 --------
 youtube_dl/extractor/pornoxo.py          |   4 +-
 youtube_dl/extractor/rentv.py            |   3 +-
 youtube_dl/extractor/rudo.py             |   4 +-
 youtube_dl/extractor/screencastomatic.py |   4 +-
 youtube_dl/extractor/sendtonews.py       |   4 +-
 youtube_dl/extractor/thisav.py           |   4 +-
 youtube_dl/extractor/tvnoe.py            |   4 +-
 youtube_dl/extractor/vidzi.py            |   4 +-
 youtube_dl/extractor/wimp.py             |   4 +-
 16 files changed, 166 insertions(+), 197 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 8ef8a8307..4e69b03d0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
++ [generic] Support complex JWPlayer embedded videos (#12030)
+
+
 version 2017.02.16
 
 Core
diff --git a/youtube_dl/extractor/archiveorg.py b/youtube_dl/extractor/archiveorg.py
index 486dff82d..e21045bed 100644
--- a/youtube_dl/extractor/archiveorg.py
+++ b/youtube_dl/extractor/archiveorg.py
@@ -1,13 +1,13 @@
 from __future__ import unicode_literals
 
-from .jwplatform import JWPlatformBaseIE
+from .common import InfoExtractor
 from ..utils import (
     unified_strdate,
     clean_html,
 )
 
 
-class ArchiveOrgIE(JWPlatformBaseIE):
+class ArchiveOrgIE(InfoExtractor):
     IE_NAME = 'archive.org'
     IE_DESC = 'archive.org videos'
     _VALID_URL = r'https?://(?:www\.)?archive\.org/(?:details|embed)/(?P<id>[^/?#]+)(?:[?].*)?$'
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9681453ca..f6ff56eda 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -40,6 +40,7 @@ from ..utils import (
     fix_xml_ampersands,
     float_or_none,
     int_or_none,
+    js_to_json,
     parse_iso8601,
     RegexNotFoundError,
     sanitize_filename,
@@ -2073,6 +2074,123 @@ class InfoExtractor(object):
                     })
         return formats
 
+    @staticmethod
+    def _find_jwplayer_data(webpage):
+        mobj = re.search(
+            r'jwplayer\((?P<quote>[\'"])[^\'" ]+(?P=quote)\)\.setup\s*\((?P<options>[^)]+)\)',
+            webpage)
+        if mobj:
+            return mobj.group('options')
+
+    def _extract_jwplayer_data(self, webpage, video_id, *args, **kwargs):
+        jwplayer_data = self._parse_json(
+            self._find_jwplayer_data(webpage), video_id,
+            transform_source=js_to_json)
+        return self._parse_jwplayer_data(
+            jwplayer_data, video_id, *args, **kwargs)
+
+    def _parse_jwplayer_data(self, jwplayer_data, video_id=None, require_title=True,
+                             m3u8_id=None, mpd_id=None, rtmp_params=None, base_url=None):
+        # JWPlayer backward compatibility: flattened playlists
+        # https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/api/config.js#L81-L96
+        if 'playlist' not in jwplayer_data:
+            jwplayer_data = {'playlist': [jwplayer_data]}
+
+        entries = []
+
+        # JWPlayer backward compatibility: single playlist item
+        # https://github.com/jwplayer/jwplayer/blob/v7.7.0/src/js/playlist/playlist.js#L10
+        if not isinstance(jwplayer_data['playlist'], list):
+            jwplayer_data['playlist'] = [jwplayer_data['playlist']]
+
+        for video_data in jwplayer_data['playlist']:
+            # JWPlayer backward compatibility: flattened sources
+            # https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/playlist/item.js#L29-L35
+            if 'sources' not in video_data:
+                video_data['sources'] = [video_data]
+
+            this_video_id = video_id or video_data['mediaid']
+
+            formats = []
+            for source in video_data['sources']:
+                source_url = self._proto_relative_url(source['file'])
+                if base_url:
+                    source_url = compat_urlparse.urljoin(base_url, source_url)
+                source_type = source.get('type') or ''
+                ext = mimetype2ext(source_type) or determine_ext(source_url)
+                if source_type == 'hls' or ext == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        source_url, this_video_id, 'mp4', 'm3u8_native', m3u8_id=m3u8_id, fatal=False))
+                elif ext == 'mpd':
+                    formats.extend(self._extract_mpd_formats(
+                        source_url, this_video_id, mpd_id=mpd_id, fatal=False))
+                # https://github.com/jwplayer/jwplayer/blob/master/src/js/providers/default.js#L67
+                elif source_type.startswith('audio') or ext in ('oga', 'aac', 'mp3', 'mpeg', 'vorbis'):
+                    formats.append({
+                        'url': source_url,
+                        'vcodec': 'none',
+                        'ext': ext,
+                    })
+                else:
+                    height = int_or_none(source.get('height'))
+                    if height is None:
+                        # Often no height is provided but there is a label in
+                        # format like 1080p.
+                        height = int_or_none(self._search_regex(
+                            r'^(\d{3,})[pP]$', source.get('label') or '',
+                            'height', default=None))
+                    a_format = {
+                        'url': source_url,
+                        'width': int_or_none(source.get('width')),
+                        'height': height,
+                        'ext': ext,
+                    }
+                    if source_url.startswith('rtmp'):
+                        a_format['ext'] = 'flv'
+
+                        # See com/longtailvideo/jwplayer/media/RTMPMediaProvider.as
+                        # of jwplayer.flash.swf
+                        rtmp_url_parts = re.split(
+                            r'((?:mp4|mp3|flv):)', source_url, 1)
+                        if len(rtmp_url_parts) == 3:
+                            rtmp_url, prefix, play_path = rtmp_url_parts
+                            a_format.update({
+                                'url': rtmp_url,
+                                'play_path': prefix + play_path,
+                            })
+                        if rtmp_params:
+                            a_format.update(rtmp_params)
+                    formats.append(a_format)
+            self._sort_formats(formats)
+
+            subtitles = {}
+            tracks = video_data.get('tracks')
+            if tracks and isinstance(tracks, list):
+                for track in tracks:
+                    if track.get('kind') != 'captions':
+                        continue
+                    track_url = urljoin(base_url, track.get('file'))
+                    if not track_url:
+                        continue
+                    subtitles.setdefault(track.get('label') or 'en', []).append({
+                        'url': self._proto_relative_url(track_url)
+                    })
+
+            entries.append({
+                'id': this_video_id,
+                'title': video_data['title'] if require_title else video_data.get('title'),
+                'description': video_data.get('description'),
+                'thumbnail': self._proto_relative_url(video_data.get('image')),
+                'timestamp': int_or_none(video_data.get('pubdate')),
+                'duration': float_or_none(jwplayer_data.get('duration') or video_data.get('duration')),
+                'subtitles': subtitles,
+                'formats': formats,
+            })
+        if len(entries) == 1:
+            return entries[0]
+        else:
+            return self.playlist_result(entries)
+
     def _live_title(self, name):
         """ Generate the title for a live video """
         now = datetime.datetime.now()
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a2b0298ec..3db31debe 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -20,6 +20,7 @@ from ..utils import (
     float_or_none,
     HEADRequest,
     is_html,
+    js_to_json,
     orderedSet,
     sanitized_Request,
     smuggle_url,
@@ -961,6 +962,16 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             }
         },
+        # Complex jwplayer
+        {
+            'url': 'http://www.indiedb.com/games/king-machine/videos',
+            'info_dict': {
+                'id': 'videos',
+                'ext': 'mp4',
+                'title': 'king machine trailer 1',
+                'thumbnail': r're:^https?://.*\.jpg$',
+            },
+        },
         # rtl.nl embed
         {
             'url': 'http://www.rtlnieuws.nl/nieuws/buitenland/aanslagen-kopenhagen',
@@ -2488,6 +2499,15 @@ class GenericIE(InfoExtractor):
                 self._sort_formats(entry['formats'])
             return self.playlist_result(entries)
 
+        jwplayer_data_str = self._find_jwplayer_data(webpage)
+        if jwplayer_data_str:
+            try:
+                jwplayer_data = self._parse_json(
+                    jwplayer_data_str, video_id, transform_source=js_to_json)
+                return self._parse_jwplayer_data(jwplayer_data, video_id)
+            except ExtractorError:
+                pass
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True
diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index aff7ab49a..33d55f770 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -4,139 +4,9 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
-from ..utils import (
-    determine_ext,
-    float_or_none,
-    int_or_none,
-    js_to_json,
-    mimetype2ext,
-    urljoin,
-)
 
 
-class JWPlatformBaseIE(InfoExtractor):
-    @staticmethod
-    def _find_jwplayer_data(webpage):
-        # TODO: Merge this with JWPlayer-related codes in generic.py
-
-        mobj = re.search(
-            r'jwplayer\((?P<quote>[\'"])[^\'" ]+(?P=quote)\)\.setup\s*\((?P<options>[^)]+)\)',
-            webpage)
-        if mobj:
-            return mobj.group('options')
-
-    def _extract_jwplayer_data(self, webpage, video_id, *args, **kwargs):
-        jwplayer_data = self._parse_json(
-            self._find_jwplayer_data(webpage), video_id,
-            transform_source=js_to_json)
-        return self._parse_jwplayer_data(
-            jwplayer_data, video_id, *args, **kwargs)
-
-    def _parse_jwplayer_data(self, jwplayer_data, video_id=None, require_title=True,
-                             m3u8_id=None, mpd_id=None, rtmp_params=None, base_url=None):
-        # JWPlayer backward compatibility: flattened playlists
-        # https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/api/config.js#L81-L96
-        if 'playlist' not in jwplayer_data:
-            jwplayer_data = {'playlist': [jwplayer_data]}
-
-        entries = []
-
-        # JWPlayer backward compatibility: single playlist item
-        # https://github.com/jwplayer/jwplayer/blob/v7.7.0/src/js/playlist/playlist.js#L10
-        if not isinstance(jwplayer_data['playlist'], list):
-            jwplayer_data['playlist'] = [jwplayer_data['playlist']]
-
-        for video_data in jwplayer_data['playlist']:
-            # JWPlayer backward compatibility: flattened sources
-            # https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/playlist/item.js#L29-L35
-            if 'sources' not in video_data:
-                video_data['sources'] = [video_data]
-
-            this_video_id = video_id or video_data['mediaid']
-
-            formats = []
-            for source in video_data['sources']:
-                source_url = self._proto_relative_url(source['file'])
-                if base_url:
-                    source_url = compat_urlparse.urljoin(base_url, source_url)
-                source_type = source.get('type') or ''
-                ext = mimetype2ext(source_type) or determine_ext(source_url)
-                if source_type == 'hls' or ext == 'm3u8':
-                    formats.extend(self._extract_m3u8_formats(
-                        source_url, this_video_id, 'mp4', 'm3u8_native', m3u8_id=m3u8_id, fatal=False))
-                elif ext == 'mpd':
-                    formats.extend(self._extract_mpd_formats(
-                        source_url, this_video_id, mpd_id=mpd_id, fatal=False))
-                # https://github.com/jwplayer/jwplayer/blob/master/src/js/providers/default.js#L67
-                elif source_type.startswith('audio') or ext in ('oga', 'aac', 'mp3', 'mpeg', 'vorbis'):
-                    formats.append({
-                        'url': source_url,
-                        'vcodec': 'none',
-                        'ext': ext,
-                    })
-                else:
-                    height = int_or_none(source.get('height'))
-                    if height is None:
-                        # Often no height is provided but there is a label in
-                        # format like 1080p.
-                        height = int_or_none(self._search_regex(
-                            r'^(\d{3,})[pP]$', source.get('label') or '',
-                            'height', default=None))
-                    a_format = {
-                        'url': source_url,
-                        'width': int_or_none(source.get('width')),
-                        'height': height,
-                        'ext': ext,
-                    }
-                    if source_url.startswith('rtmp'):
-                        a_format['ext'] = 'flv'
-
-                        # See com/longtailvideo/jwplayer/media/RTMPMediaProvider.as
-                        # of jwplayer.flash.swf
-                        rtmp_url_parts = re.split(
-                            r'((?:mp4|mp3|flv):)', source_url, 1)
-                        if len(rtmp_url_parts) == 3:
-                            rtmp_url, prefix, play_path = rtmp_url_parts
-                            a_format.update({
-                                'url': rtmp_url,
-                                'play_path': prefix + play_path,
-                            })
-                        if rtmp_params:
-                            a_format.update(rtmp_params)
-                    formats.append(a_format)
-            self._sort_formats(formats)
-
-            subtitles = {}
-            tracks = video_data.get('tracks')
-            if tracks and isinstance(tracks, list):
-                for track in tracks:
-                    if track.get('kind') != 'captions':
-                        continue
-                    track_url = urljoin(base_url, track.get('file'))
-                    if not track_url:
-                        continue
-                    subtitles.setdefault(track.get('label') or 'en', []).append({
-                        'url': self._proto_relative_url(track_url)
-                    })
-
-            entries.append({
-                'id': this_video_id,
-                'title': video_data['title'] if require_title else video_data.get('title'),
-                'description': video_data.get('description'),
-                'thumbnail': self._proto_relative_url(video_data.get('image')),
-                'timestamp': int_or_none(video_data.get('pubdate')),
-                'duration': float_or_none(jwplayer_data.get('duration') or video_data.get('duration')),
-                'subtitles': subtitles,
-                'formats': formats,
-            })
-        if len(entries) == 1:
-            return entries[0]
-        else:
-            return self.playlist_result(entries)
-
-
-class JWPlatformIE(JWPlatformBaseIE):
+class JWPlatformIE(InfoExtractor):
     _VALID_URL = r'(?:https?://content\.jwplatform\.com/(?:feeds|players|jw6)/|jwplatform:)(?P<id>[a-zA-Z0-9]{8})'
     _TEST = {
         'url': 'http://content.jwplatform.com/players/nPripu9l-ALJ3XQCI.js',
diff --git a/youtube_dl/extractor/ondemandkorea.py b/youtube_dl/extractor/ondemandkorea.py
index de1d6b08a..dcd157777 100644
--- a/youtube_dl/extractor/ondemandkorea.py
+++ b/youtube_dl/extractor/ondemandkorea.py
@@ -1,14 +1,14 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .jwplatform import JWPlatformBaseIE
+from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     js_to_json,
 )
 
 
-class OnDemandKoreaIE(JWPlatformBaseIE):
+class OnDemandKoreaIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?ondemandkorea\.com/(?P<id>[^/]+)\.html'
     _TEST = {
         'url': 'http://www.ondemandkorea.com/ask-us-anything-e43.html',
diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 7a2737032..9b413590a 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -169,50 +169,6 @@ class PornHubIE(InfoExtractor):
         comment_count = self._extract_count(
             r'All Comments\s*<span>\(([\d,.]+)\)', webpage, 'comment')
 
-        """
-        video_variables = {}
-        for video_variablename, quote, video_variable in re.findall(
-                r'(player_quality_[0-9]{3,4}p\w+)\s*=\s*(["\'])(.+?)\2;', webpage):
-            video_variables[video_variablename] = video_variable
-
-        video_urls = []
-        for encoded_video_url in re.findall(
-                r'player_quality_[0-9]{3,4}p\s*=(.+?);', webpage):
-            for varname, varval in video_variables.items():
-                encoded_video_url = encoded_video_url.replace(varname, varval)
-            video_urls.append(re.sub(r'[\s+]', '', encoded_video_url))
-
-        if webpage.find('"encrypted":true') != -1:
-            password = compat_urllib_parse_unquote_plus(
-                self._search_regex(r'"video_title":"([^"]+)', webpage, 'password'))
-            video_urls = list(map(lambda s: aes_decrypt_text(s, password, 32).decode('utf-8'), video_urls))
-
-        formats = []
-        for video_url in video_urls:
-            path = compat_urllib_parse_urlparse(video_url).path
-            extension = os.path.splitext(path)[1][1:]
-            format = path.split('/')[5].split('_')[:2]
-            format = '-'.join(format)
-
-            m = re.match(r'^(?P<height>[0-9]+)[pP]-(?P<tbr>[0-9]+)[kK]$', format)
-            if m is None:
-                height = None
-                tbr = None
-            else:
-                height = int(m.group('height'))
-                tbr = int(m.group('tbr'))
-
-            formats.append({
-                'url': video_url,
-                'ext': extension,
-                'format': format,
-                'format_id': format,
-                'tbr': tbr,
-                'height': height,
-            })
-        self._sort_formats(formats)
-        """
-
         page_params = self._parse_json(self._search_regex(
             r'page_params\.zoneDetails\[([\'"])[^\'"]+\1\]\s*=\s*(?P<data>{[^}]+})',
             webpage, 'page parameters', group='data', default='{}'),
diff --git a/youtube_dl/extractor/pornoxo.py b/youtube_dl/extractor/pornoxo.py
index 1a0cce7e0..2831368b6 100644
--- a/youtube_dl/extractor/pornoxo.py
+++ b/youtube_dl/extractor/pornoxo.py
@@ -2,13 +2,13 @@ from __future__ import unicode_literals
 
 import re
 
-from .jwplatform import JWPlatformBaseIE
+from .common import InfoExtractor
 from ..utils import (
     str_to_int,
 )
 
 
-class PornoXOIE(JWPlatformBaseIE):
+class PornoXOIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?pornoxo\.com/videos/(?P<id>\d+)/(?P<display_id>[^/]+)\.html'
     _TEST = {
         'url': 'http://www.pornoxo.com/videos/7564/striptease-from-sexy-secretary.html',
diff --git a/youtube_dl/extractor/rentv.py b/youtube_dl/extractor/rentv.py
index 422c02cff..d338b3a93 100644
--- a/youtube_dl/extractor/rentv.py
+++ b/youtube_dl/extractor/rentv.py
@@ -2,11 +2,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from .jwplatform import JWPlatformBaseIE
 from ..compat import compat_str
 
 
-class RENTVIE(JWPlatformBaseIE):
+class RENTVIE(InfoExtractor):
     _VALID_URL = r'(?:rentv:|https?://(?:www\.)?ren\.tv/(?:player|video/epizod)/)(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://ren.tv/video/epizod/118577',
diff --git a/youtube_dl/extractor/rudo.py b/youtube_dl/extractor/rudo.py
index 3bfe934d8..51644011e 100644
--- a/youtube_dl/extractor/rudo.py
+++ b/youtube_dl/extractor/rudo.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 
 import re
 
-from .jwplatform import JWPlatformBaseIE
+from .common import InfoExtractor
 from ..utils import (
     js_to_json,
     get_element_by_class,
@@ -11,7 +11,7 @@ from ..utils import (
 )
 
 
-class RudoIE(JWPlatformBaseIE):
+class RudoIE(InfoExtractor):
     _VALID_URL = r'https?://rudo\.video/vod/(?P<id>[0-9a-zA-Z]+)'
 
     _TEST = {
diff --git a/youtube_dl/extractor/screencastomatic.py b/youtube_dl/extractor/screencastomatic.py
index 94a2a37d2..b5e76c9af 100644
--- a/youtube_dl/extractor/screencastomatic.py
+++ b/youtube_dl/extractor/screencastomatic.py
@@ -1,11 +1,11 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .jwplatform import JWPlatformBaseIE
+from .common import InfoExtractor
 from ..utils import js_to_json
 
 
-class ScreencastOMaticIE(JWPlatformBaseIE):
+class ScreencastOMaticIE(InfoExtractor):
     _VALID_URL = r'https?://screencast-o-matic\.com/watch/(?P<id>[0-9a-zA-Z]+)'
     _TEST = {
         'url': 'http://screencast-o-matic.com/watch/c2lD3BeOPl',
diff --git a/youtube_dl/extractor/sendtonews.py b/youtube_dl/extractor/sendtonews.py
index 9880a5a78..9d9652949 100644
--- a/youtube_dl/extractor/sendtonews.py
+++ b/youtube_dl/extractor/sendtonews.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 
 import re
 
-from .jwplatform import JWPlatformBaseIE
+from .common import InfoExtractor
 from ..utils import (
     float_or_none,
     parse_iso8601,
@@ -14,7 +14,7 @@ from ..utils import (
 )
 
 
-class SendtoNewsIE(JWPlatformBaseIE):
+class SendtoNewsIE(InfoExtractor):
     _VALID_URL = r'https?://embed\.sendtonews\.com/player2/embedplayer\.php\?.*\bSC=(?P<id>[0-9A-Za-z-]+)'
 
     _TEST = {
diff --git a/youtube_dl/extractor/thisav.py b/youtube_dl/extractor/thisav.py
index 4473a3c77..b7b3568cb 100644
--- a/youtube_dl/extractor/thisav.py
+++ b/youtube_dl/extractor/thisav.py
@@ -3,11 +3,11 @@ from __future__ import unicode_literals
 
 import re
 
-from .jwplatform import JWPlatformBaseIE
+from .common import InfoExtractor
 from ..utils import remove_end
 
 
-class ThisAVIE(JWPlatformBaseIE):
+class ThisAVIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?thisav\.com/video/(?P<id>[0-9]+)/.*'
     _TESTS = [{
         'url': 'http://www.thisav.com/video/47734/%98%26sup1%3B%83%9E%83%82---just-fit.html',
diff --git a/youtube_dl/extractor/tvnoe.py b/youtube_dl/extractor/tvnoe.py
index 6d5c74826..1a5b76bf2 100644
--- a/youtube_dl/extractor/tvnoe.py
+++ b/youtube_dl/extractor/tvnoe.py
@@ -1,7 +1,7 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .jwplatform import JWPlatformBaseIE
+from .common import InfoExtractor
 from ..utils import (
     clean_html,
     get_element_by_class,
@@ -9,7 +9,7 @@ from ..utils import (
 )
 
 
-class TVNoeIE(JWPlatformBaseIE):
+class TVNoeIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?tvnoe\.cz/video/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.tvnoe.cz/video/10362',
diff --git a/youtube_dl/extractor/vidzi.py b/youtube_dl/extractor/vidzi.py
index 9950c62ad..1f1828fce 100644
--- a/youtube_dl/extractor/vidzi.py
+++ b/youtube_dl/extractor/vidzi.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 
 import re
 
-from .jwplatform import JWPlatformBaseIE
+from .common import InfoExtractor
 from ..utils import (
     decode_packed_codes,
     js_to_json,
@@ -12,7 +12,7 @@ from ..utils import (
 )
 
 
-class VidziIE(JWPlatformBaseIE):
+class VidziIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?vidzi\.tv/(?:embed-)?(?P<id>[0-9a-zA-Z]+)'
     _TESTS = [{
         'url': 'http://vidzi.tv/cghql9yq6emu.html',
diff --git a/youtube_dl/extractor/wimp.py b/youtube_dl/extractor/wimp.py
index 54eb51427..c022fb33e 100644
--- a/youtube_dl/extractor/wimp.py
+++ b/youtube_dl/extractor/wimp.py
@@ -1,10 +1,10 @@
 from __future__ import unicode_literals
 
+from .common import InfoExtractor
 from .youtube import YoutubeIE
-from .jwplatform import JWPlatformBaseIE
 
 
-class WimpIE(JWPlatformBaseIE):
+class WimpIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?wimp\.com/(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'http://www.wimp.com/maru-is-exhausted/',

From 4cead6a614b5a293e78dce5cd5eda7476f83985d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Feb 2017 22:02:01 +0700
Subject: [PATCH 0268/1696] [einthusan] Relax _VALID_URL (closes #12141, closes
 #12159)

---
 youtube_dl/extractor/einthusan.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/einthusan.py b/youtube_dl/extractor/einthusan.py
index 8a2a17b63..3f6268637 100644
--- a/youtube_dl/extractor/einthusan.py
+++ b/youtube_dl/extractor/einthusan.py
@@ -18,8 +18,8 @@ from ..utils import (
 
 
 class EinthusanIE(InfoExtractor):
-    _VALID_URL = r'https?://einthusan\.tv/movie/watch/(?P<id>[0-9]+)'
-    _TEST = {
+    _VALID_URL = r'https?://einthusan\.tv/movie/watch/(?P<id>[^/?#&]+)'
+    _TESTS = [{
         'url': 'https://einthusan.tv/movie/watch/9097/',
         'md5': 'ff0f7f2065031b8a2cf13a933731c035',
         'info_dict': {
@@ -29,7 +29,10 @@ class EinthusanIE(InfoExtractor):
             'description': 'md5:33ef934c82a671a94652a9b4e54d931b',
             'thumbnail': r're:^https?://.*\.jpg$',
         }
-    }
+    }, {
+        'url': 'https://einthusan.tv/movie/watch/51MZ/?lang=hindi',
+        'only_matching': True,
+    }]
 
     # reversed from jsoncrypto.prototype.decrypt() in einthusan-PGMovieWatcher.js
     def _decrypt(self, encrypted_data, video_id):

From fef51645d6c224f898ff6f44d041a458d21e8547 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 17 Feb 2017 23:13:51 +0800
Subject: [PATCH 0269/1696] [theplatform] Recognize URLs with whitespaces
 (closes #12044)

---
 ChangeLog                           | 1 +
 youtube_dl/extractor/generic.py     | 7 ++++++-
 youtube_dl/extractor/theplatform.py | 6 ++++--
 3 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 4e69b03d0..d5fe3dd5b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
++ [theplatform] Recognize URLs with whitespaces (#12044)
 + [generic] Support complex JWPlayer embedded videos (#12030)
 
 
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 3db31debe..9868ca6d0 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1501,7 +1501,12 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
             'add_ie': [VideoPressIE.ie_key()],
-        }
+        },
+        {
+            # ThePlatform embedded with whitespaces in URLs
+            'url': 'http://www.golfchannel.com/topics/shows/golftalkcentral.htm',
+            'only_matching': True,
+        },
         # {
         #     # TODO: find another test
         #     # http://schema.org/VideoObject
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 5c5987c6a..9a424b1c6 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -179,10 +179,12 @@ class ThePlatformIE(ThePlatformBaseIE, AdobePassIE):
         if m:
             return [m.group('url')]
 
+        # Are whitesapces ignored in URLs?
+        # https://github.com/rg3/youtube-dl/issues/12044
         matches = re.findall(
-            r'<(?:iframe|script)[^>]+src=(["\'])((?:https?:)?//player\.theplatform\.com/p/.+?)\1', webpage)
+            r'(?s)<(?:iframe|script)[^>]+src=(["\'])((?:https?:)?//player\.theplatform\.com/p/.+?)\1', webpage)
         if matches:
-            return list(zip(*matches))[1]
+            return [re.sub(r'\s', '', list(zip(*matches))[1][0])]
 
     @staticmethod
     def _sign_url(url, sig_key, sig_secret, life=600, include_qs=False):

From d94badc755228ee3159b9b499aa718d27fa472ed Mon Sep 17 00:00:00 2001
From: Vijay Singh <sudovijay@users.noreply.github.com>
Date: Tue, 7 Feb 2017 10:32:45 +0530
Subject: [PATCH 0270/1696] [openload] Semifix extraction (closes #10408)

just updated the code. i don't do much python still i tried to convert my code. lemme know if there is any prob with it
---
 youtube_dl/extractor/openload.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 32289d897..bd1120fd8 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -75,17 +75,20 @@ class OpenloadIE(InfoExtractor):
             '<span[^>]+id="[^"]+"[^>]*>([0-9]+)</span>',
             webpage, 'openload ID')
 
-        first_three_chars = int(float(ol_id[0:][:3]))
-        fifth_char = int(float(ol_id[3:5]))
-        urlcode = ''
-        num = 5
+        first_two_chars = int(float(ol_id[0:][:2]))
+        urlcode = {}
+        num = 2
 
         while num < len(ol_id):
-            urlcode += compat_chr(int(float(ol_id[num:][:3])) +
-                                  first_three_chars - fifth_char * int(float(ol_id[num + 3:][:2])))
+            key = int(float(ol_id[num + 3:][:2]))
+            urlcode[key] = compat_chr(int(float(ol_id[num:][:3])) - first_two_chars)
             num += 5
+        
+        sorted(urlcode, key=lambda key: urlcode[key])  
 
-        video_url = 'https://openload.co/stream/' + urlcode
+        urllink = ''.join(['%s' % (value) for (key, value) in urlcode.items()])
+
+        video_url = 'https://openload.co/stream/' + urllink
 
         title = self._og_search_title(webpage, default=None) or self._search_regex(
             r'<span[^>]+class=["\']title["\'][^>]*>([^<]+)', webpage,

From 90fad0e74cd8079246c5f3d8150650b5f65f998b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Feb 2017 22:31:16 +0700
Subject: [PATCH 0271/1696] [openload] Fix extraction (closes #12002)

---
 youtube_dl/extractor/openload.py | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index bd1120fd8..10896c442 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -76,19 +76,16 @@ class OpenloadIE(InfoExtractor):
             webpage, 'openload ID')
 
         first_two_chars = int(float(ol_id[0:][:2]))
-        urlcode = {}
+        urlcode = []
         num = 2
 
         while num < len(ol_id):
             key = int(float(ol_id[num + 3:][:2]))
-            urlcode[key] = compat_chr(int(float(ol_id[num:][:3])) - first_two_chars)
+            urlcode.append((key, compat_chr(int(float(ol_id[num:][:3])) - first_two_chars)))
             num += 5
-        
-        sorted(urlcode, key=lambda key: urlcode[key])  
 
-        urllink = ''.join(['%s' % (value) for (key, value) in urlcode.items()])
-
-        video_url = 'https://openload.co/stream/' + urllink
+        video_url = 'https://openload.co/stream/' + ''.join(
+            [value for _, value in sorted(urlcode, key=lambda x: x[0])])
 
         title = self._og_search_title(webpage, default=None) or self._search_regex(
             r'<span[^>]+class=["\']title["\'][^>]*>([^<]+)', webpage,

From c2bde5d08163ce46548ea60333750a0a74a8fe44 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?D=C3=A9stin=20Reed?= <TRox1972@noreply.github.com>
Date: Mon, 9 Jan 2017 18:22:53 +0100
Subject: [PATCH 0272/1696] [ellentv] Improve

---
 youtube_dl/extractor/ellentv.py | 23 ++++++++---------------
 1 file changed, 8 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py
index 74bbc5c51..e0a13dd76 100644
--- a/youtube_dl/extractor/ellentv.py
+++ b/youtube_dl/extractor/ellentv.py
@@ -1,13 +1,9 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import json
-
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    NO_DEFAULT,
-)
+from .kaltura import KalturaIE
+from ..utils import NO_DEFAULT
 
 
 class EllenTVIE(InfoExtractor):
@@ -65,7 +61,7 @@ class EllenTVIE(InfoExtractor):
             if partner_id and kaltura_id:
                 break
 
-        return self.url_result('kaltura:%s:%s' % (partner_id, kaltura_id), 'Kaltura')
+        return self.url_result('kaltura:%s:%s' % (partner_id, kaltura_id), KalturaIE.ie_key())
 
 
 class EllenTVClipsIE(InfoExtractor):
@@ -77,14 +73,14 @@ class EllenTVClipsIE(InfoExtractor):
             'id': 'meryl-streep-vanessa-hudgens',
             'title': 'Meryl Streep, Vanessa Hudgens',
         },
-        'playlist_mincount': 7,
+        'playlist_mincount': 5,
     }
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
 
         webpage = self._download_webpage(url, playlist_id)
-        playlist = self._extract_playlist(webpage)
+        playlist = self._extract_playlist(webpage, playlist_id)
 
         return {
             '_type': 'playlist',
@@ -93,16 +89,13 @@ class EllenTVClipsIE(InfoExtractor):
             'entries': self._extract_entries(playlist)
         }
 
-    def _extract_playlist(self, webpage):
+    def _extract_playlist(self, webpage, playlist_id):
         json_string = self._search_regex(r'playerView.addClips\(\[\{(.*?)\}\]\);', webpage, 'json')
-        try:
-            return json.loads('[{' + json_string + '}]')
-        except ValueError as ve:
-            raise ExtractorError('Failed to download JSON', cause=ve)
+        return self._parse_json('[{' + json_string + '}]', playlist_id)
 
     def _extract_entries(self, playlist):
         return [
             self.url_result(
                 'kaltura:%s:%s' % (item['kaltura_partner_id'], item['kaltura_entry_id']),
-                'Kaltura')
+                KalturaIE.ie_key(), video_id=item['kaltura_entry_id'])
             for item in playlist]

From db76c30c6ecb5d198a72f1807163c9b69771bba1 Mon Sep 17 00:00:00 2001
From: Tobias Gruetzmacher <tobias-git@23.gs>
Date: Tue, 7 Jun 2016 23:42:56 +0200
Subject: [PATCH 0273/1696] [heise] Support videos embedded in any article.

---
 youtube_dl/extractor/heise.py | 119 +++++++++++++++++++++++++---------
 1 file changed, 90 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/heise.py b/youtube_dl/extractor/heise.py
index 1629cdb8d..a5ec0fae9 100644
--- a/youtube_dl/extractor/heise.py
+++ b/youtube_dl/extractor/heise.py
@@ -11,54 +11,115 @@ from ..utils import (
 
 class HeiseIE(InfoExtractor):
     _VALID_URL = r'''(?x)
-        https?://(?:www\.)?heise\.de/video/artikel/
-        .+?(?P<id>[0-9]+)\.html(?:$|[?#])
+        https?://(?:www\.)?heise\.de/.+?(?P<id>[0-9]+)\.html(?:$|[?#])
     '''
-    _TEST = {
-        'url': (
-            'http://www.heise.de/video/artikel/Podcast-c-t-uplink-3-3-Owncloud-Tastaturen-Peilsender-Smartphone-2404147.html'
-        ),
-        'md5': 'ffed432483e922e88545ad9f2f15d30e',
-        'info_dict': {
-            'id': '2404147',
-            'ext': 'mp4',
-            'title': (
-                "Podcast: c't uplink 3.3 – Owncloud / Tastaturen / Peilsender Smartphone"
+    _TESTS = [
+        {
+            'url': (
+                'http://www.heise.de/video/artikel/Podcast-c-t-uplink-3-3-Owncloud-Tastaturen-Peilsender-Smartphone-2404147.html'
             ),
-            'format_id': 'mp4_720p',
-            'timestamp': 1411812600,
-            'upload_date': '20140927',
-            'description': 'In uplink-Episode 3.3 geht es darum, wie man sich von Cloud-Anbietern emanzipieren kann, worauf man beim Kauf einer Tastatur achten sollte und was Smartphones über uns verraten.',
-            'thumbnail': r're:^https?://.*\.jpe?g$',
-        }
-    }
+            'md5': 'ffed432483e922e88545ad9f2f15d30e',
+            'info_dict': {
+                'id': '2404147',
+                'ext': 'mp4',
+                'title': (
+                    "Podcast: c't uplink 3.3 – Owncloud / Tastaturen / Peilsender Smartphone"
+                ),
+                'format_id': 'mp4_720p',
+                'timestamp': 1411812600,
+                'upload_date': '20140927',
+                'description': 'In uplink-Episode 3.3 geht es darum, wie man sich von Cloud-Anbietern emanzipieren kann, worauf man beim Kauf einer Tastatur achten sollte und was Smartphones über uns verraten.',
+                'thumbnail': r're:^https?://.*/gallery/$',
+            }
+        },
+        {
+            'url': (
+                'http://www.heise.de/ct/artikel/c-t-uplink-3-3-Owncloud-Tastaturen-Peilsender-Smartphone-2403911.html'
+            ),
+            'md5': 'ffed432483e922e88545ad9f2f15d30e',
+            'info_dict': {
+                'id': '2403911',
+                'ext': 'mp4',
+                'title': (
+                    "c't uplink 3.3: Owncloud, Tastaturen, Peilsender Smartphone"
+                ),
+                'format_id': 'mp4_720p',
+                'timestamp': 1411803000,
+                'upload_date': '20140927',
+                'description': "In c't uplink erklären wir in dieser Woche, wie man mit Owncloud die Kontrolle über die eigenen Daten behält. Darüber hinaus erklären wir, dass zur Wahl der richtigen Tastatur mehr gehört, als man denkt und wie Smartphones uns weiter verraten.",
+                'thumbnail': r're:^https?://.*/gallery/$',
+            }
+        },
+        {
+            'url': (
+                'http://www.heise.de/newsticker/meldung/c-t-uplink-Owncloud-Tastaturen-Peilsender-Smartphone-2404251.html?wt_mc=rss.ho.beitrag.atom'
+            ),
+            'md5': 'ffed432483e922e88545ad9f2f15d30e',
+            'info_dict': {
+                'id': '2404251',
+                'ext': 'mp4',
+                'title': (
+                    "c't uplink: Owncloud, Tastaturen, Peilsender Smartphone"
+                ),
+                'format_id': 'mp4_720p',
+                'timestamp': 1411811400,
+                'upload_date': '20140927',
+                'description': 'In uplink-Episode 3.3 sprechen wir über Owncloud und wie man sich damit von Cloudanbietern emanzipieren kann. Außerdem erklären wir, woran man alles beim Kauf einer Tastatur denken sollte und was Smartphones nun über uns verraten.',
+                'thumbnail': r're:^https?://.*/gallery/$',
+            }
+        },
+        {
+            'url': (
+                'http://www.heise.de/ct/ausgabe/2016-12-Spiele-3214137.html'
+            ),
+            'md5': '0616c9297d9c989f9b2a23b483b408c3',
+            'info_dict': {
+                'id': '3214137',
+                'ext': 'mp4',
+                'title': (
+                    "c\u2019t zockt \u201eGlitchspace\u201c, \u201eThe Mind's Eclipse\u201c und \u201eWindowframe\u201c."
+                ),
+                'format_id': 'mp4_720p',
+                'timestamp': 1464011220,
+                'upload_date': '20160523',
+                'description': "Unsere Spiele-Tipps der Woche: Das Puzzle-Adventure Glitchspace, das Jump&Run-Spiel Windowframe und The Mind's Eclipse",
+                'thumbnail': r're:^https?://.*/gallery/$',
+            }
+        },
+
+    ]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         container_id = self._search_regex(
-            r'<div class="videoplayerjw".*?data-container="([0-9]+)"',
+            r'<div class="videoplayerjw"[^>]*data-container="([0-9]+)"',
             webpage, 'container ID')
         sequenz_id = self._search_regex(
-            r'<div class="videoplayerjw".*?data-sequenz="([0-9]+)"',
+            r'<div class="videoplayerjw"[^>]*data-sequenz="([0-9]+)"',
             webpage, 'sequenz ID')
         data_url = 'http://www.heise.de/videout/feed?container=%s&sequenz=%s' % (container_id, sequenz_id)
         doc = self._download_xml(data_url, video_id)
 
         info = {
             'id': video_id,
-            'thumbnail': self._og_search_thumbnail(webpage),
+            'thumbnail': doc.find('.//{http://rss.jwpcdn.com/}image').text,
             'timestamp': parse_iso8601(
-                self._html_search_meta('date', webpage)),
-            'description': self._og_search_description(webpage),
+                self._html_search_meta('date', webpage))
         }
 
-        title = self._html_search_meta('fulltitle', webpage)
-        if title:
-            info['title'] = title
-        else:
-            info['title'] = self._og_search_title(webpage)
+        title = self._html_search_meta('fulltitle', webpage, default=None)
+        if not title or title == "c't":
+            title = self._search_regex(
+                r'<div class="videoplayerjw"[^>]*data-title="([^"]+)"',
+                webpage, 'video title')
+        info['title'] = title
+
+        desc = self._og_search_description(webpage, default=None)
+        if not desc:
+            desc = self._html_search_meta('description', webpage)
+        info['description'] = desc
 
         formats = []
         for source_node in doc.findall('.//{http://rss.jwpcdn.com/}source'):

From bad4ccdb5db7c00865d433558ddfcdfdbd499343 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Feb 2017 23:09:40 +0700
Subject: [PATCH 0274/1696] [heise] Improve (closes #9725)

---
 youtube_dl/extractor/heise.py | 146 +++++++++++-----------------------
 1 file changed, 48 insertions(+), 98 deletions(-)

diff --git a/youtube_dl/extractor/heise.py b/youtube_dl/extractor/heise.py
index a5ec0fae9..382f32771 100644
--- a/youtube_dl/extractor/heise.py
+++ b/youtube_dl/extractor/heise.py
@@ -6,120 +6,58 @@ from ..utils import (
     determine_ext,
     int_or_none,
     parse_iso8601,
+    xpath_text,
 )
 
 
 class HeiseIE(InfoExtractor):
-    _VALID_URL = r'''(?x)
-        https?://(?:www\.)?heise\.de/.+?(?P<id>[0-9]+)\.html(?:$|[?#])
-    '''
-    _TESTS = [
-        {
-            'url': (
-                'http://www.heise.de/video/artikel/Podcast-c-t-uplink-3-3-Owncloud-Tastaturen-Peilsender-Smartphone-2404147.html'
-            ),
-            'md5': 'ffed432483e922e88545ad9f2f15d30e',
-            'info_dict': {
-                'id': '2404147',
-                'ext': 'mp4',
-                'title': (
-                    "Podcast: c't uplink 3.3 – Owncloud / Tastaturen / Peilsender Smartphone"
-                ),
-                'format_id': 'mp4_720p',
-                'timestamp': 1411812600,
-                'upload_date': '20140927',
-                'description': 'In uplink-Episode 3.3 geht es darum, wie man sich von Cloud-Anbietern emanzipieren kann, worauf man beim Kauf einer Tastatur achten sollte und was Smartphones über uns verraten.',
-                'thumbnail': r're:^https?://.*/gallery/$',
-            }
-        },
-        {
-            'url': (
-                'http://www.heise.de/ct/artikel/c-t-uplink-3-3-Owncloud-Tastaturen-Peilsender-Smartphone-2403911.html'
-            ),
-            'md5': 'ffed432483e922e88545ad9f2f15d30e',
-            'info_dict': {
-                'id': '2403911',
-                'ext': 'mp4',
-                'title': (
-                    "c't uplink 3.3: Owncloud, Tastaturen, Peilsender Smartphone"
-                ),
-                'format_id': 'mp4_720p',
-                'timestamp': 1411803000,
-                'upload_date': '20140927',
-                'description': "In c't uplink erklären wir in dieser Woche, wie man mit Owncloud die Kontrolle über die eigenen Daten behält. Darüber hinaus erklären wir, dass zur Wahl der richtigen Tastatur mehr gehört, als man denkt und wie Smartphones uns weiter verraten.",
-                'thumbnail': r're:^https?://.*/gallery/$',
-            }
-        },
-        {
-            'url': (
-                'http://www.heise.de/newsticker/meldung/c-t-uplink-Owncloud-Tastaturen-Peilsender-Smartphone-2404251.html?wt_mc=rss.ho.beitrag.atom'
-            ),
-            'md5': 'ffed432483e922e88545ad9f2f15d30e',
-            'info_dict': {
-                'id': '2404251',
-                'ext': 'mp4',
-                'title': (
-                    "c't uplink: Owncloud, Tastaturen, Peilsender Smartphone"
-                ),
-                'format_id': 'mp4_720p',
-                'timestamp': 1411811400,
-                'upload_date': '20140927',
-                'description': 'In uplink-Episode 3.3 sprechen wir über Owncloud und wie man sich damit von Cloudanbietern emanzipieren kann. Außerdem erklären wir, woran man alles beim Kauf einer Tastatur denken sollte und was Smartphones nun über uns verraten.',
-                'thumbnail': r're:^https?://.*/gallery/$',
-            }
-        },
-        {
-            'url': (
-                'http://www.heise.de/ct/ausgabe/2016-12-Spiele-3214137.html'
-            ),
-            'md5': '0616c9297d9c989f9b2a23b483b408c3',
-            'info_dict': {
-                'id': '3214137',
-                'ext': 'mp4',
-                'title': (
-                    "c\u2019t zockt \u201eGlitchspace\u201c, \u201eThe Mind's Eclipse\u201c und \u201eWindowframe\u201c."
-                ),
-                'format_id': 'mp4_720p',
-                'timestamp': 1464011220,
-                'upload_date': '20160523',
-                'description': "Unsere Spiele-Tipps der Woche: Das Puzzle-Adventure Glitchspace, das Jump&Run-Spiel Windowframe und The Mind's Eclipse",
-                'thumbnail': r're:^https?://.*/gallery/$',
-            }
-        },
-
-    ]
+    _VALID_URL = r'https?://(?:www\.)?heise\.de/(?:[^/]+/)+[^/]+-(?P<id>[0-9]+)\.html'
+    _TESTS = [{
+        'url': 'http://www.heise.de/video/artikel/Podcast-c-t-uplink-3-3-Owncloud-Tastaturen-Peilsender-Smartphone-2404147.html',
+        'md5': 'ffed432483e922e88545ad9f2f15d30e',
+        'info_dict': {
+            'id': '2404147',
+            'ext': 'mp4',
+            'title': "Podcast: c't uplink 3.3 – Owncloud / Tastaturen / Peilsender Smartphone",
+            'format_id': 'mp4_720p',
+            'timestamp': 1411812600,
+            'upload_date': '20140927',
+            'description': 'md5:c934cbfb326c669c2bcabcbe3d3fcd20',
+            'thumbnail': r're:^https?://.*/gallery/$',
+        }
+    }, {
+        'url': 'http://www.heise.de/ct/artikel/c-t-uplink-3-3-Owncloud-Tastaturen-Peilsender-Smartphone-2403911.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.heise.de/newsticker/meldung/c-t-uplink-Owncloud-Tastaturen-Peilsender-Smartphone-2404251.html?wt_mc=rss.ho.beitrag.atom',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.heise.de/ct/ausgabe/2016-12-Spiele-3214137.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         container_id = self._search_regex(
-            r'<div class="videoplayerjw"[^>]*data-container="([0-9]+)"',
+            r'<div class="videoplayerjw"[^>]+data-container="([0-9]+)"',
             webpage, 'container ID')
         sequenz_id = self._search_regex(
-            r'<div class="videoplayerjw"[^>]*data-sequenz="([0-9]+)"',
+            r'<div class="videoplayerjw"[^>]+data-sequenz="([0-9]+)"',
             webpage, 'sequenz ID')
-        data_url = 'http://www.heise.de/videout/feed?container=%s&sequenz=%s' % (container_id, sequenz_id)
-        doc = self._download_xml(data_url, video_id)
-
-        info = {
-            'id': video_id,
-            'thumbnail': doc.find('.//{http://rss.jwpcdn.com/}image').text,
-            'timestamp': parse_iso8601(
-                self._html_search_meta('date', webpage))
-        }
 
         title = self._html_search_meta('fulltitle', webpage, default=None)
         if not title or title == "c't":
             title = self._search_regex(
-                r'<div class="videoplayerjw"[^>]*data-title="([^"]+)"',
-                webpage, 'video title')
-        info['title'] = title
+                r'<div[^>]+class="videoplayerjw"[^>]+data-title="([^"]+)"',
+                webpage, 'title')
 
-        desc = self._og_search_description(webpage, default=None)
-        if not desc:
-            desc = self._html_search_meta('description', webpage)
-        info['description'] = desc
+        doc = self._download_xml(
+            'http://www.heise.de/videout/feed', video_id, query={
+                'container': container_id,
+                'sequenz': sequenz_id,
+            })
 
         formats = []
         for source_node in doc.findall('.//{http://rss.jwpcdn.com/}source'):
@@ -135,6 +73,18 @@ class HeiseIE(InfoExtractor):
                 'height': height,
             })
         self._sort_formats(formats)
-        info['formats'] = formats
 
-        return info
+        description = self._og_search_description(
+            webpage, default=None) or self._html_search_meta(
+            'description', webpage)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': (xpath_text(doc, './/{http://rss.jwpcdn.com/}image') or
+                          self._og_search_thumbnail(webpage)),
+            'timestamp': parse_iso8601(
+                self._html_search_meta('date', webpage)),
+            'formats': formats,
+        }

From 2c1f442c2bb4de65479f2e6c2f81c5741445184e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Feb 2017 23:18:26 +0700
Subject: [PATCH 0275/1696] [options] Add missing spaces

---
 youtube_dl/options.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 349f44778..2fea99ff2 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -298,14 +298,14 @@ def parseOpts(overrideArguments=None):
         metavar='FILTER', dest='match_filter', default=None,
         help=(
             'Generic video filter. '
-            'Specify any key (see help for -o for a list of available keys) to'
-            ' match if the key is present, '
-            '!key to check if the key is not present,'
+            'Specify any key (see help for -o for a list of available keys) to '
+            'match if the key is present, '
+            '!key to check if the key is not present, '
             'key > NUMBER (like "comment_count > 12", also works with '
             '>=, <, <=, !=, =) to compare against a number, and '
             '& to require multiple matches. '
-            'Values which are not known are excluded unless you'
-            ' put a question mark (?) after the operator.'
+            'Values which are not known are excluded unless you '
+            'put a question mark (?) after the operator. '
             'For example, to only match videos that have been liked more than '
             '100 times and disliked less than 50 times (or the dislike '
             'functionality is not available at the given service), but who '

From cf3704c132800809caacc6ce89afa87f0dfae487 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Feb 2017 23:47:54 +0700
Subject: [PATCH 0276/1696] [ChangeLog] Actualize

---
 ChangeLog | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index d5fe3dd5b..00ee0a5a9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,8 +1,13 @@
 version <unreleased>
 
 Extractors
+* [heise] Improve extraction (#9725)
+* [ellentv] Improve (#11653)
+* [openload] Fix extraction (#10408, #12002)
 + [theplatform] Recognize URLs with whitespaces (#12044)
+* [einthusan] Relax URL regular expression (#12141, #12159)
 + [generic] Support complex JWPlayer embedded videos (#12030)
+* [elpais] Improve extraction (#12139)
 
 
 version 2017.02.16

From 28e35f50702a8841b4caf072a546ff06ca63db96 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Feb 2017 23:59:56 +0700
Subject: [PATCH 0277/1696] release 2017.02.17

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 README.md                 | 4 ++--
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 06711f73b..6f1361b32 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.16*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.16**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.17*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.17**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.02.16
+[debug] youtube-dl version 2017.02.17
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 00ee0a5a9..2c90f791d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.02.17
 
 Extractors
 * [heise] Improve extraction (#9725)
diff --git a/README.md b/README.md
index 89876bd7a..c2a1a6b02 100644
--- a/README.md
+++ b/README.md
@@ -137,13 +137,13 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
     --match-filter FILTER            Generic video filter. Specify any key (see
                                      help for -o for a list of available keys)
                                      to match if the key is present, !key to
-                                     check if the key is not present,key >
+                                     check if the key is not present, key >
                                      NUMBER (like "comment_count > 12", also
                                      works with >=, <, <=, !=, =) to compare
                                      against a number, and & to require multiple
                                      matches. Values which are not known are
                                      excluded unless you put a question mark (?)
-                                     after the operator.For example, to only
+                                     after the operator. For example, to only
                                      match videos that have been liked more than
                                      100 times and disliked less than 50 times
                                      (or the dislike functionality is not
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 323e80954..530e1856b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.02.16'
+__version__ = '2017.02.17'

From 70bcc444a990ee9ca3daab6f3dc2d5d58a948ba4 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 18 Feb 2017 09:52:43 +0100
Subject: [PATCH 0278/1696] [viceland] improve info extraction and update test

---
 youtube_dl/extractor/vice.py     |  6 +++---
 youtube_dl/extractor/viceland.py | 14 +++++++-------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index 8a00c8fee..f0a7fd739 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -70,10 +70,10 @@ class ViceBaseIE(AdobePassIE):
             'url': uplynk_preplay_url,
             'id': video_id,
             'title': title,
-            'description': base.get('body'),
+            'description': base.get('body') or base.get('display_body'),
             'thumbnail': watch_hub_data.get('cover-image') or watch_hub_data.get('thumbnail'),
-            'duration': parse_duration(video_data.get('video_duration') or watch_hub_data.get('video-duration')),
-            'timestamp': int_or_none(video_data.get('created_at')),
+            'duration': int_or_none(video_data.get('video_duration')) or parse_duration(watch_hub_data.get('video-duration')),
+            'timestamp': int_or_none(video_data.get('created_at'), 1000),
             'age_limit': parse_age_limit(video_data.get('video_rating')),
             'series': video_data.get('show_title') or watch_hub_data.get('show-title'),
             'episode_number': int_or_none(episode.get('episode_number') or watch_hub_data.get('episode')),
diff --git a/youtube_dl/extractor/viceland.py b/youtube_dl/extractor/viceland.py
index 0eff055a6..87f9216b5 100644
--- a/youtube_dl/extractor/viceland.py
+++ b/youtube_dl/extractor/viceland.py
@@ -7,16 +7,16 @@ from .vice import ViceBaseIE
 class VicelandIE(ViceBaseIE):
     _VALID_URL = r'https?://(?:www\.)?viceland\.com/[^/]+/video/[^/]+/(?P<id>[a-f0-9]+)'
     _TEST = {
-        'url': 'https://www.viceland.com/en_us/video/cyberwar-trailer/57608447973ee7705f6fbd4e',
+        'url': 'https://www.viceland.com/en_us/video/trapped/588a70d0dba8a16007de7316',
         'info_dict': {
-            'id': '57608447973ee7705f6fbd4e',
+            'id': '588a70d0dba8a16007de7316',
             'ext': 'mp4',
-            'title': 'CYBERWAR (Trailer)',
-            'description': 'Tapping into the geopolitics of hacking and surveillance, Ben Makuch travels the world to meet with hackers, government officials, and dissidents to investigate the ecosystem of cyberwarfare.',
+            'title': 'TRAPPED (Series Trailer)',
+            'description': 'md5:7a8e95c2b6cd86461502a2845e581ccf',
             'age_limit': 14,
-            'timestamp': 1466008539,
-            'upload_date': '20160615',
-            'uploader_id': '11',
+            'timestamp': 1485474122,
+            'upload_date': '20170126',
+            'uploader_id': '57a204098cb727dec794c6a3',
             'uploader': 'Viceland',
         },
         'params': {

From bdabbc220c60ea6be50c9b1058405b636f70fb71 Mon Sep 17 00:00:00 2001
From: Alex Monk <krenair@gmail.com>
Date: Wed, 17 Aug 2016 21:13:28 +0100
Subject: [PATCH 0279/1696] [metacafe] Bypass family filter

If you don't send this user=ffilter: false cookie, it will 301 redirect you to a page asking about it, and then the title check will fail.
---
 youtube_dl/extractor/metacafe.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dl/extractor/metacafe.py
index 9880924e6..adbd44fd1 100644
--- a/youtube_dl/extractor/metacafe.py
+++ b/youtube_dl/extractor/metacafe.py
@@ -50,6 +50,18 @@ class MetacafeIE(InfoExtractor):
             },
             'skip': 'Page is temporarily unavailable.',
         },
+        # metacafe video with family filter
+        {
+            'url': 'http://www.metacafe.com/watch/2155630/adult_art_by_david_hart_156/',
+            'md5': 'b06082c5079bbdcde677a6291fbdf376',
+            'info_dict': {
+                'id': '2155630',
+                'ext': 'mp4',
+                'title': 'Adult Art By David Hart #156',
+                'uploader': 'hartistry',
+                'description': 'Adult Art By David Hart.  All the Art Works presented here are not in the possession of the American Artist, David John Hart.  The paintings are in collections worldwide of individuals, countries, art museums, foundations and charities.',
+            }
+        },
         # AnyClip video
         {
             'url': 'http://www.metacafe.com/watch/an-dVVXnuY7Jh77J/the_andromeda_strain_1971_stop_the_bomb_part_3/',
@@ -148,8 +160,9 @@ class MetacafeIE(InfoExtractor):
         # AnyClip videos require the flashversion cookie so that we get the link
         # to the mp4 file
         headers = {}
+        headers['Cookie'] = 'user=%7B%22ffilter%22%3Afalse%7D;';
         if video_id.startswith('an-'):
-            headers['Cookie'] = 'flashVersion=0;'
+            headers['Cookie'] += ' flashVersion=0;'
 
         # Retrieve video webpage to extract further information
         webpage = self._download_webpage(url, video_id, headers=headers)

From f75caf059eb7a1a156921124cbf4b720fea526e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Feb 2017 19:58:25 +0700
Subject: [PATCH 0280/1696] [metacafe] Improve (closes #10371)

---
 youtube_dl/extractor/metacafe.py | 38 +++++++++++---------------------
 1 file changed, 13 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dl/extractor/metacafe.py
index adbd44fd1..28f59f63c 100644
--- a/youtube_dl/extractor/metacafe.py
+++ b/youtube_dl/extractor/metacafe.py
@@ -6,12 +6,12 @@ from .common import InfoExtractor
 from ..compat import (
     compat_parse_qs,
     compat_urllib_parse_unquote,
+    compat_urllib_parse_urlencode,
 )
 from ..utils import (
     determine_ext,
     ExtractorError,
     int_or_none,
-    urlencode_postdata,
     get_element_by_attribute,
     mimetype2ext,
 )
@@ -57,10 +57,13 @@ class MetacafeIE(InfoExtractor):
             'info_dict': {
                 'id': '2155630',
                 'ext': 'mp4',
-                'title': 'Adult Art By David Hart #156',
-                'uploader': 'hartistry',
-                'description': 'Adult Art By David Hart.  All the Art Works presented here are not in the possession of the American Artist, David John Hart.  The paintings are in collections worldwide of individuals, countries, art museums, foundations and charities.',
-            }
+                'title': 'Adult Art By David Hart 156',
+                'uploader': '63346',
+                'description': 'md5:9afac8fc885252201ad14563694040fc',
+            },
+            'params': {
+                'skip_download': True,
+            },
         },
         # AnyClip video
         {
@@ -124,22 +127,6 @@ class MetacafeIE(InfoExtractor):
     def report_disclaimer(self):
         self.to_screen('Retrieving disclaimer')
 
-    def _confirm_age(self):
-        # Retrieve disclaimer
-        self.report_disclaimer()
-        self._download_webpage(self._DISCLAIMER, None, False, 'Unable to retrieve disclaimer')
-
-        # Confirm age
-        self.report_age_confirmation()
-        self._download_webpage(
-            self._FILTER_POST, None, False, 'Unable to confirm age',
-            data=urlencode_postdata({
-                'filters': '0',
-                'submit': "Continue - I'm over 18",
-            }), headers={
-                'Content-Type': 'application/x-www-form-urlencoded',
-            })
-
     def _real_extract(self, url):
         # Extract id and simplified title from URL
         video_id, display_id = re.match(self._VALID_URL, url).groups()
@@ -155,14 +142,15 @@ class MetacafeIE(InfoExtractor):
             if prefix == 'cb':
                 return self.url_result('theplatform:%s' % ext_id, 'ThePlatform')
 
-        # self._confirm_age()
+        headers = {
+            # Disable family filter
+            'Cookie': 'user=%s; ' % compat_urllib_parse_urlencode({'ffilter': False})
+        }
 
         # AnyClip videos require the flashversion cookie so that we get the link
         # to the mp4 file
-        headers = {}
-        headers['Cookie'] = 'user=%7B%22ffilter%22%3Afalse%7D;';
         if video_id.startswith('an-'):
-            headers['Cookie'] += ' flashVersion=0;'
+            headers['Cookie'] += 'flashVersion=0; '
 
         # Retrieve video webpage to extract further information
         webpage = self._download_webpage(url, video_id, headers=headers)

From a2e3286676606103601f9499154ad465037314d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Feb 2017 20:21:37 +0700
Subject: [PATCH 0281/1696] [thisav] Add support for html5 media (closes
 #11771)

---
 youtube_dl/extractor/thisav.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/thisav.py b/youtube_dl/extractor/thisav.py
index b7b3568cb..33683b139 100644
--- a/youtube_dl/extractor/thisav.py
+++ b/youtube_dl/extractor/thisav.py
@@ -10,6 +10,7 @@ from ..utils import remove_end
 class ThisAVIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?thisav\.com/video/(?P<id>[0-9]+)/.*'
     _TESTS = [{
+        # jwplayer
         'url': 'http://www.thisav.com/video/47734/%98%26sup1%3B%83%9E%83%82---just-fit.html',
         'md5': '0480f1ef3932d901f0e0e719f188f19b',
         'info_dict': {
@@ -20,6 +21,7 @@ class ThisAVIE(InfoExtractor):
             'uploader_id': 'dj7970'
         }
     }, {
+        # html5 media
         'url': 'http://www.thisav.com/video/242352/nerdy-18yo-big-ass-tattoos-and-glasses.html',
         'md5': 'ba90c076bd0f80203679e5b60bf523ee',
         'info_dict': {
@@ -48,8 +50,12 @@ class ThisAVIE(InfoExtractor):
                 }],
             }
         else:
-            info_dict = self._extract_jwplayer_data(
-                webpage, video_id, require_title=False)
+            entries = self._parse_html5_media_entries(url, webpage, video_id)
+            if entries:
+                info_dict = entries[0]
+            else:
+                info_dict = self._extract_jwplayer_data(
+                    webpage, video_id, require_title=False)
         uploader = self._html_search_regex(
             r': <a href="http://www.thisav.com/user/[0-9]+/(?:[^"]+)">([^<]+)</a>',
             webpage, 'uploader name', fatal=False)

From 02d9b82a233abcb778f3f8601b229f996fd7df94 Mon Sep 17 00:00:00 2001
From: Jakub Wilk <jwilk@jwilk.net>
Date: Wed, 11 Jan 2017 18:49:40 +0100
Subject: [PATCH 0282/1696] [tvn24] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/tvn24.py      | 47 ++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+)
 create mode 100644 youtube_dl/extractor/tvn24.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index be3688d5a..55b4782d3 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1009,6 +1009,7 @@ from .tvc import (
 )
 from .tvigle import TvigleIE
 from .tvland import TVLandIE
+from .tvn24 import TVN24IE
 from .tvnoe import TVNoeIE
 from .tvp import (
     TVPEmbedIE,
diff --git a/youtube_dl/extractor/tvn24.py b/youtube_dl/extractor/tvn24.py
new file mode 100644
index 000000000..225ee4a6a
--- /dev/null
+++ b/youtube_dl/extractor/tvn24.py
@@ -0,0 +1,47 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class TVN24IE(InfoExtractor):
+    _VALID_URL = r'http://(?:tvn24bis|(?:www|fakty)\.tvn24)\.pl/.+/(?P<id>[^/]+)\.html'
+    _TEST = {
+        'url': 'http://www.tvn24.pl/wiadomosci-z-kraju,3/oredzie-artura-andrusa,702428.html',
+        'md5': 'fbdec753d7bc29d96036808275f2130c',
+        'info_dict': {
+            'id': '1584444',
+            'ext': 'mp4',
+            'title': '"Święta mają być wesołe, dlatego, ludziska, wszyscy pod jemiołę"',
+            'description': 'Wyjątkowe orędzie Artura Andrusa, jednego z gości "Szkła kontaktowego".',
+            'thumbnail': 're:http://.*[.]jpeg',
+        }
+    }
+
+    def _real_extract(self, url):
+        page_id = self._match_id(url)
+        webpage = self._download_webpage(url, page_id)
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
+        thumbnail = self._html_search_regex(r'\bdata-poster="(.+?)"', webpage, 'data-poster')
+        share_params = self._html_search_regex(r'\bdata-share-params="(.+?)"', webpage, 'data-share-params')
+        share_params = self._parse_json(share_params, page_id)
+        video_id = share_params['id']
+        quality_data = self._html_search_regex(r'\bdata-quality="(.+?)"', webpage, 'data-quality')
+        quality_data = self._parse_json(quality_data, page_id)
+        formats = []
+        for format_id, url in quality_data.items():
+            formats.append({
+                'format_id': format_id,
+                'height': int(format_id.rstrip('p')),
+                'url': url,
+                'ext': 'mp4',
+            })
+        self._sort_formats(formats)
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'formats': formats,
+        }

From e84888b4322abd2e2a74e8a89b7942a68dd0b6a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Feb 2017 23:34:09 +0700
Subject: [PATCH 0283/1696] [tvn24] Improve extraction (closes #11679)

---
 youtube_dl/extractor/tvn24.py | 59 ++++++++++++++++++++++++++---------
 1 file changed, 44 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/tvn24.py b/youtube_dl/extractor/tvn24.py
index 225ee4a6a..12ed6039c 100644
--- a/youtube_dl/extractor/tvn24.py
+++ b/youtube_dl/extractor/tvn24.py
@@ -2,11 +2,15 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    unescapeHTML,
+)
 
 
 class TVN24IE(InfoExtractor):
-    _VALID_URL = r'http://(?:tvn24bis|(?:www|fakty)\.tvn24)\.pl/.+/(?P<id>[^/]+)\.html'
-    _TEST = {
+    _VALID_URL = r'https?://(?:(?:[^/]+)\.)?tvn24(?:bis)?\.pl/(?:[^/]+/)*(?P<id>[^/]+)\.html'
+    _TESTS = [{
         'url': 'http://www.tvn24.pl/wiadomosci-z-kraju,3/oredzie-artura-andrusa,702428.html',
         'md5': 'fbdec753d7bc29d96036808275f2130c',
         'info_dict': {
@@ -16,28 +20,53 @@ class TVN24IE(InfoExtractor):
             'description': 'Wyjątkowe orędzie Artura Andrusa, jednego z gości "Szkła kontaktowego".',
             'thumbnail': 're:http://.*[.]jpeg',
         }
-    }
+    }, {
+        'url': 'http://fakty.tvn24.pl/ogladaj-online,60/53-konferencja-bezpieczenstwa-w-monachium,716431.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://sport.tvn24.pl/pilka-nozna,105/ligue-1-kamil-glik-rozcial-glowe-monaco-tylko-remisuje-z-bastia,716522.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://tvn24bis.pl/poranek,146,m/gen-koziej-w-tvn24-bis-wracamy-do-czasow-zimnej-wojny,715660.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        page_id = self._match_id(url)
-        webpage = self._download_webpage(url, page_id)
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
         title = self._og_search_title(webpage)
-        description = self._og_search_description(webpage)
-        thumbnail = self._html_search_regex(r'\bdata-poster="(.+?)"', webpage, 'data-poster')
-        share_params = self._html_search_regex(r'\bdata-share-params="(.+?)"', webpage, 'data-share-params')
-        share_params = self._parse_json(share_params, page_id)
-        video_id = share_params['id']
-        quality_data = self._html_search_regex(r'\bdata-quality="(.+?)"', webpage, 'data-quality')
-        quality_data = self._parse_json(quality_data, page_id)
+
+        def extract_json(attr, name, fatal=True):
+            return self._parse_json(
+                self._search_regex(
+                    r'\b%s=(["\'])(?P<json>(?!\1).+?)\1' % attr, webpage,
+                    name, group='json', fatal=fatal) or '{}',
+                video_id, transform_source=unescapeHTML, fatal=fatal)
+
+        quality_data = extract_json('data-quality', 'formats')
+
         formats = []
         for format_id, url in quality_data.items():
             formats.append({
-                'format_id': format_id,
-                'height': int(format_id.rstrip('p')),
                 'url': url,
-                'ext': 'mp4',
+                'format_id': format_id,
+                'height': int_or_none(format_id.rstrip('p')),
             })
         self._sort_formats(formats)
+
+        description = self._og_search_description(webpage)
+        thumbnail = self._og_search_thumbnail(
+            webpage, default=None) or self._html_search_regex(
+            r'\bdata-poster=(["\'])(?P<url>(?!\1).+?)\1', webpage,
+            'thumbnail', group='url')
+
+        share_params = extract_json(
+            'data-share-params', 'share params', fatal=False)
+        if isinstance(share_params, dict):
+            video_id = share_params.get('id') or video_id
+
         return {
             'id': video_id,
             'title': title,

From ac33accd96279ee541952aaa4f0bb72b4f76b9ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Feb 2017 23:59:26 +0700
Subject: [PATCH 0284/1696] [options] Mention quoted string literals for
 --match-filter

---
 youtube_dl/options.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 2fea99ff2..deff54324 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -302,8 +302,10 @@ def parseOpts(overrideArguments=None):
             'match if the key is present, '
             '!key to check if the key is not present, '
             'key > NUMBER (like "comment_count > 12", also works with '
-            '>=, <, <=, !=, =) to compare against a number, and '
-            '& to require multiple matches. '
+            '>=, <, <=, !=, =) to compare against a number, '
+            'key = \'LITERAL\' (like "uploader = \'Mike Smith\'", also works with !=) '
+            'to match against a string literal '
+            'and & to require multiple matches. '
             'Values which are not known are excluded unless you '
             'put a question mark (?) after the operator. '
             'For example, to only match videos that have been liked more than '

From 049a0f4d6da55f4062658da7593363147c92f4a8 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 18 Feb 2017 21:07:09 +0100
Subject: [PATCH 0285/1696] [brightcove:legacy] restrict videoPlayer
 value(closes #12040)

---
 youtube_dl/extractor/brightcove.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 5c6e99da1..27685eed0 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -191,6 +191,10 @@ class BrightcoveLegacyIE(InfoExtractor):
         # These fields hold the id of the video
         videoPlayer = find_param('@videoPlayer') or find_param('videoId') or find_param('videoID') or find_param('@videoList')
         if videoPlayer is not None:
+            if isinstance(videoPlayer, list):
+                videoPlayer = videoPlayer[0]
+            if not (videoPlayer.isdigit() or videoPlayer.startswith('ref:')):
+                return None
             params['@videoPlayer'] = videoPlayer
         linkBase = find_param('linkBaseURL')
         if linkBase is not None:

From bf5b9d859a1f2a68fda0dc57eb839448c7571dfa Mon Sep 17 00:00:00 2001
From: Pierre Mdawar <p.mdawar@gmail.com>
Date: Mon, 17 Oct 2016 14:38:37 +0300
Subject: [PATCH 0286/1696] [utils] Introduce YoutubeDLError base class for all
 youtube-dl exceptions

---
 youtube_dl/utils.py | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 07c07be6f..3f9e592e3 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -701,7 +701,12 @@ def bug_reports_message():
     return msg
 
 
-class ExtractorError(Exception):
+class YoutubeDLError(Exception):
+    """Base exception for YoutubeDL errors."""
+    pass
+
+
+class ExtractorError(YoutubeDLError):
     """Error during info extraction."""
 
     def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None):
@@ -742,7 +747,7 @@ class RegexNotFoundError(ExtractorError):
     pass
 
 
-class DownloadError(Exception):
+class DownloadError(YoutubeDLError):
     """Download Error exception.
 
     This exception may be thrown by FileDownloader objects if they are not
@@ -756,7 +761,7 @@ class DownloadError(Exception):
         self.exc_info = exc_info
 
 
-class SameFileError(Exception):
+class SameFileError(YoutubeDLError):
     """Same File exception.
 
     This exception will be thrown by FileDownloader objects if they detect
@@ -765,7 +770,7 @@ class SameFileError(Exception):
     pass
 
 
-class PostProcessingError(Exception):
+class PostProcessingError(YoutubeDLError):
     """Post Processing exception.
 
     This exception may be raised by PostProcessor's .run() method to
@@ -773,15 +778,16 @@ class PostProcessingError(Exception):
     """
 
     def __init__(self, msg):
+        super(PostProcessingError, self).__init__(msg)
         self.msg = msg
 
 
-class MaxDownloadsReached(Exception):
+class MaxDownloadsReached(YoutubeDLError):
     """ --max-downloads limit has been reached. """
     pass
 
 
-class UnavailableVideoError(Exception):
+class UnavailableVideoError(YoutubeDLError):
     """Unavailable Format exception.
 
     This exception will be thrown when a video is requested
@@ -790,7 +796,7 @@ class UnavailableVideoError(Exception):
     pass
 
 
-class ContentTooShortError(Exception):
+class ContentTooShortError(YoutubeDLError):
     """Content Too Short exception.
 
     This exception may be raised by FileDownloader objects when a file they
@@ -799,12 +805,15 @@ class ContentTooShortError(Exception):
     """
 
     def __init__(self, downloaded, expected):
+        super(ContentTooShortError, self).__init__(
+            'Downloaded {0} bytes, expected {1} bytes'.format(downloaded, expected)
+        )
         # Both in bytes
         self.downloaded = downloaded
         self.expected = expected
 
 
-class XAttrMetadataError(Exception):
+class XAttrMetadataError(YoutubeDLError):
     def __init__(self, code=None, msg='Unknown error'):
         super(XAttrMetadataError, self).__init__(msg)
         self.code = code
@@ -820,7 +829,7 @@ class XAttrMetadataError(Exception):
             self.reason = 'NOT_SUPPORTED'
 
 
-class XAttrUnavailableError(Exception):
+class XAttrUnavailableError(YoutubeDLError):
     pass
 
 

From 773f291dcbce486fefe24e1abd29735d374d0a9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Feb 2017 18:49:58 +0700
Subject: [PATCH 0287/1696] Add experimental geo restriction bypass mechanism
 Based on faking X-Forwarded-For HTTP header

---
 youtube_dl/YoutubeDL.py        |  17 +++
 youtube_dl/__init__.py         |   2 +
 youtube_dl/extractor/common.py |  48 +++++-
 youtube_dl/options.py          |  12 ++
 youtube_dl/utils.py            | 267 +++++++++++++++++++++++++++++++++
 5 files changed, 340 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index a7bf5a1b0..ebace6b57 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -56,6 +56,8 @@ from .utils import (
     ExtractorError,
     format_bytes,
     formatSeconds,
+    GeoRestrictedError,
+    ISO3166Utils,
     locked_file,
     make_HTTPS_handler,
     MaxDownloadsReached,
@@ -272,6 +274,13 @@ class YoutubeDL(object):
                        If it returns None, the video is downloaded.
                        match_filter_func in utils.py is one example for this.
     no_color:          Do not emit color codes in output.
+    bypass_geo_restriction:
+                       Bypass geographic restriction via faking X-Forwarded-For
+                       HTTP header (experimental)
+    bypass_geo_restriction_as_country:
+                       Two-letter ISO 3166-2 country code that will be used for
+                       explicit geographic restriction bypassing via faking
+                       X-Forwarded-For HTTP header (experimental)
 
     The following options determine which downloader is picked:
     external_downloader: Executable of the external downloader to call.
@@ -707,6 +716,14 @@ class YoutubeDL(object):
                     return self.process_ie_result(ie_result, download, extra_info)
                 else:
                     return ie_result
+            except GeoRestrictedError as e:
+                msg = e.msg
+                if e.countries:
+                    msg += '\nThis video is available in %s.' % ', '.join(
+                        map(ISO3166Utils.short2full, e.countries))
+                msg += '\nYou might want to use a VPN or a proxy server (with --proxy) to workaround.'
+                self.report_error(msg)
+                break
             except ExtractorError as e:  # An error we somewhat expected
                 self.report_error(compat_str(e), e.format_traceback())
                 break
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 5c5b8094b..94f461a78 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -414,6 +414,8 @@ def _real_main(argv=None):
         'cn_verification_proxy': opts.cn_verification_proxy,
         'geo_verification_proxy': opts.geo_verification_proxy,
         'config_location': opts.config_location,
+        'bypass_geo_restriction': opts.bypass_geo_restriction,
+        'bypass_geo_restriction_as_country': opts.bypass_geo_restriction_as_country,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index f6ff56eda..96815099d 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -6,6 +6,7 @@ import hashlib
 import json
 import netrc
 import os
+import random
 import re
 import socket
 import sys
@@ -39,6 +40,8 @@ from ..utils import (
     ExtractorError,
     fix_xml_ampersands,
     float_or_none,
+    GeoRestrictedError,
+    GeoUtils,
     int_or_none,
     js_to_json,
     parse_iso8601,
@@ -320,17 +323,25 @@ class InfoExtractor(object):
     _real_extract() methods and define a _VALID_URL regexp.
     Probably, they should also be added to the list of extractors.
 
+    _BYPASS_GEO attribute may be set to False in order to disable
+    geo restriction bypass mechanisms for a particular extractor.
+    Though it won't disable explicit geo restriction bypass based on
+    country code provided with bypass_geo_restriction_as_country.
+
     Finally, the _WORKING attribute should be set to False for broken IEs
     in order to warn the users and skip the tests.
     """
 
     _ready = False
     _downloader = None
+    _x_forwarded_for_ip = None
+    _BYPASS_GEO = True
     _WORKING = True
 
     def __init__(self, downloader=None):
         """Constructor. Receives an optional downloader."""
         self._ready = False
+        self._x_forwarded_for_ip = None
         self.set_downloader(downloader)
 
     @classmethod
@@ -359,6 +370,10 @@ class InfoExtractor(object):
 
     def initialize(self):
         """Initializes an instance (authentication, etc)."""
+        if not self._x_forwarded_for_ip:
+            country_code = self._downloader.params.get('bypass_geo_restriction_as_country', None)
+            if country_code:
+                self._x_forwarded_for_ip = GeoUtils.random_ipv4(country_code)
         if not self._ready:
             self._real_initialize()
             self._ready = True
@@ -366,8 +381,22 @@ class InfoExtractor(object):
     def extract(self, url):
         """Extracts URL information and returns it in list of dicts."""
         try:
-            self.initialize()
-            return self._real_extract(url)
+            for _ in range(2):
+                try:
+                    self.initialize()
+                    return self._real_extract(url)
+                except GeoRestrictedError as e:
+                    if (not self._downloader.params.get('bypass_geo_restriction_as_country', None) and
+                            self._BYPASS_GEO and
+                            self._downloader.params.get('bypass_geo_restriction', True) and
+                            not self._x_forwarded_for_ip and
+                            e.countries):
+                        self._x_forwarded_for_ip = GeoUtils.random_ipv4(random.choice(e.countries))
+                        if self._x_forwarded_for_ip:
+                            self.report_warning(
+                                'Video is geo restricted. Retrying extraction with fake %s IP as X-Forwarded-For.' % self._x_forwarded_for_ip)
+                            continue
+                    raise
         except ExtractorError:
             raise
         except compat_http_client.IncompleteRead as e:
@@ -434,6 +463,15 @@ class InfoExtractor(object):
         if isinstance(url_or_request, (compat_str, str)):
             url_or_request = url_or_request.partition('#')[0]
 
+        # Some sites check X-Forwarded-For HTTP header in order to figure out
+        # the origin of the client behind proxy. This allows bypassing geo
+        # restriction by faking this header's value to IP that belongs to some
+        # geo unrestricted country. We will do so once we encounter any
+        # geo restriction error.
+        if self._x_forwarded_for_ip:
+            if 'X-Forwarded-For' not in headers:
+                headers['X-Forwarded-For'] = self._x_forwarded_for_ip
+
         urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal, data=data, headers=headers, query=query)
         if urlh is False:
             assert not fatal
@@ -609,10 +647,8 @@ class InfoExtractor(object):
             expected=True)
 
     @staticmethod
-    def raise_geo_restricted(msg='This video is not available from your location due to geo restriction'):
-        raise ExtractorError(
-            '%s. You might want to use --proxy to workaround.' % msg,
-            expected=True)
+    def raise_geo_restricted(msg='This video is not available from your location due to geo restriction', countries=None):
+        raise GeoRestrictedError(msg, countries=countries)
 
     # Methods for following #608
     @staticmethod
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index deff54324..2e194f6dc 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -549,6 +549,18 @@ def parseOpts(overrideArguments=None):
             'Upper bound of a range for randomized sleep before each download '
             '(maximum possible number of seconds to sleep). Must only be used '
             'along with --min-sleep-interval.'))
+    workarounds.add_option(
+        '--bypass-geo',
+        action='store_true', dest='bypass_geo_restriction', default=True,
+        help='Bypass geographic restriction via faking X-Forwarded-For HTTP header (experimental)')
+    workarounds.add_option(
+        '--no-bypass-geo',
+        action='store_false', dest='bypass_geo_restriction', default=True,
+        help='Do not bypass geographic restriction via faking X-Forwarded-For HTTP header (experimental)')
+    workarounds.add_option(
+        '--bypass-geo-as-country', metavar='CODE',
+        dest='bypass_geo_restriction_as_country', default=None,
+        help='Force bypass geographic restriction with explicitly provided two-letter ISO 3166-2 country code (experimental)')
 
     verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
     verbosity.add_option(
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 3f9e592e3..4e76b6b7b 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -23,6 +23,7 @@ import operator
 import os
 import pipes
 import platform
+import random
 import re
 import socket
 import ssl
@@ -747,6 +748,18 @@ class RegexNotFoundError(ExtractorError):
     pass
 
 
+class GeoRestrictedError(ExtractorError):
+    """Geographic restriction Error exception.
+
+    This exception may be thrown when a video is not available from your
+    geographic location due to geographic restrictions imposed by a website.
+    """
+    def __init__(self, msg, countries=None):
+        super(GeoRestrictedError, self).__init__(msg, expected=True)
+        self.msg = msg
+        self.countries = countries
+
+
 class DownloadError(YoutubeDLError):
     """Download Error exception.
 
@@ -3027,6 +3040,260 @@ class ISO3166Utils(object):
         return cls._country_map.get(code.upper())
 
 
+class GeoUtils(object):
+    # Major IPv4 address blocks per country
+    _country_ip_map = {
+        'AD': '85.94.160.0/19',
+        'AE': '94.200.0.0/13',
+        'AF': '149.54.0.0/17',
+        'AG': '209.59.64.0/18',
+        'AI': '204.14.248.0/21',
+        'AL': '46.99.0.0/16',
+        'AM': '46.70.0.0/15',
+        'AO': '105.168.0.0/13',
+        'AP': '159.117.192.0/21',
+        'AR': '181.0.0.0/12',
+        'AS': '202.70.112.0/20',
+        'AT': '84.112.0.0/13',
+        'AU': '1.128.0.0/11',
+        'AW': '181.41.0.0/18',
+        'AZ': '5.191.0.0/16',
+        'BA': '31.176.128.0/17',
+        'BB': '65.48.128.0/17',
+        'BD': '114.130.0.0/16',
+        'BE': '57.0.0.0/8',
+        'BF': '129.45.128.0/17',
+        'BG': '95.42.0.0/15',
+        'BH': '37.131.0.0/17',
+        'BI': '154.117.192.0/18',
+        'BJ': '137.255.0.0/16',
+        'BL': '192.131.134.0/24',
+        'BM': '196.12.64.0/18',
+        'BN': '156.31.0.0/16',
+        'BO': '161.56.0.0/16',
+        'BQ': '161.0.80.0/20',
+        'BR': '152.240.0.0/12',
+        'BS': '24.51.64.0/18',
+        'BT': '119.2.96.0/19',
+        'BW': '168.167.0.0/16',
+        'BY': '178.120.0.0/13',
+        'BZ': '179.42.192.0/18',
+        'CA': '99.224.0.0/11',
+        'CD': '41.243.0.0/16',
+        'CF': '196.32.200.0/21',
+        'CG': '197.214.128.0/17',
+        'CH': '85.0.0.0/13',
+        'CI': '154.232.0.0/14',
+        'CK': '202.65.32.0/19',
+        'CL': '152.172.0.0/14',
+        'CM': '165.210.0.0/15',
+        'CN': '36.128.0.0/10',
+        'CO': '181.240.0.0/12',
+        'CR': '201.192.0.0/12',
+        'CU': '152.206.0.0/15',
+        'CV': '165.90.96.0/19',
+        'CW': '190.88.128.0/17',
+        'CY': '46.198.0.0/15',
+        'CZ': '88.100.0.0/14',
+        'DE': '53.0.0.0/8',
+        'DJ': '197.241.0.0/17',
+        'DK': '87.48.0.0/12',
+        'DM': '192.243.48.0/20',
+        'DO': '152.166.0.0/15',
+        'DZ': '41.96.0.0/12',
+        'EC': '186.68.0.0/15',
+        'EE': '90.190.0.0/15',
+        'EG': '156.160.0.0/11',
+        'ER': '196.200.96.0/20',
+        'ES': '88.0.0.0/11',
+        'ET': '196.188.0.0/14',
+        'EU': '2.16.0.0/13',
+        'FI': '91.152.0.0/13',
+        'FJ': '144.120.0.0/16',
+        'FM': '119.252.112.0/20',
+        'FO': '88.85.32.0/19',
+        'FR': '90.0.0.0/9',
+        'GA': '41.158.0.0/15',
+        'GB': '25.0.0.0/8',
+        'GD': '74.122.88.0/21',
+        'GE': '31.146.0.0/16',
+        'GF': '161.22.64.0/18',
+        'GG': '62.68.160.0/19',
+        'GH': '45.208.0.0/14',
+        'GI': '85.115.128.0/19',
+        'GL': '88.83.0.0/19',
+        'GM': '160.182.0.0/15',
+        'GN': '197.149.192.0/18',
+        'GP': '104.250.0.0/19',
+        'GQ': '105.235.224.0/20',
+        'GR': '94.64.0.0/13',
+        'GT': '168.234.0.0/16',
+        'GU': '168.123.0.0/16',
+        'GW': '197.214.80.0/20',
+        'GY': '181.41.64.0/18',
+        'HK': '113.252.0.0/14',
+        'HN': '181.210.0.0/16',
+        'HR': '93.136.0.0/13',
+        'HT': '148.102.128.0/17',
+        'HU': '84.0.0.0/14',
+        'ID': '39.192.0.0/10',
+        'IE': '87.32.0.0/12',
+        'IL': '79.176.0.0/13',
+        'IM': '5.62.80.0/20',
+        'IN': '117.192.0.0/10',
+        'IO': '203.83.48.0/21',
+        'IQ': '37.236.0.0/14',
+        'IR': '2.176.0.0/12',
+        'IS': '82.221.0.0/16',
+        'IT': '79.0.0.0/10',
+        'JE': '87.244.64.0/18',
+        'JM': '72.27.0.0/17',
+        'JO': '176.29.0.0/16',
+        'JP': '126.0.0.0/8',
+        'KE': '105.48.0.0/12',
+        'KG': '158.181.128.0/17',
+        'KH': '36.37.128.0/17',
+        'KI': '103.25.140.0/22',
+        'KM': '197.255.224.0/20',
+        'KN': '198.32.32.0/19',
+        'KP': '175.45.176.0/22',
+        'KR': '175.192.0.0/10',
+        'KW': '37.36.0.0/14',
+        'KY': '64.96.0.0/15',
+        'KZ': '2.72.0.0/13',
+        'LA': '115.84.64.0/18',
+        'LB': '178.135.0.0/16',
+        'LC': '192.147.231.0/24',
+        'LI': '82.117.0.0/19',
+        'LK': '112.134.0.0/15',
+        'LR': '41.86.0.0/19',
+        'LS': '129.232.0.0/17',
+        'LT': '78.56.0.0/13',
+        'LU': '188.42.0.0/16',
+        'LV': '46.109.0.0/16',
+        'LY': '41.252.0.0/14',
+        'MA': '105.128.0.0/11',
+        'MC': '88.209.64.0/18',
+        'MD': '37.246.0.0/16',
+        'ME': '178.175.0.0/17',
+        'MF': '74.112.232.0/21',
+        'MG': '154.126.0.0/17',
+        'MH': '117.103.88.0/21',
+        'MK': '77.28.0.0/15',
+        'ML': '154.118.128.0/18',
+        'MM': '37.111.0.0/17',
+        'MN': '49.0.128.0/17',
+        'MO': '60.246.0.0/16',
+        'MP': '202.88.64.0/20',
+        'MQ': '109.203.224.0/19',
+        'MR': '41.188.64.0/18',
+        'MS': '208.90.112.0/22',
+        'MT': '46.11.0.0/16',
+        'MU': '105.16.0.0/12',
+        'MV': '27.114.128.0/18',
+        'MW': '105.234.0.0/16',
+        'MX': '187.192.0.0/11',
+        'MY': '175.136.0.0/13',
+        'MZ': '197.218.0.0/15',
+        'NA': '41.182.0.0/16',
+        'NC': '101.101.0.0/18',
+        'NE': '197.214.0.0/18',
+        'NF': '203.17.240.0/22',
+        'NG': '105.112.0.0/12',
+        'NI': '186.76.0.0/15',
+        'NL': '145.96.0.0/11',
+        'NO': '84.208.0.0/13',
+        'NP': '36.252.0.0/15',
+        'NR': '203.98.224.0/19',
+        'NU': '49.156.48.0/22',
+        'NZ': '49.224.0.0/14',
+        'OM': '5.36.0.0/15',
+        'PA': '186.72.0.0/15',
+        'PE': '186.160.0.0/14',
+        'PF': '123.50.64.0/18',
+        'PG': '124.240.192.0/19',
+        'PH': '49.144.0.0/13',
+        'PK': '39.32.0.0/11',
+        'PL': '83.0.0.0/11',
+        'PM': '70.36.0.0/20',
+        'PR': '66.50.0.0/16',
+        'PS': '188.161.0.0/16',
+        'PT': '85.240.0.0/13',
+        'PW': '202.124.224.0/20',
+        'PY': '181.120.0.0/14',
+        'QA': '37.210.0.0/15',
+        'RE': '139.26.0.0/16',
+        'RO': '79.112.0.0/13',
+        'RS': '178.220.0.0/14',
+        'RU': '5.136.0.0/13',
+        'RW': '105.178.0.0/15',
+        'SA': '188.48.0.0/13',
+        'SB': '202.1.160.0/19',
+        'SC': '154.192.0.0/11',
+        'SD': '154.96.0.0/13',
+        'SE': '78.64.0.0/12',
+        'SG': '152.56.0.0/14',
+        'SI': '188.196.0.0/14',
+        'SK': '78.98.0.0/15',
+        'SL': '197.215.0.0/17',
+        'SM': '89.186.32.0/19',
+        'SN': '41.82.0.0/15',
+        'SO': '197.220.64.0/19',
+        'SR': '186.179.128.0/17',
+        'SS': '105.235.208.0/21',
+        'ST': '197.159.160.0/19',
+        'SV': '168.243.0.0/16',
+        'SX': '190.102.0.0/20',
+        'SY': '5.0.0.0/16',
+        'SZ': '41.84.224.0/19',
+        'TC': '65.255.48.0/20',
+        'TD': '154.68.128.0/19',
+        'TG': '196.168.0.0/14',
+        'TH': '171.96.0.0/13',
+        'TJ': '85.9.128.0/18',
+        'TK': '27.96.24.0/21',
+        'TL': '180.189.160.0/20',
+        'TM': '95.85.96.0/19',
+        'TN': '197.0.0.0/11',
+        'TO': '175.176.144.0/21',
+        'TR': '78.160.0.0/11',
+        'TT': '186.44.0.0/15',
+        'TV': '202.2.96.0/19',
+        'TW': '120.96.0.0/11',
+        'TZ': '156.156.0.0/14',
+        'UA': '93.72.0.0/13',
+        'UG': '154.224.0.0/13',
+        'US': '3.0.0.0/8',
+        'UY': '167.56.0.0/13',
+        'UZ': '82.215.64.0/18',
+        'VA': '212.77.0.0/19',
+        'VC': '24.92.144.0/20',
+        'VE': '186.88.0.0/13',
+        'VG': '172.103.64.0/18',
+        'VI': '146.226.0.0/16',
+        'VN': '14.160.0.0/11',
+        'VU': '202.80.32.0/20',
+        'WF': '117.20.32.0/21',
+        'WS': '202.4.32.0/19',
+        'YE': '134.35.0.0/16',
+        'YT': '41.242.116.0/22',
+        'ZA': '41.0.0.0/11',
+        'ZM': '165.56.0.0/13',
+        'ZW': '41.85.192.0/19',
+    }
+
+    @classmethod
+    def random_ipv4(cls, code):
+        block = cls._country_ip_map.get(code.upper())
+        if not block:
+            return None
+        addr, preflen = block.split('/')
+        addr_min = compat_struct_unpack('!L', socket.inet_aton(addr))[0]
+        addr_max = addr_min | (0xffffffff >> int(preflen))
+        return socket.inet_ntoa(
+            compat_struct_pack('!I', random.randint(addr_min, addr_max)))
+
+
 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
     def __init__(self, proxies=None):
         # Set default handlers

From d392005a795a6cf85fda3c0f982254f8a2731e94 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Feb 2017 18:51:16 +0700
Subject: [PATCH 0288/1696] [dramafever] Improve geo restriction detection and
 use geo bypass mechanism

---
 youtube_dl/extractor/dramafever.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index bcd9fe2a0..755db806a 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -116,8 +116,9 @@ class DramaFeverIE(DramaFeverBaseIE):
                 'http://www.dramafever.com/amp/episode/feed.json?guid=%s' % video_id)
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError):
-                raise ExtractorError(
-                    'Currently unavailable in your country.', expected=True)
+                self.raise_geo_restricted(
+                    msg='Currently unavailable in your country',
+                    countries=['US', 'CA'])
             raise
 
         series_id, episode_number = video_id.split('.')

From e633f21a96f37a96e8ef0fd4d6c1e4d3c0b41fbf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Feb 2017 18:51:33 +0700
Subject: [PATCH 0289/1696] [go] Improve geo restriction detection and use geo
 bypass mechanism

---
 youtube_dl/extractor/go.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/go.py b/youtube_dl/extractor/go.py
index f28e6fbf5..ec902c670 100644
--- a/youtube_dl/extractor/go.py
+++ b/youtube_dl/extractor/go.py
@@ -101,6 +101,10 @@ class GoIE(AdobePassIE):
                         video_id, data=urlencode_postdata(data), headers=self.geo_verification_headers())
                     errors = entitlement.get('errors', {}).get('errors', [])
                     if errors:
+                        for error in errors:
+                            if error.get('code') == 1002:
+                                self.raise_geo_restricted(
+                                    error['message'], countries=['US'])
                         error_message = ', '.join([error['message'] for error in errors])
                         raise ExtractorError('%s said: %s' % (self.IE_NAME, error_message), expected=True)
                     asset_url += '?' + entitlement['uplynkData']['sessionKey']

From 28200e654b8051cadca12e51bd57f77e1ff0a4ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Feb 2017 18:51:51 +0700
Subject: [PATCH 0290/1696] [itv] Improve geo restriction detection and use geo
 bypass mechanism

---
 youtube_dl/extractor/itv.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py
index b0d860452..aabde15f3 100644
--- a/youtube_dl/extractor/itv.py
+++ b/youtube_dl/extractor/itv.py
@@ -98,7 +98,10 @@ class ITVIE(InfoExtractor):
             headers=headers, data=etree.tostring(req_env))
         playlist = xpath_element(resp_env, './/Playlist')
         if playlist is None:
+            fault_code = xpath_text(resp_env, './/faultcode')
             fault_string = xpath_text(resp_env, './/faultstring')
+            if fault_code == 'InvalidGeoRegion':
+                self.raise_geo_restricted(msg=fault_string, countries=['GB'])
             raise ExtractorError('%s said: %s' % (self.IE_NAME, fault_string))
         title = xpath_text(playlist, 'EpisodeTitle', fatal=True)
         video_element = xpath_element(playlist, 'VideoEntries/Video', fatal=True)

From ff4007891fde74212eb0898bb04c14b2de92ed03 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Feb 2017 18:51:59 +0700
Subject: [PATCH 0291/1696] [nrk] Improve geo restriction detection and use geo
 bypass mechanism

---
 youtube_dl/extractor/nrk.py | 36 +++++-------------------------------
 1 file changed, 5 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index fc3c0cd3c..78ece33e1 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -1,7 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import random
 import re
 
 from .common import InfoExtractor
@@ -15,25 +14,6 @@ from ..utils import (
 
 
 class NRKBaseIE(InfoExtractor):
-    _faked_ip = None
-
-    def _download_webpage_handle(self, *args, **kwargs):
-        # NRK checks X-Forwarded-For HTTP header in order to figure out the
-        # origin of the client behind proxy. This allows to bypass geo
-        # restriction by faking this header's value to some Norway IP.
-        # We will do so once we encounter any geo restriction error.
-        if self._faked_ip:
-            # NB: str is intentional
-            kwargs.setdefault(str('headers'), {})['X-Forwarded-For'] = self._faked_ip
-        return super(NRKBaseIE, self)._download_webpage_handle(*args, **kwargs)
-
-    def _fake_ip(self):
-        # Use fake IP from 37.191.128.0/17 in order to workaround geo
-        # restriction
-        def octet(lb=0, ub=255):
-            return random.randint(lb, ub)
-        self._faked_ip = '37.191.%d.%d' % (octet(128), octet())
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
@@ -44,8 +24,6 @@ class NRKBaseIE(InfoExtractor):
         title = data.get('fullTitle') or data.get('mainTitle') or data['title']
         video_id = data.get('id') or video_id
 
-        http_headers = {'X-Forwarded-For': self._faked_ip} if self._faked_ip else {}
-
         entries = []
 
         conviva = data.get('convivaStatistics') or {}
@@ -90,7 +68,6 @@ class NRKBaseIE(InfoExtractor):
                     'duration': duration,
                     'subtitles': subtitles,
                     'formats': formats,
-                    'http_headers': http_headers,
                 })
 
         if not entries:
@@ -107,19 +84,16 @@ class NRKBaseIE(InfoExtractor):
                 }]
 
         if not entries:
-            message_type = data.get('messageType', '')
-            # Can be ProgramIsGeoBlocked or ChannelIsGeoBlocked*
-            if 'IsGeoBlocked' in message_type and not self._faked_ip:
-                self.report_warning(
-                    'Video is geo restricted, trying to fake IP')
-                self._fake_ip()
-                return self._real_extract(url)
-
             MESSAGES = {
                 'ProgramRightsAreNotReady': 'Du kan dessverre ikke se eller høre programmet',
                 'ProgramRightsHasExpired': 'Programmet har gått ut',
                 'ProgramIsGeoBlocked': 'NRK har ikke rettigheter til å vise dette programmet utenfor Norge',
             }
+            message_type = data.get('messageType', '')
+            # Can be ProgramIsGeoBlocked or ChannelIsGeoBlocked*
+            if 'IsGeoBlocked' in message_type:
+                self.raise_geo_restricted(
+                    msg=MESSAGES.get('ProgramIsGeoBlocked'), countries=['NO'])
             raise ExtractorError(
                 '%s said: %s' % (self.IE_NAME, MESSAGES.get(
                     message_type, message_type)),

From 01b1aa9ff408ce15b8bbea08dbc190f3282141a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Feb 2017 18:52:11 +0700
Subject: [PATCH 0292/1696] [ondemandkorea] Improve geo restriction detection
 and use geo bypass mechanism

---
 youtube_dl/extractor/ondemandkorea.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ondemandkorea.py b/youtube_dl/extractor/ondemandkorea.py
index dcd157777..0c85d549e 100644
--- a/youtube_dl/extractor/ondemandkorea.py
+++ b/youtube_dl/extractor/ondemandkorea.py
@@ -35,7 +35,8 @@ class OnDemandKoreaIE(InfoExtractor):
 
         if 'msg_block_01.png' in webpage:
             self.raise_geo_restricted(
-                'This content is not available in your region')
+                msg='This content is not available in your region',
+                countries=['US', 'CA'])
 
         if 'This video is only available to ODK PLUS members.' in webpage:
             raise ExtractorError(

From 8ab8066cf08352ad336c3ff594d0ac27f6c809c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Feb 2017 18:52:20 +0700
Subject: [PATCH 0293/1696] [pbs] Improve geo restriction detection and use geo
 bypass mechanism

---
 youtube_dl/extractor/pbs.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 6baed773f..64f47bae3 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -489,11 +489,12 @@ class PBSIE(InfoExtractor):
                 headers=self.geo_verification_headers())
 
             if redirect_info['status'] == 'error':
+                message = self._ERRORS.get(
+                    redirect_info['http_code'], redirect_info['message'])
+                if redirect_info['http_code'] == 403:
+                    self.raise_geo_restricted(msg=message, countries=['US'])
                 raise ExtractorError(
-                    '%s said: %s' % (
-                        self.IE_NAME,
-                        self._ERRORS.get(redirect_info['http_code'], redirect_info['message'])),
-                    expected=True)
+                    '%s said: %s' % (self.IE_NAME, message), expected=True)
 
             format_url = redirect_info.get('url')
             if not format_url:

From 04d906eae3071e37049cfcd2a02e9079b72a265c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Feb 2017 18:52:33 +0700
Subject: [PATCH 0294/1696] [svt] Improve geo restriction detection and use geo
 bypass mechanism

---
 youtube_dl/extractor/svt.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/svt.py b/youtube_dl/extractor/svt.py
index 10cf80885..f2a2200bf 100644
--- a/youtube_dl/extractor/svt.py
+++ b/youtube_dl/extractor/svt.py
@@ -38,7 +38,8 @@ class SVTBaseIE(InfoExtractor):
                     'url': vurl,
                 })
         if not formats and video_info.get('rights', {}).get('geoBlockedSweden'):
-            self.raise_geo_restricted('This video is only available in Sweden')
+            self.raise_geo_restricted(
+                'This video is only available in Sweden', countries=['SE'])
         self._sort_formats(formats)
 
         subtitles = {}

From 89cc7fe7705b6534f434b514265a0507b70ef40f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Feb 2017 18:52:42 +0700
Subject: [PATCH 0295/1696] [vbox7] Improve geo restriction detection and use
 geo bypass mechanism

---
 youtube_dl/extractor/vbox7.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vbox7.py b/youtube_dl/extractor/vbox7.py
index bef639462..f86d804c1 100644
--- a/youtube_dl/extractor/vbox7.py
+++ b/youtube_dl/extractor/vbox7.py
@@ -78,7 +78,7 @@ class Vbox7IE(InfoExtractor):
         video_url = video['src']
 
         if '/na.mp4' in video_url:
-            self.raise_geo_restricted()
+            self.raise_geo_restricted(countries=['BG'])
 
         uploader = video.get('uploader')
 

From 71631862f4de5a10223642ebdbd5e10db374d270 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Feb 2017 18:55:23 +0700
Subject: [PATCH 0296/1696] [srgssr] Improve geo restriction detection

---
 youtube_dl/extractor/srgssr.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/srgssr.py b/youtube_dl/extractor/srgssr.py
index 319a48a7a..a35a0a538 100644
--- a/youtube_dl/extractor/srgssr.py
+++ b/youtube_dl/extractor/srgssr.py
@@ -14,6 +14,7 @@ from ..utils import (
 
 class SRGSSRIE(InfoExtractor):
     _VALID_URL = r'(?:https?://tp\.srgssr\.ch/p(?:/[^/]+)+\?urn=urn|srgssr):(?P<bu>srf|rts|rsi|rtr|swi):(?:[^:]+:)?(?P<type>video|audio):(?P<id>[0-9a-f\-]{36}|\d+)'
+    _BYPASS_GEO = False
 
     _ERRORS = {
         'AGERATING12': 'To protect children under the age of 12, this video is only available between 8 p.m. and 6 a.m.',
@@ -40,8 +41,11 @@ class SRGSSRIE(InfoExtractor):
             media_id)[media_type.capitalize()]
 
         if media_data.get('block') and media_data['block'] in self._ERRORS:
-            raise ExtractorError('%s said: %s' % (
-                self.IE_NAME, self._ERRORS[media_data['block']]), expected=True)
+            message = self._ERRORS[media_data['block']]
+            if media_data['block'] == 'GEOBLOCK':
+                self.raise_geo_restricted(msg=message, countries=['CH'])
+            raise ExtractorError(
+                '%s said: %s' % (self.IE_NAME, message), expected=True)
 
         return media_data
 

From 80b59020e02e9c61f74f8f8f8891f9745667edb7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Feb 2017 18:55:31 +0700
Subject: [PATCH 0297/1696] [vgtv] Improve geo restriction detection

---
 youtube_dl/extractor/vgtv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 8a574bc26..1709fd6bb 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -14,6 +14,7 @@ from ..utils import (
 
 class VGTVIE(XstreamIE):
     IE_DESC = 'VGTV, BTTV, FTV, Aftenposten and Aftonbladet'
+    _BYPASS_GEO = False
 
     _HOST_TO_APPNAME = {
         'vgtv.no': 'vgtv',
@@ -217,7 +218,7 @@ class VGTVIE(XstreamIE):
             properties = try_get(
                 data, lambda x: x['streamConfiguration']['properties'], list)
             if properties and 'geoblocked' in properties:
-                raise self.raise_geo_restricted()
+                raise self.raise_geo_restricted(countries=['NO'])
 
         self._sort_formats(info['formats'])
 

From 5d3fbf77d96ade64c645b6942979c0b99aa4d775 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Feb 2017 18:55:39 +0700
Subject: [PATCH 0298/1696] [viki] Improve geo restriction detection

---
 youtube_dl/extractor/viki.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 9c48701c1..68a74e246 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -27,6 +27,7 @@ class VikiBaseIE(InfoExtractor):
     _APP_VERSION = '2.2.5.1428709186'
     _APP_SECRET = '-$iJ}@p7!G@SyU/je1bEyWg}upLu-6V6-Lg9VD(]siH,r.,m-r|ulZ,U4LC/SeR)'
 
+    _BYPASS_GEO = False
     _NETRC_MACHINE = 'viki'
 
     _token = None
@@ -77,8 +78,11 @@ class VikiBaseIE(InfoExtractor):
     def _check_errors(self, data):
         for reason, status in data.get('blocking', {}).items():
             if status and reason in self._ERRORS:
+                message = self._ERRORS[reason]
+                if reason == 'geo':
+                    self.raise_geo_restricted(msg=message)
                 raise ExtractorError('%s said: %s' % (
-                    self.IE_NAME, self._ERRORS[reason]), expected=True)
+                    self.IE_NAME, message), expected=True)
 
     def _real_initialize(self):
         self._login()

From 18a0defab063523cd76a30be2dd5a80e9f9172d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Feb 2017 20:26:43 +0700
Subject: [PATCH 0299/1696] [utils] Make random_ipv4 return unicode string

---
 youtube_dl/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 4e76b6b7b..cbf7639c5 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3290,8 +3290,8 @@ class GeoUtils(object):
         addr, preflen = block.split('/')
         addr_min = compat_struct_unpack('!L', socket.inet_aton(addr))[0]
         addr_max = addr_min | (0xffffffff >> int(preflen))
-        return socket.inet_ntoa(
-            compat_struct_pack('!I', random.randint(addr_min, addr_max)))
+        return compat_str(socket.inet_ntoa(
+            compat_struct_pack('!I', random.randint(addr_min, addr_max))))
 
 
 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):

From 0016b84e16965a07c52946c4672363153e8b18a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Feb 2017 21:06:07 +0700
Subject: [PATCH 0300/1696] Add faked X-Forwarded-For to formats' HTTP headers

---
 youtube_dl/YoutubeDL.py        | 14 ++++++++++++++
 youtube_dl/extractor/common.py |  5 ++++-
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index ebace6b57..1c04e46c1 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -864,8 +864,14 @@ class YoutubeDL(object):
             if self.params.get('playlistrandom', False):
                 random.shuffle(entries)
 
+            x_forwarded_for = ie_result.get('__x_forwarded_for_ip')
+
             for i, entry in enumerate(entries, 1):
                 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
+                # This __x_forwarded_for_ip thing is a bit ugly but requires
+                # minimal changes
+                if x_forwarded_for:
+                    entry['__x_forwarded_for_ip'] = x_forwarded_for
                 extra = {
                     'n_entries': n_entries,
                     'playlist': playlist,
@@ -1250,6 +1256,11 @@ class YoutubeDL(object):
         if cookies:
             res['Cookie'] = cookies
 
+        if 'X-Forwarded-For' not in res:
+            x_forwarded_for_ip = info_dict.get('__x_forwarded_for_ip')
+            if x_forwarded_for_ip:
+                res['X-Forwarded-For'] = x_forwarded_for_ip
+
         return res
 
     def _calc_cookies(self, info_dict):
@@ -1392,6 +1403,9 @@ class YoutubeDL(object):
             full_format_info = info_dict.copy()
             full_format_info.update(format)
             format['http_headers'] = self._calc_headers(full_format_info)
+        # Remove private housekeeping stuff
+        if '__x_forwarded_for_ip' in info_dict:
+            del info_dict['__x_forwarded_for_ip']
 
         # TODO Central sorting goes here
 
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 96815099d..c1f7f28a0 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -384,7 +384,10 @@ class InfoExtractor(object):
             for _ in range(2):
                 try:
                     self.initialize()
-                    return self._real_extract(url)
+                    ie_result = self._real_extract(url)
+                    if self._x_forwarded_for_ip:
+                        ie_result['__x_forwarded_for_ip'] = self._x_forwarded_for_ip
+                    return ie_result
                 except GeoRestrictedError as e:
                     if (not self._downloader.params.get('bypass_geo_restriction_as_country', None) and
                             self._BYPASS_GEO and

From 0a840f584c3f1fedb6957c05587dec697143f2d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Feb 2017 01:53:41 +0700
Subject: [PATCH 0301/1696] Rename bypass geo restriction options

---
 youtube_dl/YoutubeDL.py        |  5 ++---
 youtube_dl/__init__.py         |  4 ++--
 youtube_dl/extractor/common.py |  8 ++++----
 youtube_dl/options.py          | 12 ++++++------
 4 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 1c04e46c1..68000dea2 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -274,10 +274,9 @@ class YoutubeDL(object):
                        If it returns None, the video is downloaded.
                        match_filter_func in utils.py is one example for this.
     no_color:          Do not emit color codes in output.
-    bypass_geo_restriction:
-                       Bypass geographic restriction via faking X-Forwarded-For
+    geo_bypass:        Bypass geographic restriction via faking X-Forwarded-For
                        HTTP header (experimental)
-    bypass_geo_restriction_as_country:
+    geo_bypass_country:
                        Two-letter ISO 3166-2 country code that will be used for
                        explicit geographic restriction bypassing via faking
                        X-Forwarded-For HTTP header (experimental)
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 94f461a78..f91d29a7b 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -414,8 +414,8 @@ def _real_main(argv=None):
         'cn_verification_proxy': opts.cn_verification_proxy,
         'geo_verification_proxy': opts.geo_verification_proxy,
         'config_location': opts.config_location,
-        'bypass_geo_restriction': opts.bypass_geo_restriction,
-        'bypass_geo_restriction_as_country': opts.bypass_geo_restriction_as_country,
+        'geo_bypass': opts.geo_bypass,
+        'geo_bypass_country': opts.geo_bypass_country,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index c1f7f28a0..6eb6a25b8 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -326,7 +326,7 @@ class InfoExtractor(object):
     _BYPASS_GEO attribute may be set to False in order to disable
     geo restriction bypass mechanisms for a particular extractor.
     Though it won't disable explicit geo restriction bypass based on
-    country code provided with bypass_geo_restriction_as_country.
+    country code provided with geo_bypass_country.
 
     Finally, the _WORKING attribute should be set to False for broken IEs
     in order to warn the users and skip the tests.
@@ -371,7 +371,7 @@ class InfoExtractor(object):
     def initialize(self):
         """Initializes an instance (authentication, etc)."""
         if not self._x_forwarded_for_ip:
-            country_code = self._downloader.params.get('bypass_geo_restriction_as_country', None)
+            country_code = self._downloader.params.get('geo_bypass_country', None)
             if country_code:
                 self._x_forwarded_for_ip = GeoUtils.random_ipv4(country_code)
         if not self._ready:
@@ -389,9 +389,9 @@ class InfoExtractor(object):
                         ie_result['__x_forwarded_for_ip'] = self._x_forwarded_for_ip
                     return ie_result
                 except GeoRestrictedError as e:
-                    if (not self._downloader.params.get('bypass_geo_restriction_as_country', None) and
+                    if (not self._downloader.params.get('geo_bypass_country', None) and
                             self._BYPASS_GEO and
-                            self._downloader.params.get('bypass_geo_restriction', True) and
+                            self._downloader.params.get('geo_bypass', True) and
                             not self._x_forwarded_for_ip and
                             e.countries):
                         self._x_forwarded_for_ip = GeoUtils.random_ipv4(random.choice(e.countries))
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 2e194f6dc..ae3f50754 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -550,16 +550,16 @@ def parseOpts(overrideArguments=None):
             '(maximum possible number of seconds to sleep). Must only be used '
             'along with --min-sleep-interval.'))
     workarounds.add_option(
-        '--bypass-geo',
-        action='store_true', dest='bypass_geo_restriction', default=True,
+        '--geo-bypass',
+        action='store_true', dest='geo_bypass', default=True,
         help='Bypass geographic restriction via faking X-Forwarded-For HTTP header (experimental)')
     workarounds.add_option(
-        '--no-bypass-geo',
-        action='store_false', dest='bypass_geo_restriction', default=True,
+        '--no-geo-bypass',
+        action='store_false', dest='geo_bypass', default=True,
         help='Do not bypass geographic restriction via faking X-Forwarded-For HTTP header (experimental)')
     workarounds.add_option(
-        '--bypass-geo-as-country', metavar='CODE',
-        dest='bypass_geo_restriction_as_country', default=None,
+        '--geo-bypass-country', metavar='CODE',
+        dest='geo_bypass_country', default=None,
         help='Force bypass geographic restriction with explicitly provided two-letter ISO 3166-2 country code (experimental)')
 
     verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')

From 4248dad92bd87650c791194276296b148f668e68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Feb 2017 03:53:23 +0700
Subject: [PATCH 0302/1696] Improve geo bypass mechanism * Rename options to
 preffixly match with --geo-verification-proxy * Introduce _GEO_COUNTRIES for
 extractors * Implement faking IP right away for sites with known geo
 restriction

---
 youtube_dl/extractor/common.py        | 59 +++++++++++++++++++--------
 youtube_dl/extractor/dramafever.py    |  3 +-
 youtube_dl/extractor/go.py            |  3 +-
 youtube_dl/extractor/itv.py           |  4 +-
 youtube_dl/extractor/nrk.py           |  4 +-
 youtube_dl/extractor/ondemandkorea.py |  3 +-
 youtube_dl/extractor/pbs.py           |  5 ++-
 youtube_dl/extractor/srgssr.py        |  6 ++-
 youtube_dl/extractor/svt.py           |  4 +-
 youtube_dl/extractor/vbox7.py         |  3 +-
 youtube_dl/extractor/vgtv.py          |  5 ++-
 youtube_dl/extractor/viki.py          |  2 +-
 youtube_dl/utils.py                   |  2 +-
 13 files changed, 72 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 6eb6a25b8..272da74b6 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -323,10 +323,15 @@ class InfoExtractor(object):
     _real_extract() methods and define a _VALID_URL regexp.
     Probably, they should also be added to the list of extractors.
 
-    _BYPASS_GEO attribute may be set to False in order to disable
+    _GEO_BYPASS attribute may be set to False in order to disable
     geo restriction bypass mechanisms for a particular extractor.
     Though it won't disable explicit geo restriction bypass based on
-    country code provided with geo_bypass_country.
+    country code provided with geo_bypass_country. (experimental)
+
+    _GEO_COUNTRIES attribute may contain a list of presumably geo unrestricted
+    countries for this extractor. One of these countries will be used by
+    geo restriction bypass mechanism right away in order to bypass
+    geo restriction, of course, if the mechanism is not disabled. (experimental)
 
     Finally, the _WORKING attribute should be set to False for broken IEs
     in order to warn the users and skip the tests.
@@ -335,7 +340,8 @@ class InfoExtractor(object):
     _ready = False
     _downloader = None
     _x_forwarded_for_ip = None
-    _BYPASS_GEO = True
+    _GEO_BYPASS = True
+    _GEO_COUNTRIES = None
     _WORKING = True
 
     def __init__(self, downloader=None):
@@ -370,14 +376,28 @@ class InfoExtractor(object):
 
     def initialize(self):
         """Initializes an instance (authentication, etc)."""
-        if not self._x_forwarded_for_ip:
-            country_code = self._downloader.params.get('geo_bypass_country', None)
-            if country_code:
-                self._x_forwarded_for_ip = GeoUtils.random_ipv4(country_code)
+        self.__initialize_geo_bypass()
         if not self._ready:
             self._real_initialize()
             self._ready = True
 
+    def __initialize_geo_bypass(self):
+        if not self._x_forwarded_for_ip:
+            country_code = self._downloader.params.get('geo_bypass_country', None)
+            # If there is no explicit country for geo bypass specified and
+            # the extractor is known to be geo restricted let's fake IP
+            # as X-Forwarded-For right away.
+            if (not country_code and
+                    self._GEO_BYPASS and
+                    self._downloader.params.get('geo_bypass', True) and
+                    self._GEO_COUNTRIES):
+                country_code = random.choice(self._GEO_COUNTRIES)
+            if country_code:
+                self._x_forwarded_for_ip = GeoUtils.random_ipv4(country_code)
+                if self._downloader.params.get('verbose', False):
+                    self._downloader.to_stdout(
+                        '[debug] Using fake %s IP as X-Forwarded-For.' % self._x_forwarded_for_ip)
+
     def extract(self, url):
         """Extracts URL information and returns it in list of dicts."""
         try:
@@ -389,16 +409,8 @@ class InfoExtractor(object):
                         ie_result['__x_forwarded_for_ip'] = self._x_forwarded_for_ip
                     return ie_result
                 except GeoRestrictedError as e:
-                    if (not self._downloader.params.get('geo_bypass_country', None) and
-                            self._BYPASS_GEO and
-                            self._downloader.params.get('geo_bypass', True) and
-                            not self._x_forwarded_for_ip and
-                            e.countries):
-                        self._x_forwarded_for_ip = GeoUtils.random_ipv4(random.choice(e.countries))
-                        if self._x_forwarded_for_ip:
-                            self.report_warning(
-                                'Video is geo restricted. Retrying extraction with fake %s IP as X-Forwarded-For.' % self._x_forwarded_for_ip)
-                            continue
+                    if self.__maybe_fake_ip_and_retry(e.countries):
+                        continue
                     raise
         except ExtractorError:
             raise
@@ -407,6 +419,19 @@ class InfoExtractor(object):
         except (KeyError, StopIteration) as e:
             raise ExtractorError('An extractor error has occurred.', cause=e)
 
+    def __maybe_fake_ip_and_retry(self, countries):
+        if (not self._downloader.params.get('geo_bypass_country', None) and
+                self._GEO_BYPASS and
+                self._downloader.params.get('geo_bypass', True) and
+                not self._x_forwarded_for_ip and
+                countries):
+            self._x_forwarded_for_ip = GeoUtils.random_ipv4(random.choice(countries))
+            if self._x_forwarded_for_ip:
+                self.report_warning(
+                    'Video is geo restricted. Retrying extraction with fake %s IP as X-Forwarded-For.' % self._x_forwarded_for_ip)
+                return True
+        return False
+
     def set_downloader(self, downloader):
         """Sets the downloader for this IE."""
         self._downloader = downloader
diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 755db806a..e7abc8889 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -20,6 +20,7 @@ from ..utils import (
 class DramaFeverBaseIE(AMPIE):
     _LOGIN_URL = 'https://www.dramafever.com/accounts/login/'
     _NETRC_MACHINE = 'dramafever'
+    _GEO_COUNTRIES = ['US', 'CA']
 
     _CONSUMER_SECRET = 'DA59dtVXYLxajktV'
 
@@ -118,7 +119,7 @@ class DramaFeverIE(DramaFeverBaseIE):
             if isinstance(e.cause, compat_HTTPError):
                 self.raise_geo_restricted(
                     msg='Currently unavailable in your country',
-                    countries=['US', 'CA'])
+                    countries=self._GEO_COUNTRIES)
             raise
 
         series_id, episode_number = video_id.split('.')
diff --git a/youtube_dl/extractor/go.py b/youtube_dl/extractor/go.py
index ec902c670..b205bfc7c 100644
--- a/youtube_dl/extractor/go.py
+++ b/youtube_dl/extractor/go.py
@@ -37,6 +37,7 @@ class GoIE(AdobePassIE):
         }
     }
     _VALID_URL = r'https?://(?:(?P<sub_domain>%s)\.)?go\.com/(?:[^/]+/)*(?:vdka(?P<id>\w+)|season-\d+/\d+-(?P<display_id>[^/?#]+))' % '|'.join(_SITE_INFO.keys())
+    _GEO_COUNTRIES = ['US']
     _TESTS = [{
         'url': 'http://abc.go.com/shows/castle/video/most-recent/vdka0_g86w5onx',
         'info_dict': {
@@ -104,7 +105,7 @@ class GoIE(AdobePassIE):
                         for error in errors:
                             if error.get('code') == 1002:
                                 self.raise_geo_restricted(
-                                    error['message'], countries=['US'])
+                                    error['message'], countries=self._GEO_COUNTRIES)
                         error_message = ', '.join([error['message'] for error in errors])
                         raise ExtractorError('%s said: %s' % (self.IE_NAME, error_message), expected=True)
                     asset_url += '?' + entitlement['uplynkData']['sessionKey']
diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py
index aabde15f3..021c6b278 100644
--- a/youtube_dl/extractor/itv.py
+++ b/youtube_dl/extractor/itv.py
@@ -24,6 +24,7 @@ from ..utils import (
 
 class ITVIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?itv\.com/hub/[^/]+/(?P<id>[0-9a-zA-Z]+)'
+    _GEO_COUNTRIES = ['GB']
     _TEST = {
         'url': 'http://www.itv.com/hub/mr-bean-animated-series/2a2936a0053',
         'info_dict': {
@@ -101,7 +102,8 @@ class ITVIE(InfoExtractor):
             fault_code = xpath_text(resp_env, './/faultcode')
             fault_string = xpath_text(resp_env, './/faultstring')
             if fault_code == 'InvalidGeoRegion':
-                self.raise_geo_restricted(msg=fault_string, countries=['GB'])
+                self.raise_geo_restricted(
+                    msg=fault_string, countries=self._GEO_COUNTRIES)
             raise ExtractorError('%s said: %s' % (self.IE_NAME, fault_string))
         title = xpath_text(playlist, 'EpisodeTitle', fatal=True)
         video_element = xpath_element(playlist, 'VideoEntries/Video', fatal=True)
diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 78ece33e1..13af9ed1f 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -14,6 +14,7 @@ from ..utils import (
 
 
 class NRKBaseIE(InfoExtractor):
+    _GEO_COUNTRIES = ['NO']
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
@@ -93,7 +94,8 @@ class NRKBaseIE(InfoExtractor):
             # Can be ProgramIsGeoBlocked or ChannelIsGeoBlocked*
             if 'IsGeoBlocked' in message_type:
                 self.raise_geo_restricted(
-                    msg=MESSAGES.get('ProgramIsGeoBlocked'), countries=['NO'])
+                    msg=MESSAGES.get('ProgramIsGeoBlocked'),
+                    countries=self._GEO_COUNTRIES)
             raise ExtractorError(
                 '%s said: %s' % (self.IE_NAME, MESSAGES.get(
                     message_type, message_type)),
diff --git a/youtube_dl/extractor/ondemandkorea.py b/youtube_dl/extractor/ondemandkorea.py
index 0c85d549e..df1ce3c1d 100644
--- a/youtube_dl/extractor/ondemandkorea.py
+++ b/youtube_dl/extractor/ondemandkorea.py
@@ -10,6 +10,7 @@ from ..utils import (
 
 class OnDemandKoreaIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?ondemandkorea\.com/(?P<id>[^/]+)\.html'
+    _GEO_COUNTRIES = ['US', 'CA']
     _TEST = {
         'url': 'http://www.ondemandkorea.com/ask-us-anything-e43.html',
         'info_dict': {
@@ -36,7 +37,7 @@ class OnDemandKoreaIE(InfoExtractor):
         if 'msg_block_01.png' in webpage:
             self.raise_geo_restricted(
                 msg='This content is not available in your region',
-                countries=['US', 'CA'])
+                countries=self._GEO_COUNTRIES)
 
         if 'This video is only available to ODK PLUS members.' in webpage:
             raise ExtractorError(
diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 64f47bae3..3e51b4dd7 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -193,6 +193,8 @@ class PBSIE(InfoExtractor):
         )
     ''' % '|'.join(list(zip(*_STATIONS))[0])
 
+    _GEO_COUNTRIES = ['US']
+
     _TESTS = [
         {
             'url': 'http://www.pbs.org/tpt/constitution-usa-peter-sagal/watch/a-more-perfect-union/',
@@ -492,7 +494,8 @@ class PBSIE(InfoExtractor):
                 message = self._ERRORS.get(
                     redirect_info['http_code'], redirect_info['message'])
                 if redirect_info['http_code'] == 403:
-                    self.raise_geo_restricted(msg=message, countries=['US'])
+                    self.raise_geo_restricted(
+                        msg=message, countries=self._GEO_COUNTRIES)
                 raise ExtractorError(
                     '%s said: %s' % (self.IE_NAME, message), expected=True)
 
diff --git a/youtube_dl/extractor/srgssr.py b/youtube_dl/extractor/srgssr.py
index a35a0a538..bb73eb1d5 100644
--- a/youtube_dl/extractor/srgssr.py
+++ b/youtube_dl/extractor/srgssr.py
@@ -14,7 +14,8 @@ from ..utils import (
 
 class SRGSSRIE(InfoExtractor):
     _VALID_URL = r'(?:https?://tp\.srgssr\.ch/p(?:/[^/]+)+\?urn=urn|srgssr):(?P<bu>srf|rts|rsi|rtr|swi):(?:[^:]+:)?(?P<type>video|audio):(?P<id>[0-9a-f\-]{36}|\d+)'
-    _BYPASS_GEO = False
+    _GEO_BYPASS = False
+    _GEO_COUNTRIES = ['CH']
 
     _ERRORS = {
         'AGERATING12': 'To protect children under the age of 12, this video is only available between 8 p.m. and 6 a.m.',
@@ -43,7 +44,8 @@ class SRGSSRIE(InfoExtractor):
         if media_data.get('block') and media_data['block'] in self._ERRORS:
             message = self._ERRORS[media_data['block']]
             if media_data['block'] == 'GEOBLOCK':
-                self.raise_geo_restricted(msg=message, countries=['CH'])
+                self.raise_geo_restricted(
+                    msg=message, countries=self._GEO_COUNTRIES)
             raise ExtractorError(
                 '%s said: %s' % (self.IE_NAME, message), expected=True)
 
diff --git a/youtube_dl/extractor/svt.py b/youtube_dl/extractor/svt.py
index f2a2200bf..9e2c9fcc6 100644
--- a/youtube_dl/extractor/svt.py
+++ b/youtube_dl/extractor/svt.py
@@ -13,6 +13,7 @@ from ..utils import (
 
 
 class SVTBaseIE(InfoExtractor):
+    _GEO_COUNTRIES = ['SE']
     def _extract_video(self, video_info, video_id):
         formats = []
         for vr in video_info['videoReferences']:
@@ -39,7 +40,8 @@ class SVTBaseIE(InfoExtractor):
                 })
         if not formats and video_info.get('rights', {}).get('geoBlockedSweden'):
             self.raise_geo_restricted(
-                'This video is only available in Sweden', countries=['SE'])
+                'This video is only available in Sweden',
+                countries=self._GEO_COUNTRIES)
         self._sort_formats(formats)
 
         subtitles = {}
diff --git a/youtube_dl/extractor/vbox7.py b/youtube_dl/extractor/vbox7.py
index f86d804c1..8152acefd 100644
--- a/youtube_dl/extractor/vbox7.py
+++ b/youtube_dl/extractor/vbox7.py
@@ -20,6 +20,7 @@ class Vbox7IE(InfoExtractor):
                         )
                         (?P<id>[\da-fA-F]+)
                     '''
+    _GEO_COUNTRIES = ['BG']
     _TESTS = [{
         'url': 'http://vbox7.com/play:0946fff23c',
         'md5': 'a60f9ab3a3a2f013ef9a967d5f7be5bf',
@@ -78,7 +79,7 @@ class Vbox7IE(InfoExtractor):
         video_url = video['src']
 
         if '/na.mp4' in video_url:
-            self.raise_geo_restricted(countries=['BG'])
+            self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
 
         uploader = video.get('uploader')
 
diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 1709fd6bb..0f8c156a7 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 class VGTVIE(XstreamIE):
     IE_DESC = 'VGTV, BTTV, FTV, Aftenposten and Aftonbladet'
-    _BYPASS_GEO = False
+    _GEO_BYPASS = False
 
     _HOST_TO_APPNAME = {
         'vgtv.no': 'vgtv',
@@ -218,7 +218,8 @@ class VGTVIE(XstreamIE):
             properties = try_get(
                 data, lambda x: x['streamConfiguration']['properties'], list)
             if properties and 'geoblocked' in properties:
-                raise self.raise_geo_restricted(countries=['NO'])
+                raise self.raise_geo_restricted(
+                    countries=[host.rpartition('.')[-1].partition('/')[0].upper()])
 
         self._sort_formats(info['formats'])
 
diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 68a74e246..e9c8bf824 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -27,7 +27,7 @@ class VikiBaseIE(InfoExtractor):
     _APP_VERSION = '2.2.5.1428709186'
     _APP_SECRET = '-$iJ}@p7!G@SyU/je1bEyWg}upLu-6V6-Lg9VD(]siH,r.,m-r|ulZ,U4LC/SeR)'
 
-    _BYPASS_GEO = False
+    _GEO_BYPASS = False
     _NETRC_MACHINE = 'viki'
 
     _token = None
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index cbf7639c5..17b83794a 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3291,7 +3291,7 @@ class GeoUtils(object):
         addr_min = compat_struct_unpack('!L', socket.inet_aton(addr))[0]
         addr_max = addr_min | (0xffffffff >> int(preflen))
         return compat_str(socket.inet_ntoa(
-            compat_struct_pack('!I', random.randint(addr_min, addr_max))))
+            compat_struct_pack('!L', random.randint(addr_min, addr_max))))
 
 
 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):

From 0aa10994f452b4ca978baf124df0cb2239d49305 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Feb 2017 03:58:17 +0700
Subject: [PATCH 0303/1696] [options] Move geo restriction related options to
 separate section

---
 youtube_dl/options.py | 37 +++++++++++++++++++------------------
 1 file changed, 19 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index ae3f50754..2c880d06a 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -228,17 +228,29 @@ def parseOpts(overrideArguments=None):
         action='store_const', const='::', dest='source_address',
         help='Make all connections via IPv6',
     )
-    network.add_option(
+
+    geo = optparse.OptionGroup(parser, 'Geo Restriction')
+    geo.add_option(
         '--geo-verification-proxy',
         dest='geo_verification_proxy', default=None, metavar='URL',
         help='Use this proxy to verify the IP address for some geo-restricted sites. '
-        'The default proxy specified by --proxy (or none, if the options is not present) is used for the actual downloading.'
-    )
-    network.add_option(
+        'The default proxy specified by --proxy (or none, if the options is not present) is used for the actual downloading.')
+    geo.add_option(
         '--cn-verification-proxy',
         dest='cn_verification_proxy', default=None, metavar='URL',
-        help=optparse.SUPPRESS_HELP,
-    )
+        help=optparse.SUPPRESS_HELP)
+    geo.add_option(
+        '--geo-bypass',
+        action='store_true', dest='geo_bypass', default=True,
+        help='Bypass geographic restriction via faking X-Forwarded-For HTTP header (experimental)')
+    geo.add_option(
+        '--no-geo-bypass',
+        action='store_false', dest='geo_bypass', default=True,
+        help='Do not bypass geographic restriction via faking X-Forwarded-For HTTP header (experimental)')
+    geo.add_option(
+        '--geo-bypass-country', metavar='CODE',
+        dest='geo_bypass_country', default=None,
+        help='Force bypass geographic restriction with explicitly provided two-letter ISO 3166-2 country code (experimental)')
 
     selection = optparse.OptionGroup(parser, 'Video Selection')
     selection.add_option(
@@ -549,18 +561,6 @@ def parseOpts(overrideArguments=None):
             'Upper bound of a range for randomized sleep before each download '
             '(maximum possible number of seconds to sleep). Must only be used '
             'along with --min-sleep-interval.'))
-    workarounds.add_option(
-        '--geo-bypass',
-        action='store_true', dest='geo_bypass', default=True,
-        help='Bypass geographic restriction via faking X-Forwarded-For HTTP header (experimental)')
-    workarounds.add_option(
-        '--no-geo-bypass',
-        action='store_false', dest='geo_bypass', default=True,
-        help='Do not bypass geographic restriction via faking X-Forwarded-For HTTP header (experimental)')
-    workarounds.add_option(
-        '--geo-bypass-country', metavar='CODE',
-        dest='geo_bypass_country', default=None,
-        help='Force bypass geographic restriction with explicitly provided two-letter ISO 3166-2 country code (experimental)')
 
     verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
     verbosity.add_option(
@@ -848,6 +848,7 @@ def parseOpts(overrideArguments=None):
 
     parser.add_option_group(general)
     parser.add_option_group(network)
+    parser.add_option_group(geo)
     parser.add_option_group(selection)
     parser.add_option_group(downloader)
     parser.add_option_group(filesystem)

From 553f6dbac7afac84994eae18f551799f807d1503 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Feb 2017 04:18:22 +0700
Subject: [PATCH 0304/1696] [downloader/dash] Honor HTTP headers when
 downloading fragments For example,
 https://www.oppetarkiv.se/video/1196142/natten-ar-dagens-mor

---
 youtube_dl/downloader/dash.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index 8437dde30..e2ddc369e 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -43,7 +43,10 @@ class DashSegmentsFD(FragmentFD):
             count = 0
             while count <= fragment_retries:
                 try:
-                    success = ctx['dl'].download(target_filename, {'url': segment_url})
+                    success = ctx['dl'].download(target_filename, {
+                        'url': segment_url,
+                        'http_headers': info_dict.get('http_headers'),
+                    })
                     if not success:
                         return False
                     down, target_sanitized = sanitize_open(target_filename, 'rb')

From de64e23c5663ceb4f62264077a7993d13ace0d6c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Feb 2017 04:18:36 +0700
Subject: [PATCH 0305/1696] [downloader/ism] Honor HTTP headers when
 downloading fragments

---
 youtube_dl/downloader/ism.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/ism.py b/youtube_dl/downloader/ism.py
index 93cac5e98..63a636cb7 100644
--- a/youtube_dl/downloader/ism.py
+++ b/youtube_dl/downloader/ism.py
@@ -238,7 +238,10 @@ class IsmFD(FragmentFD):
             count = 0
             while count <= fragment_retries:
                 try:
-                    success = ctx['dl'].download(target_filename, {'url': segment_url})
+                    success = ctx['dl'].download(target_filename, {
+                        'url': segment_url,
+                        'http_headers': info_dict.get('http_headers'),
+                    })
                     if not success:
                         return False
                     down, target_sanitized = sanitize_open(target_filename, 'rb')

From f1a78ee4ef3bfd8e7ff06a3014d96c3cf11b4d9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Feb 2017 06:16:00 +0700
Subject: [PATCH 0306/1696] [tv4] Switch to hls3 protocol (closes #12177)

---
 youtube_dl/extractor/tv4.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tv4.py b/youtube_dl/extractor/tv4.py
index ad79db92b..7918e3d86 100644
--- a/youtube_dl/extractor/tv4.py
+++ b/youtube_dl/extractor/tv4.py
@@ -80,7 +80,7 @@ class TV4IE(InfoExtractor):
         subtitles = {}
         formats = []
         # http formats are linked with unresolvable host
-        for kind in ('hls', ''):
+        for kind in ('hls3', ''):
             data = self._download_json(
                 'https://prima.tv4play.se/api/web/asset/%s/play.json' % video_id,
                 video_id, 'Downloading sources JSON', query={

From c58b7ffef43f60fa6a183c849cfdca42e36eae0c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Feb 2017 06:24:38 +0700
Subject: [PATCH 0307/1696] [tv4] Bypass geo restriction and improve detection

---
 youtube_dl/extractor/tv4.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/tv4.py b/youtube_dl/extractor/tv4.py
index 7918e3d86..7aeb2c620 100644
--- a/youtube_dl/extractor/tv4.py
+++ b/youtube_dl/extractor/tv4.py
@@ -24,6 +24,7 @@ class TV4IE(InfoExtractor):
                 sport/|
             )
         )(?P<id>[0-9]+)'''
+    _GEO_COUNTRIES = ['SE']
     _TESTS = [
         {
             'url': 'http://www.tv4.se/kalla-fakta/klipp/kalla-fakta-5-english-subtitles-2491650',
@@ -71,10 +72,6 @@ class TV4IE(InfoExtractor):
             'http://www.tv4play.se/player/assets/%s.json' % video_id,
             video_id, 'Downloading video info JSON')
 
-        # If is_geo_restricted is true, it doesn't necessarily mean we can't download it
-        if info.get('is_geo_restricted'):
-            self.report_warning('This content might not be available in your country due to licensing restrictions.')
-
         title = info['title']
 
         subtitles = {}
@@ -113,6 +110,10 @@ class TV4IE(InfoExtractor):
                                 'url': manifest_url,
                                 'ext': 'vtt',
                             }]})
+
+        if not formats and info.get('is_geo_restricted'):
+            self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
+
         self._sort_formats(formats)
 
         return {

From 8936f68a0ba3284c88ec619fb4cc22eb0499e7f3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 21 Oct 2015 00:37:28 +0800
Subject: [PATCH 0308/1696] [travis] Run tests in parallel

[test_download] Print test names in case of network errors

[test_download] Add comments for nose parameters

[test_download] Modify outtmpl to prevent info JSON filename conflicts

Thanks @jaimeMF for the idea.

[travis] Only download tests should be run in parallel
---
 devscripts/run_tests.sh |  4 +++-
 test/test_download.py   | 11 ++++++++---
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/devscripts/run_tests.sh b/devscripts/run_tests.sh
index 7f4c1e083..c60807215 100755
--- a/devscripts/run_tests.sh
+++ b/devscripts/run_tests.sh
@@ -3,6 +3,7 @@
 DOWNLOAD_TESTS="age_restriction|download|subtitles|write_annotations|iqiyi_sdk_interpreter"
 
 test_set=""
+multiprocess_args=""
 
 case "$YTDL_TEST_SET" in
     core)
@@ -10,10 +11,11 @@ case "$YTDL_TEST_SET" in
     ;;
     download)
         test_set="-I test_(?!$DOWNLOAD_TESTS).+\.py"
+        multiprocess_args="--processes=4 --process-timeout=540"
     ;;
     *)
         break
     ;;
 esac
 
-nosetests test --verbose $test_set
+nosetests test --verbose $test_set $multiprocess_args
diff --git a/test/test_download.py b/test/test_download.py
index 463952989..30034f978 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -65,6 +65,10 @@ defs = gettestcases()
 
 
 class TestDownload(unittest.TestCase):
+    # Parallel testing in nosetests. See
+    # http://nose.readthedocs.org/en/latest/doc_tests/test_multiprocess/multiprocess.html
+    _multiprocess_shared_ = True
+
     maxDiff = None
 
     def setUp(self):
@@ -73,7 +77,7 @@ class TestDownload(unittest.TestCase):
 # Dynamically generate tests
 
 
-def generator(test_case):
+def generator(test_case, tname):
 
     def test_template(self):
         ie = youtube_dl.extractor.get_info_extractor(test_case['name'])
@@ -102,6 +106,7 @@ def generator(test_case):
                 return
 
         params = get_params(test_case.get('params', {}))
+        params['outtmpl'] = tname + '_' + params['outtmpl']
         if is_playlist and 'playlist' not in test_case:
             params.setdefault('extract_flat', 'in_playlist')
             params.setdefault('skip_download', True)
@@ -146,7 +151,7 @@ def generator(test_case):
                         raise
 
                     if try_num == RETRIES:
-                        report_warning('Failed due to network errors, skipping...')
+                        report_warning('%s failed due to network errors, skipping...' % tname)
                         return
 
                     print('Retrying: {0} failed tries\n\n##########\n\n'.format(try_num))
@@ -221,12 +226,12 @@ def generator(test_case):
 
 # And add them to TestDownload
 for n, test_case in enumerate(defs):
-    test_method = generator(test_case)
     tname = 'test_' + str(test_case['name'])
     i = 1
     while hasattr(TestDownload, tname):
         tname = 'test_%s_%d' % (test_case['name'], i)
         i += 1
+    test_method = generator(test_case, tname)
     test_method.__name__ = str(tname)
     setattr(TestDownload, test_method.__name__, test_method)
     del test_method

From 983e9b774643fc588fbfb51d314381025ffac248 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Feb 2017 00:59:31 +0700
Subject: [PATCH 0309/1696] [nrk] Update _API_HOST and relax _VALID_URL

---
 youtube_dl/extractor/nrk.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 13af9ed1f..7b98626f2 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -164,12 +164,12 @@ class NRKIE(NRKBaseIE):
                             https?://
                                 (?:
                                     (?:www\.)?nrk\.no/video/PS\*|
-                                    v8-psapi\.nrk\.no/mediaelement/
+                                    v8[-.]psapi\.nrk\.no/mediaelement/
                                 )
                             )
-                            (?P<id>[^/?#&]+)
+                            (?P<id>[^?#&]+)
                         '''
-    _API_HOST = 'v8.psapi.nrk.no'
+    _API_HOST = 'v8-psapi.nrk.no'
     _TESTS = [{
         # video
         'url': 'http://www.nrk.no/video/PS*150533',
@@ -195,6 +195,9 @@ class NRKIE(NRKBaseIE):
     }, {
         'url': 'nrk:ecc1b952-96dc-4a98-81b9-5296dc7a98d9',
         'only_matching': True,
+    }, {
+        'url': 'nrk:clip/7707d5a3-ebe7-434a-87d5-a3ebe7a34a70',
+        'only_matching': True,
     }, {
         'url': 'https://v8-psapi.nrk.no/mediaelement/ecc1b952-96dc-4a98-81b9-5296dc7a98d9',
         'only_matching': True,

From 8ffb8e63fe2853f9e51420ba224db428f1241c35 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Feb 2017 01:00:53 +0700
Subject: [PATCH 0310/1696] [prosiebensat1] Throw ExtractionError on
 unsupported page type (closes #12180)

---
 youtube_dl/extractor/prosiebensat1.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 5091d8456..1245309a7 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -424,3 +424,6 @@ class ProSiebenSat1IE(ProSiebenSat1BaseIE):
             return self._extract_clip(url, webpage)
         elif page_type == 'playlist':
             return self._extract_playlist(url, webpage)
+        else:
+            raise ExtractorError(
+                'Unsupported page type %s' % page_type, expected=True)

From c78dd3549155d4cb8f70707c1b4085f9f974db2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Feb 2017 02:25:39 +0700
Subject: [PATCH 0311/1696] [nrk] PEP 8

---
 youtube_dl/extractor/nrk.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 7b98626f2..7fe79cb53 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -15,6 +15,7 @@ from ..utils import (
 
 class NRKBaseIE(InfoExtractor):
     _GEO_COUNTRIES = ['NO']
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From 6d4c259765de86bdb8a10e71bfbc7b6e196f6967 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Feb 2017 02:25:55 +0700
Subject: [PATCH 0312/1696] [svt] PEP 8

---
 youtube_dl/extractor/svt.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/svt.py b/youtube_dl/extractor/svt.py
index 9e2c9fcc6..1b5afb73e 100644
--- a/youtube_dl/extractor/svt.py
+++ b/youtube_dl/extractor/svt.py
@@ -14,6 +14,7 @@ from ..utils import (
 
 class SVTBaseIE(InfoExtractor):
     _GEO_COUNTRIES = ['SE']
+
     def _extract_video(self, video_info, video_id):
         formats = []
         for vr in video_info['videoReferences']:

From 2cc7fcd338e8690a5c211b95fb9e0dcdc5d98ef5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Feb 2017 03:06:52 +0700
Subject: [PATCH 0313/1696] [commonmistakes] Disable UnicodeBOM extractor test
 for python 3.2

---
 youtube_dl/extractor/commonmistakes.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/commonmistakes.py b/youtube_dl/extractor/commonmistakes.py
index d3ed4a9a4..79f7a9cd1 100644
--- a/youtube_dl/extractor/commonmistakes.py
+++ b/youtube_dl/extractor/commonmistakes.py
@@ -1,5 +1,7 @@
 from __future__ import unicode_literals
 
+import sys
+
 from .common import InfoExtractor
 from ..utils import ExtractorError
 
@@ -33,7 +35,9 @@ class UnicodeBOMIE(InfoExtractor):
         IE_DESC = False
         _VALID_URL = r'(?P<bom>\ufeff)(?P<id>.*)$'
 
-        _TESTS = [{
+        # Disable test for python 3.2 since BOM is broken in re in this version
+        # (see https://github.com/rg3/youtube-dl/issues/9751)
+        _TESTS = [] if (3, 0) < sys.version_info <= (3, 3) else [{
             'url': '\ufeffhttp://www.youtube.com/watch?v=BaW_jenozKc',
             'only_matching': True,
         }]

From 82f662182b9ade630b37af81ebf8ae7ae6468898 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Feb 2017 23:16:14 +0700
Subject: [PATCH 0314/1696] [iprima] Modernize

---
 youtube_dl/extractor/iprima.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index 0fe576883..7afa2def0 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -8,7 +8,6 @@ from .common import InfoExtractor
 from ..utils import (
     determine_ext,
     js_to_json,
-    sanitized_Request,
 )
 
 
@@ -38,11 +37,13 @@ class IPrimaIE(InfoExtractor):
 
         video_id = self._search_regex(r'data-product="([^"]+)">', webpage, 'real id')
 
-        req = sanitized_Request(
-            'http://play.iprima.cz/prehravac/init?_infuse=1'
-            '&_ts=%s&productId=%s' % (round(time.time()), video_id))
-        req.add_header('Referer', url)
-        playerpage = self._download_webpage(req, video_id, note='Downloading player')
+        playerpage = self._download_webpage(
+            'http://play.iprima.cz/prehravac/init',
+            video_id, note='Downloading player', query={
+                '_infuse': 1,
+                '_ts': round(time.time()),
+                'productId': video_id,
+            }, headers={'Referer': url})
 
         formats = []
 

From da42ff066811490064e0c3039b9db5c0e9a69f58 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Feb 2017 23:17:19 +0700
Subject: [PATCH 0315/1696] [iprima] Improve geo restriction detection and
 disable geo bypass

---
 youtube_dl/extractor/iprima.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index 7afa2def0..a29e6a5ba 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -13,6 +13,7 @@ from ..utils import (
 
 class IPrimaIE(InfoExtractor):
     _VALID_URL = r'https?://play\.iprima\.cz/(?:.+/)?(?P<id>[^?#]+)'
+    _GEO_BYPASS = False
 
     _TESTS = [{
         'url': 'http://play.iprima.cz/gondici-s-r-o-33',
@@ -28,6 +29,10 @@ class IPrimaIE(InfoExtractor):
     }, {
         'url': 'http://play.iprima.cz/particka/particka-92',
         'only_matching': True,
+    }, {
+        # geo restricted
+        'url': 'http://play.iprima.cz/closer-nove-pripady/closer-nove-pripady-iv-1',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -83,7 +88,7 @@ class IPrimaIE(InfoExtractor):
                 extract_formats(src)
 
         if not formats and '>GEO_IP_NOT_ALLOWED<' in playerpage:
-            self.raise_geo_restricted()
+            self.raise_geo_restricted(countries=['CZ'])
 
         self._sort_formats(formats)
 

From 3ccdde8cb76cacb7b2b64469ca51d3b1877da1f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Feb 2017 23:21:15 +0700
Subject: [PATCH 0316/1696] [extractor/common] Emphasize geo bypass APIs are
 experimental

---
 youtube_dl/extractor/common.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 272da74b6..1ae264722 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -333,6 +333,9 @@ class InfoExtractor(object):
     geo restriction bypass mechanism right away in order to bypass
     geo restriction, of course, if the mechanism is not disabled. (experimental)
 
+    NB: both these geo attributes are experimental and may change in future
+    or be completely removed.
+
     Finally, the _WORKING attribute should be set to False for broken IEs
     in order to warn the users and skip the tests.
     """

From 6926304472d4598f095abc7115ca0f36068271d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 21 Feb 2017 00:54:43 +0700
Subject: [PATCH 0317/1696] [spankbang] Make uploader optional (closes #12193)

---
 youtube_dl/extractor/spankbang.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index 123c33ac3..3394c7e6b 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -23,6 +23,10 @@ class SpankBangIE(InfoExtractor):
         # 480p only
         'url': 'http://spankbang.com/1vt0/video/solvane+gangbang',
         'only_matching': True,
+    }, {
+        # no uploader
+        'url': 'http://spankbang.com/lklg/video/sex+with+anyone+wedding+edition+2',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -48,7 +52,7 @@ class SpankBangIE(InfoExtractor):
         thumbnail = self._og_search_thumbnail(webpage)
         uploader = self._search_regex(
             r'class="user"[^>]*><img[^>]+>([^<]+)',
-            webpage, 'uploader', fatal=False)
+            webpage, 'uploader', default=None)
 
         age_limit = self._rta_search(webpage)
 

From 890d44b005c3073442064a847f2e0204619a8b47 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 20 Feb 2017 19:00:06 +0100
Subject: [PATCH 0318/1696] [adobepass] add support for Time Warner
 Cable(closes #12191)

---
 youtube_dl/extractor/adobepass.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
index 12eeab271..4d655bd5e 100644
--- a/youtube_dl/extractor/adobepass.py
+++ b/youtube_dl/extractor/adobepass.py
@@ -31,6 +31,11 @@ MSO_INFO = {
         'username_field': 'user',
         'password_field': 'passwd',
     },
+    'TWC': {
+        'name': 'Time Warner Cable | Spectrum',
+        'username_field': 'Ecom_User_ID',
+        'password_field': 'Ecom_Password',
+    },
     'thr030': {
         'name': '3 Rivers Communications'
     },

From e469ab25280433781881d0c3ea6fd423ac5fea71 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 21 Feb 2017 14:38:00 +0100
Subject: [PATCH 0319/1696] [ninecninemedia] use geo bypass mechanism

---
 youtube_dl/extractor/ninecninemedia.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/ninecninemedia.py b/youtube_dl/extractor/ninecninemedia.py
index ec4d675e2..d9943fc2c 100644
--- a/youtube_dl/extractor/ninecninemedia.py
+++ b/youtube_dl/extractor/ninecninemedia.py
@@ -19,6 +19,7 @@ class NineCNineMediaBaseIE(InfoExtractor):
 
 class NineCNineMediaStackIE(NineCNineMediaBaseIE):
     IE_NAME = '9c9media:stack'
+    _GEO_COUNTRIES = ['CA']
     _VALID_URL = r'9c9media:stack:(?P<destination_code>[^:]+):(?P<content_id>\d+):(?P<content_package>\d+):(?P<id>\d+)'
 
     def _real_extract(self, url):

From e39b5d4ab83de7a466c6d4c9528d385758566b22 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 21 Feb 2017 23:00:43 +0700
Subject: [PATCH 0320/1696] [extractor/common] Allow calling
 _initialize_geo_bypass from extractors (#11970)

---
 youtube_dl/extractor/common.py | 27 +++++++++++++++++++++++----
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 1ae264722..86aff3312 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -379,12 +379,31 @@ class InfoExtractor(object):
 
     def initialize(self):
         """Initializes an instance (authentication, etc)."""
-        self.__initialize_geo_bypass()
+        self._initialize_geo_bypass(self._GEO_COUNTRIES)
         if not self._ready:
             self._real_initialize()
             self._ready = True
 
-    def __initialize_geo_bypass(self):
+    def _initialize_geo_bypass(self, countries):
+        """
+        Initialize geo restriction bypass mechanism.
+
+        This method is used to initialize geo bypass mechanism based on faking
+        X-Forwarded-For HTTP header. A random country from provided country list
+        is selected and a random IP brlonging to this country is generated. This
+        IP will be passed as X-Forwarded-For HTTP header in all subsequent
+        HTTP requests.
+        Method does nothing if no countries are specified.
+
+        This method will be used for initial geo bypass mechanism initialization
+        during the instance initialization with _GEO_COUNTRIES.
+
+        You may also manually call it from extractor's code if geo countries
+        information is not available beforehand (e.g. obtained during
+        extraction) or due to some another reason.
+        """
+        if not countries:
+            return
         if not self._x_forwarded_for_ip:
             country_code = self._downloader.params.get('geo_bypass_country', None)
             # If there is no explicit country for geo bypass specified and
@@ -393,8 +412,8 @@ class InfoExtractor(object):
             if (not country_code and
                     self._GEO_BYPASS and
                     self._downloader.params.get('geo_bypass', True) and
-                    self._GEO_COUNTRIES):
-                country_code = random.choice(self._GEO_COUNTRIES)
+                    countries):
+                country_code = random.choice(countries)
             if country_code:
                 self._x_forwarded_for_ip = GeoUtils.random_ipv4(country_code)
                 if self._downloader.params.get('verbose', False):

From dc0a869e5ee7a75218a759706bb11f17c4de6b72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 21 Feb 2017 23:05:31 +0700
Subject: [PATCH 0321/1696] [extractor/common] Fix typo

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 86aff3312..6d4789d96 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -390,7 +390,7 @@ class InfoExtractor(object):
 
         This method is used to initialize geo bypass mechanism based on faking
         X-Forwarded-For HTTP header. A random country from provided country list
-        is selected and a random IP brlonging to this country is generated. This
+        is selected and a random IP belonging to this country is generated. This
         IP will be passed as X-Forwarded-For HTTP header in all subsequent
         HTTP requests.
         Method does nothing if no countries are specified.

From 336a76551b92db1c040cbf3c4a9b1857e125ad45 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 21 Feb 2017 23:09:41 +0700
Subject: [PATCH 0322/1696] [extractor/common] Do not quit
 _initialize_geo_bypass on empty countries

---
 youtube_dl/extractor/common.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 6d4789d96..a34fbbc9b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -393,7 +393,6 @@ class InfoExtractor(object):
         is selected and a random IP belonging to this country is generated. This
         IP will be passed as X-Forwarded-For HTTP header in all subsequent
         HTTP requests.
-        Method does nothing if no countries are specified.
 
         This method will be used for initial geo bypass mechanism initialization
         during the instance initialization with _GEO_COUNTRIES.
@@ -402,8 +401,6 @@ class InfoExtractor(object):
         information is not available beforehand (e.g. obtained during
         extraction) or due to some another reason.
         """
-        if not countries:
-            return
         if not self._x_forwarded_for_ip:
             country_code = self._downloader.params.get('geo_bypass_country', None)
             # If there is no explicit country for geo bypass specified and

From eea0716cae1290fe08faea89e24a58ec91098638 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 21 Feb 2017 23:14:33 +0700
Subject: [PATCH 0323/1696] [extractor/common] Print origin country for fake IP

---
 youtube_dl/extractor/common.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a34fbbc9b..4252d6825 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -415,7 +415,8 @@ class InfoExtractor(object):
                 self._x_forwarded_for_ip = GeoUtils.random_ipv4(country_code)
                 if self._downloader.params.get('verbose', False):
                     self._downloader.to_stdout(
-                        '[debug] Using fake %s IP as X-Forwarded-For.' % self._x_forwarded_for_ip)
+                        '[debug] Using fake IP %s (%s) as X-Forwarded-For.'
+                        % (self._x_forwarded_for_ip, country_code.upper()))
 
     def extract(self, url):
         """Extracts URL information and returns it in list of dicts."""
@@ -444,10 +445,12 @@ class InfoExtractor(object):
                 self._downloader.params.get('geo_bypass', True) and
                 not self._x_forwarded_for_ip and
                 countries):
-            self._x_forwarded_for_ip = GeoUtils.random_ipv4(random.choice(countries))
+            country_code = random.choice(countries)
+            self._x_forwarded_for_ip = GeoUtils.random_ipv4(country_code)
             if self._x_forwarded_for_ip:
                 self.report_warning(
-                    'Video is geo restricted. Retrying extraction with fake %s IP as X-Forwarded-For.' % self._x_forwarded_for_ip)
+                    'Video is geo restricted. Retrying extraction with fake IP %s (%s) as X-Forwarded-For.'
+                    % (self._x_forwarded_for_ip, country_code.upper()))
                 return True
         return False
 

From 159aaaa9d09ce5843ec843d6e10030e229709e17 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 21 Feb 2017 23:46:58 +0700
Subject: [PATCH 0324/1696] [ChangeLog] Actualize

---
 ChangeLog | 46 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 2c90f791d..2b02994e0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,49 @@
+version <unreleased>
+
+Core
+* [extractor/common] Allow calling _initialize_geo_bypass from extractors
+  (#11970)
++ [adobepass] Add support for Time Warner Cable (#12191)
++ [travis] Run tests in parallel
++ [downloader/ism] Honor HTTP headers when downloading fragments
++ [downloader/dash] Honor HTTP headers when downloading fragments
++ [utils] Add GeoUtils class for working with geo tools and GeoUtils.random_ipv4
++ Add option --geo-bypass-country for explicit geo bypass on behalf of
+  specified country
++ Add options to control geo bypass mechanism --geo-bypass and --no-geo-bypass
++ Add experimental geo restriction bypass mechanism based on faking
+  X-Forwarded-For HTTP header
++ [utils] Introduce GeoRestrictedError for geo restricted videos
++ [utils] Introduce YoutubeDLError base class for all youtube-dl exceptions
+
+Extractors
++ [ninecninemedia] Use geo bypass mechanism
+* [spankbang] Make uploader optional (#12193)
++ [iprima] Improve geo restriction detection and disable geo bypass
+* [iprima] Modernize
+* [commonmistakes] Disable UnicodeBOM extractor test for python 3.2
++ [prosiebensat1] Throw ExtractionError on unsupported page type (#12180)
+* [nrk] Update _API_HOST and relax _VALID_URL
++ [tv4] Bypass geo restriction and improve detection
+* [tv4] Switch to hls3 protocol (#12177)
++ [viki] Improve geo restriction detection
++ [vgtv] Improve geo restriction detection
++ [srgssr] Improve geo restriction detection
++ [vbox7] Improve geo restriction detection and use geo bypass mechanism
++ [svt] Improve geo restriction detection and use geo bypass mechanism
++ [pbs] Improve geo restriction detection and use geo bypass mechanism
++ [ondemandkorea] Improve geo restriction detection and use geo bypass mechanism
++ [nrk] Improve geo restriction detection and use geo bypass mechanism
++ [itv] Improve geo restriction detection and use geo bypass mechanism
++ [go] Improve geo restriction detection and use geo bypass mechanism
++ [dramafever] Improve geo restriction detection and use geo bypass mechanism
+* [brightcove:legacy] Restrict videoPlayer value (#12040)
++ [tvn24] Add support for tvn24.pl and tvn24bis.pl (#11679)
++ [thisav] Add support for HTML5 media (#11771)
+* [metacafe] Bypass family filter (#10371)
+* [viceland] Improve info extraction
+
+
 version 2017.02.17
 
 Extractors

From 8c6c88c7dae595d5cb7d5926eb00fbaf40103f8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 21 Feb 2017 23:48:24 +0700
Subject: [PATCH 0325/1696] release 2017.02.21

---
 .github/ISSUE_TEMPLATE.md |  6 +++---
 ChangeLog                 |  2 +-
 README.md                 | 34 +++++++++++++++++++++++-----------
 docs/supportedsites.md    |  1 +
 youtube_dl/version.py     |  2 +-
 5 files changed, 29 insertions(+), 16 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 6f1361b32..8b6d14fa2 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.17*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.17**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.21*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.21**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.02.17
+[debug] youtube-dl version 2017.02.21
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 2b02994e0..a479d274f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.02.21
 
 Core
 * [extractor/common] Allow calling _initialize_geo_bypass from extractors
diff --git a/README.md b/README.md
index c2a1a6b02..1eccfd287 100644
--- a/README.md
+++ b/README.md
@@ -99,11 +99,21 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
     --source-address IP              Client-side IP address to bind to
     -4, --force-ipv4                 Make all connections via IPv4
     -6, --force-ipv6                 Make all connections via IPv6
+
+## Geo Restriction:
     --geo-verification-proxy URL     Use this proxy to verify the IP address for
                                      some geo-restricted sites. The default
                                      proxy specified by --proxy (or none, if the
                                      options is not present) is used for the
                                      actual downloading.
+    --geo-bypass                     Bypass geographic restriction via faking
+                                     X-Forwarded-For HTTP header (experimental)
+    --no-geo-bypass                  Do not bypass geographic restriction via
+                                     faking X-Forwarded-For HTTP header
+                                     (experimental)
+    --geo-bypass-country CODE        Force bypass geographic restriction with
+                                     explicitly provided two-letter ISO 3166-2
+                                     country code (experimental)
 
 ## Video Selection:
     --playlist-start NUMBER          Playlist video to start at (default is 1)
@@ -140,17 +150,19 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
                                      check if the key is not present, key >
                                      NUMBER (like "comment_count > 12", also
                                      works with >=, <, <=, !=, =) to compare
-                                     against a number, and & to require multiple
-                                     matches. Values which are not known are
-                                     excluded unless you put a question mark (?)
-                                     after the operator. For example, to only
-                                     match videos that have been liked more than
-                                     100 times and disliked less than 50 times
-                                     (or the dislike functionality is not
-                                     available at the given service), but who
-                                     also have a description, use --match-filter
-                                     "like_count > 100 & dislike_count <? 50 &
-                                     description" .
+                                     against a number, key = 'LITERAL' (like
+                                     "uploader = 'Mike Smith'", also works with
+                                     !=) to match against a string literal and &
+                                     to require multiple matches. Values which
+                                     are not known are excluded unless you put a
+                                     question mark (?) after the operator. For
+                                     example, to only match videos that have
+                                     been liked more than 100 times and disliked
+                                     less than 50 times (or the dislike
+                                     functionality is not available at the given
+                                     service), but who also have a description,
+                                     use --match-filter "like_count > 100 &
+                                     dislike_count <? 50 & description" .
     --no-playlist                    Download only the video, if the URL refers
                                      to a video and a playlist.
     --yes-playlist                   Download the playlist, if the URL refers to
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 5a436e8f7..1eb9c2cdd 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -804,6 +804,7 @@
  - **TVCArticle**
  - **tvigle**: Интернет-телевидение Tvigle.ru
  - **tvland.com**
+ - **TVN24**
  - **TVNoe**
  - **tvp**: Telewizja Polska
  - **tvp:embed**: Telewizja Polska
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 530e1856b..a85aebaa3 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.02.17'
+__version__ = '2017.02.21'

From 3444844b04ae482edc5a353d9125b45ba47cd8d8 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 21 Feb 2017 17:47:14 +0100
Subject: [PATCH 0326/1696] [limelight] extract PlaylistService errors

---
 youtube_dl/extractor/limelight.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index a3712665b..422be2528 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -4,11 +4,13 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_HTTPError
 from ..utils import (
     determine_ext,
     float_or_none,
     int_or_none,
     unsmuggle_url,
+    ExtractorError,
 )
 
 
@@ -20,9 +22,17 @@ class LimelightBaseIE(InfoExtractor):
         headers = {}
         if referer:
             headers['Referer'] = referer
-        return self._download_json(
-            self._PLAYLIST_SERVICE_URL % (self._PLAYLIST_SERVICE_PATH, item_id, method),
-            item_id, 'Downloading PlaylistService %s JSON' % method, fatal=fatal, headers=headers)
+        try:
+            return self._download_json(
+                self._PLAYLIST_SERVICE_URL % (self._PLAYLIST_SERVICE_PATH, item_id, method),
+                item_id, 'Downloading PlaylistService %s JSON' % method, fatal=fatal, headers=headers)
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+                error = self._parse_json(e.cause.read().decode(), item_id)['detail']['contentAccessPermission']
+                if error == 'CountryDisabled':
+                    self.raise_geo_restricted()
+                raise ExtractorError(error, expected=True)
+            raise
 
     def _call_api(self, organization_id, item_id, method):
         return self._download_json(
@@ -213,6 +223,7 @@ class LimelightMediaIE(LimelightBaseIE):
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})
         video_id = self._match_id(url)
+        self._initialize_geo_bypass(smuggled_data.get('geo_countries'))
 
         pc, mobile, metadata = self._extract(
             video_id, 'getPlaylistByMediaId',

From 33dc173cdc84efbc1f794033480af0e4af459891 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 21 Feb 2017 17:50:36 +0100
Subject: [PATCH 0327/1696] [telequebec] use geo bypass mechanism

---
 youtube_dl/extractor/telequebec.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/telequebec.py b/youtube_dl/extractor/telequebec.py
index 4043fcb92..82d73c31d 100644
--- a/youtube_dl/extractor/telequebec.py
+++ b/youtube_dl/extractor/telequebec.py
@@ -2,7 +2,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    smuggle_url,
+)
 
 
 class TeleQuebecIE(InfoExtractor):
@@ -28,7 +31,7 @@ class TeleQuebecIE(InfoExtractor):
         return {
             '_type': 'url_transparent',
             'id': media_id,
-            'url': 'limelight:media:' + media_data['streamInfo']['sourceId'],
+            'url': smuggle_url('limelight:media:' + media_data['streamInfo']['sourceId'], {'geo_countries': ['CA']}),
             'title': media_data['title'],
             'description': media_data.get('descriptions', [{'text': None}])[0].get('text'),
             'duration': int_or_none(media_data.get('durationInMilliseconds'), 1000),

From 86466a8b6f313d0d8c80823e8e61215ac16046a5 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 21 Feb 2017 17:50:53 +0100
Subject: [PATCH 0328/1696] [cbc] use geo bypass mechanism

---
 youtube_dl/extractor/cbc.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/cbc.py b/youtube_dl/extractor/cbc.py
index cf678e7f8..4d93c7744 100644
--- a/youtube_dl/extractor/cbc.py
+++ b/youtube_dl/extractor/cbc.py
@@ -195,6 +195,7 @@ class CBCPlayerIE(InfoExtractor):
 class CBCWatchBaseIE(InfoExtractor):
     _device_id = None
     _device_token = None
+    _GEO_COUNTRIES = ['CA']
     _API_BASE_URL = 'https://api-cbc.cloud.clearleap.com/cloffice/client/'
     _NS_MAP = {
         'media': 'http://search.yahoo.com/mrss/',

From 7345d6d465c4889ae06672a11f8b6e491b0b7fe4 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 21 Feb 2017 17:51:40 +0100
Subject: [PATCH 0329/1696] [tfo] Improve geo restriction detection and use geo
 bypass mechanism

---
 youtube_dl/extractor/tfo.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tfo.py b/youtube_dl/extractor/tfo.py
index 6f1eeac57..0e2370cd8 100644
--- a/youtube_dl/extractor/tfo.py
+++ b/youtube_dl/extractor/tfo.py
@@ -8,10 +8,12 @@ from ..utils import (
     HEADRequest,
     ExtractorError,
     int_or_none,
+    clean_html,
 )
 
 
 class TFOIE(InfoExtractor):
+    _GEO_COUNTRIES = ['CA']
     _VALID_URL = r'https?://(?:www\.)?tfo\.org/(?:en|fr)/(?:[^/]+/){2}(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.tfo.org/en/universe/tfo-247/100463871/video-game-hackathon',
@@ -36,7 +38,9 @@ class TFOIE(InfoExtractor):
                 'X-tfo-session': self._get_cookies('http://www.tfo.org/')['tfo-session'].value,
             })
         if infos.get('success') == 0:
-            raise ExtractorError('%s said: %s' % (self.IE_NAME, infos['msg']), expected=True)
+            if infos.get('code') == 'ErrGeoBlocked':
+                self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, clean_html(infos['msg'])), expected=True)
         video_data = infos['data']
 
         return {

From fc320a40d97ed0c439a8d2ace9cfad94a74ff635 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 21 Feb 2017 18:14:55 +0100
Subject: [PATCH 0330/1696] Revert "[cbc] use geo bypass mechanism"

This reverts commit 86466a8b6f313d0d8c80823e8e61215ac16046a5.
---
 youtube_dl/extractor/cbc.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/cbc.py b/youtube_dl/extractor/cbc.py
index 4d93c7744..cf678e7f8 100644
--- a/youtube_dl/extractor/cbc.py
+++ b/youtube_dl/extractor/cbc.py
@@ -195,7 +195,6 @@ class CBCPlayerIE(InfoExtractor):
 class CBCWatchBaseIE(InfoExtractor):
     _device_id = None
     _device_token = None
-    _GEO_COUNTRIES = ['CA']
     _API_BASE_URL = 'https://api-cbc.cloud.clearleap.com/cloffice/client/'
     _NS_MAP = {
         'media': 'http://search.yahoo.com/mrss/',

From 31615ac279ac60fbd3925995de2eed69a4b3976a Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 21 Feb 2017 19:36:39 +0100
Subject: [PATCH 0331/1696] [viewster] use geo verifcation headers

---
 youtube_dl/extractor/viewster.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 52dd95e2f..fcf0cb100 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -86,7 +86,9 @@ class ViewsterIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         # Get 'api_token' cookie
-        self._request_webpage(HEADRequest('http://www.viewster.com/'), video_id)
+        self._request_webpage(
+            HEADRequest('http://www.viewster.com/'),
+            video_id, headers=self.geo_verification_headers())
         cookies = self._get_cookies('http://www.viewster.com/')
         self._AUTH_TOKEN = compat_urllib_parse_unquote(cookies['api_token'].value)
 

From abd29a2cedaab096e920ed5be9c480921cfacf0a Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 21 Feb 2017 19:37:26 +0100
Subject: [PATCH 0332/1696] [crackle] use geo bypass mechanism

---
 youtube_dl/extractor/crackle.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/crackle.py b/youtube_dl/extractor/crackle.py
index 377fb45e9..f919ed208 100644
--- a/youtube_dl/extractor/crackle.py
+++ b/youtube_dl/extractor/crackle.py
@@ -6,6 +6,7 @@ from ..utils import int_or_none
 
 
 class CrackleIE(InfoExtractor):
+    _GEO_COUNTRIES = ['US']
     _VALID_URL = r'(?:crackle:|https?://(?:(?:www|m)\.)?crackle\.com/(?:playlist/\d+/|(?:[^/]+/)+))(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.crackle.com/comedians-in-cars-getting-coffee/2498934',

From 139d8ac106dc173d27cf20361b649c0dbc5f9b67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Feb 2017 01:50:34 +0700
Subject: [PATCH 0333/1696] [setup] Add python 3.6 classifier

---
 setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup.py b/setup.py
index ce6dd1870..b8c3e0925 100644
--- a/setup.py
+++ b/setup.py
@@ -130,6 +130,7 @@ setup(
         'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
     ],
 
     cmdclass={'build_lazy_extractors': build_lazy_extractors},

From 0d427c83047778d2984df5594b96f119ec7f8771 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Feb 2017 01:51:27 +0700
Subject: [PATCH 0334/1696] [setup] Actualize maintainer info

---
 setup.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index b8c3e0925..67d6633ed 100644
--- a/setup.py
+++ b/setup.py
@@ -107,8 +107,8 @@ setup(
     url='https://github.com/rg3/youtube-dl',
     author='Ricardo Garcia',
     author_email='ytdl@yt-dl.org',
-    maintainer='Philipp Hagemeister',
-    maintainer_email='phihag@phihag.de',
+    maintainer='Sergey M.',
+    maintainer_email='dstftw@gmail.com',
     packages=[
         'youtube_dl',
         'youtube_dl.extractor', 'youtube_dl.downloader',

From 71e9577b94a4792a330e9bdab4674c6893ea5bac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Feb 2017 21:19:52 +0700
Subject: [PATCH 0335/1696] [24video] Add support for 24video.tube (closes
 #12217)

---
 youtube_dl/extractor/twentyfourvideo.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twentyfourvideo.py b/youtube_dl/extractor/twentyfourvideo.py
index a983ebf05..f3541b654 100644
--- a/youtube_dl/extractor/twentyfourvideo.py
+++ b/youtube_dl/extractor/twentyfourvideo.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class TwentyFourVideoIE(InfoExtractor):
     IE_NAME = '24video'
-    _VALID_URL = r'https?://(?:www\.)?24video\.(?:net|me|xxx|sex)/(?:video/(?:view|xml)/|player/new24_play\.swf\?id=)(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?24video\.(?:net|me|xxx|sex|tube)/(?:video/(?:view|xml)/|player/new24_play\.swf\?id=)(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.24video.net/video/view/1044982',
@@ -37,6 +37,9 @@ class TwentyFourVideoIE(InfoExtractor):
     }, {
         'url': 'http://www.24video.me/video/view/1044982',
         'only_matching': True,
+    }, {
+        'url': 'http://www.24video.tube/video/view/2363750',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From a86e4160880e0747c5a0b774fe8d3b0cbb2990ae Mon Sep 17 00:00:00 2001
From: Tobias Florek <me@ibotty.net>
Date: Wed, 22 Feb 2017 15:28:09 +0100
Subject: [PATCH 0336/1696] [vidzi] Add support for vidzi.cc

---
 youtube_dl/extractor/vidzi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vidzi.py b/youtube_dl/extractor/vidzi.py
index 1f1828fce..b642caf22 100644
--- a/youtube_dl/extractor/vidzi.py
+++ b/youtube_dl/extractor/vidzi.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class VidziIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?vidzi\.tv/(?:embed-)?(?P<id>[0-9a-zA-Z]+)'
+    _VALID_URL = r'https?://(?:www\.)?vidzi\.(?:tv|cc)/(?:embed-)?(?P<id>[0-9a-zA-Z]+)'
     _TESTS = [{
         'url': 'http://vidzi.tv/cghql9yq6emu.html',
         'md5': '4f16c71ca0c8c8635ab6932b5f3f1660',

From 58ad6995cd3974eb9cff64f3fa8d34ec68cb6a03 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Feb 2017 21:29:53 +0700
Subject: [PATCH 0337/1696] [vidzi] Add test for #12213

---
 youtube_dl/extractor/vidzi.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/vidzi.py b/youtube_dl/extractor/vidzi.py
index b642caf22..d0556297e 100644
--- a/youtube_dl/extractor/vidzi.py
+++ b/youtube_dl/extractor/vidzi.py
@@ -29,6 +29,9 @@ class VidziIE(InfoExtractor):
     }, {
         'url': 'http://vidzi.tv/embed-4z2yb0rzphe9-600x338.html',
         'skip_download': True,
+    }, {
+        'url': 'http://vidzi.cc/cghql9yq6emu.html',
+        'skip_download': True,
     }]
 
     def _real_extract(self, url):

From 527ef85fe9bc65ed676ab855ee386c7cce8716ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Feb 2017 21:49:30 +0700
Subject: [PATCH 0338/1696] [dailymotion] Make comment count optional (closes
 #12209) Not served anymore

---
 youtube_dl/extractor/dailymotion.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 31bf5faf6..b312401dc 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -66,7 +66,6 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                 'uploader_id': 'xijv66',
                 'age_limit': 0,
                 'view_count': int,
-                'comment_count': int,
             }
         },
         # Vevo video
@@ -140,7 +139,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
         view_count = str_to_int(view_count_str)
         comment_count = int_or_none(self._search_regex(
             r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserComments:(\d+)"',
-            webpage, 'comment count', fatal=False))
+            webpage, 'comment count', default=None))
 
         player_v5 = self._search_regex(
             [r'buildPlayer\(({.+?})\);\n',  # See https://github.com/rg3/youtube-dl/issues/7826

From b5869560a44caaa06b317302425fb472169c2d28 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 23 Feb 2017 00:08:45 +0800
Subject: [PATCH 0339/1696] [crunchyroll] Fix descriptions with double quotes
 (closes #12124)

---
 ChangeLog                           |  6 ++++++
 youtube_dl/extractor/crunchyroll.py | 23 +++++++++++++++++++----
 2 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index a479d274f..e57e7fece 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [crunchyroll] Fix descriptions with double quotes (#12124)
+
+
 version 2017.02.21
 
 Core
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 109d1c5a8..d2b87442d 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -123,7 +123,7 @@ class CrunchyrollIE(CrunchyrollBaseIE):
         'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513',
         'info_dict': {
             'id': '645513',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Wanna be the Strongest in the World Episode 1 – An Idol-Wrestler is Born!',
             'description': 'md5:2d17137920c64f2f49981a7797d275ef',
             'thumbnail': 'http://img1.ak.crunchyroll.com/i/spire1-tmb/20c6b5e10f1a47b10516877d3c039cae1380951166_full.jpg',
@@ -192,6 +192,21 @@ class CrunchyrollIE(CrunchyrollBaseIE):
         # geo-restricted (US), 18+ maturity wall, non-premium available
         'url': 'http://www.crunchyroll.com/cosplay-complex-ova/episode-1-the-birth-of-the-cosplay-club-565617',
         'only_matching': True,
+    }, {
+        # A description with double quotes
+        'url': 'http://www.crunchyroll.com/11eyes/episode-1-piros-jszaka-red-night-535080',
+        'info_dict': {
+            'id': '535080',
+            'ext': 'mp4',
+            'title': '11eyes Episode 1 – Piros éjszaka - Red Night',
+            'description': 'Kakeru and Yuka are thrown into an alternate nightmarish world they call "Red Night".',
+            'uploader': 'Marvelous AQL Inc.',
+            'upload_date': '20091021',
+        },
+        'params': {
+            # Just test metadata extraction
+            'skip_download': True,
+        },
     }]
 
     _FORMAT_IDS = {
@@ -362,9 +377,9 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             r'(?s)<h1[^>]*>((?:(?!<h1).)*?<span[^>]+itemprop=["\']title["\'][^>]*>(?:(?!<h1).)+?)</h1>',
             webpage, 'video_title')
         video_title = re.sub(r' {2,}', ' ', video_title)
-        video_description = self._html_search_regex(
-            r'<script[^>]*>\s*.+?\[media_id=%s\].+?"description"\s*:\s*"([^"]+)' % video_id,
-            webpage, 'description', default=None)
+        video_description = self._parse_json(self._html_search_regex(
+            r'<script[^>]*>\s*.+?\[media_id=%s\].+?({.+?"description"\s*:.+?})\);' % video_id,
+            webpage, 'description', default='{}'), video_id).get('description')
         if video_description:
             video_description = lowercase_escape(video_description.replace(r'\r\n', '\n'))
         video_upload_date = self._html_search_regex(

From 63a29b6118d147404b5fff63d82f098c4bfa3ffc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Feb 2017 23:45:01 +0700
Subject: [PATCH 0340/1696] [ChangeLog] Actualize

---
 ChangeLog | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index e57e7fece..14db5ac13 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -2,6 +2,15 @@ version <unreleased>
 
 Extractors
 * [crunchyroll] Fix descriptions with double quotes (#12124)
+* [dailymotion] Make comment count optional (#12209)
++ [vidzi] Add support for vidzi.cc (#12213)
++ [24video] Add support for 24video.tube (#12217)
++ [crackle] Use geo bypass mechanism
++ [viewster] Use geo verification headers
++ [tfo] Improve geo restriction detection and use geo bypass mechanism
++ [telequebec] Use geo bypass mechanism
++ [limelight] Extract PlaylistService errors and improve geo restriction
+  detection
 
 
 version 2017.02.21

From 345b24538b24772c6c5917439e62c510437fce04 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Feb 2017 23:50:42 +0700
Subject: [PATCH 0341/1696] release 2017.02.22

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 8b6d14fa2..923f28276 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.21*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.21**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.22*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.22**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.02.21
+[debug] youtube-dl version 2017.02.22
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 14db5ac13..cff065171 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.02.22
 
 Extractors
 * [crunchyroll] Fix descriptions with double quotes (#12124)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a85aebaa3..fce1b8558 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.02.21'
+__version__ = '2017.02.22'

From bc61c80c143d8faed3e264c2487204924acd1eb6 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 23 Feb 2017 11:47:06 +0100
Subject: [PATCH 0342/1696] [leeco] raise GeoRestrictedError and use geo bypass
 mechanism

---
 youtube_dl/extractor/leeco.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/leeco.py b/youtube_dl/extractor/leeco.py
index 4321f90c8..9eda956d2 100644
--- a/youtube_dl/extractor/leeco.py
+++ b/youtube_dl/extractor/leeco.py
@@ -30,7 +30,7 @@ from ..utils import (
 class LeIE(InfoExtractor):
     IE_DESC = '乐视网'
     _VALID_URL = r'https?://(?:www\.le\.com/ptv/vplay|(?:sports\.le|(?:www\.)?lesports)\.com/(?:match|video))/(?P<id>\d+)\.html'
-
+    _GEO_COUNTRIES = ['CN']
     _URL_TEMPLATE = 'http://www.le.com/ptv/vplay/%s.html'
 
     _TESTS = [{
@@ -126,10 +126,9 @@ class LeIE(InfoExtractor):
         if playstatus['status'] == 0:
             flag = playstatus['flag']
             if flag == 1:
-                msg = 'Country %s auth error' % playstatus['country']
+                self.raise_geo_restricted()
             else:
-                msg = 'Generic error. flag = %d' % flag
-            raise ExtractorError(msg, expected=True)
+                raise ExtractorError('Generic error. flag = %d' % flag, expected=True)
 
     def _real_extract(self, url):
         media_id = self._match_id(url)

From c59f7036101b5349b3b02a8bd700eff507012e3f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 23 Feb 2017 11:49:35 +0100
Subject: [PATCH 0343/1696] [sohu] raise GeoRestrictedError

---
 youtube_dl/extractor/sohu.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 30760ca06..7da12cef8 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -108,12 +108,11 @@ class SohuIE(InfoExtractor):
         if vid_data['play'] != 1:
             if vid_data.get('status') == 12:
                 raise ExtractorError(
-                    'Sohu said: There\'s something wrong in the video.',
+                    '%s said: There\'s something wrong in the video.' % self.IE_NAME,
                     expected=True)
             else:
-                raise ExtractorError(
-                    'Sohu said: The video is only licensed to users in Mainland China.',
-                    expected=True)
+                self.raise_geo_restricted(
+                    '%s said: The video is only licensed to users in Mainland China.' % self.IE_NAME)
 
         formats_json = {}
         for format_id in ('nor', 'high', 'super', 'ori', 'h2644k', 'h2654k'):

From 30eaa3a7023a3c03c62eb481f6415cb0599e0da5 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 23 Feb 2017 11:50:04 +0100
Subject: [PATCH 0344/1696] [mgtv] fix extraction

---
 youtube_dl/extractor/mgtv.py | 50 +++++++++++++++++-------------------
 1 file changed, 24 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/mgtv.py b/youtube_dl/extractor/mgtv.py
index 659ede8c2..d53d96aae 100644
--- a/youtube_dl/extractor/mgtv.py
+++ b/youtube_dl/extractor/mgtv.py
@@ -2,16 +2,17 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import int_or_none
 
 
 class MGTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?mgtv\.com/v/(?:[^/]+/)*(?P<id>\d+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?mgtv\.com/(v|b)/(?:[^/]+/)*(?P<id>\d+)\.html'
     IE_DESC = '芒果TV'
 
     _TESTS = [{
         'url': 'http://www.mgtv.com/v/1/290525/f/3116640.html',
-        'md5': '1bdadcf760a0b90946ca68ee9a2db41a',
+        'md5': 'b1ffc0fc163152acf6beaa81832c9ee7',
         'info_dict': {
             'id': '3116640',
             'ext': 'mp4',
@@ -21,48 +22,45 @@ class MGTVIE(InfoExtractor):
             'thumbnail': r're:^https?://.*\.jpg$',
         },
     }, {
-        # no tbr extracted from stream_url
-        'url': 'http://www.mgtv.com/v/1/1/f/3324755.html',
+        'url': 'http://www.mgtv.com/b/301817/3826653.html',
         'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         api_data = self._download_json(
-            'http://v.api.mgtv.com/player/video', video_id,
+            'http://pcweb.api.mgtv.com/player/video', video_id,
             query={'video_id': video_id},
             headers=self.geo_verification_headers())['data']
         info = api_data['info']
+        title = info['title'].strip()
+        stream_domain = api_data['stream_domain'][0]
 
         formats = []
         for idx, stream in enumerate(api_data['stream']):
-            stream_url = stream.get('url')
-            if not stream_url:
+            stream_path = stream.get('url')
+            if not stream_path:
+                continue
+            format_data = self._download_json(
+                stream_domain + stream_path, video_id,
+                note='Download video info for format #%d' % idx)
+            format_url = format_data.get('info')
+            if not format_url:
                 continue
             tbr = int_or_none(self._search_regex(
-                r'(\d+)\.mp4', stream_url, 'tbr', default=None))
-
-            def extract_format(stream_url, format_id, idx, query={}):
-                format_info = self._download_json(
-                    stream_url, video_id,
-                    note='Download video info for format %s' % (format_id or '#%d' % idx),
-                    query=query)
-                return {
-                    'format_id': format_id,
-                    'url': format_info['info'],
-                    'ext': 'mp4',
-                    'tbr': tbr,
-                }
-
-            formats.append(extract_format(
-                stream_url, 'hls-%d' % tbr if tbr else None, idx * 2))
-            formats.append(extract_format(stream_url.replace(
-                '/playlist.m3u8', ''), 'http-%d' % tbr if tbr else None, idx * 2 + 1, {'pno': 1031}))
+                r'_(\d+)_mp4/', format_url, 'tbr', default=None))
+            formats.append({
+                'format_id': compat_str(tbr or idx),
+                'url': format_url,
+                'ext': 'mp4',
+                'tbr': tbr,
+                'protocol': 'm3u8_native',
+            })
         self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'title': info['title'].strip(),
+            'title': title,
             'formats': formats,
             'description': info.get('desc'),
             'duration': int_or_none(info.get('duration')),

From 9e03aa75c779b79da79353ef1ecc4520ad06d6d0 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 23 Feb 2017 11:54:43 +0100
Subject: [PATCH 0345/1696] [crunchyroll] extract playlist entries ids

---
 youtube_dl/extractor/crunchyroll.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index d2b87442d..a1fc6a756 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -534,11 +534,11 @@ class CrunchyrollShowPlaylistIE(CrunchyrollBaseIE):
             r'(?s)<h1[^>]*>\s*<span itemprop="name">(.*?)</span>',
             webpage, 'title')
         episode_paths = re.findall(
-            r'(?s)<li id="showview_videos_media_[0-9]+"[^>]+>.*?<a href="([^"]+)"',
+            r'(?s)<li id="showview_videos_media_(\d+)"[^>]+>.*?<a href="([^"]+)"',
             webpage)
         entries = [
-            self.url_result('http://www.crunchyroll.com' + ep, 'Crunchyroll')
-            for ep in episode_paths
+            self.url_result('http://www.crunchyroll.com' + ep, 'Crunchyroll', ep_id)
+            for ep_id, ep in episode_paths
         ]
         entries.reverse()
 

From ada77fa544e185a8cd7c3e5d6374e0b6995557a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Feb 2017 18:02:04 +0700
Subject: [PATCH 0346/1696] [instagram] Add support for multi video posts
 (closes #12226)

---
 youtube_dl/extractor/instagram.py | 54 +++++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index 98f408c18..c1921cbcf 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     get_element_by_attribute,
     int_or_none,
@@ -50,6 +51,33 @@ class InstagramIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        # multi video post
+        'url': 'https://www.instagram.com/p/BQ0eAlwhDrw/',
+        'playlist': [{
+            'info_dict': {
+                'id': 'BQ0dSaohpPW',
+                'ext': 'mp4',
+                'title': 'Video 1',
+            },
+        }, {
+            'info_dict': {
+                'id': 'BQ0dTpOhuHT',
+                'ext': 'mp4',
+                'title': 'Video 2',
+            },
+        }, {
+            'info_dict': {
+                'id': 'BQ0dT7RBFeF',
+                'ext': 'mp4',
+                'title': 'Video 3',
+            },
+        }],
+        'info_dict': {
+            'id': 'BQ0eAlwhDrw',
+            'title': 'Post by instagram',
+            'description': 'md5:0f9203fc6a2ce4d228da5754bcf54957',
+        },
     }, {
         'url': 'https://instagram.com/p/-Cmh1cukG2/',
         'only_matching': True,
@@ -113,6 +141,32 @@ class InstagramIE(InfoExtractor):
                     'timestamp': int_or_none(comment.get('created_at')),
                 } for comment in media.get(
                     'comments', {}).get('nodes', []) if comment.get('text')]
+                if not video_url:
+                    edges = try_get(
+                        media, lambda x: x['edge_sidecar_to_children']['edges'],
+                        list) or []
+                    if edges:
+                        entries = []
+                        for edge_num, edge in enumerate(edges, start=1):
+                            node = try_get(edge, lambda x: x['node'], dict)
+                            if not node:
+                                continue
+                            node_video_url = try_get(node, lambda x: x['video_url'], compat_str)
+                            if not node_video_url:
+                                continue
+                            entries.append({
+                                'id': node.get('shortcode') or node['id'],
+                                'title': 'Video %d' % edge_num,
+                                'url': node_video_url,
+                                'thumbnail': node.get('display_url'),
+                                'width': int_or_none(try_get(node, lambda x: x['dimensions']['width'])),
+                                'height': int_or_none(try_get(node, lambda x: x['dimensions']['height'])),
+                                'view_count': int_or_none(node.get('video_view_count')),
+                            })
+                        return self.playlist_result(
+                            entries, video_id,
+                            'Post by %s' % uploader_id if uploader_id else None,
+                            description)
 
         if not video_url:
             video_url = self._og_search_video_url(webpage, secure=False)

From d5fd9a3be305aa8fead8fb70aae64703afe49e43 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Feb 2017 18:44:04 +0700
Subject: [PATCH 0347/1696] [skylinewebcams] Add extractor (closes #12221)

---
 youtube_dl/extractor/extractors.py     |  1 +
 youtube_dl/extractor/skylinewebcams.py | 42 ++++++++++++++++++++++++++
 2 files changed, 43 insertions(+)
 create mode 100644 youtube_dl/extractor/skylinewebcams.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 55b4782d3..83a170fa7 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -852,6 +852,7 @@ from .shared import (
 from .showroomlive import ShowRoomLiveIE
 from .sina import SinaIE
 from .sixplay import SixPlayIE
+from .skylinewebcams import SkylineWebcamsIE
 from .skynewsarabia import (
     SkyNewsArabiaIE,
     SkyNewsArabiaArticleIE,
diff --git a/youtube_dl/extractor/skylinewebcams.py b/youtube_dl/extractor/skylinewebcams.py
new file mode 100644
index 000000000..5b4aaac6f
--- /dev/null
+++ b/youtube_dl/extractor/skylinewebcams.py
@@ -0,0 +1,42 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class SkylineWebcamsIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?skylinewebcams\.com/[^/]+/webcam/(?:[^/]+/)+(?P<id>[^/]+)\.html'
+    _TEST = {
+        'url': 'https://www.skylinewebcams.com/it/webcam/italia/lazio/roma/scalinata-piazza-di-spagna-barcaccia.html',
+        'info_dict': {
+            'id': 'scalinata-piazza-di-spagna-barcaccia',
+            'ext': 'mp4',
+            'title': 're:^Live Webcam Scalinata di Piazza di Spagna - La Barcaccia [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'description': 'Roma, veduta sulla Scalinata di Piazza di Spagna e sulla Barcaccia',
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        stream_url = self._search_regex(
+            r'url\s*:\s*(["\'])(?P<url>(?:https?:)?//.+?\.m3u8.*?)\1', webpage,
+            'stream url', group='url')
+
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
+
+        return {
+            'id': video_id,
+            'url': stream_url,
+            'ext': 'mp4',
+            'title': self._live_title(title),
+            'description': description,
+            'is_live': True,
+        }

From 0f3d41b44d84869e7f4e809692ce71567b3f7130 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Feb 2017 19:48:54 +0700
Subject: [PATCH 0348/1696] [devscripts/run_tests] Exclude youtube lists tests
 from core build

---
 devscripts/run_tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/devscripts/run_tests.sh b/devscripts/run_tests.sh
index c60807215..6ba26720d 100755
--- a/devscripts/run_tests.sh
+++ b/devscripts/run_tests.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-DOWNLOAD_TESTS="age_restriction|download|subtitles|write_annotations|iqiyi_sdk_interpreter"
+DOWNLOAD_TESTS="age_restriction|download|subtitles|write_annotations|iqiyi_sdk_interpreter|youtube_lists"
 
 test_set=""
 multiprocess_args=""

From 28572a1a0b27ba3ccedac5d8d093f925dfb7485f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Mar 2016 23:07:25 +0600
Subject: [PATCH 0349/1696] [compat] Add compat_numeric_types

---
 youtube_dl/compat.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 718902019..b257e2e81 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2760,6 +2760,10 @@ else:
     compat_kwargs = lambda kwargs: kwargs
 
 
+compat_numeric_types = ((int, float, long, complex) if sys.version_info[0] < 3
+                        else (int, float, complex))
+
+
 if sys.version_info < (2, 7):
     def compat_socket_create_connection(address, timeout, source_address=None):
         host, port = address
@@ -2895,6 +2899,7 @@ __all__ = [
     'compat_input',
     'compat_itertools_count',
     'compat_kwargs',
+    'compat_numeric_types',
     'compat_ord',
     'compat_os_name',
     'compat_parse_qs',

From d0d9ade4860fd44a07f5513d13b66233fdca0e89 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Mar 2016 03:52:42 +0600
Subject: [PATCH 0350/1696] [YoutubeDL] Add support for string formatting
 operations in output template

---
 test/test_YoutubeDL.py  | 14 ++++++++++++++
 youtube_dl/YoutubeDL.py | 36 +++++++++++++++++++++++++++++++++++-
 2 files changed, 49 insertions(+), 1 deletion(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 2cfcf743a..8491a88bd 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -526,6 +526,7 @@ class TestYoutubeDL(unittest.TestCase):
             'id': '1234',
             'ext': 'mp4',
             'width': None,
+            'height': 1080,
         }
 
         def fname(templ):
@@ -535,6 +536,19 @@ class TestYoutubeDL(unittest.TestCase):
         self.assertEqual(fname('%(id)s-%(width)s.%(ext)s'), '1234-NA.mp4')
         # Replace missing fields with 'NA'
         self.assertEqual(fname('%(uploader_date)s-%(id)s.%(ext)s'), 'NA-1234.mp4')
+        self.assertEqual(fname('%(height)d.%(ext)s'), '1080.mp4')
+        self.assertEqual(fname('%(height)6d.%(ext)s'), '  1080.mp4')
+        self.assertEqual(fname('%(height)-6d.%(ext)s'), '1080  .mp4')
+        self.assertEqual(fname('%(height)06d.%(ext)s'), '001080.mp4')
+        self.assertEqual(fname('%(height) 06d.%(ext)s'), ' 01080.mp4')
+        self.assertEqual(fname('%(height)   06d.%(ext)s'), ' 01080.mp4')
+        self.assertEqual(fname('%(height)0 6d.%(ext)s'), ' 01080.mp4')
+        self.assertEqual(fname('%(height)0   6d.%(ext)s'), ' 01080.mp4')
+        self.assertEqual(fname('%(height)   0   6d.%(ext)s'), ' 01080.mp4')
+        self.assertEqual(fname('%%(height)06d.%(ext)s'), '%(height)06d.mp4')
+        self.assertEqual(fname('%(width)06d.%(ext)s'), 'NA.mp4')
+        self.assertEqual(fname('%(width)06d.%%(ext)s'), 'NA.%(ext)s')
+        self.assertEqual(fname('%%(width)06d.%(ext)s'), '%(width)06d.mp4')
 
     def test_format_note(self):
         ydl = YoutubeDL()
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 68000dea2..bdaf06e62 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -33,6 +33,7 @@ from .compat import (
     compat_get_terminal_size,
     compat_http_client,
     compat_kwargs,
+    compat_numeric_types,
     compat_os_name,
     compat_str,
     compat_tokenize_tokenize,
@@ -609,12 +610,45 @@ class YoutubeDL(object):
                 compat_str(v),
                 restricted=self.params.get('restrictfilenames'),
                 is_id=(k == 'id'))
-            template_dict = dict((k, sanitize(k, v))
+            template_dict = dict((k, v if isinstance(v, compat_numeric_types) else sanitize(k, v))
                                  for k, v in template_dict.items()
                                  if v is not None and not isinstance(v, (list, tuple, dict)))
             template_dict = collections.defaultdict(lambda: 'NA', template_dict)
 
             outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
+
+            NUMERIC_FIELDS = set((
+                'width', 'height', 'tbr', 'abr', 'asr', 'vbr', 'fps', 'filesize', 'filesize_approx',
+                'upload_year', 'upload_month', 'upload_day',
+                'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
+                'average_rating', 'comment_count', 'age_limit',
+                'start_time', 'end_time',
+                'chapter_number', 'season_number', 'episode_number',
+            ))
+
+            # Missing numeric fields used together with integer presentation types
+            # in format specification will break the argument substitution since
+            # string 'NA' is returned for missing fields. We will patch output
+            # template for missing fields to meet string presentation type.
+            for numeric_field in NUMERIC_FIELDS:
+                if numeric_field not in template_dict:
+                    # As of [1] format syntax is:
+                    #  %[mapping_key][conversion_flags][minimum_width][.precision][length_modifier]type
+                    # 1. https://docs.python.org/2/library/stdtypes.html#string-formatting
+                    FORMAT_RE = r'''(?x)
+                        (?<!%)
+                        %
+                        \({0}\)  # mapping key
+                        (?:[#0\-+ ]+)?  # conversion flags (optional)
+                        (?:\d+)?  # minimum field width (optional)
+                        (?:\.\d+)?  # precision (optional)
+                        [hlL]?  # length modifier (optional)
+                        [diouxXeEfFgGcrs%]  # conversion type
+                    '''
+                    outtmpl = re.sub(
+                        FORMAT_RE.format(numeric_field),
+                        r'%({0})s'.format(numeric_field), outtmpl)
+
             tmpl = compat_expanduser(outtmpl)
             filename = tmpl % template_dict
             # Temporary fix for #4787

From 89db639dfe02b291bbf901973ca00d6e60fc1dce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Feb 2017 22:01:09 +0700
Subject: [PATCH 0351/1696] [YoutubeDL] Rewrite outtmpl for playlist_index and
 autonumber for backward compatibility

---
 youtube_dl/YoutubeDL.py | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index bdaf06e62..fc5e67828 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -594,10 +594,7 @@ class YoutubeDL(object):
             autonumber_size = self.params.get('autonumber_size')
             if autonumber_size is None:
                 autonumber_size = 5
-            autonumber_templ = '%0' + str(autonumber_size) + 'd'
-            template_dict['autonumber'] = autonumber_templ % (self.params.get('autonumber_start', 1) - 1 + self._num_downloads)
-            if template_dict.get('playlist_index') is not None:
-                template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index'])
+            template_dict['autonumber'] = self.params.get('autonumber_start', 1) - 1 + self._num_downloads
             if template_dict.get('resolution') is None:
                 if template_dict.get('width') and template_dict.get('height'):
                     template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
@@ -617,6 +614,20 @@ class YoutubeDL(object):
 
             outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
 
+            # For fields playlist_index and autonumber convert all occurrences
+            # of %(field)s to %(field)0Nd for backward compatibility
+            field_size_compat_map = {
+                'playlist_index': len(str(template_dict['n_entries'])),
+                'autonumber': autonumber_size,
+            }
+            FIELD_SIZE_COMPAT_RE = r'(?<!%)%\((?P<field>autonumber|playlist_index)\)s'
+            mobj = re.search(FIELD_SIZE_COMPAT_RE, outtmpl)
+            if mobj:
+                outtmpl = re.sub(
+                    FIELD_SIZE_COMPAT_RE,
+                    r'%%(\1)0%dd' % field_size_compat_map[mobj.group('field')],
+                    outtmpl)
+
             NUMERIC_FIELDS = set((
                 'width', 'height', 'tbr', 'abr', 'asr', 'vbr', 'fps', 'filesize', 'filesize_approx',
                 'upload_year', 'upload_month', 'upload_day',
@@ -624,6 +635,7 @@ class YoutubeDL(object):
                 'average_rating', 'comment_count', 'age_limit',
                 'start_time', 'end_time',
                 'chapter_number', 'season_number', 'episode_number',
+                'playlist_index',
             ))
 
             # Missing numeric fields used together with integer presentation types

From b3175982c31a61ff4184d666b0bdb6dd34213365 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Feb 2017 22:01:57 +0700
Subject: [PATCH 0352/1696] [YoutubeDL] Add more numeric fields for NA
 substitution in outtmpl

---
 youtube_dl/YoutubeDL.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index fc5e67828..56a8691eb 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -635,6 +635,7 @@ class YoutubeDL(object):
                 'average_rating', 'comment_count', 'age_limit',
                 'start_time', 'end_time',
                 'chapter_number', 'season_number', 'episode_number',
+                'track_number', 'disc_number', 'release_year',
                 'playlist_index',
             ))
 

From fafc2bf5a92b8397148e47e0c9b46fb4d9212075 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Feb 2017 22:11:16 +0700
Subject: [PATCH 0353/1696] [options] Deprecate --autonumber-size

---
 youtube_dl/options.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 2c880d06a..5a11dddf9 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -680,7 +680,9 @@ def parseOpts(overrideArguments=None):
     filesystem.add_option(
         '--autonumber-size',
         dest='autonumber_size', metavar='NUMBER', default=5, type=int,
-        help='Specify the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given (default is %default)')
+        help='[deprecated; use output template with %(autonumber)0Nd, where N in the number of digits] '
+             'Specify the number of digits in %(autonumber)s when it is present '
+             'in output filename template or --auto-number option is given (default is %default)')
     filesystem.add_option(
         '--autonumber-start',
         dest='autonumber_start', metavar='NUMBER', default=1, type=int,

From 050f143c1286ddafcb6966a0f679c5bbaceecca5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Feb 2017 23:00:13 +0700
Subject: [PATCH 0354/1696] [README.md] Clarify sequence types in output
 template and document numeric string formatting operations

---
 README.md | 138 +++++++++++++++++++++++++++---------------------------
 1 file changed, 70 insertions(+), 68 deletions(-)

diff --git a/README.md b/README.md
index 1eccfd287..b4364ef61 100644
--- a/README.md
+++ b/README.md
@@ -486,87 +486,89 @@ The `-o` option allows users to indicate a template for the output file names.
 
 **tl;dr:** [navigate me to examples](#output-template-examples).
 
-The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "http://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences have the format `%(NAME)s`. To clarify, that is a percent symbol followed by a name in parentheses, followed by a lowercase S. Allowed names are:
+The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "http://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences may be formatted according to [python string formatting operations](https://docs.python.org/2/library/stdtypes.html#string-formatting). For example, `%(NAME)s` or `%(NAME)05d`. To clarify, that is a percent symbol followed by a name in parentheses, followed by a formatting operations. Allowed names along with sequence type are:
 
- - `id`: Video identifier
- - `title`: Video title
- - `url`: Video URL
- - `ext`: Video filename extension
- - `alt_title`: A secondary title of the video
- - `display_id`: An alternative identifier for the video
- - `uploader`: Full name of the video uploader
- - `license`: License name the video is licensed under
- - `creator`: The creator of the video
- - `release_date`: The date (YYYYMMDD) when the video was released
- - `timestamp`: UNIX timestamp of the moment the video became available
- - `upload_date`: Video upload date (YYYYMMDD)
- - `uploader_id`: Nickname or id of the video uploader
- - `location`: Physical location where the video was filmed
- - `duration`: Length of the video in seconds
- - `view_count`: How many users have watched the video on the platform
- - `like_count`: Number of positive ratings of the video
- - `dislike_count`: Number of negative ratings of the video
- - `repost_count`: Number of reposts of the video
- - `average_rating`: Average rating give by users, the scale used depends on the webpage
- - `comment_count`: Number of comments on the video
- - `age_limit`: Age restriction for the video (years)
- - `format`: A human-readable description of the format 
- - `format_id`: Format code specified by `--format`
- - `format_note`: Additional info about the format
- - `width`: Width of the video
- - `height`: Height of the video
- - `resolution`: Textual description of width and height
- - `tbr`: Average bitrate of audio and video in KBit/s
- - `abr`: Average audio bitrate in KBit/s
- - `acodec`: Name of the audio codec in use
- - `asr`: Audio sampling rate in Hertz
- - `vbr`: Average video bitrate in KBit/s
- - `fps`: Frame rate
- - `vcodec`: Name of the video codec in use
- - `container`: Name of the container format
- - `filesize`: The number of bytes, if known in advance
- - `filesize_approx`: An estimate for the number of bytes
- - `protocol`: The protocol that will be used for the actual download
- - `extractor`: Name of the extractor
- - `extractor_key`: Key name of the extractor
- - `epoch`: Unix epoch when creating the file
- - `autonumber`: Five-digit number that will be increased with each download, starting at zero
- - `playlist`: Name or id of the playlist that contains the video
- - `playlist_index`: Index of the video in the playlist padded with leading zeros according to the total length of the playlist
- - `playlist_id`: Playlist identifier
- - `playlist_title`: Playlist title
+ - `id` (string): Video identifier
+ - `title` (string): Video title
+ - `url` (string): Video URL
+ - `ext` (string): Video filename extension
+ - `alt_title` (string): A secondary title of the video
+ - `display_id` (string): An alternative identifier for the video
+ - `uploader` (string): Full name of the video uploader
+ - `license` (string): License name the video is licensed under
+ - `creator` (string): The creator of the video
+ - `release_date` (string): The date (YYYYMMDD) when the video was released
+ - `timestamp` (numeric): UNIX timestamp of the moment the video became available
+ - `upload_date` (string): Video upload date (YYYYMMDD)
+ - `uploader_id` (string): Nickname or id of the video uploader
+ - `location` (string): Physical location where the video was filmed
+ - `duration` (numeric): Length of the video in seconds
+ - `view_count` (numeric): How many users have watched the video on the platform
+ - `like_count` (numeric): Number of positive ratings of the video
+ - `dislike_count` (numeric): Number of negative ratings of the video
+ - `repost_count` (numeric): Number of reposts of the video
+ - `average_rating` (numeric): Average rating give by users, the scale used depends on the webpage
+ - `comment_count` (numeric): Number of comments on the video
+ - `age_limit` (numeric): Age restriction for the video (years)
+ - `format` (string): A human-readable description of the format 
+ - `format_id` (string): Format code specified by `--format`
+ - `format_note` (string): Additional info about the format
+ - `width` (numeric): Width of the video
+ - `height` (numeric): Height of the video
+ - `resolution` (string): Textual description of width and height
+ - `tbr` (numeric): Average bitrate of audio and video in KBit/s
+ - `abr` (numeric): Average audio bitrate in KBit/s
+ - `acodec` (string): Name of the audio codec in use
+ - `asr` (numeric): Audio sampling rate in Hertz
+ - `vbr` (numeric): Average video bitrate in KBit/s
+ - `fps` (numeric): Frame rate
+ - `vcodec` (string): Name of the video codec in use
+ - `container` (string): Name of the container format
+ - `filesize` (numeric): The number of bytes, if known in advance
+ - `filesize_approx` (numeric): An estimate for the number of bytes
+ - `protocol` (string): The protocol that will be used for the actual download
+ - `extractor` (string): Name of the extractor
+ - `extractor_key` (string): Key name of the extractor
+ - `epoch` (numeric): Unix epoch when creating the file
+ - `autonumber` (numeric): Five-digit number that will be increased with each download, starting at zero
+ - `playlist` (string): Name or id of the playlist that contains the video
+ - `playlist_index` (numeric): Index of the video in the playlist padded with leading zeros according to the total length of the playlist
+ - `playlist_id` (string): Playlist identifier
+ - `playlist_title` (string): Playlist title
 
 
 Available for the video that belongs to some logical chapter or section:
- - `chapter`: Name or title of the chapter the video belongs to
- - `chapter_number`: Number of the chapter the video belongs to
- - `chapter_id`: Id of the chapter the video belongs to
+ - `chapter` (string): Name or title of the chapter the video belongs to
+ - `chapter_number` (numeric): Number of the chapter the video belongs to
+ - `chapter_id` (string): Id of the chapter the video belongs to
 
 Available for the video that is an episode of some series or programme:
- - `series`: Title of the series or programme the video episode belongs to
- - `season`: Title of the season the video episode belongs to
- - `season_number`: Number of the season the video episode belongs to
- - `season_id`: Id of the season the video episode belongs to
- - `episode`: Title of the video episode
- - `episode_number`: Number of the video episode within a season
- - `episode_id`: Id of the video episode
+ - `series` (string): Title of the series or programme the video episode belongs to
+ - `season` (string): Title of the season the video episode belongs to
+ - `season_number` (numeric): Number of the season the video episode belongs to
+ - `season_id` (string): Id of the season the video episode belongs to
+ - `episode` (string): Title of the video episode
+ - `episode_number` (numeric): Number of the video episode within a season
+ - `episode_id` (string): Id of the video episode
 
 Available for the media that is a track or a part of a music album:
- - `track`: Title of the track
- - `track_number`: Number of the track within an album or a disc
- - `track_id`: Id of the track
- - `artist`: Artist(s) of the track
- - `genre`: Genre(s) of the track
- - `album`: Title of the album the track belongs to
- - `album_type`: Type of the album
- - `album_artist`: List of all artists appeared on the album
- - `disc_number`: Number of the disc or other physical medium the track belongs to
- - `release_year`: Year (YYYY) when the album was released
+ - `track` (string): Title of the track
+ - `track_number` (numeric): Number of the track within an album or a disc
+ - `track_id` (string): Id of the track
+ - `artist` (string): Artist(s) of the track
+ - `genre` (string): Genre(s) of the track
+ - `album` (string): Title of the album the track belongs to
+ - `album_type` (string): Type of the album
+ - `album_artist` (string): List of all artists appeared on the album
+ - `disc_number` (numeric): Number of the disc or other physical medium the track belongs to
+ - `release_year` (numeric): Year (YYYY) when the album was released
 
 Each aforementioned sequence when referenced in an output template will be replaced by the actual value corresponding to the sequence name. Note that some of the sequences are not guaranteed to be present since they depend on the metadata obtained by a particular extractor. Such sequences will be replaced with `NA`.
 
 For example for `-o %(title)s-%(id)s.%(ext)s` and an mp4 video with title `youtube-dl test video` and id `BaW_jenozKcj`, this will result in a `youtube-dl test video-BaW_jenozKcj.mp4` file created in the current directory.
 
+For numeric sequences you can use numeric related formatting, for example, `%(view_count)05d` will result in a string with view count padded with zeros up to 5 characters, like in `00042`.
+
 Output templates can also contain arbitrary hierarchical path, e.g. `-o '%(playlist)s/%(playlist_index)s - %(title)s.%(ext)s'` which will result in downloading each video in a directory corresponding to this path template. Any missing directory will be automatically created for you.
 
 To use percent literals in an output template use `%%`. To output to stdout use `-o -`.

From 8e1409fd805b3b5e3731da66a2101494643a06ea Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 23 Feb 2017 18:42:06 +0100
Subject: [PATCH 0355/1696] [go] sign all uplynk urls and use geo bypass only
 for free videos(closes #12087)(closes #12210)

---
 youtube_dl/extractor/go.py | 81 ++++++++++++++++++++++----------------
 1 file changed, 48 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/go.py b/youtube_dl/extractor/go.py
index b205bfc7c..21ed846b2 100644
--- a/youtube_dl/extractor/go.py
+++ b/youtube_dl/extractor/go.py
@@ -37,7 +37,6 @@ class GoIE(AdobePassIE):
         }
     }
     _VALID_URL = r'https?://(?:(?P<sub_domain>%s)\.)?go\.com/(?:[^/]+/)*(?:vdka(?P<id>\w+)|season-\d+/\d+-(?P<display_id>[^/?#]+))' % '|'.join(_SITE_INFO.keys())
-    _GEO_COUNTRIES = ['US']
     _TESTS = [{
         'url': 'http://abc.go.com/shows/castle/video/most-recent/vdka0_g86w5onx',
         'info_dict': {
@@ -79,44 +78,60 @@ class GoIE(AdobePassIE):
             ext = determine_ext(asset_url)
             if ext == 'm3u8':
                 video_type = video_data.get('type')
-                if video_type == 'lf':
-                    data = {
-                        'video_id': video_data['id'],
-                        'video_type': video_type,
-                        'brand': brand,
-                        'device': '001',
-                    }
-                    if video_data.get('accesslevel') == '1':
-                        requestor_id = site_info['requestor_id']
-                        resource = self._get_mvpd_resource(
-                            requestor_id, title, video_id, None)
-                        auth = self._extract_mvpd_auth(
-                            url, video_id, requestor_id, resource)
-                        data.update({
-                            'token': auth,
-                            'token_type': 'ap',
-                            'adobe_requestor_id': requestor_id,
-                        })
-                    entitlement = self._download_json(
-                        'https://api.entitlement.watchabc.go.com/vp2/ws-secure/entitlement/2020/authorize.json',
-                        video_id, data=urlencode_postdata(data), headers=self.geo_verification_headers())
-                    errors = entitlement.get('errors', {}).get('errors', [])
-                    if errors:
-                        for error in errors:
-                            if error.get('code') == 1002:
-                                self.raise_geo_restricted(
-                                    error['message'], countries=self._GEO_COUNTRIES)
-                        error_message = ', '.join([error['message'] for error in errors])
-                        raise ExtractorError('%s said: %s' % (self.IE_NAME, error_message), expected=True)
-                    asset_url += '?' + entitlement['uplynkData']['sessionKey']
+                data = {
+                    'video_id': video_data['id'],
+                    'video_type': video_type,
+                    'brand': brand,
+                    'device': '001',
+                }
+                if video_data.get('accesslevel') == '1':
+                    requestor_id = site_info['requestor_id']
+                    resource = self._get_mvpd_resource(
+                        requestor_id, title, video_id, None)
+                    auth = self._extract_mvpd_auth(
+                        url, video_id, requestor_id, resource)
+                    data.update({
+                        'token': auth,
+                        'token_type': 'ap',
+                        'adobe_requestor_id': requestor_id,
+                    })
+                else:
+                    self._initialize_geo_bypass(['US'])
+                entitlement = self._download_json(
+                    'https://api.entitlement.watchabc.go.com/vp2/ws-secure/entitlement/2020/authorize.json',
+                    video_id, data=urlencode_postdata(data), headers=self.geo_verification_headers())
+                errors = entitlement.get('errors', {}).get('errors', [])
+                if errors:
+                    for error in errors:
+                        if error.get('code') == 1002:
+                            self.raise_geo_restricted(
+                                error['message'], countries=['US'])
+                    error_message = ', '.join([error['message'] for error in errors])
+                    raise ExtractorError('%s said: %s' % (self.IE_NAME, error_message), expected=True)
+                asset_url += '?' + entitlement['uplynkData']['sessionKey']
                 formats.extend(self._extract_m3u8_formats(
                     asset_url, video_id, 'mp4', m3u8_id=format_id or 'hls', fatal=False))
             else:
-                formats.append({
+                f = {
                     'format_id': format_id,
                     'url': asset_url,
                     'ext': ext,
-                })
+                }
+                if re.search(r'(?:/mp4/source/|_source\.mp4)', asset_url):
+                    f.update({
+                        'format_id': ('%s-' % format_id if format_id else '') + 'SOURCE',
+                        'preference': 1,
+                    })
+                else:
+                    mobj = re.search(r'/(\d+)x(\d+)/', asset_url)
+                    if mobj:
+                        height = int(mobj.group(2))
+                        f.update({
+                            'format_id': ('%s-' % format_id if format_id else '') + '%dP' % height,
+                            'width': int(mobj.group(1)),
+                            'height': height,
+                        })
+                formats.append(f)
         self._sort_formats(formats)
 
         subtitles = {}

From 19f3821821ada01fbf9b466402bc1d0366b3edb2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Feb 2017 02:09:13 +0700
Subject: [PATCH 0356/1696] [devscripts/make_lazy_extractors] Fix making lazy
 extractors on python 3 under Windows

---
 devscripts/make_lazy_extractors.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py
index 19114d30d..0a1762dbc 100644
--- a/devscripts/make_lazy_extractors.py
+++ b/devscripts/make_lazy_extractors.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals, print_function
 
 from inspect import getsource
+import io
 import os
 from os.path import dirname as dirn
 import sys
@@ -95,5 +96,5 @@ module_contents.append(
 
 module_src = '\n'.join(module_contents) + '\n'
 
-with open(lazy_extractors_filename, 'wt') as f:
+with io.open(lazy_extractors_filename, 'wt', encoding='utf-8') as f:
     f.write(module_src)

From 0f57447de790fb2434cdd80d819876859ac4fcc0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Feb 2017 04:56:58 +0700
Subject: [PATCH 0357/1696] [postprocessor/ffmpeg] Add mising space (closes
 #12232)

---
 youtube_dl/postprocessor/ffmpeg.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 1881f4849..96ddb3b36 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -536,8 +536,7 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
             ext = sub['ext']
             if ext == new_ext:
                 self._downloader.to_screen(
-                    '[ffmpeg] Subtitle file for %s is already in the requested'
-                    'format' % new_ext)
+                    '[ffmpeg] Subtitle file for %s is already in the requested format' % new_ext)
                 continue
             old_file = subtitles_filename(filename, lang, ext)
             sub_filenames.append(old_file)

From f2980fddeb587f113afe15cc3ecf4bfc3911ca67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Feb 2017 05:01:31 +0700
Subject: [PATCH 0358/1696] [lynda:course] Add webpage extraction fallback
 (closes #12238)

---
 youtube_dl/extractor/lynda.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index da94eab56..d2f75296a 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -260,9 +260,24 @@ class LyndaCourseIE(LyndaBaseIE):
         course_path = mobj.group('coursepath')
         course_id = mobj.group('courseid')
 
+        item_template = 'https://www.lynda.com/%s/%%s-4.html' % course_path
+
         course = self._download_json(
             'https://www.lynda.com/ajax/player?courseId=%s&type=course' % course_id,
-            course_id, 'Downloading course JSON')
+            course_id, 'Downloading course JSON', fatal=False)
+
+        if not course:
+            webpage = self._download_webpage(url, course_id)
+            entries = [
+                self.url_result(
+                    item_template % video_id, ie=LyndaIE.ie_key(),
+                    video_id=video_id)
+                for video_id in re.findall(
+                    r'data-video-id=["\'](\d+)', webpage)]
+            return self.playlist_result(
+                entries, course_id,
+                self._og_search_title(webpage, fatal=False),
+                self._og_search_description(webpage))
 
         if course.get('Status') == 'NotFound':
             raise ExtractorError(
@@ -283,7 +298,7 @@ class LyndaCourseIE(LyndaBaseIE):
                 if video_id:
                     entries.append({
                         '_type': 'url_transparent',
-                        'url': 'https://www.lynda.com/%s/%s-4.html' % (course_path, video_id),
+                        'url': item_template % video_id,
                         'ie_key': LyndaIE.ie_key(),
                         'chapter': chapter.get('Title'),
                         'chapter_number': int_or_none(chapter.get('ChapterIndex')),

From be5df5ee311b3ad027f5d59fe077969babd0aa53 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Feb 2017 06:04:27 +0700
Subject: [PATCH 0359/1696] Suppress help for all deprecated options and print
 warning when used

---
 youtube_dl/YoutubeDL.py | 14 ++++++++++++--
 youtube_dl/__init__.py  |  3 +++
 youtube_dl/options.py   | 12 +++++-------
 3 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 56a8691eb..f7254560c 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -328,11 +328,21 @@ class YoutubeDL(object):
         self.params.update(params)
         self.cache = Cache(self)
 
-        if self.params.get('cn_verification_proxy') is not None:
-            self.report_warning('--cn-verification-proxy is deprecated. Use --geo-verification-proxy instead.')
+        def check_deprecated(param, option, suggestion):
+            if self.params.get(param) is not None:
+                self.report_warning(
+                    '%s is deprecated. Use %s instead.' % (option, suggestion))
+                return True
+            return False
+
+        if check_deprecated('cn_verification_proxy', '--cn-verification-proxy', '--geo-verification-proxy'):
             if self.params.get('geo_verification_proxy') is None:
                 self.params['geo_verification_proxy'] = self.params['cn_verification_proxy']
 
+        check_deprecated('autonumber_size', '--autonumber-size', 'output template with %(autonumber)0Nd, where N in the number of digits')
+        check_deprecated('autonumber', '--auto-number', '-o "%(autonumber)s-%(title)s.%(ext)s"')
+        check_deprecated('usetitle', '--title', '-o "%(title)s-%(id)s.%(ext)s"')
+
         if params.get('bidi_workaround', False):
             try:
                 import pty
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index f91d29a7b..0c401baa6 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -416,6 +416,9 @@ def _real_main(argv=None):
         'config_location': opts.config_location,
         'geo_bypass': opts.geo_bypass,
         'geo_bypass_country': opts.geo_bypass_country,
+        # just for deprecation check
+        'autonumber': opts.autonumber if opts.autonumber is True else None,
+        'usetitle': opts.usetitle if opts.usetitle is True else None,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 5a11dddf9..8b51d3c6f 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -679,10 +679,8 @@ def parseOpts(overrideArguments=None):
         help=('Output filename template, see the "OUTPUT TEMPLATE" for all the info'))
     filesystem.add_option(
         '--autonumber-size',
-        dest='autonumber_size', metavar='NUMBER', default=5, type=int,
-        help='[deprecated; use output template with %(autonumber)0Nd, where N in the number of digits] '
-             'Specify the number of digits in %(autonumber)s when it is present '
-             'in output filename template or --auto-number option is given (default is %default)')
+        dest='autonumber_size', metavar='NUMBER', type=int,
+        help=optparse.SUPPRESS_HELP)
     filesystem.add_option(
         '--autonumber-start',
         dest='autonumber_start', metavar='NUMBER', default=1, type=int,
@@ -694,15 +692,15 @@ def parseOpts(overrideArguments=None):
     filesystem.add_option(
         '-A', '--auto-number',
         action='store_true', dest='autonumber', default=False,
-        help='[deprecated; use -o "%(autonumber)s-%(title)s.%(ext)s" ] Number downloaded files starting from 00000')
+        help=optparse.SUPPRESS_HELP)
     filesystem.add_option(
         '-t', '--title',
         action='store_true', dest='usetitle', default=False,
-        help='[deprecated] Use title in file name (default)')
+        help=optparse.SUPPRESS_HELP)
     filesystem.add_option(
         '-l', '--literal', default=False,
         action='store_true', dest='usetitle',
-        help='[deprecated] Alias of --title')
+        help=optparse.SUPPRESS_HELP)
     filesystem.add_option(
         '-w', '--no-overwrites',
         action='store_true', dest='nooverwrites', default=False,

From f2f7961820155b6db4a70f083f011cd014c9b51c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Feb 2017 06:07:41 +0700
Subject: [PATCH 0360/1696] [ChangeLog] Actualize

---
 ChangeLog | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index cff065171..6f63723fc 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,23 @@
+version <unreleased>
+
+Core
+* [options] Hide deprecated options from --help
+* [options] Deprecate --autonumber-size
++ [YoutubeDL] Add support for string formatting operations in output template
+  (#5185, #5748, #6841, #9929, #9966 #9978, #12189)
+
+Extractors
++ [lynda:course] Add webpage extraction fallback (#12238)
+* [go] Sign all uplynk URLs and use geo bypass only for free videos
+  (#12087, #12210)
++ [skylinewebcams] Add support for skylinewebcams.com (#12221)
++ [instagram] Add support for multi video posts (#12226)
++ [crunchyroll] Extract playlist entries ids
+* [mgtv] Fix extraction
++ [sohu] Raise GeoRestrictedError
++ [leeco] Raise GeoRestrictedError and use geo bypass mechanism
+
+
 version 2017.02.22
 
 Extractors

From 6b097cff278c93de0665bf681729d75121a98eed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Feb 2017 06:09:15 +0700
Subject: [PATCH 0361/1696] release 2017.02.24

---
 .github/ISSUE_TEMPLATE.md |  6 +++---
 ChangeLog                 |  2 +-
 README.md                 | 10 ----------
 docs/supportedsites.md    |  1 +
 youtube_dl/version.py     |  2 +-
 5 files changed, 6 insertions(+), 15 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 923f28276..743b796ac 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.22*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.22**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.24*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.24**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.02.22
+[debug] youtube-dl version 2017.02.24
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 6f63723fc..6034c9ea4 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.02.24
 
 Core
 * [options] Hide deprecated options from --help
diff --git a/README.md b/README.md
index b4364ef61..0fc5984dc 100644
--- a/README.md
+++ b/README.md
@@ -217,21 +217,11 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
     --id                             Use only video ID in file name
     -o, --output TEMPLATE            Output filename template, see the "OUTPUT
                                      TEMPLATE" for all the info
-    --autonumber-size NUMBER         Specify the number of digits in
-                                     %(autonumber)s when it is present in output
-                                     filename template or --auto-number option
-                                     is given (default is 5)
     --autonumber-start NUMBER        Specify the start value for %(autonumber)s
                                      (default is 1)
     --restrict-filenames             Restrict filenames to only ASCII
                                      characters, and avoid "&" and spaces in
                                      filenames
-    -A, --auto-number                [deprecated; use -o
-                                     "%(autonumber)s-%(title)s.%(ext)s" ] Number
-                                     downloaded files starting from 00000
-    -t, --title                      [deprecated] Use title in file name
-                                     (default)
-    -l, --literal                    [deprecated] Alias of --title
     -w, --no-overwrites              Do not overwrite files
     -c, --continue                   Force resume of partially downloaded files.
                                      By default, youtube-dl will resume
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 1eb9c2cdd..f97397331 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -680,6 +680,7 @@
  - **Shared**: shared.sx
  - **ShowRoomLive**
  - **Sina**
+ - **SkylineWebcams**
  - **skynewsarabia:article**
  - **skynewsarabia:video**
  - **SkySports**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index fce1b8558..129447e10 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.02.22'
+__version__ = '2017.02.24'

From 42dcdbe11cd738e6b196bc9c14b746a71d61de5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Feb 2017 10:52:41 +0700
Subject: [PATCH 0362/1696] [ivi] Raise GeoRestrictedError

---
 youtube_dl/extractor/ivi.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py
index 3d3c15024..cb51cef2d 100644
--- a/youtube_dl/extractor/ivi.py
+++ b/youtube_dl/extractor/ivi.py
@@ -16,6 +16,8 @@ class IviIE(InfoExtractor):
     IE_DESC = 'ivi.ru'
     IE_NAME = 'ivi'
     _VALID_URL = r'https?://(?:www\.)?ivi\.ru/(?:watch/(?:[^/]+/)?|video/player\?.*?videoId=)(?P<id>\d+)'
+    _GEO_BYPASS = False
+    _GEO_COUNTRIES = ['RU']
 
     _TESTS = [
         # Single movie
@@ -91,7 +93,11 @@ class IviIE(InfoExtractor):
 
         if 'error' in video_json:
             error = video_json['error']
-            if error['origin'] == 'NoRedisValidData':
+            origin = error['origin']
+            if origin == 'NotAllowedForLocation':
+                self.raise_geo_restricted(
+                    msg=error['message'], countries=self._GEO_COUNTRIES)
+            elif origin == 'NoRedisValidData':
                 raise ExtractorError('Video %s does not exist' % video_id, expected=True)
             raise ExtractorError(
                 'Unable to download video %s: %s' % (video_id, error['message']),

From 39e7277ed16c1647d636c766d57870121f5f2d68 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 24 Feb 2017 11:21:13 +0100
Subject: [PATCH 0363/1696] [openload] fix extraction(closes #10408)

---
 youtube_dl/extractor/openload.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 10896c442..fc7ff43a6 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -72,16 +72,21 @@ class OpenloadIE(InfoExtractor):
             raise ExtractorError('File not found', expected=True)
 
         ol_id = self._search_regex(
-            '<span[^>]+id="[^"]+"[^>]*>([0-9]+)</span>',
+            '<span[^>]+id="[^"]+"[^>]*>([0-9A-Za-z]+)</span>',
             webpage, 'openload ID')
 
-        first_two_chars = int(float(ol_id[0:][:2]))
+        first_char = int(ol_id[0])
         urlcode = []
-        num = 2
+        num = 1
 
         while num < len(ol_id):
-            key = int(float(ol_id[num + 3:][:2]))
-            urlcode.append((key, compat_chr(int(float(ol_id[num:][:3])) - first_two_chars)))
+            i = ord(ol_id[num])
+            key = 0
+            if i <= 90:
+                key = i - 65
+            elif i >= 97:
+                key = 25 + i - 97
+            urlcode.append((key, compat_chr(int(ol_id[num + 2:num + 5]) // int(ol_id[num + 1]) - first_char)))
             num += 5
 
         video_url = 'https://openload.co/stream/' + ''.join(

From 68f17a9c2df07828d364421e59ede26981aa7756 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 24 Feb 2017 12:27:56 +0100
Subject: [PATCH 0364/1696] [tubitv] use geo bypass mechanism

---
 youtube_dl/extractor/tubitv.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/tubitv.py b/youtube_dl/extractor/tubitv.py
index 3a37df2e8..c44018aec 100644
--- a/youtube_dl/extractor/tubitv.py
+++ b/youtube_dl/extractor/tubitv.py
@@ -16,6 +16,7 @@ class TubiTvIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?tubitv\.com/video/(?P<id>[0-9]+)'
     _LOGIN_URL = 'http://tubitv.com/login'
     _NETRC_MACHINE = 'tubitv'
+    _GEO_COUNTRIES = ['US']
     _TEST = {
         'url': 'http://tubitv.com/video/283829/the_comedian_at_the_friday',
         'md5': '43ac06be9326f41912dc64ccf7a80320',

From 51ed496307a8a98134f25e8128c317663ccd4355 Mon Sep 17 00:00:00 2001
From: Thomas Christlieb <thomaschristlieb@hotmail.com>
Date: Fri, 24 Feb 2017 15:08:45 +0100
Subject: [PATCH 0365/1696] [thescene] Fix extraction (closes #12235)

---
 youtube_dl/extractor/thescene.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/thescene.py b/youtube_dl/extractor/thescene.py
index ce1326c03..6c16d66ed 100644
--- a/youtube_dl/extractor/thescene.py
+++ b/youtube_dl/extractor/thescene.py
@@ -32,7 +32,7 @@ class TheSceneIE(InfoExtractor):
         player = self._download_webpage(player_url, display_id)
         info = self._parse_json(
             self._search_regex(
-                r'(?m)var\s+video\s+=\s+({.+?});$', player, 'info json'),
+                r'(?m)video\s*:\s*({.+?}),$', player, 'info json'),
             display_id)
 
         qualities_order = qualities(('low', 'high'))
@@ -40,7 +40,7 @@ class TheSceneIE(InfoExtractor):
             'format_id': '{0}-{1}'.format(f['type'].split('/')[0], f['quality']),
             'url': f['src'],
             'quality': qualities_order(f['quality']),
-        } for f in info['sources'][0]]
+        } for f in info['sources']]
         self._sort_formats(formats)
 
         return {

From 441d7a32e563b3985c58af5ab80d76ad943f0c07 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Feb 2017 21:22:29 +0700
Subject: [PATCH 0366/1696] [thescene] Extract more metadata

---
 youtube_dl/extractor/thescene.py | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/thescene.py b/youtube_dl/extractor/thescene.py
index 6c16d66ed..b8504f0eb 100644
--- a/youtube_dl/extractor/thescene.py
+++ b/youtube_dl/extractor/thescene.py
@@ -3,7 +3,10 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 
 from ..compat import compat_urlparse
-from ..utils import qualities
+from ..utils import (
+    int_or_none,
+    qualities,
+)
 
 
 class TheSceneIE(InfoExtractor):
@@ -16,6 +19,11 @@ class TheSceneIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Narciso Rodriguez: Spring 2013 Ready-to-Wear',
             'display_id': 'narciso-rodriguez-spring-2013-ready-to-wear',
+            'duration': 127,
+            'series': 'Style.com Fashion Shows',
+            'season': 'Ready To Wear Spring 2013',
+            'tags': list,
+            'categories': list,
         },
     }
 
@@ -35,6 +43,9 @@ class TheSceneIE(InfoExtractor):
                 r'(?m)video\s*:\s*({.+?}),$', player, 'info json'),
             display_id)
 
+        video_id = info['id']
+        title = info['title']
+
         qualities_order = qualities(('low', 'high'))
         formats = [{
             'format_id': '{0}-{1}'.format(f['type'].split('/')[0], f['quality']),
@@ -44,9 +55,14 @@ class TheSceneIE(InfoExtractor):
         self._sort_formats(formats)
 
         return {
-            'id': info['id'],
+            'id': video_id,
             'display_id': display_id,
-            'title': info['title'],
+            'title': title,
             'formats': formats,
             'thumbnail': info.get('poster_frame'),
+            'duration': int_or_none(info.get('duration')),
+            'series': info.get('series_title'),
+            'season': info.get('season_title'),
+            'tags': info.get('tags'),
+            'categories': info.get('categories'),
         }

From f3bc281239bafa971195eefd30773d152bfdb10c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Feb 2017 21:48:34 +0700
Subject: [PATCH 0367/1696] [noco] Swtich login URL to https (closes #12246)

---
 youtube_dl/extractor/noco.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index 70ff2ab36..fc0624dd0 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -23,7 +23,7 @@ from ..utils import (
 
 class NocoIE(InfoExtractor):
     _VALID_URL = r'https?://(?:(?:www\.)?noco\.tv/emission/|player\.noco\.tv/\?idvideo=)(?P<id>\d+)'
-    _LOGIN_URL = 'http://noco.tv/do.php'
+    _LOGIN_URL = 'https://noco.tv/do.php'
     _API_URL_TEMPLATE = 'https://api.noco.tv/1.1/%s?ts=%s&tk=%s'
     _SUB_LANG_TEMPLATE = '&sub_lang=%s'
     _NETRC_MACHINE = 'noco'

From ad3033037ce6b6809c3d06f2074d36691ba10cbe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Feb 2017 21:50:13 +0700
Subject: [PATCH 0368/1696] [noco] Modernize

---
 youtube_dl/extractor/noco.py | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index fc0624dd0..8b83e1f76 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -69,16 +69,17 @@ class NocoIE(InfoExtractor):
         if username is None:
             return
 
-        login_form = {
-            'a': 'login',
-            'cookie': '1',
-            'username': username,
-            'password': password,
-        }
-        request = sanitized_Request(self._LOGIN_URL, urlencode_postdata(login_form))
-        request.add_header('Content-Type', 'application/x-www-form-urlencoded; charset=UTF-8')
-
-        login = self._download_json(request, None, 'Logging in as %s' % username)
+        login = self._download_json(
+            self._LOGIN_URL, None, 'Logging in as %s' % username,
+            data=urlencode_postdata({
+                'a': 'login',
+                'cookie': '1',
+                'username': username,
+                'password': password,
+            }),
+            headers={
+                'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
+            })
 
         if 'erreur' in login:
             raise ExtractorError('Unable to login: %s' % clean_html(login['erreur']), expected=True)

From 5d3a51e1b939ada71b3ec6ae7e004ccb6e0861c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Feb 2017 21:57:39 +0700
Subject: [PATCH 0369/1696] [ChangeLog] Actualize

---
 ChangeLog | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 6034c9ea4..0225af42a 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,15 @@
+version <unreleased>
+
+Extractors
+* [noco] Modernize
+* [noco] Switch login URL to https (#12246)
++ [thescene] Extract more metadata
+* [thescene] Fix extraction (#12235)
++ [tubitv] Use geo bypass mechanism
+* [openload] Fix extraction (#10408)
++ [ivi] Raise GeoRestrictedError
+
+
 version 2017.02.24
 
 Core

From 204efc850905a5b78c86f884b94210644784b9c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Feb 2017 21:59:39 +0700
Subject: [PATCH 0370/1696] release 2017.02.24.1

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 743b796ac..564108122 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.24*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.24**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.24.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.24.1**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.02.24
+[debug] youtube-dl version 2017.02.24.1
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 0225af42a..add8a6758 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.02.24.1
 
 Extractors
 * [noco] Modernize
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 129447e10..fe7462eac 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.02.24'
+__version__ = '2017.02.24.1'

From 231bcd0b6b7737e6b0484ce8aaa4a14de442a1ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Feb 2017 02:51:53 +0700
Subject: [PATCH 0371/1696] [amcnetworks] Relax _VALID_URL (#12127)

---
 youtube_dl/extractor/amcnetworks.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/amcnetworks.py b/youtube_dl/extractor/amcnetworks.py
index b71d1a093..3a0ec6776 100644
--- a/youtube_dl/extractor/amcnetworks.py
+++ b/youtube_dl/extractor/amcnetworks.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class AMCNetworksIE(ThePlatformIE):
-    _VALID_URL = r'https?://(?:www\.)?(?:amc|bbcamerica|ifc|wetv)\.com/(?:movies/|shows/[^/]+/(?:full-episodes/)?[^/]+/episode-\d+(?:-(?:[^/]+/)?|/))(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:amc|bbcamerica|ifc|wetv)\.com/(?:movies|shows(?:/[^/]+)+)/(?P<id>[^/?#]+)'
     _TESTS = [{
         'url': 'http://www.ifc.com/shows/maron/season-04/episode-01/step-1',
         'md5': '',
@@ -44,6 +44,12 @@ class AMCNetworksIE(ThePlatformIE):
     }, {
         'url': 'http://www.bbcamerica.com/shows/doctor-who/full-episodes/the-power-of-the-daleks/episode-01-episode-1-color-version',
         'only_matching': True,
+    }, {
+        'url': 'http://www.wetv.com/shows/mama-june-from-not-to-hot/full-episode/season-01/thin-tervention',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.wetv.com/shows/la-hair/videos/season-05/episode-09-episode-9-2/episode-9-sneak-peek-3',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From eeb0a9568442a4dbbf3478579abe2696fbe890e2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 25 Feb 2017 18:40:05 +0800
Subject: [PATCH 0372/1696] [extractor/common] Add 'preference' to
 _parse_html5_media_entries

Some websites, like NJPWorld, put different qualities on different
player pages.
---
 youtube_dl/extractor/common.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 4252d6825..eb3c091aa 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2010,7 +2010,7 @@ class InfoExtractor(object):
                 })
         return formats
 
-    def _parse_html5_media_entries(self, base_url, webpage, video_id, m3u8_id=None, m3u8_entry_protocol='m3u8', mpd_id=None):
+    def _parse_html5_media_entries(self, base_url, webpage, video_id, m3u8_id=None, m3u8_entry_protocol='m3u8', mpd_id=None, preference=None):
         def absolute_url(video_url):
             return compat_urlparse.urljoin(base_url, video_url)
 
@@ -2032,7 +2032,8 @@ class InfoExtractor(object):
                 is_plain_url = False
                 formats = self._extract_m3u8_formats(
                     full_url, video_id, ext='mp4',
-                    entry_protocol=m3u8_entry_protocol, m3u8_id=m3u8_id)
+                    entry_protocol=m3u8_entry_protocol, m3u8_id=m3u8_id,
+                    preference=preference)
             elif ext == 'mpd':
                 is_plain_url = False
                 formats = self._extract_mpd_formats(

From db182c63fb4a58974e425a56d235131fd9efc531 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 25 Feb 2017 18:44:39 +0800
Subject: [PATCH 0373/1696] [njpwworld] Add new extractor (closes #11561)

---
 ChangeLog                          |  6 +++
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/njpwworld.py  | 83 ++++++++++++++++++++++++++++++
 3 files changed, 90 insertions(+)
 create mode 100644 youtube_dl/extractor/njpwworld.py

diff --git a/ChangeLog b/ChangeLog
index add8a6758..e0e1f52d0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
++ [njpwworld] Add new extractor (#11561)
+
+
 version 2017.02.24.1
 
 Extractors
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 83a170fa7..703bc5d34 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -637,6 +637,7 @@ from .ninecninemedia import (
 from .ninegag import NineGagIE
 from .ninenow import NineNowIE
 from .nintendo import NintendoIE
+from .njpwworld import NJPWWorldIE
 from .nobelprize import NobelPrizeIE
 from .noco import NocoIE
 from .normalboots import NormalbootsIE
diff --git a/youtube_dl/extractor/njpwworld.py b/youtube_dl/extractor/njpwworld.py
new file mode 100644
index 000000000..f5e3f6815
--- /dev/null
+++ b/youtube_dl/extractor/njpwworld.py
@@ -0,0 +1,83 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import (
+    get_element_by_class,
+    urlencode_postdata,
+)
+
+
+class NJPWWorldIE(InfoExtractor):
+    _VALID_URL = r'https?://njpwworld\.com/p/(?P<id>[a-z0-9_]+)'
+    IE_DESC = '新日本プロレスワールド'
+    _NETRC_MACHINE = 'njpwworld'
+
+    _TEST = {
+        'url': 'http://njpwworld.com/p/s_series_00155_1_9/',
+        'info_dict': {
+            'id': 's_series_00155_1_9',
+            'ext': 'mp4',
+            'title': '第9試合　ランディ・サベージ　vs　リック・スタイナー',
+            'tags': list,
+        },
+        'params': {
+            'skip_download': True,  # AES-encrypted m3u8
+        },
+        'skip': 'Requires login',
+    }
+
+    def _real_initialize(self):
+        self._login()
+
+    def _login(self):
+        username, password = self._get_login_info()
+        # No authentication to be performed
+        if not username:
+            return True
+
+        webpage, urlh = self._download_webpage_handle(
+            'https://njpwworld.com/auth/login', None,
+            note='Logging in', errnote='Unable to login',
+            data=urlencode_postdata({'login_id': username, 'pw': password}))
+        # /auth/login will return 302 for successful logins
+        if urlh.geturl() == 'https://njpwworld.com/auth/login':
+            self.report_warning('unable to login')
+            return False
+
+        return True
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        formats = []
+        for player_url, kind in re.findall(r'<a[^>]+href="(/player[^"]+)".+?<img[^>]+src="[^"]+qf_btn_([^".]+)', webpage):
+            player_url = compat_urlparse.urljoin(url, player_url)
+
+            player_page = self._download_webpage(
+                player_url, video_id, note='Downloading player page')
+
+            entries = self._parse_html5_media_entries(
+                player_url, player_page, video_id, m3u8_id='hls-%s' % kind,
+                m3u8_entry_protocol='m3u8_native',
+                preference=2 if 'hq' in kind else 1)
+            formats.extend(entries[0]['formats'])
+
+        self._sort_formats(formats)
+
+        post_content = get_element_by_class('post-content', webpage)
+        tags = re.findall(
+            r'<li[^>]+class="tag-[^"]+"><a[^>]*>([^<]+)</a></li>', post_content
+        ) if post_content else None
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage),
+            'formats': formats,
+            'tags': tags,
+        }

From 831217291ac05ad75ef16fd6d9985e255489c1e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Feb 2017 19:44:31 +0700
Subject: [PATCH 0374/1696] [compat] Use try except for compat_numeric_types

---
 youtube_dl/compat.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index b257e2e81..0c119e417 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2760,8 +2760,10 @@ else:
     compat_kwargs = lambda kwargs: kwargs
 
 
-compat_numeric_types = ((int, float, long, complex) if sys.version_info[0] < 3
-                        else (int, float, complex))
+try:
+    compat_numeric_types = (int, float, long, complex)
+except NameError:  # Python 3
+    compat_numeric_types = (int, float, complex)
 
 
 if sys.version_info < (2, 7):

From 922ab7840b01bd108887849063572cffa855cdc1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Feb 2017 20:16:40 +0700
Subject: [PATCH 0375/1696] [etonline] Add extractor (closes #12236)

---
 youtube_dl/extractor/etonline.py   | 39 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  1 +
 2 files changed, 40 insertions(+)
 create mode 100644 youtube_dl/extractor/etonline.py

diff --git a/youtube_dl/extractor/etonline.py b/youtube_dl/extractor/etonline.py
new file mode 100644
index 000000000..17d7cfec6
--- /dev/null
+++ b/youtube_dl/extractor/etonline.py
@@ -0,0 +1,39 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class ETOnlineIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?etonline\.com/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'http://www.etonline.com/tv/211130_dove_cameron_liv_and_maddie_emotional_episode_series_finale/',
+        'info_dict': {
+            'id': '211130_dove_cameron_liv_and_maddie_emotional_episode_series_finale',
+            'title': 'md5:a21ec7d3872ed98335cbd2a046f34ee6',
+            'description': 'md5:8b94484063f463cca709617c79618ccd',
+        },
+        'playlist_count': 2,
+    }, {
+        'url': 'http://www.etonline.com/media/video/here_are_the_stars_who_love_bringing_their_moms_as_dates_to_the_oscars-211359/',
+        'only_matching': True,
+    }]
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/1242911076001/default_default/index.html?videoId=ref:%s'
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        entries = [
+            self.url_result(
+                self.BRIGHTCOVE_URL_TEMPLATE % video_id, 'BrightcoveNew', video_id)
+            for video_id in re.findall(
+                r'site\.brightcove\s*\([^,]+,\s*["\'](title_\d+)', webpage)]
+
+        return self.playlist_result(
+            entries, playlist_id,
+            self._og_search_title(webpage, fatal=False),
+            self._og_search_description(webpage))
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 703bc5d34..9f5aaf1d8 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -288,6 +288,7 @@ from .espn import (
     ESPNArticleIE,
 )
 from .esri import EsriVideoIE
+from .etonline import ETOnlineIE
 from .europa import EuropaIE
 from .everyonesmixtape import EveryonesMixtapeIE
 from .expotv import ExpoTVIE

From 103f8c8d36170d5cb489420db0e8fe383b1f93dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Feb 2017 20:38:21 +0700
Subject: [PATCH 0376/1696] [xhamster] Capture and output videoClosed error
 (#12263)

---
 youtube_dl/extractor/xhamster.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 36a8c9840..7b6703714 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -5,6 +5,7 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     dict_get,
+    ExtractorError,
     int_or_none,
     parse_duration,
     unified_strdate,
@@ -57,6 +58,10 @@ class XHamsterIE(InfoExtractor):
     }, {
         'url': 'https://xhamster.com/movies/2272726/amber_slayed_by_the_knight.html',
         'only_matching': True,
+    }, {
+        # This video is visible for marcoalfa123456's friends only
+        'url': 'https://it.xhamster.com/movies/7263980/la_mia_vicina.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -78,6 +83,12 @@ class XHamsterIE(InfoExtractor):
         mrss_url = '%s://xhamster.com/movies/%s/%s.html' % (proto, video_id, seo)
         webpage = self._download_webpage(mrss_url, video_id)
 
+        error = self._html_search_regex(
+            r'<div[^>]+id=["\']videoClosed["\'][^>]*>(.+?)</div>',
+            webpage, 'error', default=None)
+        if error:
+            raise ExtractorError(error, expected=True)
+
         title = self._html_search_regex(
             [r'<h1[^>]*>([^<]+)</h1>',
              r'<meta[^>]+itemprop=".*?caption.*?"[^>]+content="(.+?)"',

From d374d943f3d9eca4b2052cfc830905d299a0688a Mon Sep 17 00:00:00 2001
From: Pratyush Singh <singh.pratyush96@gmail.com>
Date: Sat, 24 Dec 2016 21:05:41 +0530
Subject: [PATCH 0377/1696] [downloader/common] Limit displaying 2 digits after
 decimal point in sleep interval message

---
 youtube_dl/downloader/common.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 3dc144b4e..2c4470a95 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -347,7 +347,10 @@ class FileDownloader(object):
         if min_sleep_interval:
             max_sleep_interval = self.params.get('max_sleep_interval', min_sleep_interval)
             sleep_interval = random.uniform(min_sleep_interval, max_sleep_interval)
-            self.to_screen('[download] Sleeping %s seconds...' % sleep_interval)
+            self.to_screen(
+                '[download] Sleeping %s seconds...' % (
+                    int(sleep_interval) if sleep_interval.is_integer()
+                    else '%.2f' % sleep_interval))
             time.sleep(sleep_interval)
 
         return self.real_download(filename, info_dict)

From 5fc8d893612285ddba082f381006337573b36afa Mon Sep 17 00:00:00 2001
From: Ricardo Constantino <wiiaboo@gmail.com>
Date: Sat, 18 Feb 2017 17:42:31 +0000
Subject: [PATCH 0378/1696] [freshlive] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/freshlive.py  | 64 ++++++++++++++++++++++++++++++
 2 files changed, 65 insertions(+)
 create mode 100644 youtube_dl/extractor/freshlive.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 9f5aaf1d8..860b8f422 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -339,6 +339,7 @@ from .francetv import (
 )
 from .freesound import FreesoundIE
 from .freespeech import FreespeechIE
+from .freshlive import FreshliveIE
 from .funimation import FunimationIE
 from .funnyordie import FunnyOrDieIE
 from .fusion import FusionIE
diff --git a/youtube_dl/extractor/freshlive.py b/youtube_dl/extractor/freshlive.py
new file mode 100644
index 000000000..113f8f4fa
--- /dev/null
+++ b/youtube_dl/extractor/freshlive.py
@@ -0,0 +1,64 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from ..utils import (
+    int_or_none,
+    parse_iso8601
+)
+
+class FreshliveIE(InfoExtractor):
+    _VALID_URL = r'https?://freshlive\.tv/(?P<streamer>[^/]+)/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'https://freshlive.tv/satotv/74712',
+        'md5': '224f50d268b6b9f94e4198deccd55d6d',
+        'info_dict': {
+            'description': 'テスト',
+            'duration': 1511,
+            'id': '74712',
+            'ext': 'mp4',
+            'timestamp': 1483621764,
+            'title': 'テスト',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'upload_date': '20170105',
+            'uploader': 'サトTV',
+            'uploader_id': 'satotv',
+            'view_count': int,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        options = self._parse_json(
+            self._search_regex(
+                r'window\.__CONTEXT__\s*=\s*({.+?});\s*</script>',
+                webpage, 'initial context'),
+            video_id)
+
+        programs = options['context']['dispatcher']['stores']['ProgramStore']['programs']
+        info = programs.get(video_id, {})
+
+        video_url = info.get('liveStreamUrl') or info.get('archiveStreamUrl')
+        if not video_url:
+            raise ExtractorError('%s not a valid broadcast ID' % video_id, expected=True)
+
+        formats = self._extract_m3u8_formats(
+            video_url, video_id, ext='mp4', m3u8_id='hls')
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'title': info.get('title'),
+            'description': info.get('description'),
+            'duration': int_or_none(info.get('airTime')),
+            'is_live': int_or_none(info.get('airTime')) == None,
+            'thumbnail': info.get('thumbnailUrl'),
+            'uploader': info.get('channel', {}).get('title'),
+            'uploader_id': info.get('channel', {}).get('code'),
+            'uploader_url': info.get('channel', {}).get('permalink'),
+            'timestamp': parse_iso8601(info.get('startAt')),
+            'view_count': int_or_none(info.get('viewCount')),
+        }
\ No newline at end of file

From e498758b9cfa983d05b10a185ecdef480a93cf30 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Feb 2017 22:56:42 +0700
Subject: [PATCH 0379/1696] [freshlive] Fix issues and improve (closes #12175)

---
 youtube_dl/extractor/extractors.py |  2 +-
 youtube_dl/extractor/freshlive.py  | 64 ++++++++++++++++++++----------
 2 files changed, 43 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 860b8f422..58139ee4e 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -339,7 +339,7 @@ from .francetv import (
 )
 from .freesound import FreesoundIE
 from .freespeech import FreespeechIE
-from .freshlive import FreshliveIE
+from .freshlive import FreshLiveIE
 from .funimation import FunimationIE
 from .funnyordie import FunnyOrDieIE
 from .fusion import FusionIE
diff --git a/youtube_dl/extractor/freshlive.py b/youtube_dl/extractor/freshlive.py
index 113f8f4fa..a90f9156c 100644
--- a/youtube_dl/extractor/freshlive.py
+++ b/youtube_dl/extractor/freshlive.py
@@ -2,34 +2,40 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-
+from ..compat import compat_str
 from ..utils import (
+    ExtractorError,
     int_or_none,
-    parse_iso8601
+    try_get,
+    unified_timestamp,
 )
 
-class FreshliveIE(InfoExtractor):
-    _VALID_URL = r'https?://freshlive\.tv/(?P<streamer>[^/]+)/(?P<id>[0-9]+)'
+
+class FreshLiveIE(InfoExtractor):
+    _VALID_URL = r'https?://freshlive\.tv/[^/]+/(?P<id>\d+)'
     _TEST = {
         'url': 'https://freshlive.tv/satotv/74712',
-        'md5': '224f50d268b6b9f94e4198deccd55d6d',
+        'md5': '9f0cf5516979c4454ce982df3d97f352',
         'info_dict': {
-            'description': 'テスト',
-            'duration': 1511,
             'id': '74712',
             'ext': 'mp4',
-            'timestamp': 1483621764,
             'title': 'テスト',
+            'description': 'テスト',
             'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 1511,
+            'timestamp': 1483619655,
             'upload_date': '20170105',
             'uploader': 'サトTV',
             'uploader_id': 'satotv',
             'view_count': int,
+            'comment_count': int,
+            'is_live': False,
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
 
         options = self._parse_json(
@@ -38,27 +44,41 @@ class FreshliveIE(InfoExtractor):
                 webpage, 'initial context'),
             video_id)
 
-        programs = options['context']['dispatcher']['stores']['ProgramStore']['programs']
-        info = programs.get(video_id, {})
+        info = options['context']['dispatcher']['stores']['ProgramStore']['programs'][video_id]
 
-        video_url = info.get('liveStreamUrl') or info.get('archiveStreamUrl')
-        if not video_url:
-            raise ExtractorError('%s not a valid broadcast ID' % video_id, expected=True)
+        title = info['title']
+
+        if info.get('status') == 'upcoming':
+            raise ExtractorError('Stream %s is upcoming' % video_id, expected=True)
+
+        stream_url = info.get('liveStreamUrl') or info['archiveStreamUrl']
+
+        is_live = info.get('liveStreamUrl') is not None
 
         formats = self._extract_m3u8_formats(
-            video_url, video_id, ext='mp4', m3u8_id='hls')
+            stream_url, video_id, ext='mp4',
+            entry_protocol='m3u8' if is_live else 'm3u8_native',
+            m3u8_id='hls')
+
+        if is_live:
+            title = self._live_title(title)
 
         return {
             'id': video_id,
             'formats': formats,
-            'title': info.get('title'),
+            'title': title,
             'description': info.get('description'),
-            'duration': int_or_none(info.get('airTime')),
-            'is_live': int_or_none(info.get('airTime')) == None,
             'thumbnail': info.get('thumbnailUrl'),
-            'uploader': info.get('channel', {}).get('title'),
-            'uploader_id': info.get('channel', {}).get('code'),
-            'uploader_url': info.get('channel', {}).get('permalink'),
-            'timestamp': parse_iso8601(info.get('startAt')),
+            'duration': int_or_none(info.get('airTime')),
+            'timestamp': unified_timestamp(info.get('createdAt')),
+            'uploader': try_get(
+                info, lambda x: x['channel']['title'], compat_str),
+            'uploader_id': try_get(
+                info, lambda x: x['channel']['code'], compat_str),
+            'uploader_url': try_get(
+                info, lambda x: x['channel']['permalink'], compat_str),
             'view_count': int_or_none(info.get('viewCount')),
-        }
\ No newline at end of file
+            'comment_count': int_or_none(info.get('commentCount')),
+            'tags': info.get('tags', []),
+            'is_live': is_live,
+        }

From 9d0c08a02c55136221621e8b8dda5860211af8dd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 25 Feb 2017 01:40:12 +0800
Subject: [PATCH 0380/1696] [vevo] Fix videos with the new streams/streamsV3
 format (closes #11719)

---
 ChangeLog                    |  2 ++
 youtube_dl/extractor/vevo.py | 22 +++++++++++++++++-----
 2 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index e0e1f52d0..701afd57a 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,8 @@
 version <unreleased>
 
 Extractors
+* [vevo] Fix extraction for videos with the new streams/streamsV3 format
+  (#11719)
 + [njpwworld] Add new extractor (#11561)
 
 
diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index c4e37f694..9aa38bc5a 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -17,12 +17,12 @@ from ..utils import (
 
 
 class VevoBaseIE(InfoExtractor):
-    def _extract_json(self, webpage, video_id, item):
+    def _extract_json(self, webpage, video_id):
         return self._parse_json(
             self._search_regex(
                 r'window\.__INITIAL_STORE__\s*=\s*({.+?});\s*</script>',
                 webpage, 'initial store'),
-            video_id)['default'][item]
+            video_id)
 
 
 class VevoIE(VevoBaseIE):
@@ -139,6 +139,11 @@ class VevoIE(VevoBaseIE):
         # no genres available
         'url': 'http://www.vevo.com/watch/INS171400764',
         'only_matching': True,
+    }, {
+        # Another case available only via the webpage; using streams/streamsV3 formats
+        # Geo-restricted to Netherlands/Germany
+        'url': 'http://www.vevo.com/watch/boostee/pop-corn-clip-officiel/FR1A91600909',
+        'only_matching': True,
     }]
     _VERSIONS = {
         0: 'youtube',  # only in AuthenticateVideo videoVersions
@@ -193,7 +198,14 @@ class VevoIE(VevoBaseIE):
         # https://github.com/rg3/youtube-dl/issues/9366)
         if not video_versions:
             webpage = self._download_webpage(url, video_id)
-            video_versions = self._extract_json(webpage, video_id, 'streams')[video_id][0]
+            json_data = self._extract_json(webpage, video_id)
+            if 'streams' in json_data.get('default', {}):
+                video_versions = json_data['default']['streams'][video_id][0]
+            else:
+                video_versions = [
+                    value
+                    for key, value in json_data['apollo']['data'].items()
+                    if key.startswith('%s.streams' % video_id)]
 
         uploader = None
         artist = None
@@ -207,7 +219,7 @@ class VevoIE(VevoBaseIE):
 
         formats = []
         for video_version in video_versions:
-            version = self._VERSIONS.get(video_version['version'])
+            version = self._VERSIONS.get(video_version.get('version'), 'generic')
             version_url = video_version.get('url')
             if not version_url:
                 continue
@@ -339,7 +351,7 @@ class VevoPlaylistIE(VevoBaseIE):
             if video_id:
                 return self.url_result('vevo:%s' % video_id, VevoIE.ie_key())
 
-        playlists = self._extract_json(webpage, playlist_id, '%ss' % playlist_kind)
+        playlists = self._extract_json(webpage, playlist_id)['default']['%ss' % playlist_kind]
 
         playlist = (list(playlists.values())[0]
                     if playlist_kind == 'playlist' else playlists[playlist_id])

From b3aec47665104223578181c71cc90112f5b17fce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Feb 2017 23:27:14 +0700
Subject: [PATCH 0381/1696] [tvigle] Raise GeoRestrictedError

---
 youtube_dl/extractor/tvigle.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tvigle.py b/youtube_dl/extractor/tvigle.py
index f3817ab28..3475ef4c3 100644
--- a/youtube_dl/extractor/tvigle.py
+++ b/youtube_dl/extractor/tvigle.py
@@ -17,6 +17,9 @@ class TvigleIE(InfoExtractor):
     IE_DESC = 'Интернет-телевидение Tvigle.ru'
     _VALID_URL = r'https?://(?:www\.)?(?:tvigle\.ru/(?:[^/]+/)+(?P<display_id>[^/]+)/$|cloud\.tvigle\.ru/video/(?P<id>\d+))'
 
+    _GEO_BYPASS = False
+    _GEO_COUNTRIES = ['RU']
+
     _TESTS = [
         {
             'url': 'http://www.tvigle.ru/video/sokrat/',
@@ -72,8 +75,13 @@ class TvigleIE(InfoExtractor):
 
         error_message = item.get('errorMessage')
         if not videos and error_message:
-            raise ExtractorError(
-                '%s returned error: %s' % (self.IE_NAME, error_message), expected=True)
+            if item.get('isGeoBlocked') is True:
+                self.raise_geo_restricted(
+                    msg=error_message, countries=self._GEO_COUNTRIES)
+            else:
+                raise ExtractorError(
+                    '%s returned error: %s' % (self.IE_NAME, error_message),
+                    expected=True)
 
         title = item['title']
         description = item.get('description')

From a5cf17989b04e559fda9a2731a3b33e881c5cc3c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 26 Feb 2017 17:24:54 +0800
Subject: [PATCH 0382/1696] [MDR] Relax _VALID_URL and playerURL matching and
 update _TESTS

Ref: #12169
---
 ChangeLog                   |  1 +
 youtube_dl/extractor/mdr.py | 20 ++++++++++++++++----
 2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 701afd57a..4009acf37 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
++ [MDR] Recognize more URL patterns (#12169)
 * [vevo] Fix extraction for videos with the new streams/streamsV3 format
   (#11719)
 + [njpwworld] Add new extractor (#11561)
diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py
index 6e4290aad..322e5b45a 100644
--- a/youtube_dl/extractor/mdr.py
+++ b/youtube_dl/extractor/mdr.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 class MDRIE(InfoExtractor):
     IE_DESC = 'MDR.DE and KiKA'
-    _VALID_URL = r'https?://(?:www\.)?(?:mdr|kika)\.de/(?:.*)/[a-z]+-?(?P<id>\d+)(?:_.+?)?\.html'
+    _VALID_URL = r'https?://(?:www\.)?(?:mdr|kika)\.de/(?:.*)/[a-z-]+-?(?P<id>\d+)(?:_.+?)?\.html'
 
     _TESTS = [{
         # MDR regularly deletes its videos
@@ -31,6 +31,7 @@ class MDRIE(InfoExtractor):
             'duration': 250,
             'uploader': 'MITTELDEUTSCHER RUNDFUNK',
         },
+        'skip': '404 not found',
     }, {
         'url': 'http://www.kika.de/baumhaus/videos/video19636.html',
         'md5': '4930515e36b06c111213e80d1e4aad0e',
@@ -41,6 +42,7 @@ class MDRIE(InfoExtractor):
             'duration': 134,
             'uploader': 'KIKA',
         },
+        'skip': '404 not found',
     }, {
         'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/videos/video8182.html',
         'md5': '5fe9c4dd7d71e3b238f04b8fdd588357',
@@ -49,11 +51,21 @@ class MDRIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Beutolomäus und der geheime Weihnachtswunsch',
             'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd',
-            'timestamp': 1450950000,
-            'upload_date': '20151224',
+            'timestamp': 1482541200,
+            'upload_date': '20161224',
             'duration': 4628,
             'uploader': 'KIKA',
         },
+    }, {
+        # audio with alternative playerURL pattern
+        'url': 'http://www.mdr.de/kultur/videos-und-audios/audio-radio/operation-mindfuck-robert-wilson100.html',
+        'info_dict': {
+            'id': '100',
+            'ext': 'mp4',
+            'title': 'Feature: Operation Mindfuck - Robert Anton Wilson',
+            'duration': 3239,
+            'uploader': 'MITTELDEUTSCHER RUNDFUNK',
+        },
     }, {
         'url': 'http://www.kika.de/baumhaus/sendungen/video19636_zc-fea7f8a0_zs-4bf89c60.html',
         'only_matching': True,
@@ -71,7 +83,7 @@ class MDRIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         data_url = self._search_regex(
-            r'(?:dataURL|playerXml(?:["\'])?)\s*:\s*(["\'])(?P<url>.+/(?:video|audio)-?[0-9]+-avCustom\.xml)\1',
+            r'(?:dataURL|playerXml(?:["\'])?)\s*:\s*(["\'])(?P<url>.+?-avCustom\.xml)\1',
             webpage, 'data url', group='url').replace(r'\/', '/')
 
         doc = self._download_xml(

From 8878789f1117b59186ecc6bf82f462201166a26a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Feb 2017 16:50:57 +0700
Subject: [PATCH 0383/1696] [dailymotion] Raise GeoRestrictedError

---
 youtube_dl/extractor/dailymotion.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index b312401dc..246efde43 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -282,9 +282,14 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
         }
 
     def _check_error(self, info):
+        error = info.get('error')
         if info.get('error') is not None:
+            title = error['title']
+            # See https://developer.dailymotion.com/api#access-error
+            if error.get('code') == 'DM007':
+                self.raise_geo_restricted(msg=title)
             raise ExtractorError(
-                '%s said: %s' % (self.IE_NAME, info['error']['title']), expected=True)
+                '%s said: %s' % (self.IE_NAME, title), expected=True)
 
     def _get_subtitles(self, video_id, webpage):
         try:

From fd5c4aab5958a2a086072488913cc190ff028bc3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Feb 2017 16:51:21 +0700
Subject: [PATCH 0384/1696] [youtube] Raise GeoRestrictedError

---
 youtube_dl/extractor/youtube.py | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index dec02804b..b3c3cd5b2 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -47,7 +47,6 @@ from ..utils import (
     unsmuggle_url,
     uppercase_escape,
     urlencode_postdata,
-    ISO3166Utils,
 )
 
 
@@ -371,6 +370,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
     }
     _SUBTITLE_FORMATS = ('ttml', 'vtt')
 
+    _GEO_BYPASS = False
+
     IE_NAME = 'youtube'
     _TESTS = [
         {
@@ -917,7 +918,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             # itag 212
             'url': '1t24XAntNCY',
             'only_matching': True,
-        }
+        },
+        {
+            # geo restricted to JP
+            'url': 'sJL6WA-aGkQ',
+            'only_matching': True,
+        },
     ]
 
     def __init__(self, *args, **kwargs):
@@ -1376,11 +1382,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         if 'token' not in video_info:
             if 'reason' in video_info:
                 if 'The uploader has not made this video available in your country.' in video_info['reason']:
-                    regions_allowed = self._html_search_meta('regionsAllowed', video_webpage, default=None)
-                    if regions_allowed:
-                        raise ExtractorError('YouTube said: This video is available in %s only' % (
-                            ', '.join(map(ISO3166Utils.short2full, regions_allowed.split(',')))),
-                            expected=True)
+                    regions_allowed = self._html_search_meta(
+                        'regionsAllowed', video_webpage, default=None)
+                    countries = regions_allowed.split(',') if regions_allowed else None
+                    self.raise_geo_restricted(
+                        msg=video_info['reason'][0], countries=countries)
                 raise ExtractorError(
                     'YouTube said: %s' % video_info['reason'][0],
                     expected=True, video_id=video_id)
@@ -2126,6 +2132,10 @@ class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
             'id': 'UUs0ifCMCm1icqRbqhUINa0w',
             'title': 'Uploads from Deus Ex',
         },
+    }, {
+        # geo restricted to JP
+        'url': 'https://www.youtube.com/user/kananishinoSMEJ',
+        'only_matching': True,
     }]
 
     @classmethod

From 7fd465525695bb589fa8932e1e36f38ad511735b Mon Sep 17 00:00:00 2001
From: xbe <xbe@users.noreply.github.com>
Date: Sun, 26 Feb 2017 03:08:10 -0800
Subject: [PATCH 0385/1696] [crunchyroll] Extract uploader name that's not a
 link

Provide the Crunchyroll extractor with the ability to extract uploader
names that aren't links. Add a test for this new functionality.
This fixes #12267.
---
 youtube_dl/extractor/crunchyroll.py | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index a1fc6a756..9c6cf00ca 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -207,6 +207,21 @@ class CrunchyrollIE(CrunchyrollBaseIE):
             # Just test metadata extraction
             'skip_download': True,
         },
+    }, {
+        # make sure we can extract an uploader name that's not a link
+        'url': 'http://www.crunchyroll.com/hakuoki-reimeiroku/episode-1-dawn-of-the-divine-warriors-606899',
+        'info_dict': {
+            'id': '606899',
+            'ext': 'mp4',
+            'title': 'Hakuoki Reimeiroku Episode 1 – Dawn of the Divine Warriors',
+            'description': 'Ryunosuke was left to die, but Serizawa-san asked him a simple question "Do you want to live?"',
+            'uploader': 'Geneon Entertainment',
+            'upload_date': '20120717',
+        },
+        'params': {
+            # just test metadata extraction
+            'skip_download': True,
+        },
     }]
 
     _FORMAT_IDS = {
@@ -388,8 +403,9 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         if video_upload_date:
             video_upload_date = unified_strdate(video_upload_date)
         video_uploader = self._html_search_regex(
-            r'<a[^>]+href="/publisher/[^"]+"[^>]*>([^<]+)</a>', webpage,
-            'video_uploader', fatal=False)
+            # try looking for both an uploader that's a link and one that's not
+            [r'<a[^>]+href="/publisher/[^"]+"[^>]*>([^<]+)</a>', r'<div>\s*Publisher:\s*<span>\s*(.+?)\s*</span>\s*</div>'],
+            webpage, 'video_uploader', fatal=False)
 
         available_fmts = []
         for a, fmt in re.findall(r'(<a[^>]+token=["\']showmedia\.([0-9]{3,4})p["\'][^>]+>)', webpage):

From fdeea72611b2c2c29a9a34e91ae4bd9b8dfc1f64 Mon Sep 17 00:00:00 2001
From: Alex Seiler <seileralex@gmail.com>
Date: Sun, 26 Feb 2017 15:05:52 +0100
Subject: [PATCH 0386/1696] [cda] Decode URL (fixes #12255)

---
 youtube_dl/extractor/cda.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/cda.py b/youtube_dl/extractor/cda.py
index ae7af2f0e..1ee35b501 100755
--- a/youtube_dl/extractor/cda.py
+++ b/youtube_dl/extractor/cda.py
@@ -1,6 +1,7 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import codecs
 import re
 
 from .common import InfoExtractor
@@ -96,6 +97,10 @@ class CDAIE(InfoExtractor):
             if not video or 'file' not in video:
                 self.report_warning('Unable to extract %s version information' % version)
                 return
+            if video['file'].startswith('uggc'):
+                video['file'] = codecs.decode(video['file'], 'rot_13')
+                if video['file'].endswith('adc.mp4'):
+                    video['file'] = video['file'].replace('adc.mp4', '.mp4')
             f = {
                 'url': video['file'],
             }

From 892b47ab6c9147e785c562229e6dab305fffba61 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 27 Feb 2017 21:34:33 +0800
Subject: [PATCH 0387/1696] [scivee] Remove extractor (#9315)

The Wikipedia page is changed from active to down:
https://en.wikipedia.org/w/index.php?title=SciVee&diff=prev&oldid=723161154

Some other interesting bits:

$ nslookup www.scivee.tv
Server:         8.8.8.8
Address:        8.8.8.8#53

Non-authoritative answer:
www.scivee.tv   canonical name = scivee.rcsb.org.
Name:   scivee.rcsb.org
Address: 132.249.231.211

$ nslookup rcsb.org
Server:         8.8.8.8
Address:        8.8.8.8#53

Non-authoritative answer:
Name:   rcsb.org
Address: 132.249.231.77

Both IPs are from UCSD. I guess it's maintained by a lab and they don't
maintain it anymore.
---
 ChangeLog                          |  1 +
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/scivee.py     | 57 ------------------------------
 3 files changed, 1 insertion(+), 58 deletions(-)
 delete mode 100644 youtube_dl/extractor/scivee.py

diff --git a/ChangeLog b/ChangeLog
index 4009acf37..4ed9cb4e0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+- [scivee] Remove extractor (#9315)
 + [MDR] Recognize more URL patterns (#12169)
 * [vevo] Fix extraction for videos with the new streams/streamsV3 format
   (#11719)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 58139ee4e..d09104096 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -838,7 +838,6 @@ from .safari import (
 from .sapo import SapoIE
 from .savefrom import SaveFromIE
 from .sbs import SBSIE
-from .scivee import SciVeeIE
 from .screencast import ScreencastIE
 from .screencastomatic import ScreencastOMaticIE
 from .scrippsnetworks import ScrippsNetworksWatchIE
diff --git a/youtube_dl/extractor/scivee.py b/youtube_dl/extractor/scivee.py
deleted file mode 100644
index b1ca12fde..000000000
--- a/youtube_dl/extractor/scivee.py
+++ /dev/null
@@ -1,57 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import int_or_none
-
-
-class SciVeeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?scivee\.tv/node/(?P<id>\d+)'
-
-    _TEST = {
-        'url': 'http://www.scivee.tv/node/62352',
-        'md5': 'b16699b74c9e6a120f6772a44960304f',
-        'info_dict': {
-            'id': '62352',
-            'ext': 'mp4',
-            'title': 'Adam Arkin at the 2014 DOE JGI Genomics of Energy & Environment Meeting',
-            'description': 'md5:81f1710638e11a481358fab1b11059d7',
-        },
-        'skip': 'Not accessible from Travis CI server',
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
-        # annotations XML is malformed
-        annotations = self._download_webpage(
-            'http://www.scivee.tv/assets/annotations/%s' % video_id, video_id, 'Downloading annotations')
-
-        title = self._html_search_regex(r'<title>([^<]+)</title>', annotations, 'title')
-        description = self._html_search_regex(r'<abstract>([^<]+)</abstract>', annotations, 'abstract', fatal=False)
-        filesize = int_or_none(self._html_search_regex(
-            r'<filesize>([^<]+)</filesize>', annotations, 'filesize', fatal=False))
-
-        formats = [
-            {
-                'url': 'http://www.scivee.tv/assets/audio/%s' % video_id,
-                'ext': 'mp3',
-                'format_id': 'audio',
-            },
-            {
-                'url': 'http://www.scivee.tv/assets/video/%s' % video_id,
-                'ext': 'mp4',
-                'format_id': 'video',
-                'filesize': filesize,
-            },
-        ]
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': 'http://www.scivee.tv/assets/videothumb/%s' % video_id,
-            'formats': formats,
-        }

From 0e879f432afe5b9a04a06cbc697cf28d08ac5518 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Feb 2017 22:22:43 +0700
Subject: [PATCH 0388/1696] [youtube:channel] Remove duplicate test

---
 youtube_dl/extractor/youtube.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index b3c3cd5b2..7053e5512 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2132,10 +2132,6 @@ class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
             'id': 'UUs0ifCMCm1icqRbqhUINa0w',
             'title': 'Uploads from Deus Ex',
         },
-    }, {
-        # geo restricted to JP
-        'url': 'https://www.youtube.com/user/kananishinoSMEJ',
-        'only_matching': True,
     }]
 
     @classmethod
@@ -2236,7 +2232,7 @@ class YoutubeUserIE(YoutubeChannelIE):
         'url': 'https://www.youtube.com/gametrailers',
         'only_matching': True,
     }, {
-        # This channel is not available.
+        # This channel is not available, geo restricted to JP
         'url': 'https://www.youtube.com/user/kananishinoSMEJ/videos',
         'only_matching': True,
     }]

From 0dc5a86a329314f551f86c2ef3202342b7506667 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Feb 2017 22:43:19 +0700
Subject: [PATCH 0389/1696] [npo] Add support for hetklokhuis.nl (closes
 #12293)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/npo.py        | 44 ++++++++++++++++++++++--------
 2 files changed, 34 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d09104096..0910b7b05 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -669,6 +669,7 @@ from .npo import (
     NPORadioIE,
     NPORadioFragmentIE,
     SchoolTVIE,
+    HetKlokhuisIE,
     VPROIE,
     WNLIE,
 )
diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 962437145..09e8d9987 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -416,7 +416,21 @@ class NPORadioFragmentIE(InfoExtractor):
         }
 
 
-class SchoolTVIE(InfoExtractor):
+class NPODataMidEmbedIE(InfoExtractor):
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        video_id = self._search_regex(
+            r'data-mid=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video_id', group='id')
+        return {
+            '_type': 'url_transparent',
+            'ie_key': 'NPO',
+            'url': 'npo:%s' % video_id,
+            'display_id': display_id
+        }
+
+
+class SchoolTVIE(NPODataMidEmbedIE):
     IE_NAME = 'schooltv'
     _VALID_URL = r'https?://(?:www\.)?schooltv\.nl/video/(?P<id>[^/?#&]+)'
 
@@ -435,17 +449,25 @@ class SchoolTVIE(InfoExtractor):
         }
     }
 
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-        video_id = self._search_regex(
-            r'data-mid=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video_id', group='id')
-        return {
-            '_type': 'url_transparent',
-            'ie_key': 'NPO',
-            'url': 'npo:%s' % video_id,
-            'display_id': display_id
+
+class HetKlokhuisIE(NPODataMidEmbedIE):
+    IE_NAME = 'schooltv'
+    _VALID_URL = r'https?://(?:www\.)?hetklokhuis.nl/[^/]+/\d+/(?P<id>[^/?#&]+)'
+
+    _TEST = {
+        'url': 'http://hetklokhuis.nl/tv-uitzending/3471/Zwaartekrachtsgolven',
+        'info_dict': {
+            'id': 'VPWON_1260528',
+            'display_id': 'Zwaartekrachtsgolven',
+            'ext': 'm4v',
+            'title': 'Het Klokhuis: Zwaartekrachtsgolven',
+            'description': 'md5:c94f31fb930d76c2efa4a4a71651dd48',
+            'upload_date': '20170223',
+        },
+        'params': {
+            'skip_download': True
         }
+    }
 
 
 class NPOPlaylistBaseIE(NPOIE):

From f264c62334fdd31a7620b4fdefb822e1bae6bd77 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Feb 2017 23:10:00 +0700
Subject: [PATCH 0390/1696] [npo] Add support for zapp.nl

---
 youtube_dl/extractor/npo.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 09e8d9987..7c2c93f27 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -51,7 +51,8 @@ class NPOIE(NPOBaseIE):
                             (?:
                                 npo\.nl/(?!live|radio)(?:[^/]+/){2}|
                                 ntr\.nl/(?:[^/]+/){2,}|
-                                omroepwnl\.nl/video/fragment/[^/]+__
+                                omroepwnl\.nl/video/fragment/[^/]+__|
+                                zapp\.nl/[^/]+/(?:gemist|filmpjes)/
                             )
                         )
                         (?P<id>[^/?#]+)
@@ -140,6 +141,14 @@ class NPOIE(NPOBaseIE):
                 'upload_date': '20150508',
                 'duration': 462,
             },
+        },
+        {
+            'url': 'http://www.zapp.nl/de-bzt-show/gemist/KN_1687547',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://www.zapp.nl/de-bzt-show/filmpjes/POMS_KN_7315118',
+            'only_matching': True,
         }
     ]
 

From dbc01fdb6f4b4c58469ffb75d00a179f5af5cdcb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Feb 2017 23:10:29 +0700
Subject: [PATCH 0391/1696] [hetklokhuis] Fix IE_NAME

---
 youtube_dl/extractor/npo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 7c2c93f27..b53c29993 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -460,7 +460,7 @@ class SchoolTVIE(NPODataMidEmbedIE):
 
 
 class HetKlokhuisIE(NPODataMidEmbedIE):
-    IE_NAME = 'schooltv'
+    IE_NAME = 'hetklokhuis'
     _VALID_URL = r'https?://(?:www\.)?hetklokhuis.nl/[^/]+/\d+/(?P<id>[^/?#&]+)'
 
     _TEST = {

From 18abb743762ce5b9b2ffd4d9d5e01b62621cc62e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Feb 2017 23:13:51 +0700
Subject: [PATCH 0392/1696] [npo] Relax _VALID_URL for zapp.nl

---
 youtube_dl/extractor/npo.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index b53c29993..50473d777 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -52,7 +52,7 @@ class NPOIE(NPOBaseIE):
                                 npo\.nl/(?!live|radio)(?:[^/]+/){2}|
                                 ntr\.nl/(?:[^/]+/){2,}|
                                 omroepwnl\.nl/video/fragment/[^/]+__|
-                                zapp\.nl/[^/]+/(?:gemist|filmpjes)/
+                                zapp\.nl/[^/]+/[^/]+/
                             )
                         )
                         (?P<id>[^/?#]+)
@@ -149,6 +149,10 @@ class NPOIE(NPOBaseIE):
         {
             'url': 'http://www.zapp.nl/de-bzt-show/filmpjes/POMS_KN_7315118',
             'only_matching': True,
+        },
+        {
+            'url': 'http://www.zapp.nl/beste-vrienden-quiz/extra-video-s/WO_NTR_1067990',
+            'only_matching': True,
         }
     ]
 

From c6184bcf7b58476b659a92290711e4c58faff277 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Feb 2017 23:24:03 +0700
Subject: [PATCH 0393/1696] [ChangeLog] Actualize

---
 ChangeLog | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 4ed9cb4e0..4cb897f7f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,11 +1,27 @@
 version <unreleased>
 
+Core
+* [downloader/common] Limit displaying 2 digits after decimal point in sleep
+  interval message (#12183)
++ [extractor/common] Add preference to _parse_html5_media_entries
+
 Extractors
++ [npo] Add support for zapp.nl
++ [npo] Add support for hetklokhuis.nl (#12293)
 - [scivee] Remove extractor (#9315)
-+ [MDR] Recognize more URL patterns (#12169)
++ [cda] Decode download URL (#12255)
++ [crunchyroll] Improve uploader extraction (#12267)
++ [youtube] Raise GeoRestrictedError
++ [dailymotion] Raise GeoRestrictedError
++ [mdr] Recognize more URL patterns (#12169)
++ [tvigle] Raise GeoRestrictedError
 * [vevo] Fix extraction for videos with the new streams/streamsV3 format
   (#11719)
-+ [njpwworld] Add new extractor (#11561)
++ [freshlive] Add support for freshlive.tv (#12175)
++ [xhamster] Capture and output videoClosed error (#12263)
++ [etonline] Add support for etonline.com (#12236)
++ [njpwworld] Add support for njpwworld.com (#11561)
+* [amcnetworks] Relax URL regular expression (#12127)
 
 
 version 2017.02.24.1

From ef48a1175dc4e28b07c55ae7277d8196abec7ace Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Feb 2017 23:26:07 +0700
Subject: [PATCH 0394/1696] release 2017.02.27

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 5 ++++-
 youtube_dl/version.py     | 2 +-
 4 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 564108122..6374f7c25 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.24.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.24.1**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.27*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.27**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.02.24.1
+[debug] youtube-dl version 2017.02.27
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 4cb897f7f..949ea1810 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.02.27
 
 Core
 * [downloader/common] Limit displaying 2 digits after decimal point in sleep
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index f97397331..1b01c6d9d 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -239,6 +239,7 @@
  - **ESPN**
  - **ESPNArticle**
  - **EsriVideo**
+ - **ETOnline**
  - **Europa**
  - **EveryonesMixtape**
  - **ExpoTV**
@@ -274,6 +275,7 @@
  - **francetvinfo.fr**
  - **Freesound**
  - **freespeech.org**
+ - **FreshLive**
  - **Funimation**
  - **FunnyOrDie**
  - **Fusion**
@@ -310,6 +312,7 @@
  - **HellPorno**
  - **Helsinki**: helsinki.fi
  - **HentaiStigma**
+ - **hetklokhuis**
  - **hgtv.com:show**
  - **HistoricFilms**
  - **history:topic**: History.com Topic
@@ -511,6 +514,7 @@
  - **Nintendo**
  - **njoy**: N-JOY
  - **njoy:embed**
+ - **NJPWWorld**: 新日本プロレスワールド
  - **NobelPrize**
  - **Noco**
  - **Normalboots**
@@ -666,7 +670,6 @@
  - **savefrom.net**
  - **SBS**: sbs.com.au
  - **schooltv**
- - **SciVee**
  - **screen.yahoo:search**: Yahoo screen search
  - **Screencast**
  - **ScreencastOMatic**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index fe7462eac..261218b80 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.02.24.1'
+__version__ = '2017.02.27'

From f4c68ba372655c8ff4d6b1cfecfa129796159afd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 28 Feb 2017 21:40:22 +0800
Subject: [PATCH 0395/1696] [douyu] Fix extraction and update _TESTS

They've switched from flv to hls

Closes #12301
---
 ChangeLog                       |  6 ++++
 youtube_dl/extractor/douyutv.py | 53 ++++++---------------------------
 2 files changed, 15 insertions(+), 44 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 949ea1810..f9986ae61 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [douyu] Fix extraction (#12301)
+
+
 version 2017.02.27
 
 Core
diff --git a/youtube_dl/extractor/douyutv.py b/youtube_dl/extractor/douyutv.py
index 911594413..9a83fb31a 100644
--- a/youtube_dl/extractor/douyutv.py
+++ b/youtube_dl/extractor/douyutv.py
@@ -1,15 +1,7 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import hashlib
-import time
-import uuid
-
 from .common import InfoExtractor
-from ..compat import (
-    compat_str,
-    compat_urllib_parse_urlencode,
-)
 from ..utils import (
     ExtractorError,
     unescapeHTML,
@@ -24,8 +16,8 @@ class DouyuTVIE(InfoExtractor):
         'info_dict': {
             'id': '17732',
             'display_id': 'iseven',
-            'ext': 'flv',
-            'title': 're:^清晨醒脑！T-ara根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'ext': 'mp4',
+            'title': 're:^清晨醒脑！T-ARA根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
             'description': r're:.*m7show@163\.com.*',
             'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': '7师傅',
@@ -39,7 +31,7 @@ class DouyuTVIE(InfoExtractor):
         'info_dict': {
             'id': '85982',
             'display_id': '85982',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 're:^小漠从零单排记！——CSOL2躲猫猫 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
             'description': 'md5:746a2f7a253966a06755a912f0acc0d2',
             'thumbnail': r're:^https?://.*\.jpg$',
@@ -55,8 +47,8 @@ class DouyuTVIE(InfoExtractor):
         'info_dict': {
             'id': '17732',
             'display_id': '17732',
-            'ext': 'flv',
-            'title': 're:^清晨醒脑！T-ara根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'ext': 'mp4',
+            'title': 're:^清晨醒脑！T-ARA根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
             'description': r're:.*m7show@163\.com.*',
             'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': '7师傅',
@@ -96,45 +88,18 @@ class DouyuTVIE(InfoExtractor):
         if room.get('show_status') == '2':
             raise ExtractorError('Live stream is offline', expected=True)
 
-        tt = compat_str(int(time.time() / 60))
-        did = uuid.uuid4().hex.upper()
-
-        sign_content = ''.join((room_id, did, self._API_KEY, tt))
-        sign = hashlib.md5((sign_content).encode('utf-8')).hexdigest()
-
-        flv_data = compat_urllib_parse_urlencode({
-            'cdn': 'ws',
-            'rate': '0',
-            'tt': tt,
-            'did': did,
-            'sign': sign,
-        })
-
-        video_info = self._download_json(
-            'http://www.douyu.com/lapi/live/getPlay/%s' % room_id, video_id,
-            data=flv_data, note='Downloading video info',
-            headers={'Content-Type': 'application/x-www-form-urlencoded'})
-
-        error_code = video_info.get('error', 0)
-        if error_code is not 0:
-            raise ExtractorError(
-                '%s reported error %i' % (self.IE_NAME, error_code),
-                expected=True)
-
-        base_url = video_info['data']['rtmp_url']
-        live_path = video_info['data']['rtmp_live']
-
-        video_url = '%s/%s' % (base_url, live_path)
+        formats = self._extract_m3u8_formats(
+            room['hls_url'], video_id, ext='mp4')
 
         title = self._live_title(unescapeHTML(room['room_name']))
-        description = room.get('notice')
+        description = room.get('show_details')
         thumbnail = room.get('room_src')
         uploader = room.get('nickname')
 
         return {
             'id': room_id,
             'display_id': video_id,
-            'url': video_url,
+            'formats': formats,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,

From c9619f0a17927086c49e4b443202be296d734a76 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 27 Feb 2017 18:47:47 +0800
Subject: [PATCH 0396/1696] [aes] Add aes_cbc_encrypt

Used in daisuki.net (#4738)
---
 test/test_aes.py  |  9 ++++++++-
 youtube_dl/aes.py | 28 ++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/test/test_aes.py b/test/test_aes.py
index 54078a66d..78a28751b 100644
--- a/test/test_aes.py
+++ b/test/test_aes.py
@@ -8,7 +8,7 @@ import sys
 import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from youtube_dl.aes import aes_decrypt, aes_encrypt, aes_cbc_decrypt, aes_decrypt_text
+from youtube_dl.aes import aes_decrypt, aes_encrypt, aes_cbc_decrypt, aes_cbc_encrypt, aes_decrypt_text
 from youtube_dl.utils import bytes_to_intlist, intlist_to_bytes
 import base64
 
@@ -34,6 +34,13 @@ class TestAES(unittest.TestCase):
         decrypted = intlist_to_bytes(aes_cbc_decrypt(data, self.key, self.iv))
         self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
 
+    def test_cbc_encrypt(self):
+        data = bytes_to_intlist(self.secret_msg)
+        encrypted = intlist_to_bytes(aes_cbc_encrypt(data, self.key, self.iv))
+        self.assertEqual(
+            encrypted,
+            b"\x97\x92+\xe5\x0b\xc3\x18\x91ky9m&\xb3\xb5@\xe6'\xc2\x96.\xc8u\x88\xab9-[\x9e|\xf1\xcd")
+
     def test_decrypt_text(self):
         password = intlist_to_bytes(self.key).decode('utf-8')
         encrypted = base64.b64encode(
diff --git a/youtube_dl/aes.py b/youtube_dl/aes.py
index b8ff45481..c5bb3c4ef 100644
--- a/youtube_dl/aes.py
+++ b/youtube_dl/aes.py
@@ -60,6 +60,34 @@ def aes_cbc_decrypt(data, key, iv):
     return decrypted_data
 
 
+def aes_cbc_encrypt(data, key, iv):
+    """
+    Encrypt with aes in CBC mode. Using PKCS#7 padding
+
+    @param {int[]} data        cleartext
+    @param {int[]} key         16/24/32-Byte cipher key
+    @param {int[]} iv          16-Byte IV
+    @returns {int[]}           encrypted data
+    """
+    expanded_key = key_expansion(key)
+    block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES))
+
+    encrypted_data = []
+    previous_cipher_block = iv
+    for i in range(block_count):
+        block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
+        remaining_length = BLOCK_SIZE_BYTES - len(block)
+        block += [remaining_length] * remaining_length
+        mixed_block = xor(block, previous_cipher_block)
+
+        encrypted_block = aes_encrypt(mixed_block, expanded_key)
+        encrypted_data += encrypted_block
+
+        previous_cipher_block = encrypted_block
+
+    return encrypted_data
+
+
 def key_expansion(data):
     """
     Generate key schedule

From f48409c7ac186fa38bbeb2df2b210e37a18eb04b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 27 Feb 2017 18:50:19 +0800
Subject: [PATCH 0397/1696] [utils] Add pkcs1pad

Used in daisuki.net (#4738)
---
 test/test_utils.py  |  9 +++++++++
 youtube_dl/utils.py | 15 +++++++++++++++
 2 files changed, 24 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 3cdb21d40..aefd94518 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -52,6 +52,7 @@ from youtube_dl.utils import (
     parse_filesize,
     parse_count,
     parse_iso8601,
+    pkcs1pad,
     read_batch_urls,
     sanitize_filename,
     sanitize_path,
@@ -1104,6 +1105,14 @@ The first line
             ohdave_rsa_encrypt(b'aa111222', e, N),
             '726664bd9a23fd0c70f9f1b84aab5e3905ce1e45a584e9cbcf9bcc7510338fc1986d6c599ff990d923aa43c51c0d9013cd572e13bc58f4ae48f2ed8c0b0ba881')
 
+    def test_pkcs1pad(self):
+        data = [1, 2, 3]
+        padded_data = pkcs1pad(data, 32)
+        self.assertEqual(padded_data[:2], [0, 2])
+        self.assertEqual(padded_data[28:], [0, 1, 2, 3])
+
+        self.assertRaises(ValueError, pkcs1pad, data, 8)
+
     def test_encode_base_n(self):
         self.assertEqual(encode_base_n(0, 30), '0')
         self.assertEqual(encode_base_n(80, 30), '2k')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 17b83794a..8bd075eaf 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3336,6 +3336,21 @@ def ohdave_rsa_encrypt(data, exponent, modulus):
     return '%x' % encrypted
 
 
+def pkcs1pad(data, length):
+    """
+    Padding input data with PKCS#1 scheme
+
+    @param {int[]} data        input data
+    @param {int}   length      target length
+    @returns {int[]}           padded data
+    """
+    if len(data) > length - 11:
+        raise ValueError('Input data too long for PKCS#1 padding')
+
+    pseudo_random = [random.randint(0, 254) for _ in range(length - len(data) - 3)]
+    return [0, 2] + pseudo_random + [0] + data
+
+
 def encode_base_n(num, n, table=None):
     FULL_TABLE = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
     if not table:

From 0a5445ddbeb8d391bbac92e5fe9074c6aa2e1565 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 28 Feb 2017 19:16:55 +0800
Subject: [PATCH 0398/1696] [utils] Add bytes_to_long() and long_to_bytes()

Used in daisuki.net (#4738)

Both are adapted from public domain PyCrypto:
https://github.com/dlitz/pycrypto/blob/master/lib/Crypto/Util/number.py
---
 youtube_dl/utils.py | 51 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 8bd075eaf..807183f4a 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3319,6 +3319,57 @@ class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
             self, req, proxy, type)
 
 
+# Both long_to_bytes and bytes_to_long are adapted from PyCrypto, which is
+# released into Public Domain
+# https://github.com/dlitz/pycrypto/blob/master/lib/Crypto/Util/number.py#L387
+
+def long_to_bytes(n, blocksize=0):
+    """long_to_bytes(n:long, blocksize:int) : string
+    Convert a long integer to a byte string.
+
+    If optional blocksize is given and greater than zero, pad the front of the
+    byte string with binary zeros so that the length is a multiple of
+    blocksize.
+    """
+    # after much testing, this algorithm was deemed to be the fastest
+    s = b''
+    n = int(n)
+    while n > 0:
+        s = compat_struct_pack('>I', n & 0xffffffff) + s
+        n = n >> 32
+    # strip off leading zeros
+    for i in range(len(s)):
+        if s[i] != b'\000'[0]:
+            break
+    else:
+        # only happens when n == 0
+        s = b'\000'
+        i = 0
+    s = s[i:]
+    # add back some pad bytes.  this could be done more efficiently w.r.t. the
+    # de-padding being done above, but sigh...
+    if blocksize > 0 and len(s) % blocksize:
+        s = (blocksize - len(s) % blocksize) * b'\000' + s
+    return s
+
+
+def bytes_to_long(s):
+    """bytes_to_long(string) : long
+    Convert a byte string to a long integer.
+
+    This is (essentially) the inverse of long_to_bytes().
+    """
+    acc = 0
+    length = len(s)
+    if length % 4:
+        extra = (4 - length % 4)
+        s = b'\000' * extra + s
+        length = length + extra
+    for i in range(0, length, 4):
+        acc = (acc << 32) + compat_struct_unpack('>I', s[i:i + 4])[0]
+    return acc
+
+
 def ohdave_rsa_encrypt(data, exponent, modulus):
     '''
     Implement OHDave's RSA algorithm. See http://www.ohdave.com/rsa/

From 9bd05b5a18c535f5517158d505af5dced498d23e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 27 Feb 2017 18:56:45 +0800
Subject: [PATCH 0399/1696] [daisuki] Add new extractor (closes #4738)

---
 ChangeLog                          |   1 +
 youtube_dl/extractor/daisuki.py    | 144 +++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |   4 +
 3 files changed, 149 insertions(+)
 create mode 100644 youtube_dl/extractor/daisuki.py

diff --git a/ChangeLog b/ChangeLog
index f9986ae61..401c5885e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
++ [daisuki] Add new extractor (#2486, #3186, #4738, #6175, #7776, #10060)
 * [douyu] Fix extraction (#12301)
 
 
diff --git a/youtube_dl/extractor/daisuki.py b/youtube_dl/extractor/daisuki.py
new file mode 100644
index 000000000..6fd09faf1
--- /dev/null
+++ b/youtube_dl/extractor/daisuki.py
@@ -0,0 +1,144 @@
+from __future__ import unicode_literals
+
+import base64
+import json
+import random
+import re
+
+from .common import InfoExtractor
+from ..aes import (
+    aes_cbc_decrypt,
+    aes_cbc_encrypt,
+)
+from ..utils import (
+    bytes_to_intlist,
+    bytes_to_long,
+    clean_html,
+    ExtractorError,
+    intlist_to_bytes,
+    get_element_by_id,
+    js_to_json,
+    int_or_none,
+    long_to_bytes,
+    pkcs1pad,
+    remove_end,
+)
+
+
+class DaisukiIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?daisuki\.net/[^/]+/[^/]+/[^/]+/watch\.[^.]+\.(?P<id>\d+)\.html'
+
+    _TEST = {
+        'url': 'http://www.daisuki.net/tw/en/anime/watch.TheIdolMasterCG.11213.html',
+        'info_dict': {
+            'id': '11213',
+            'ext': 'mp4',
+            'title': '#01 Who is in the pumpkin carriage? - THE IDOLM@STER CINDERELLA GIRLS',
+            'creator': 'BANDAI NAMCO Entertainment',
+        },
+        'params': {
+            'skip_download': True,  # AES-encrypted HLS stream
+        },
+    }
+
+    # The public key in PEM format can be found in clientlibs_anime_watch.min.js
+    _RSA_KEY = (0xc5524c25e8e14b366b3754940beeb6f96cb7e2feef0b932c7659a0c5c3bf173d602464c2df73d693b513ae06ff1be8f367529ab30bf969c5640522181f2a0c51ea546ae120d3d8d908595e4eff765b389cde080a1ef7f1bbfb07411cc568db73b7f521cedf270cbfbe0ddbc29b1ac9d0f2d8f4359098caffee6d07915020077d, 65537)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        flashvars = self._parse_json(self._search_regex(
+            r'(?s)var\s+flashvars\s*=\s*({.+?});', webpage, 'flashvars'),
+            video_id, transform_source=js_to_json)
+
+        iv = [0] * 16
+
+        data = {}
+        for key in ('device_cd', 'mv_id', 'ss1_prm', 'ss2_prm', 'ss3_prm', 'ss_id'):
+            data[key] = flashvars.get(key, '')
+
+        encrypted_rtn = None
+
+        # Some AES keys are rejected. Try it with different AES keys
+        for idx in range(5):
+            aes_key = [random.randint(0, 254) for _ in range(32)]
+            padded_aeskey = intlist_to_bytes(pkcs1pad(aes_key, 128))
+
+            n, e = self._RSA_KEY
+            encrypted_aeskey = long_to_bytes(pow(bytes_to_long(padded_aeskey), e, n))
+            init_data = self._download_json('http://www.daisuki.net/bin/bgn/init', video_id, query={
+                's': flashvars.get('s', ''),
+                'c': flashvars.get('ss3_prm', ''),
+                'e': url,
+                'd': base64.b64encode(intlist_to_bytes(aes_cbc_encrypt(
+                    bytes_to_intlist(json.dumps(data)),
+                    aes_key, iv))).decode('ascii'),
+                'a': base64.b64encode(encrypted_aeskey).decode('ascii'),
+            }, note='Downloading JSON metadata' + (' (try #%d)' % (idx + 1) if idx > 0 else ''))
+
+            if 'rtn' in init_data:
+                encrypted_rtn = init_data['rtn']
+                break
+
+            self._sleep(5, video_id)
+
+        if encrypted_rtn is None:
+            raise ExtractorError('Failed to fetch init data')
+
+        rtn = self._parse_json(
+            intlist_to_bytes(aes_cbc_decrypt(bytes_to_intlist(
+                base64.b64decode(encrypted_rtn)),
+                aes_key, iv)).decode('utf-8').rstrip('\0'),
+            video_id)
+
+        formats = self._extract_m3u8_formats(
+            rtn['play_url'], video_id, ext='mp4', entry_protocol='m3u8_native')
+
+        title = remove_end(self._og_search_title(webpage), ' - DAISUKI')
+
+        creator = self._html_search_regex(
+            r'Creator\s*:\s*([^<]+)', webpage, 'creator', fatal=False)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'creator': creator,
+        }
+
+
+class DaisukiPlaylistIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)daisuki\.net/[^/]+/[^/]+/[^/]+/detail\.(?P<id>[a-zA-Z0-9]+)\.html'
+
+    _TEST = {
+        'url': 'http://www.daisuki.net/tw/en/anime/detail.TheIdolMasterCG.html',
+        'info_dict': {
+            'id': 'TheIdolMasterCG',
+            'title': 'THE IDOLM@STER CINDERELLA GIRLS',
+            'description': 'md5:0f2c028a9339f7a2c7fbf839edc5c5d8',
+        },
+        'playlist_count': 26,
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        episode_pattern = r'''(?sx)
+            <img[^>]+delay="[^"]+/(\d+)/movie\.jpg".+?
+            <p[^>]+class=".*?\bepisodeNumber\b.*?">(?:<a[^>]+>)?([^<]+)'''
+        entries = [{
+            '_type': 'url_transparent',
+            'url': url.replace('detail', 'watch').replace('.html', '.' + movie_id + '.html'),
+            'episode_id': episode_id,
+            'episode_number': int_or_none(episode_id),
+        } for movie_id, episode_id in re.findall(episode_pattern, webpage)]
+
+        playlist_title = remove_end(
+            self._og_search_title(webpage, fatal=False), ' - Anime - DAISUKI')
+        playlist_description = clean_html(get_element_by_id('synopsisTxt', webpage))
+
+        return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 0910b7b05..e251d8478 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -227,6 +227,10 @@ from .dailymotion import (
     DailymotionUserIE,
     DailymotionCloudIE,
 )
+from .daisuki import (
+    DaisukiIE,
+    DaisukiPlaylistIE,
+)
 from .daum import (
     DaumIE,
     DaumClipIE,

From 7c4aa6fd6fd6fadf1cf1942c279cd5c0ff5ae498 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 28 Feb 2017 22:29:01 +0800
Subject: [PATCH 0400/1696] [daisuki] Add subtitles (#4738)

---
 youtube_dl/extractor/daisuki.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/daisuki.py b/youtube_dl/extractor/daisuki.py
index 6fd09faf1..58cc98666 100644
--- a/youtube_dl/extractor/daisuki.py
+++ b/youtube_dl/extractor/daisuki.py
@@ -34,6 +34,11 @@ class DaisukiIE(InfoExtractor):
             'id': '11213',
             'ext': 'mp4',
             'title': '#01 Who is in the pumpkin carriage? - THE IDOLM@STER CINDERELLA GIRLS',
+            'subtitles': {
+                'mul': [{
+                    'ext': 'ttml',
+                }],
+            },
             'creator': 'BANDAI NAMCO Entertainment',
         },
         'params': {
@@ -101,10 +106,20 @@ class DaisukiIE(InfoExtractor):
         creator = self._html_search_regex(
             r'Creator\s*:\s*([^<]+)', webpage, 'creator', fatal=False)
 
+        subtitles = {}
+        caption_url = rtn.get('caption_url')
+        if caption_url:
+            # mul: multiple languages
+            subtitles['mul'] = [{
+                'url': caption_url,
+                'ext': 'ttml',
+            }]
+
         return {
             'id': video_id,
             'title': title,
             'formats': formats,
+            'subtitles': subtitles,
             'creator': creator,
         }
 

From 87dadd456a138c3107ff6254bd03ed832cf2e6d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 28 Feb 2017 23:06:47 +0700
Subject: [PATCH 0401/1696] [youtube:playlist] Recognize another playlist
 pattern (closes #11928, closes #12286)

---
 youtube_dl/extractor/youtube.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 7053e5512..81c793921 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1851,7 +1851,7 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
                         (?:
                             youtube\.com/
                             (?:
-                               (?:course|view_play_list|my_playlists|artist|playlist|watch|embed/videoseries)
+                               (?:course|view_play_list|my_playlists|artist|playlist|watch|embed/(?:videoseries|[0-9A-Za-z_-]{11}))
                                \? (?:.*?[&;])*? (?:p|a|list)=
                             |  p/
                             )|
@@ -1924,6 +1924,13 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
             'title': 'JODA15',
             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
         }
+    }, {
+        'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
+        'playlist_mincount': 485,
+        'info_dict': {
+            'title': '2017 華語最新單曲 (2/24更新)',
+            'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
+        }
     }, {
         'note': 'Embedded SWF player',
         'url': 'https://www.youtube.com/p/YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ?hl=en_US&fs=1&rel=0',
@@ -2072,7 +2079,7 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
         # Check if it's a video-specific URL
         query_dict = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
         video_id = query_dict.get('v', [None])[0] or self._search_regex(
-            r'(?:^|//)youtu\.be/([0-9A-Za-z_-]{11})', url,
+            r'(?:(?:^|//)youtu\.be/|youtube\.com/embed/(?!videoseries))([0-9A-Za-z_-]{11})', url,
             'video id', default=None)
         if video_id:
             if self._downloader.params.get('noplaylist'):

From 948519b35dec420c3b3ca5369bcba1dead31fcc6 Mon Sep 17 00:00:00 2001
From: Alex Seiler <seileralex@gmail.com>
Date: Fri, 17 Feb 2017 04:49:25 +0100
Subject: [PATCH 0402/1696] [azmedien:showplaylist] Add support for all
 episodes playlists

---
 youtube_dl/extractor/azmedien.py   | 49 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  1 +
 2 files changed, 50 insertions(+)

diff --git a/youtube_dl/extractor/azmedien.py b/youtube_dl/extractor/azmedien.py
index cbc3ed564..817fc6d57 100644
--- a/youtube_dl/extractor/azmedien.py
+++ b/youtube_dl/extractor/azmedien.py
@@ -5,6 +5,7 @@ import re
 from .common import InfoExtractor
 from .kaltura import KalturaIE
 from ..utils import (
+    get_element_by_class,
     get_element_by_id,
     strip_or_none,
     urljoin,
@@ -170,3 +171,51 @@ class AZMedienPlaylistIE(AZMedienBaseIE):
                 'video-title', webpage)), group='title')
 
         return self.playlist_result(entries, show_id, title)
+
+
+class AZMedienShowPlaylistIE(AZMedienBaseIE):
+    IE_DESC = 'AZ Medien Show playlists'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:www\.)?
+                        (?P<id>
+                        (?:
+                            telezueri\.ch|
+                            telebaern\.tv|
+                            telem1\.ch
+                        )/
+                        (?:
+                            all-episodes|
+                            alle-episoden
+                        )
+                        /[^/]+
+                        )
+                    '''
+
+    _TEST = {
+        'url': 'http://www.telezueri.ch/all-episodes/astrotalk',
+        'info_dict': {
+            'id': 'telezueri.ch/all-episodes/astrotalk',
+            'title': 'TeleZüri: AstroTalk - alle episoden',
+            'description': 'md5:4c0f7e7d741d906004266e295ceb4a26',
+        },
+        'playlist_mincount': 13,
+        'params': {
+            'skip_download': True,
+        }
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        webpage = self._download_webpage(url, playlist_id)
+        episodes = get_element_by_class('search-mobile-box', webpage)
+        entries = [self.url_result(
+            urljoin(url, m.group('url'))) for m in re.finditer(
+                r'<a[^>]+href=(["\'])(?P<url>.+?)\1', episodes)]
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
+        return self.playlist_result(
+            entries,
+            playlist_id=playlist_id,
+            playlist_title=title,
+            playlist_description=description)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index e251d8478..b1613a9d3 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -83,6 +83,7 @@ from .awaan import (
 from .azmedien import (
     AZMedienIE,
     AZMedienPlaylistIE,
+    AZMedienShowPlaylistIE,
 )
 from .azubu import AzubuIE, AzubuLiveIE
 from .baidu import BaiduVideoIE

From 43b38424a9ca2ce962036b17462d59b8acbf6dd8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 28 Feb 2017 23:37:33 +0700
Subject: [PATCH 0403/1696] [azmedien:showplaylist] Improve (closes #12160)

---
 youtube_dl/extractor/azmedien.py | 24 ++++++++----------------
 1 file changed, 8 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/azmedien.py b/youtube_dl/extractor/azmedien.py
index 817fc6d57..f4e07d901 100644
--- a/youtube_dl/extractor/azmedien.py
+++ b/youtube_dl/extractor/azmedien.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
@@ -174,11 +175,10 @@ class AZMedienPlaylistIE(AZMedienBaseIE):
 
 
 class AZMedienShowPlaylistIE(AZMedienBaseIE):
-    IE_DESC = 'AZ Medien Show playlists'
+    IE_DESC = 'AZ Medien show playlists'
     _VALID_URL = r'''(?x)
                     https?://
                         (?:www\.)?
-                        (?P<id>
                         (?:
                             telezueri\.ch|
                             telebaern\.tv|
@@ -187,22 +187,18 @@ class AZMedienShowPlaylistIE(AZMedienBaseIE):
                         (?:
                             all-episodes|
                             alle-episoden
-                        )
-                        /[^/]+
-                        )
+                        )/
+                        (?P<id>[^/?#&]+)
                     '''
 
     _TEST = {
         'url': 'http://www.telezueri.ch/all-episodes/astrotalk',
         'info_dict': {
-            'id': 'telezueri.ch/all-episodes/astrotalk',
+            'id': 'astrotalk',
             'title': 'TeleZüri: AstroTalk - alle episoden',
             'description': 'md5:4c0f7e7d741d906004266e295ceb4a26',
         },
         'playlist_mincount': 13,
-        'params': {
-            'skip_download': True,
-        }
     }
 
     def _real_extract(self, url):
@@ -211,11 +207,7 @@ class AZMedienShowPlaylistIE(AZMedienBaseIE):
         episodes = get_element_by_class('search-mobile-box', webpage)
         entries = [self.url_result(
             urljoin(url, m.group('url'))) for m in re.finditer(
-                r'<a[^>]+href=(["\'])(?P<url>.+?)\1', episodes)]
-        title = self._og_search_title(webpage)
+                r'<a[^>]+href=(["\'])(?P<url>(?:(?!\1).)+)\1', episodes)]
+        title = self._og_search_title(webpage, fatal=False)
         description = self._og_search_description(webpage)
-        return self.playlist_result(
-            entries,
-            playlist_id=playlist_id,
-            playlist_title=title,
-            playlist_description=description)
+        return self.playlist_result(entries, playlist_id, title, description)

From 11bae9cdde8ed75b355d5e68ef57dae504ad94ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 28 Feb 2017 23:49:24 +0700
Subject: [PATCH 0404/1696] [ChangeLog] Actualize

---
 ChangeLog | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 401c5885e..8d33c055b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,15 @@
 version <unreleased>
 
+Core
++ [utils] Add bytes_to_long and long_to_bytes
++ [utils] Add pkcs1pad
++ [aes] Add aes_cbc_encrypt
+
 Extractors
-+ [daisuki] Add new extractor (#2486, #3186, #4738, #6175, #7776, #10060)
++ [azmedien:showplaylist] Add support for show playlists (#12160)
++ [youtube:playlist] Recognize another playlist pattern (#11928, #12286)
++ [daisuki] Add support for daisuki.net (#2486, #3186, #4738, #6175, #7776,
+  #10060)
 * [douyu] Fix extraction (#12301)
 
 

From 1dc24093f81f349c22e6bda13cb05c26ac3266f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 28 Feb 2017 23:59:22 +0700
Subject: [PATCH 0405/1696] release 2017.02.28

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 3 +++
 youtube_dl/version.py     | 2 +-
 4 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 6374f7c25..c907ef931 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.27*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.27**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.28*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.28**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.02.27
+[debug] youtube-dl version 2017.02.28
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 8d33c055b..2a1fd9018 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.02.28
 
 Core
 + [utils] Add bytes_to_long and long_to_bytes
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 1b01c6d9d..a08e00fce 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -78,6 +78,7 @@
  - **awaan:video**
  - **AZMedien**: AZ Medien videos
  - **AZMedienPlaylist**: AZ Medien playlists
+ - **AZMedienShowPlaylist**: AZ Medien show playlists
  - **Azubu**
  - **AzubuLive**
  - **BaiduVideo**: 百度视频
@@ -191,6 +192,8 @@
  - **dailymotion:playlist**
  - **dailymotion:user**
  - **DailymotionCloud**
+ - **Daisuki**
+ - **DaisukiPlaylist**
  - **daum.net**
  - **daum.net:clip**
  - **daum.net:playlist**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 261218b80..340e23bf8 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.02.27'
+__version__ = '2017.02.28'

From aa9cc2ecbfea6c82944b4e07f3e93c904f1ff421 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 1 Mar 2017 05:03:14 +0700
Subject: [PATCH 0406/1696] [npo] Adapt to app.php API (closes #12311)

---
 youtube_dl/extractor/npo.py | 79 ++++++++++++++++---------------------
 1 file changed, 35 insertions(+), 44 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 50473d777..89082c189 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -17,27 +17,9 @@ from ..utils import (
 
 class NPOBaseIE(InfoExtractor):
     def _get_token(self, video_id):
-        token_page = self._download_webpage(
-            'http://ida.omroep.nl/npoplayer/i.js',
-            video_id, note='Downloading token')
-        token = self._search_regex(
-            r'npoplayer\.token = "(.+?)"', token_page, 'token')
-        # Decryption algorithm extracted from http://npoplayer.omroep.nl/csjs/npoplayer-min.js
-        token_l = list(token)
-        first = second = None
-        for i in range(5, len(token_l) - 4):
-            if token_l[i].isdigit():
-                if first is None:
-                    first = i
-                elif second is None:
-                    second = i
-        if first is None or second is None:
-            first = 12
-            second = 13
-
-        token_l[first], token_l[second] = token_l[second], token_l[first]
-
-        return ''.join(token_l)
+        return self._download_json(
+            'http://ida.omroep.nl/app.php/auth', video_id,
+            note='Downloading token')['token']
 
 
 class NPOIE(NPOBaseIE):
@@ -187,32 +169,41 @@ class NPOIE(NPOBaseIE):
         pubopties = metadata.get('pubopties')
         if pubopties:
             quality = qualities(['adaptive', 'wmv_sb', 'h264_sb', 'wmv_bb', 'h264_bb', 'wvc1_std', 'h264_std'])
-            for format_id in pubopties:
-                format_info = self._download_json(
-                    'http://ida.omroep.nl/odi/?prid=%s&puboptions=%s&adaptive=yes&token=%s'
-                    % (video_id, format_id, token),
-                    video_id, 'Downloading %s JSON' % format_id)
-                if format_info.get('error_code', 0) or format_info.get('errorcode', 0):
+            items = self._download_json(
+                'http://ida.omroep.nl/app.php/%s' % video_id,
+                'Downloading formats JSON', query={
+                    'adaptive': 'yes',
+                    'token': token,
+                })['items'][0]
+            for num, item in enumerate(items):
+                item_url = item.get('url')
+                if not item_url:
                     continue
-                streams = format_info.get('streams')
-                if streams:
-                    try:
-                        video_info = self._download_json(
-                            streams[0] + '&type=json',
-                            video_id, 'Downloading %s stream JSON' % format_id)
-                    except ExtractorError as ee:
-                        if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 404:
-                            error = (self._parse_json(ee.cause.read().decode(), video_id, fatal=False) or {}).get('errorstring')
-                            if error:
-                                raise ExtractorError(error, expected=True)
-                        raise
-                else:
-                    video_info = format_info
-                video_url = video_info.get('url')
+                format_id = self._search_regex(
+                    r'video/ida/([^/]+)', item_url, 'format id',
+                    default=None)
+                try:
+                    stream_info = self._download_json(
+                        item_url + '&type=json', video_id,
+                        'Downloading %s stream JSON' % item.get('label') or format_id or num)
+                except ExtractorError as ee:
+                    if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 404:
+                        error = (self._parse_json(
+                            ee.cause.read().decode(), video_id,
+                            fatal=False) or {}).get('errorstring')
+                        if error:
+                            raise ExtractorError(error, expected=True)
+                    raise
+                if stream_info.get('error_code', 0) or stream_info.get('errorcode', 0):
+                    continue
+                video_url = stream_info.get('url')
                 if not video_url:
                     continue
-                if format_id == 'adaptive':
-                    formats.extend(self._extract_m3u8_formats(video_url, video_id, 'mp4'))
+                if stream_info.get('family') == 'adaptive':
+                    formats.extend(self._extract_m3u8_formats(
+                        video_url, video_id, ext='mp4',
+                        entry_protocol='m3u8_native', m3u8_id='hls',
+                        fatal=False))
                 else:
                     formats.append({
                         'url': video_url,

From 83e8fce628e810e2a5639ef9a21be839526512fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 1 Mar 2017 22:14:46 +0700
Subject: [PATCH 0407/1696] [npo] Improve extraction and update tests

---
 youtube_dl/extractor/npo.py | 410 ++++++++++++++++++------------------
 1 file changed, 201 insertions(+), 209 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 89082c189..be10fc486 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -3,15 +3,19 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..compat import (
+    compat_HTTPError,
+    compat_str,
+)
 from ..utils import (
+    determine_ext,
+    ExtractorError,
     fix_xml_ampersands,
     orderedSet,
     parse_duration,
     qualities,
     strip_jsonp,
     unified_strdate,
-    ExtractorError,
 )
 
 
@@ -40,103 +44,101 @@ class NPOIE(NPOBaseIE):
                         (?P<id>[^/?#]+)
                 '''
 
-    _TESTS = [
-        {
-            'url': 'http://www.npo.nl/nieuwsuur/22-06-2014/VPWON_1220719',
-            'md5': '4b3f9c429157ec4775f2c9cb7b911016',
-            'info_dict': {
-                'id': 'VPWON_1220719',
-                'ext': 'm4v',
-                'title': 'Nieuwsuur',
-                'description': 'Dagelijks tussen tien en elf: nieuws, sport en achtergronden.',
-                'upload_date': '20140622',
-            },
+    _TESTS = [{
+        'url': 'http://www.npo.nl/nieuwsuur/22-06-2014/VPWON_1220719',
+        'md5': '4b3f9c429157ec4775f2c9cb7b911016',
+        'info_dict': {
+            'id': 'VPWON_1220719',
+            'ext': 'm4v',
+            'title': 'Nieuwsuur',
+            'description': 'Dagelijks tussen tien en elf: nieuws, sport en achtergronden.',
+            'upload_date': '20140622',
         },
-        {
-            'url': 'http://www.npo.nl/de-mega-mike-mega-thomas-show/27-02-2009/VARA_101191800',
-            'md5': 'da50a5787dbfc1603c4ad80f31c5120b',
-            'info_dict': {
-                'id': 'VARA_101191800',
-                'ext': 'm4v',
-                'title': 'De Mega Mike & Mega Thomas show: The best of.',
-                'description': 'md5:3b74c97fc9d6901d5a665aac0e5400f4',
-                'upload_date': '20090227',
-                'duration': 2400,
-            },
+    }, {
+        'url': 'http://www.npo.nl/de-mega-mike-mega-thomas-show/27-02-2009/VARA_101191800',
+        'md5': 'da50a5787dbfc1603c4ad80f31c5120b',
+        'info_dict': {
+            'id': 'VARA_101191800',
+            'ext': 'm4v',
+            'title': 'De Mega Mike & Mega Thomas show: The best of.',
+            'description': 'md5:3b74c97fc9d6901d5a665aac0e5400f4',
+            'upload_date': '20090227',
+            'duration': 2400,
         },
-        {
-            'url': 'http://www.npo.nl/tegenlicht/25-02-2013/VPWON_1169289',
-            'md5': 'f8065e4e5a7824068ed3c7e783178f2c',
-            'info_dict': {
-                'id': 'VPWON_1169289',
-                'ext': 'm4v',
-                'title': 'Tegenlicht: De toekomst komt uit Afrika',
-                'description': 'md5:52cf4eefbc96fffcbdc06d024147abea',
-                'upload_date': '20130225',
-                'duration': 3000,
-            },
+    }, {
+        'url': 'http://www.npo.nl/tegenlicht/25-02-2013/VPWON_1169289',
+        'md5': 'f8065e4e5a7824068ed3c7e783178f2c',
+        'info_dict': {
+            'id': 'VPWON_1169289',
+            'ext': 'm4v',
+            'title': 'Tegenlicht: Zwart geld. De toekomst komt uit Afrika',
+            'description': 'md5:52cf4eefbc96fffcbdc06d024147abea',
+            'upload_date': '20130225',
+            'duration': 3000,
         },
-        {
-            'url': 'http://www.npo.nl/de-nieuwe-mens-deel-1/21-07-2010/WO_VPRO_043706',
-            'info_dict': {
-                'id': 'WO_VPRO_043706',
-                'ext': 'wmv',
-                'title': 'De nieuwe mens - Deel 1',
-                'description': 'md5:518ae51ba1293ffb80d8d8ce90b74e4b',
-                'duration': 4680,
-            },
-            'params': {
-                # mplayer mms download
-                'skip_download': True,
-            }
+    }, {
+        'url': 'http://www.npo.nl/de-nieuwe-mens-deel-1/21-07-2010/WO_VPRO_043706',
+        'info_dict': {
+            'id': 'WO_VPRO_043706',
+            'ext': 'm4v',
+            'title': 'De nieuwe mens - Deel 1',
+            'description': 'md5:518ae51ba1293ffb80d8d8ce90b74e4b',
+            'duration': 4680,
         },
-        # non asf in streams
-        {
-            'url': 'http://www.npo.nl/hoe-gaat-europa-verder-na-parijs/10-01-2015/WO_NOS_762771',
-            'md5': 'b3da13de374cbe2d5332a7e910bef97f',
-            'info_dict': {
-                'id': 'WO_NOS_762771',
-                'ext': 'mp4',
-                'title': 'Hoe gaat Europa verder na Parijs?',
-            },
-        },
-        {
-            'url': 'http://www.ntr.nl/Aap-Poot-Pies/27/detail/Aap-poot-pies/VPWON_1233944#content',
-            'md5': '01c6a2841675995da1f0cf776f03a9c3',
-            'info_dict': {
-                'id': 'VPWON_1233944',
-                'ext': 'm4v',
-                'title': 'Aap, poot, pies',
-                'description': 'md5:c9c8005d1869ae65b858e82c01a91fde',
-                'upload_date': '20150508',
-                'duration': 599,
-            },
-        },
-        {
-            'url': 'http://www.omroepwnl.nl/video/fragment/vandaag-de-dag-verkiezingen__POMS_WNL_853698',
-            'md5': 'd30cd8417b8b9bca1fdff27428860d08',
-            'info_dict': {
-                'id': 'POW_00996502',
-                'ext': 'm4v',
-                'title': '''"Dit is wel een 'landslide'..."''',
-                'description': 'md5:f8d66d537dfb641380226e31ca57b8e8',
-                'upload_date': '20150508',
-                'duration': 462,
-            },
-        },
-        {
-            'url': 'http://www.zapp.nl/de-bzt-show/gemist/KN_1687547',
-            'only_matching': True,
-        },
-        {
-            'url': 'http://www.zapp.nl/de-bzt-show/filmpjes/POMS_KN_7315118',
-            'only_matching': True,
-        },
-        {
-            'url': 'http://www.zapp.nl/beste-vrienden-quiz/extra-video-s/WO_NTR_1067990',
-            'only_matching': True,
+        'params': {
+            'skip_download': True,
         }
-    ]
+    }, {
+        # non asf in streams
+        'url': 'http://www.npo.nl/hoe-gaat-europa-verder-na-parijs/10-01-2015/WO_NOS_762771',
+        'info_dict': {
+            'id': 'WO_NOS_762771',
+            'ext': 'mp4',
+            'title': 'Hoe gaat Europa verder na Parijs?',
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }, {
+        'url': 'http://www.ntr.nl/Aap-Poot-Pies/27/detail/Aap-poot-pies/VPWON_1233944#content',
+        'info_dict': {
+            'id': 'VPWON_1233944',
+            'ext': 'm4v',
+            'title': 'Aap, poot, pies',
+            'description': 'md5:c9c8005d1869ae65b858e82c01a91fde',
+            'upload_date': '20150508',
+            'duration': 599,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }, {
+        'url': 'http://www.omroepwnl.nl/video/fragment/vandaag-de-dag-verkiezingen__POMS_WNL_853698',
+        'info_dict': {
+            'id': 'POW_00996502',
+            'ext': 'm4v',
+            'title': '''"Dit is wel een 'landslide'..."''',
+            'description': 'md5:f8d66d537dfb641380226e31ca57b8e8',
+            'upload_date': '20150508',
+            'duration': 462,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }, {
+        'url': 'http://www.zapp.nl/de-bzt-show/gemist/KN_1687547',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.zapp.nl/de-bzt-show/filmpjes/POMS_KN_7315118',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.zapp.nl/beste-vrienden-quiz/extra-video-s/WO_NTR_1067990',
+        'only_matching': True,
+    }, {
+        # live stream
+        'url': 'npo:LI_NL1_4188102',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -165,79 +167,115 @@ class NPOIE(NPOBaseIE):
         token = self._get_token(video_id)
 
         formats = []
+        urls = set()
 
-        pubopties = metadata.get('pubopties')
-        if pubopties:
-            quality = qualities(['adaptive', 'wmv_sb', 'h264_sb', 'wmv_bb', 'h264_bb', 'wvc1_std', 'h264_std'])
-            items = self._download_json(
-                'http://ida.omroep.nl/app.php/%s' % video_id,
-                'Downloading formats JSON', query={
-                    'adaptive': 'yes',
-                    'token': token,
-                })['items'][0]
-            for num, item in enumerate(items):
-                item_url = item.get('url')
-                if not item_url:
-                    continue
-                format_id = self._search_regex(
-                    r'video/ida/([^/]+)', item_url, 'format id',
-                    default=None)
-                try:
-                    stream_info = self._download_json(
-                        item_url + '&type=json', video_id,
-                        'Downloading %s stream JSON' % item.get('label') or format_id or num)
-                except ExtractorError as ee:
-                    if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 404:
-                        error = (self._parse_json(
-                            ee.cause.read().decode(), video_id,
-                            fatal=False) or {}).get('errorstring')
-                        if error:
-                            raise ExtractorError(error, expected=True)
-                    raise
-                if stream_info.get('error_code', 0) or stream_info.get('errorcode', 0):
+        quality = qualities(['adaptive', 'wmv_sb', 'h264_sb', 'wmv_bb', 'h264_bb', 'wvc1_std', 'h264_std'])
+        items = self._download_json(
+            'http://ida.omroep.nl/app.php/%s' % video_id, video_id,
+            'Downloading formats JSON', query={
+                'adaptive': 'yes',
+                'token': token,
+            })['items'][0]
+        for num, item in enumerate(items):
+            item_url = item.get('url')
+            if not item_url or item_url in urls:
+                continue
+            urls.add(item_url)
+            format_id = self._search_regex(
+                r'video/ida/([^/]+)', item_url, 'format id',
+                default=None)
+
+            def add_format_url(format_url):
+                formats.append({
+                    'url': format_url,
+                    'format_id': format_id,
+                    'quality': quality(format_id),
+                })
+
+            # Example: http://www.npo.nl/de-nieuwe-mens-deel-1/21-07-2010/WO_VPRO_043706
+            if item.get('contentType') == 'url':
+                add_format_url(item_url)
+                continue
+
+            try:
+                stream_info = self._download_json(
+                    item_url + '&type=json', video_id,
+                    'Downloading %s stream JSON'
+                    % item.get('label') or format_id or num)
+            except ExtractorError as ee:
+                if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 404:
+                    error = (self._parse_json(
+                        ee.cause.read().decode(), video_id,
+                        fatal=False) or {}).get('errorstring')
+                    if error:
+                        raise ExtractorError(error, expected=True)
+                raise
+            # Stream URL instead of JSON, example: npo:LI_NL1_4188102
+            if isinstance(stream_info, compat_str):
+                if not stream_info.startswith('http'):
                     continue
+                video_url = stream_info
+            # JSON
+            else:
                 video_url = stream_info.get('url')
-                if not video_url:
+            if not video_url or video_url in urls:
+                continue
+            urls.add(item_url)
+            if determine_ext(video_url) == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    video_url, video_id, ext='mp4',
+                    entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
+            else:
+                add_format_url(video_url)
+
+        is_live = metadata.get('medium') == 'live'
+
+        if not is_live:
+            for num, stream in enumerate(metadata.get('streams', [])):
+                stream_url = stream.get('url')
+                if not stream_url or stream_url in urls:
                     continue
-                if stream_info.get('family') == 'adaptive':
+                urls.add(stream_url)
+                # smooth streaming is not supported
+                stream_type = stream.get('type', '').lower()
+                if stream_type in ['ss', 'ms']:
+                    continue
+                if stream_type == 'hds':
+                    f4m_formats = self._extract_f4m_formats(
+                        stream_url, video_id, fatal=False)
+                    # f4m downloader downloads only piece of live stream
+                    for f4m_format in f4m_formats:
+                        f4m_format['preference'] = -1
+                    formats.extend(f4m_formats)
+                elif stream_type == 'hls':
                     formats.extend(self._extract_m3u8_formats(
-                        video_url, video_id, ext='mp4',
-                        entry_protocol='m3u8_native', m3u8_id='hls',
-                        fatal=False))
-                else:
+                        stream_url, video_id, ext='mp4', fatal=False))
+                # Example: http://www.npo.nl/de-nieuwe-mens-deel-1/21-07-2010/WO_VPRO_043706
+                elif '.asf' in stream_url:
+                    asx = self._download_xml(
+                        stream_url, video_id,
+                        'Downloading stream %d ASX playlist' % num,
+                        transform_source=fix_xml_ampersands, fatal=False)
+                    if not asx:
+                        continue
+                    ref = asx.find('./ENTRY/Ref')
+                    if ref is None:
+                        continue
+                    video_url = ref.get('href')
+                    if not video_url or video_url in urls:
+                        continue
+                    urls.add(video_url)
                     formats.append({
                         'url': video_url,
-                        'format_id': format_id,
-                        'quality': quality(format_id),
+                        'ext': stream.get('formaat', 'asf'),
+                        'quality': stream.get('kwaliteit'),
+                        'preference': -10,
                     })
-
-        streams = metadata.get('streams')
-        if streams:
-            for i, stream in enumerate(streams):
-                stream_url = stream.get('url')
-                if not stream_url:
-                    continue
-                if '.asf' not in stream_url:
+                else:
                     formats.append({
                         'url': stream_url,
                         'quality': stream.get('kwaliteit'),
                     })
-                    continue
-                asx = self._download_xml(
-                    stream_url, video_id,
-                    'Downloading stream %d ASX playlist' % i,
-                    transform_source=fix_xml_ampersands)
-                ref = asx.find('./ENTRY/Ref')
-                if ref is None:
-                    continue
-                video_url = ref.get('href')
-                if not video_url:
-                    continue
-                formats.append({
-                    'url': video_url,
-                    'ext': stream.get('formaat', 'asf'),
-                    'quality': stream.get('kwaliteit'),
-                })
 
         self._sort_formats(formats)
 
@@ -250,28 +288,28 @@ class NPOIE(NPOBaseIE):
 
         return {
             'id': video_id,
-            'title': title,
+            'title': self._live_title(title) if is_live else title,
             'description': metadata.get('info'),
             'thumbnail': metadata.get('images', [{'url': None}])[-1]['url'],
             'upload_date': unified_strdate(metadata.get('gidsdatum')),
             'duration': parse_duration(metadata.get('tijdsduur')),
             'formats': formats,
             'subtitles': subtitles,
+            'is_live': is_live,
         }
 
 
 class NPOLiveIE(NPOBaseIE):
     IE_NAME = 'npo.nl:live'
-    _VALID_URL = r'https?://(?:www\.)?npo\.nl/live/(?P<id>.+)'
+    _VALID_URL = r'https?://(?:www\.)?npo\.nl/live/(?P<id>[^/?#&]+)'
 
     _TEST = {
         'url': 'http://www.npo.nl/live/npo-1',
         'info_dict': {
-            'id': 'LI_NEDERLAND1_136692',
+            'id': 'LI_NL1_4188102',
             'display_id': 'npo-1',
             'ext': 'mp4',
-            'title': 're:^Nederland 1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
-            'description': 'Livestream',
+            'title': 're:^NPO 1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
             'is_live': True,
         },
         'params': {
@@ -287,58 +325,12 @@ class NPOLiveIE(NPOBaseIE):
         live_id = self._search_regex(
             r'data-prid="([^"]+)"', webpage, 'live id')
 
-        metadata = self._download_json(
-            'http://e.omroep.nl/metadata/%s' % live_id,
-            display_id, transform_source=strip_jsonp)
-
-        token = self._get_token(display_id)
-
-        formats = []
-
-        streams = metadata.get('streams')
-        if streams:
-            for stream in streams:
-                stream_type = stream.get('type').lower()
-                # smooth streaming is not supported
-                if stream_type in ['ss', 'ms']:
-                    continue
-                stream_info = self._download_json(
-                    'http://ida.omroep.nl/aapi/?stream=%s&token=%s&type=jsonp'
-                    % (stream.get('url'), token),
-                    display_id, 'Downloading %s JSON' % stream_type)
-                if stream_info.get('error_code', 0) or stream_info.get('errorcode', 0):
-                    continue
-                stream_url = self._download_json(
-                    stream_info['stream'], display_id,
-                    'Downloading %s URL' % stream_type,
-                    'Unable to download %s URL' % stream_type,
-                    transform_source=strip_jsonp, fatal=False)
-                if not stream_url:
-                    continue
-                if stream_type == 'hds':
-                    f4m_formats = self._extract_f4m_formats(stream_url, display_id)
-                    # f4m downloader downloads only piece of live stream
-                    for f4m_format in f4m_formats:
-                        f4m_format['preference'] = -1
-                    formats.extend(f4m_formats)
-                elif stream_type == 'hls':
-                    formats.extend(self._extract_m3u8_formats(stream_url, display_id, 'mp4'))
-                else:
-                    formats.append({
-                        'url': stream_url,
-                        'preference': -10,
-                    })
-
-        self._sort_formats(formats)
-
         return {
+            '_type': 'url_transparent',
+            'url': 'npo:%s' % live_id,
+            'ie_key': NPOIE.ie_key(),
             'id': live_id,
             'display_id': display_id,
-            'title': self._live_title(metadata['titel']),
-            'description': metadata['info'],
-            'thumbnail': metadata.get('images', [{'url': None}])[-1]['url'],
-            'formats': formats,
-            'is_live': True,
         }
 
 

From 4b8a984c67cdc1b2bfde77398d74096406db9644 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 1 Mar 2017 22:21:13 +0700
Subject: [PATCH 0408/1696] [npo] Add support for audio

---
 youtube_dl/extractor/npo.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index be10fc486..38fefe492 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -125,6 +125,18 @@ class NPOIE(NPOBaseIE):
         'params': {
             'skip_download': True,
         }
+    }, {
+        # audio
+        'url': 'http://www.npo.nl/jouw-stad-rotterdam/29-01-2017/RBX_FUNX_6683215/RBX_FUNX_7601437',
+        'info_dict': {
+            'id': 'RBX_FUNX_6683215',
+            'ext': 'mp3',
+            'title': 'Jouw Stad Rotterdam',
+            'description': 'md5:db251505244f097717ec59fabc372d9f',
+        },
+        'params': {
+            'skip_download': True,
+        }
     }, {
         'url': 'http://www.zapp.nl/de-bzt-show/gemist/KN_1687547',
         'only_matching': True,
@@ -193,7 +205,7 @@ class NPOIE(NPOBaseIE):
                 })
 
             # Example: http://www.npo.nl/de-nieuwe-mens-deel-1/21-07-2010/WO_VPRO_043706
-            if item.get('contentType') == 'url':
+            if item.get('contentType') in ('url', 'audio'):
                 add_format_url(item_url)
                 continue
 
@@ -201,7 +213,7 @@ class NPOIE(NPOBaseIE):
                 stream_info = self._download_json(
                     item_url + '&type=json', video_id,
                     'Downloading %s stream JSON'
-                    % item.get('label') or format_id or num)
+                    % item.get('label') or item.get('format') or format_id or num)
             except ExtractorError as ee:
                 if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 404:
                     error = (self._parse_json(

From 40df485f554ec3fff81ca988b5bf961d54d8e41b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 1 Mar 2017 23:03:36 +0700
Subject: [PATCH 0409/1696] [YoutubeDL] Don't sanitize identifiers (closes
 #12317)

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index f7254560c..13a3a909e 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -616,7 +616,7 @@ class YoutubeDL(object):
             sanitize = lambda k, v: sanitize_filename(
                 compat_str(v),
                 restricted=self.params.get('restrictfilenames'),
-                is_id=(k == 'id'))
+                is_id=(k == 'id' or k.endswith('_id')))
             template_dict = dict((k, v if isinstance(v, compat_numeric_types) else sanitize(k, v))
                                  for k, v in template_dict.items()
                                  if v is not None and not isinstance(v, (list, tuple, dict)))

From 158af5242e983312c0c1e7590faa9844136e338f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 1 Mar 2017 23:04:02 +0700
Subject: [PATCH 0410/1696] [utils] Carry long doc string

---
 youtube_dl/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 807183f4a..8738aa249 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -473,7 +473,8 @@ def timeconvert(timestr):
 def sanitize_filename(s, restricted=False, is_id=False):
     """Sanitizes a string so it could be used as part of a filename.
     If restricted is set, use a stricter subset of allowed characters.
-    Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
+    Set is_id if this is not an arbitrary string, but an ID that should be kept
+    if possible.
     """
     def replace_insane(char):
         if restricted and char in ACCENT_CHARS:

From af5049f128655cfec8978f17b04e5d88bd91d37f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 2 Mar 2017 02:14:42 +0800
Subject: [PATCH 0411/1696] [adobepass] Add Charter Spectrum (#11465)

Thanks @tv21 for the fix!
---
 youtube_dl/extractor/adobepass.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
index 4d655bd5e..d4816abf5 100644
--- a/youtube_dl/extractor/adobepass.py
+++ b/youtube_dl/extractor/adobepass.py
@@ -36,6 +36,11 @@ MSO_INFO = {
         'username_field': 'Ecom_User_ID',
         'password_field': 'Ecom_Password',
     },
+    'Charter_Direct': {
+        'name': 'Charter Spectrum',
+        'username_field': 'IDToken1',
+        'password_field': 'IDToken2',
+    },
     'thr030': {
         'name': '3 Rivers Communications'
     },

From c9612c04872656125108980dc61d71ba6b3a4f89 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 2 Mar 2017 16:59:12 +0800
Subject: [PATCH 0412/1696] [youtube] Mark errors about rental videos as
 expected

Closes #12324
---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 81c793921..caa048249 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1454,7 +1454,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         # Check for "rental" videos
         if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
-            raise ExtractorError('"rental" videos not supported')
+            raise ExtractorError('"rental" videos not supported. See https://github.com/rg3/youtube-dl/issues/359 for more information.', expected=True)
 
         # Start extracting information
         self.report_information_extraction(video_id)

From 11bb6ad1a5ae3767535720e7dfebb823e8167088 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 2 Mar 2017 20:49:39 +0800
Subject: [PATCH 0413/1696] [facebook] Fix extraction (closes #12323)

Almost all videos now use the pagelet type 'permalink_video_pagelet'
---
 ChangeLog                        | 6 ++++++
 youtube_dl/extractor/facebook.py | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 2a1fd9018..448d7f521 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [facebook] Fix extraction (#12323)
+
+
 version 2017.02.28
 
 Core
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 70b8c95c5..6315d40c5 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -303,7 +303,7 @@ class FacebookIE(InfoExtractor):
         if not video_data:
             server_js_data = self._parse_json(
                 self._search_regex(
-                    r'bigPipe\.onPageletArrive\(({.+?})\)\s*;\s*}\s*\)\s*,\s*["\']onPageletArrive\s+(?:stream_pagelet|pagelet_group_mall)',
+                    r'bigPipe\.onPageletArrive\(({.+?})\)\s*;\s*}\s*\)\s*,\s*["\']onPageletArrive\s+(?:stream_pagelet|pagelet_group_mall|permalink_video_pagelet)',
                     webpage, 'js data', default='{}'),
                 video_id, transform_source=js_to_json, fatal=False)
             if server_js_data:

From 28d15b73f8704c6d00efa14948da29843fdb76e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 2 Mar 2017 22:29:56 +0700
Subject: [PATCH 0414/1696] [ChangeLog] Actualize

---
 ChangeLog | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 448d7f521..e3e0f4544 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,14 @@
 version <unreleased>
 
+Core
++ [adobepass] Add support for Charter Spectrum (#11465)
+* [YoutubeDL] Don't sanitize identifiers in output template (#12317)
+
 Extractors
-* [facebook] Fix extraction (#12323)
+* [facebook] Fix extraction (#12323, #12330)
+* [youtube] Mark errors about rental videos as expected (#12324)
++ [npo] Add support for audio
+* [npo] Adapt to app.php API (#12311, #12320)
 
 
 version 2017.02.28

From 250eea6821a5715e2ee7cade8539fcd42177603d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 2 Mar 2017 22:33:22 +0700
Subject: [PATCH 0415/1696] release 2017.03.02

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index c907ef931..fec4152e3 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.02.28*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.02.28**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.03.02*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.03.02**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.02.28
+[debug] youtube-dl version 2017.03.02
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index e3e0f4544..e53fb7767 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.03.02
 
 Core
 + [adobepass] Add support for Charter Spectrum (#11465)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 340e23bf8..f4c8d3d5f 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.02.28'
+__version__ = '2017.03.02'

From 4d345bf17b3040ebfedd079e656e1ca658749187 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 2 Mar 2017 23:53:46 +0700
Subject: [PATCH 0416/1696] [ruutu] Disable DASH formats (closes #12322) Due to
 causing out of sync issue

---
 youtube_dl/extractor/ruutu.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
index 20d01754a..6c09df25a 100644
--- a/youtube_dl/extractor/ruutu.py
+++ b/youtube_dl/extractor/ruutu.py
@@ -82,6 +82,9 @@ class RuutuIE(InfoExtractor):
                         formats.extend(self._extract_f4m_formats(
                             video_url, video_id, f4m_id='hds', fatal=False))
                     elif ext == 'mpd':
+                        # video-only and audio-only streams are of different
+                        # duration resulting in out of sync issue
+                        continue
                         formats.extend(self._extract_mpd_formats(
                             video_url, video_id, mpd_id='dash', fatal=False))
                     else:

From 9bae185ba6be0b8ffe1d2168937c9d3274c5b60f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 3 Mar 2017 22:16:00 +0700
Subject: [PATCH 0417/1696] [24video] Use original host for requests (closes
 #12339)

---
 youtube_dl/extractor/twentyfourvideo.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/twentyfourvideo.py b/youtube_dl/extractor/twentyfourvideo.py
index f3541b654..7af11659f 100644
--- a/youtube_dl/extractor/twentyfourvideo.py
+++ b/youtube_dl/extractor/twentyfourvideo.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     parse_iso8601,
@@ -12,7 +14,7 @@ from ..utils import (
 
 class TwentyFourVideoIE(InfoExtractor):
     IE_NAME = '24video'
-    _VALID_URL = r'https?://(?:www\.)?24video\.(?:net|me|xxx|sex|tube)/(?:video/(?:view|xml)/|player/new24_play\.swf\?id=)(?P<id>\d+)'
+    _VALID_URL = r'https?://(?P<host>(?:www\.)?24video\.(?:net|me|xxx|sex|tube))/(?:video/(?:view|xml)/|player/new24_play\.swf\?id=)(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.24video.net/video/view/1044982',
@@ -43,10 +45,12 @@ class TwentyFourVideoIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        host = mobj.group('host')
 
         webpage = self._download_webpage(
-            'http://www.24video.sex/video/view/%s' % video_id, video_id)
+            'http://%s/video/view/%s' % (host, video_id), video_id)
 
         title = self._og_search_title(webpage)
         description = self._html_search_regex(
@@ -72,11 +76,11 @@ class TwentyFourVideoIE(InfoExtractor):
 
         # Sets some cookies
         self._download_xml(
-            r'http://www.24video.sex/video/xml/%s?mode=init' % video_id,
+            r'http://%s/video/xml/%s?mode=init' % (host, video_id),
             video_id, 'Downloading init XML')
 
         video_xml = self._download_xml(
-            'http://www.24video.sex/video/xml/%s?mode=play' % video_id,
+            'http://%s/video/xml/%s?mode=play' % (host, video_id),
             video_id, 'Downloading video XML')
 
         video = xpath_element(video_xml, './/video', 'video', fatal=True)

From 692fa200cae38a7e37f646118a268ad408c8ab95 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 3 Mar 2017 22:28:34 +0700
Subject: [PATCH 0418/1696] [go] Relax _VALID_URL (closes #12341)

---
 youtube_dl/extractor/go.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/go.py b/youtube_dl/extractor/go.py
index 21ed846b2..4c9be47b4 100644
--- a/youtube_dl/extractor/go.py
+++ b/youtube_dl/extractor/go.py
@@ -36,7 +36,7 @@ class GoIE(AdobePassIE):
             'requestor_id': 'DisneyXD',
         }
     }
-    _VALID_URL = r'https?://(?:(?P<sub_domain>%s)\.)?go\.com/(?:[^/]+/)*(?:vdka(?P<id>\w+)|season-\d+/\d+-(?P<display_id>[^/?#]+))' % '|'.join(_SITE_INFO.keys())
+    _VALID_URL = r'https?://(?:(?P<sub_domain>%s)\.)?go\.com/(?:[^/]+/)*(?:vdka(?P<id>\w+)|(?:[^/]+/)*(?P<display_id>[^/?#]+))' % '|'.join(_SITE_INFO.keys())
     _TESTS = [{
         'url': 'http://abc.go.com/shows/castle/video/most-recent/vdka0_g86w5onx',
         'info_dict': {
@@ -52,6 +52,12 @@ class GoIE(AdobePassIE):
     }, {
         'url': 'http://abc.go.com/shows/after-paradise/video/most-recent/vdka3335601',
         'only_matching': True,
+    }, {
+        'url': 'http://abc.go.com/shows/the-catch/episode-guide/season-01/10-the-wedding',
+        'only_matching': True,
+    }, {
+        'url': 'http://abc.go.com/shows/world-news-tonight/episode-guide/2017-02/17-021717-intense-stand-off-between-man-with-rifle-and-police-in-oakland',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From d02d4fa0a90f3182d65504508105e8d86886c6ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 3 Mar 2017 22:49:48 +0700
Subject: [PATCH 0419/1696] [brightcove:new] Raise GeoRestrictedError

---
 youtube_dl/extractor/brightcove.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 27685eed0..f8605be82 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -544,8 +544,10 @@ class BrightcoveNewIE(InfoExtractor):
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
                 json_data = self._parse_json(e.cause.read().decode(), video_id)[0]
-                raise ExtractorError(
-                    json_data.get('message') or json_data['error_code'], expected=True)
+                message = json_data.get('message') or json_data['error_code']
+                if json_data.get('error_subcode') == 'CLIENT_GEO':
+                    self.raise_geo_restricted(msg=message)
+                raise ExtractorError(message, expected=True)
             raise
 
         title = json_data['name'].strip()

From cbb127568a6182df2c5a2d65426de523f1f7b43f Mon Sep 17 00:00:00 2001
From: Olivier Bilodeau <olivier@bottomlesspit.org>
Date: Thu, 15 Dec 2016 20:14:04 -0500
Subject: [PATCH 0420/1696] [vrak] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/vrak.py       | 68 ++++++++++++++++++++++++++++++
 2 files changed, 69 insertions(+)
 create mode 100644 youtube_dl/extractor/vrak.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index b1613a9d3..0ac42138a 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1165,6 +1165,7 @@ from .voicerepublic import VoiceRepublicIE
 from .voxmedia import VoxMediaIE
 from .vporn import VpornIE
 from .vrt import VRTIE
+from .vrak import VrakIE
 from .vube import VubeIE
 from .vuclip import VuClipIE
 from .vvvvid import VVVVIDIE
diff --git a/youtube_dl/extractor/vrak.py b/youtube_dl/extractor/vrak.py
new file mode 100644
index 000000000..692e2fcfc
--- /dev/null
+++ b/youtube_dl/extractor/vrak.py
@@ -0,0 +1,68 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+from .brightcove import BrightcoveNewIE
+
+
+class VrakIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?vrak\.tv/videos\?.*?target=(?P<id>[0-9\.]+).*'
+    _TEST = {
+        'url': 'http://www.vrak.tv/videos?target=1.2240923&filtre=emission&id=1.1806721',
+        'md5': 'c5d5ce237bca3b1e990ce1b48d1f0948',
+        'info_dict': {
+            'id': '5231040869001',
+            'ext': 'mp4',
+            'title': 'Référendums américains, animés japonais et hooligans russes',
+            'upload_date': '20161201',
+            'description': 'This video file has been uploaded automatically using Oprah. It should be updated with real description soon.',
+            'timestamp': 1480628425,
+            'uploader_id': '2890187628001',
+        }
+    }
+
+    def _real_extract(self, url):
+        url_id = self._match_id(url)
+        webpage = self._download_webpage(url, url_id)
+
+        result = {}
+        result['title'] = self._html_search_regex(
+            r'<h3 class="videoTitle">(.+?)</h3>', webpage, 'title')
+
+        # Inspired from BrightcoveNewIE._extract_url()
+        entries = []
+        for account_id, player_id, _, video_id in re.findall(
+                # account_id, player_id and embed from:
+                #   <div class="video-player [...]
+                #     data-publisher-id="2890187628001"
+                #     data-player-id="VkSnGw3cx"
+                # video id is extracted from weird CMS Java/Javascript notation:
+                #   RW java.lang.String value = '5231040869001';
+                # Need to use backtrack to pin to a ref since video is in grid
+                # layout with others
+                r'''(?sx)
+                    <div[^>]+
+                        data-publisher-id=["\'](\d+)["\']
+                        [^>]*
+                        data-player-id=["\']([^"\']+)["\']
+                        [^>]*
+                        refId&quot;:&quot;([^&]+)&quot;
+                        [^>]*
+                        >.*?
+                    </div>.*?
+                    RW\ java\.lang\.String\ value\ =\ \'brightcove\.article\.\d+\.\3\'
+                    [^>]*
+                    RW\ java\.lang\.String\ value\ =\ \'(\d+)\'
+                ''', webpage):
+
+            entries.append(
+                'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
+                % (account_id, player_id, 'default', video_id))
+
+        if entries:
+            result = self.url_result(entries[0], BrightcoveNewIE.ie_key())
+
+        return result

From d16f27ca272cb10f4de87814665402b9737175ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 3 Mar 2017 23:57:01 +0700
Subject: [PATCH 0421/1696] [brightcove:new] Add ability to smuggle
 geo_countries into URL

---
 youtube_dl/extractor/brightcove.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index f8605be82..66c8cb219 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -515,6 +515,9 @@ class BrightcoveNewIE(InfoExtractor):
         return entries
 
     def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
+        self._initialize_geo_bypass(smuggled_data.get('geo_countries'))
+
         account_id, player_id, embed, video_id = re.match(self._VALID_URL, url).groups()
 
         webpage = self._download_webpage(

From 4d058c9862ebcb1fb7ebd988cf053fde200913cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 3 Mar 2017 23:58:16 +0700
Subject: [PATCH 0422/1696] [vrak] Improve and update test (closes #11452)

---
 youtube_dl/extractor/vrak.py | 104 +++++++++++++++++++----------------
 1 file changed, 58 insertions(+), 46 deletions(-)

diff --git a/youtube_dl/extractor/vrak.py b/youtube_dl/extractor/vrak.py
index 692e2fcfc..daa247cce 100644
--- a/youtube_dl/extractor/vrak.py
+++ b/youtube_dl/extractor/vrak.py
@@ -4,65 +4,77 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-
 from .brightcove import BrightcoveNewIE
+from ..utils import (
+    int_or_none,
+    parse_age_limit,
+    smuggle_url,
+    unescapeHTML,
+)
 
 
 class VrakIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?vrak\.tv/videos\?.*?target=(?P<id>[0-9\.]+).*'
+    _VALID_URL = r'https?://(?:www\.)?vrak\.tv/videos\?.*?\btarget=(?P<id>[\d.]+)'
     _TEST = {
-        'url': 'http://www.vrak.tv/videos?target=1.2240923&filtre=emission&id=1.1806721',
-        'md5': 'c5d5ce237bca3b1e990ce1b48d1f0948',
+        'url': 'http://www.vrak.tv/videos?target=1.2306782&filtre=emission&id=1.1806721',
         'info_dict': {
-            'id': '5231040869001',
+            'id': '5345661243001',
             'ext': 'mp4',
-            'title': 'Référendums américains, animés japonais et hooligans russes',
-            'upload_date': '20161201',
-            'description': 'This video file has been uploaded automatically using Oprah. It should be updated with real description soon.',
-            'timestamp': 1480628425,
+            'title': 'Obésité, film de hockey et Roseline Filion',
+            'timestamp': 1488492126,
+            'upload_date': '20170302',
             'uploader_id': '2890187628001',
-        }
+            'creator': 'VRAK.TV',
+            'age_limit': 8,
+            'series': 'ALT (Actualité Légèrement Tordue)',
+            'episode': 'Obésité, film de hockey et Roseline Filion',
+            'tags': list,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/2890187628001/default_default/index.html?videoId=%s'
 
     def _real_extract(self, url):
-        url_id = self._match_id(url)
-        webpage = self._download_webpage(url, url_id)
+        video_id = self._match_id(url)
 
-        result = {}
-        result['title'] = self._html_search_regex(
-            r'<h3 class="videoTitle">(.+?)</h3>', webpage, 'title')
+        webpage = self._download_webpage(url, video_id)
 
-        # Inspired from BrightcoveNewIE._extract_url()
-        entries = []
-        for account_id, player_id, _, video_id in re.findall(
-                # account_id, player_id and embed from:
-                #   <div class="video-player [...]
-                #     data-publisher-id="2890187628001"
-                #     data-player-id="VkSnGw3cx"
-                # video id is extracted from weird CMS Java/Javascript notation:
-                #   RW java.lang.String value = '5231040869001';
-                # Need to use backtrack to pin to a ref since video is in grid
-                # layout with others
-                r'''(?sx)
-                    <div[^>]+
-                        data-publisher-id=["\'](\d+)["\']
-                        [^>]*
-                        data-player-id=["\']([^"\']+)["\']
-                        [^>]*
-                        refId&quot;:&quot;([^&]+)&quot;
-                        [^>]*
-                        >.*?
-                    </div>.*?
-                    RW\ java\.lang\.String\ value\ =\ \'brightcove\.article\.\d+\.\3\'
-                    [^>]*
-                    RW\ java\.lang\.String\ value\ =\ \'(\d+)\'
-                ''', webpage):
+        title = self._html_search_regex(
+            r'<h\d\b[^>]+\bclass=["\']videoTitle["\'][^>]*>([^<]+)',
+            webpage, 'title', default=None) or self._og_search_title(webpage)
 
-            entries.append(
-                'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
-                % (account_id, player_id, 'default', video_id))
+        content = self._parse_json(
+            self._search_regex(
+                r'data-player-options-content=(["\'])(?P<content>{.+?})\1',
+                webpage, 'content', default='{}', group='content'),
+            video_id, transform_source=unescapeHTML)
 
-        if entries:
-            result = self.url_result(entries[0], BrightcoveNewIE.ie_key())
+        ref_id = content.get('refId') or self._search_regex(
+            r'refId&quot;:&quot;([^&]+)&quot;', webpage, 'ref id')
 
-        return result
+        brightcove_id = self._search_regex(
+            r'''(?x)
+                java\.lang\.String\s+value\s*=\s*["']brightcove\.article\.\d+\.%s
+                [^>]*
+                java\.lang\.String\s+value\s*=\s*["'](\d+)
+            ''' % re.escape(ref_id), webpage, 'brightcove id')
+
+        return {
+            '_type': 'url_transparent',
+            'ie_key': BrightcoveNewIE.ie_key(),
+            'url': smuggle_url(
+                self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id,
+                {'geo_countries': ['CA']}),
+            'id': brightcove_id,
+            'description': content.get('description'),
+            'creator': content.get('brand'),
+            'age_limit': parse_age_limit(content.get('rating')),
+            'series': content.get('showName') or content.get(
+                'episodeName'),  # this is intentional
+            'season_number': int_or_none(content.get('seasonNumber')),
+            'episode': title,
+            'episode_number': int_or_none(content.get('episodeNumber')),
+            'tags': content.get('tags', []),
+        }

From bc82f22879e222a1ade35fd8ebd7bb535f9166dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Mar 2017 00:42:51 +0700
Subject: [PATCH 0423/1696] [rutube] Relax _VALID_URL

---
 youtube_dl/extractor/rutube.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index fd1df925b..663b75583 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -17,7 +17,7 @@ from ..utils import (
 class RutubeIE(InfoExtractor):
     IE_NAME = 'rutube'
     IE_DESC = 'Rutube videos'
-    _VALID_URL = r'https?://rutube\.ru/(?:video|play/embed)/(?P<id>[\da-z]{32})'
+    _VALID_URL = r'https?://rutube\.ru/(?:video|(?:play/)?embed)/(?P<id>[\da-z]{32})'
 
     _TESTS = [{
         'url': 'http://rutube.ru/video/3eac3b4561676c17df9132a9a1e62e3e/',
@@ -39,6 +39,9 @@ class RutubeIE(InfoExtractor):
     }, {
         'url': 'http://rutube.ru/play/embed/a10e53b86e8f349080f718582ce4c661',
         'only_matching': True,
+    }, {
+        'url': 'http://rutube.ru/embed/a10e53b86e8f349080f718582ce4c661',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From eb3079b6ce54b63b4cc609198382b6db2cbb6f5f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Mar 2017 00:46:33 +0700
Subject: [PATCH 0424/1696] [generic] Add support for rutube embeds

---
 youtube_dl/extractor/generic.py | 24 ++++++++++++++++++++++++
 youtube_dl/extractor/rutube.py  |  6 ++++++
 2 files changed, 30 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 9868ca6d0..ebab9509d 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -84,6 +84,7 @@ from .twentymin import TwentyMinutenIE
 from .ustream import UstreamIE
 from .openload import OpenloadIE
 from .videopress import VideoPressIE
+from .rutube import RutubeIE
 
 
 class GenericIE(InfoExtractor):
@@ -1502,6 +1503,23 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': [VideoPressIE.ie_key()],
         },
+        {
+            # Rutube embed
+            'url': 'http://magazzino.friday.ru/videos/vipuski/kazan-2',
+            'info_dict': {
+                'id': '9b3d5bee0a8740bf70dfd29d3ea43541',
+                'ext': 'flv',
+                'title': 'Магаззино: Казань 2',
+                'description': 'md5:99bccdfac2269f0e8fdbc4bbc9db184a',
+                'uploader': 'Магаззино',
+                'upload_date': '20170228',
+                'uploader_id': '996642',
+            },
+            'params': {
+                'skip_download': True,
+            },
+            'add_ie': [RutubeIE.ie_key()],
+        },
         {
             # ThePlatform embedded with whitespaces in URLs
             'url': 'http://www.golfchannel.com/topics/shows/golftalkcentral.htm',
@@ -2480,6 +2498,12 @@ class GenericIE(InfoExtractor):
             return _playlist_from_matches(
                 videopress_urls, ie=VideoPressIE.ie_key())
 
+        # Look for Rutube embeds
+        rutube_urls = RutubeIE._extract_urls(webpage)
+        if rutube_urls:
+            return _playlist_from_matches(
+                rutube_urls, ie=RutubeIE.ie_key())
+
         # Looking for http://schema.org/VideoObject
         json_ld = self._search_json_ld(
             webpage, video_id, default={}, expected_type='VideoObject')
diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index 663b75583..889fa7628 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -44,6 +44,12 @@ class RutubeIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_urls(webpage):
+        return [mobj.group('url') for mobj in re.finditer(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//rutube\.ru/embed/[\da-z]{32}.*?)\1',
+            webpage)]
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         video = self._download_json(

From 6f4e4132d8ef835635059d08206ca9bc6fd5dd98 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 4 Mar 2017 23:23:18 +0800
Subject: [PATCH 0425/1696] [douyutv] Switch to the PC API to escape the 5-min
 limitation

Thanks @spacemeowx2 for the algo.

Ref: https://gist.github.com/spacemeowx2/629b1d131bd7e240a7d28742048e80fc

Closes #12316
---
 ChangeLog                       |  6 ++++++
 youtube_dl/extractor/douyutv.py | 31 +++++++++++++++++++++----------
 2 files changed, 27 insertions(+), 10 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index e53fb7767..13ccb0f8f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [douyutv] Switch to the PC API to escape the 5-min limitation (#12316)
+
+
 version 2017.03.02
 
 Core
diff --git a/youtube_dl/extractor/douyutv.py b/youtube_dl/extractor/douyutv.py
index 9a83fb31a..82d8a042f 100644
--- a/youtube_dl/extractor/douyutv.py
+++ b/youtube_dl/extractor/douyutv.py
@@ -1,6 +1,9 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import time
+import hashlib
+
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
@@ -16,7 +19,7 @@ class DouyuTVIE(InfoExtractor):
         'info_dict': {
             'id': '17732',
             'display_id': 'iseven',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 're:^清晨醒脑！T-ARA根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
             'description': r're:.*m7show@163\.com.*',
             'thumbnail': r're:^https?://.*\.jpg$',
@@ -31,7 +34,7 @@ class DouyuTVIE(InfoExtractor):
         'info_dict': {
             'id': '85982',
             'display_id': '85982',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 're:^小漠从零单排记！——CSOL2躲猫猫 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
             'description': 'md5:746a2f7a253966a06755a912f0acc0d2',
             'thumbnail': r're:^https?://.*\.jpg$',
@@ -47,7 +50,7 @@ class DouyuTVIE(InfoExtractor):
         'info_dict': {
             'id': '17732',
             'display_id': '17732',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 're:^清晨醒脑！T-ARA根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
             'description': r're:.*m7show@163\.com.*',
             'thumbnail': r're:^https?://.*\.jpg$',
@@ -66,10 +69,6 @@ class DouyuTVIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    # Decompile core.swf in webpage by ffdec "Search SWFs in memory". core.swf
-    # is encrypted originally, but ffdec can dump memory to get the decrypted one.
-    _API_KEY = 'A12Svb&%1UUmf@hC'
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
@@ -80,6 +79,7 @@ class DouyuTVIE(InfoExtractor):
             room_id = self._html_search_regex(
                 r'"room_id\\?"\s*:\s*(\d+),', page, 'room id')
 
+        # Grab metadata from mobile API
         room = self._download_json(
             'http://m.douyu.com/html5/live?roomId=%s' % room_id, video_id,
             note='Downloading room info')['data']
@@ -88,8 +88,19 @@ class DouyuTVIE(InfoExtractor):
         if room.get('show_status') == '2':
             raise ExtractorError('Live stream is offline', expected=True)
 
-        formats = self._extract_m3u8_formats(
-            room['hls_url'], video_id, ext='mp4')
+        # Grab the URL from PC client API
+        # The m3u8 url from mobile API requires re-authentication every 5 minutes
+        tt = int(time.time())
+        signContent = 'lapi/live/thirdPart/getPlay/%s?aid=pcclient&rate=0&time=%d9TUk5fjjUjg9qIMH3sdnh' % (room_id, tt)
+        sign = hashlib.md5(signContent.encode('ascii')).hexdigest()
+        video_url = self._download_json(
+            'http://coapi.douyucdn.cn/lapi/live/thirdPart/getPlay/' + room_id,
+            video_id, note='Downloading video URL info',
+            query={'rate': 0}, headers={
+                'auth': sign,
+                'time': str(tt),
+                'aid': 'pcclient'
+            })['data']['live_url']
 
         title = self._live_title(unescapeHTML(room['room_name']))
         description = room.get('show_details')
@@ -99,7 +110,7 @@ class DouyuTVIE(InfoExtractor):
         return {
             'id': room_id,
             'display_id': video_id,
-            'formats': formats,
+            'url': video_url,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,

From 64b7ccef3e3144a50f2cc01772a5ea5e81d4494d Mon Sep 17 00:00:00 2001
From: Juanjo Benages <juanjo@benages.eu>
Date: Thu, 2 Feb 2017 17:33:09 +0100
Subject: [PATCH 0426/1696] [redbulltv] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/redbull.py    | 50 ++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+)
 create mode 100644 youtube_dl/extractor/redbull.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 0ac42138a..e42a069b5 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -793,6 +793,7 @@ from .rai import (
 )
 from .rbmaradio import RBMARadioIE
 from .rds import RDSIE
+from .redbull import RedBullIE
 from .redtube import RedTubeIE
 from .regiotv import RegioTVIE
 from .rentv import (
diff --git a/youtube_dl/extractor/redbull.py b/youtube_dl/extractor/redbull.py
new file mode 100644
index 000000000..e3d978a53
--- /dev/null
+++ b/youtube_dl/extractor/redbull.py
@@ -0,0 +1,50 @@
+# coding: utf-8
+from __future__ import unicode_literals
+import re
+
+from .common import InfoExtractor
+
+
+class RedBullIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?redbull\.tv/video/(?P<id>AP-\w+)'
+    _TEST = {
+        'url': 'https://www.redbull.tv/video/AP-1Q756YYX51W11/abc-of-wrc',
+        'md5': '78e860f631d7a846e712fab8c5fe2c38',
+        'info_dict': {
+            'id': 'AP-1Q756YYX51W11',
+            'ext': 'mp4',
+            'title': 'ABC of...WRC',
+            'description': 'Buckle up for a crash course in the terminology, rules, drivers, and courses of the World Rally Championship.'
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+
+        access_token = self._download_json(
+            'http://api-v2.redbull.tv/start?build=4.0.9&category=smartphone&os_version=23&os_family=android',
+            video_id, note='Downloading  access token',
+        )['auth']['access_token']
+
+        info = self._download_json(
+            'https://api-v2.redbull.tv/views/%s' % video_id,
+            video_id, note='Downloading video information',
+            headers={'Authorization': 'Bearer ' + access_token}
+        )['blocks'][0]['top'][0]
+
+        m3u8_url = info['video_product']['url']
+        title = info['title']
+
+        formats = self._extract_m3u8_formats(
+            m3u8_url, video_id, 'mp4', 'm3u8_native',
+            m3u8_id='hls')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'description': info.get('short_description'),
+            'genre': info.get('genre'),
+            'duration': info.get('duration')
+        }

From 054a587de8fe2860bcb93a19f8c628b4ddd9ad56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Mar 2017 23:25:09 +0700
Subject: [PATCH 0427/1696] [redbulltv] Improve extraction (closes #11948,
 closes #3919)

---
 youtube_dl/extractor/extractors.py |   2 +-
 youtube_dl/extractor/redbull.py    |  50 --------------
 youtube_dl/extractor/redbulltv.py  | 106 +++++++++++++++++++++++++++++
 3 files changed, 107 insertions(+), 51 deletions(-)
 delete mode 100644 youtube_dl/extractor/redbull.py
 create mode 100644 youtube_dl/extractor/redbulltv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index e42a069b5..caf1dc766 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -793,7 +793,7 @@ from .rai import (
 )
 from .rbmaradio import RBMARadioIE
 from .rds import RDSIE
-from .redbull import RedBullIE
+from .redbulltv import RedBullTVIE
 from .redtube import RedTubeIE
 from .regiotv import RegioTVIE
 from .rentv import (
diff --git a/youtube_dl/extractor/redbull.py b/youtube_dl/extractor/redbull.py
deleted file mode 100644
index e3d978a53..000000000
--- a/youtube_dl/extractor/redbull.py
+++ /dev/null
@@ -1,50 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-import re
-
-from .common import InfoExtractor
-
-
-class RedBullIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?redbull\.tv/video/(?P<id>AP-\w+)'
-    _TEST = {
-        'url': 'https://www.redbull.tv/video/AP-1Q756YYX51W11/abc-of-wrc',
-        'md5': '78e860f631d7a846e712fab8c5fe2c38',
-        'info_dict': {
-            'id': 'AP-1Q756YYX51W11',
-            'ext': 'mp4',
-            'title': 'ABC of...WRC',
-            'description': 'Buckle up for a crash course in the terminology, rules, drivers, and courses of the World Rally Championship.'
-        }
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
-        access_token = self._download_json(
-            'http://api-v2.redbull.tv/start?build=4.0.9&category=smartphone&os_version=23&os_family=android',
-            video_id, note='Downloading  access token',
-        )['auth']['access_token']
-
-        info = self._download_json(
-            'https://api-v2.redbull.tv/views/%s' % video_id,
-            video_id, note='Downloading video information',
-            headers={'Authorization': 'Bearer ' + access_token}
-        )['blocks'][0]['top'][0]
-
-        m3u8_url = info['video_product']['url']
-        title = info['title']
-
-        formats = self._extract_m3u8_formats(
-            m3u8_url, video_id, 'mp4', 'm3u8_native',
-            m3u8_id='hls')
-
-        return {
-            'id': video_id,
-            'title': title,
-            'formats': formats,
-            'description': info.get('short_description'),
-            'genre': info.get('genre'),
-            'duration': info.get('duration')
-        }
diff --git a/youtube_dl/extractor/redbulltv.py b/youtube_dl/extractor/redbulltv.py
new file mode 100644
index 000000000..5c73d5bca
--- /dev/null
+++ b/youtube_dl/extractor/redbulltv.py
@@ -0,0 +1,106 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    float_or_none,
+    int_or_none,
+    try_get,
+    unified_timestamp,
+)
+
+
+class RedBullTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?redbull\.tv/(?:video|film)/(?P<id>AP-\w+)'
+    _TESTS = [{
+        # film
+        'url': 'https://www.redbull.tv/video/AP-1Q756YYX51W11/abc-of-wrc',
+        'md5': '78e860f631d7a846e712fab8c5fe2c38',
+        'info_dict': {
+            'id': 'AP-1Q756YYX51W11',
+            'ext': 'mp4',
+            'title': 'ABC of...WRC',
+            'description': 'md5:5c7ed8f4015c8492ecf64b6ab31e7d31',
+            'duration': 1582.04,
+            'timestamp': 1488405786,
+            'upload_date': '20170301',
+        },
+    }, {
+        # episode
+        'url': 'https://www.redbull.tv/video/AP-1PMT5JCWH1W11/grime?playlist=shows:shows-playall:web',
+        'info_dict': {
+            'id': 'AP-1PMT5JCWH1W11',
+            'ext': 'mp4',
+            'title': 'Grime - Hashtags S2 E4',
+            'description': 'md5:334b741c8c1ce65be057eab6773c1cf5',
+            'duration': 904.6,
+            'timestamp': 1487290093,
+            'upload_date': '20170217',
+            'series': 'Hashtags',
+            'season_number': 2,
+            'episode_number': 4,
+        },
+    }, {
+        'url': 'https://www.redbull.tv/film/AP-1MSKKF5T92111/in-motion',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        access_token = self._download_json(
+            'https://api-v2.redbull.tv/start', video_id,
+            note='Downloading access token', query={
+                'build': '4.0.9',
+                'category': 'smartphone',
+                'os_version': 23,
+                'os_family': 'android',
+            })['auth']['access_token']
+
+        info = self._download_json(
+            'https://api-v2.redbull.tv/views/%s' % video_id,
+            video_id, note='Downloading video information',
+            headers={'Authorization': 'Bearer ' + access_token}
+        )['blocks'][0]['top'][0]
+
+        video = info['video_product']
+
+        title = info['title'].strip()
+        m3u8_url = video['url']
+
+        formats = self._extract_m3u8_formats(
+            m3u8_url, video_id, 'mp4', entry_protocol='m3u8_native',
+            m3u8_id='hls')
+
+        subtitles = {}
+        for _, captions in (try_get(
+                video, lambda x: x['attachments']['captions'],
+                dict) or {}).items():
+            if not captions or not isinstance(captions, list):
+                continue
+            for caption in captions:
+                caption_url = caption.get('url')
+                if not caption_url:
+                    continue
+                subtitles.setdefault(caption.get('lang') or 'en', []).append({
+                    'url': caption_url,
+                    'ext': caption.get('format'),
+                })
+
+        subheading = info.get('subheading')
+        if subheading:
+            title += ' - %s' % subheading
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': info.get('long_description') or info.get(
+                'short_description'),
+            'duration': float_or_none(video.get('duration'), scale=1000),
+            'timestamp': unified_timestamp(info.get('published')),
+            'series': info.get('show_title'),
+            'season_number': int_or_none(info.get('season_number')),
+            'episode_number': int_or_none(info.get('episode_number')),
+            'formats': formats,
+            'subtitles': subtitles,
+        }

From a3ba8a7acfa2db3a8c90000d377c25d14bdad290 Mon Sep 17 00:00:00 2001
From: Lars Vierbergen <vierbergenlars@gmail.com>
Date: Sat, 4 Mar 2017 17:47:19 +0100
Subject: [PATCH 0428/1696] [vier] Add support for vijf.be

vier.be and vijf.be run on the same CMS and are property of the same company,
so the same extractor can be used for both of them.
---
 youtube_dl/extractor/vier.py | 33 +++++++++++++++++++++++++++------
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py
index d26fb49b3..5086f591e 100644
--- a/youtube_dl/extractor/vier.py
+++ b/youtube_dl/extractor/vier.py
@@ -9,7 +9,7 @@ from .common import InfoExtractor
 
 class VierIE(InfoExtractor):
     IE_NAME = 'vier'
-    _VALID_URL = r'https?://(?:www\.)?vier\.be/(?:[^/]+/videos/(?P<display_id>[^/]+)(?:/(?P<id>\d+))?|video/v3/embed/(?P<embed_id>\d+))'
+    _VALID_URL = r'https?://(?:www\.)?(?P<site>vier|vijf)\.be/(?:[^/]+/videos/(?P<display_id>[^/]+)(?:/(?P<id>\d+))?|video/v3/embed/(?P<embed_id>\d+))'
     _TESTS = [{
         'url': 'http://www.vier.be/planb/videos/het-wordt-warm-de-moestuin/16129',
         'info_dict': {
@@ -23,6 +23,19 @@ class VierIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.vijf.be/temptationisland/videos/zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas/2561614',
+        'info_dict': {
+            'id': '2561614',
+            'display_id': 'zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas',
+            'ext': 'mp4',
+            'title': 'ZO grappig: Temptation Island hosts moeten kiezen tussen onmogelijke dilemma\'s',
+            'description': 'Het spel is simpel: Annelien Coorevits en Rick Brandsteder krijgen telkens 2 dilemma\'s voorgeschoteld en ze MOETEN een keuze maken.',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
         'url': 'http://www.vier.be/planb/videos/mieren-herders-van-de-bladluizen',
         'only_matching': True,
@@ -35,6 +48,7 @@ class VierIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         embed_id = mobj.group('embed_id')
         display_id = mobj.group('display_id') or embed_id
+        site = mobj.group('site')
 
         webpage = self._download_webpage(url, display_id)
 
@@ -43,7 +57,7 @@ class VierIE(InfoExtractor):
             webpage, 'video id')
         application = self._search_regex(
             [r'data-application="([^"]+)"', r'"application"\s*:\s*"([^"]+)"'],
-            webpage, 'application', default='vier_vod')
+            webpage, 'application', default=site + '_vod')
         filename = self._search_regex(
             [r'data-filename="([^"]+)"', r'"filename"\s*:\s*"([^"]+)"'],
             webpage, 'filename')
@@ -68,13 +82,19 @@ class VierIE(InfoExtractor):
 
 class VierVideosIE(InfoExtractor):
     IE_NAME = 'vier:videos'
-    _VALID_URL = r'https?://(?:www\.)?vier\.be/(?P<program>[^/]+)/videos(?:\?.*\bpage=(?P<page>\d+)|$)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<site>vier|vijf)\.be/(?P<program>[^/]+)/videos(?:\?.*\bpage=(?P<page>\d+)|$)'
     _TESTS = [{
         'url': 'http://www.vier.be/demoestuin/videos',
         'info_dict': {
             'id': 'demoestuin',
         },
         'playlist_mincount': 153,
+    }, {
+        'url': 'http://www.vijf.be/temptationisland/videos',
+        'info_dict': {
+            'id': 'temptationisland',
+        },
+        'playlist_mincount': 159,
     }, {
         'url': 'http://www.vier.be/demoestuin/videos?page=6',
         'info_dict': {
@@ -92,6 +112,7 @@ class VierVideosIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         program = mobj.group('program')
+        site = mobj.group('site')
 
         page_id = mobj.group('page')
         if page_id:
@@ -105,13 +126,13 @@ class VierVideosIE(InfoExtractor):
         entries = []
         for current_page_id in itertools.count(start_page):
             current_page = self._download_webpage(
-                'http://www.vier.be/%s/videos?page=%d' % (program, current_page_id),
+                'http://www.%s.be/%s/videos?page=%d' % (site, program, current_page_id),
                 program,
                 'Downloading page %d' % (current_page_id + 1))
             page_entries = [
-                self.url_result('http://www.vier.be' + video_url, 'Vier')
+                self.url_result('http://www.' + site + '.be' + video_url, 'Vier')
                 for video_url in re.findall(
-                    r'<h3><a href="(/[^/]+/videos/[^/]+(?:/\d+)?)">', current_page)]
+                    r'<h[23]><a href="(/[^/]+/videos/[^/]+(?:/\d+)?)">', current_page)]
             entries.extend(page_entries)
             if page_id or '>Meer<' not in current_page:
                 break

From 30f8f142d4563df9cf47b0164adbb2c9e0130c5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Mar 2017 23:57:03 +0700
Subject: [PATCH 0429/1696] Credit @ThomasChr for #12015 and #12245

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 247c0ea13..3d8db1ee2 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -202,3 +202,4 @@ Fabian Stahl
 Bagira
 Odd Stråbø
 Philip Herzog
+Thomas Christlieb

From 466274fe9a8fae0a5f5e0358f48e54f569c10c5b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Mar 2017 23:58:12 +0700
Subject: [PATCH 0430/1696] Credit @p2004a for vodpl (#12122)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 3d8db1ee2..2a0938f3b 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -203,3 +203,4 @@ Bagira
 Odd Stråbø
 Philip Herzog
 Thomas Christlieb
+Marek Rusinowski

From f24c1e558456021d9a8704c9964c6a704e8b73ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Mar 2017 23:59:49 +0700
Subject: [PATCH 0431/1696] Credit @TobiX for #9725

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 2a0938f3b..0716d2ad6 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -204,3 +204,4 @@ Odd Stråbø
 Philip Herzog
 Thomas Christlieb
 Marek Rusinowski
+Tobias Gruetzmacher

From 6f211dc936dcd1f8ab6f178f8696f5edf2f385a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Mar 2017 00:01:17 +0700
Subject: [PATCH 0432/1696] Credit @obilodeau for vrak (#11452)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 0716d2ad6..4f383e214 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -205,3 +205,4 @@ Philip Herzog
 Thomas Christlieb
 Marek Rusinowski
 Tobias Gruetzmacher
+Olivier Bilodeau

From bcefc59279d9d4d7034e0f25a3fcf561a456766d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Mar 2017 00:02:30 +0700
Subject: [PATCH 0433/1696] Credit @vierbergenlars for vijf.be (#12304)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 4f383e214..fd1b8c9ea 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -206,3 +206,4 @@ Thomas Christlieb
 Marek Rusinowski
 Tobias Gruetzmacher
 Olivier Bilodeau
+Lars Vierbergen

From c64c03be35adae05740058e449c205120f89910d Mon Sep 17 00:00:00 2001
From: Xiao Di Guan <xdg@puxlit.net>
Date: Thu, 16 Feb 2017 17:46:54 +1100
Subject: [PATCH 0434/1696] [twitch] Add basic support for two-factor
 authentication

---
 youtube_dl/extractor/twitch.py | 79 ++++++++++++++++++++--------------
 1 file changed, 47 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index bbba394b0..096a2ac9d 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -12,7 +12,6 @@ from ..compat import (
     compat_str,
     compat_urllib_parse_urlencode,
     compat_urllib_parse_urlparse,
-    compat_urlparse,
 )
 from ..utils import (
     clean_html,
@@ -24,6 +23,7 @@ from ..utils import (
     parse_iso8601,
     update_url_query,
     urlencode_postdata,
+    urljoin,
 )
 
 
@@ -32,7 +32,7 @@ class TwitchBaseIE(InfoExtractor):
 
     _API_BASE = 'https://api.twitch.tv'
     _USHER_BASE = 'https://usher.ttvnw.net'
-    _LOGIN_URL = 'http://www.twitch.tv/login'
+    _LOGIN_URL = 'https://www.twitch.tv/login'
     _CLIENT_ID = 'jzkbprff40iqj646a697cyrvl0zt2m6'
     _NETRC_MACHINE = 'twitch'
 
@@ -64,6 +64,36 @@ class TwitchBaseIE(InfoExtractor):
             raise ExtractorError(
                 'Unable to login. Twitch said: %s' % message, expected=True)
 
+        def post_login_form(page, urlh, note, data):
+            form = self._hidden_inputs(page)
+            form.update(data)
+
+            page_url = urlh.geturl()
+            post_url = self._search_regex(
+                r'<form[^>]+action=(["\'])(?P<url>.+?)\1', page,
+                'post url', default=page_url, group='url')
+            post_url = urljoin(page_url, post_url)
+
+            headers = {'Referer': page_url}
+
+            try:
+                response = self._download_json(
+                    post_url, None, note,
+                    data=urlencode_postdata(form),
+                    headers=headers)
+            except ExtractorError as e:
+                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
+                    response = self._parse_json(
+                        e.cause.read().decode('utf-8'), None)
+                    fail(response['message'])
+                raise
+
+            if response.get('redirect'):
+                redirect_url = urljoin(post_url, response['redirect'])
+                return self._download_webpage_handle(
+                    redirect_url, None, 'Downloading login redirect page',
+                    headers=headers)
+
         login_page, handle = self._download_webpage_handle(
             self._LOGIN_URL, None, 'Downloading login page')
 
@@ -71,40 +101,25 @@ class TwitchBaseIE(InfoExtractor):
         if 'blacklist_message' in login_page:
             fail(clean_html(login_page))
 
-        login_form = self._hidden_inputs(login_page)
-
-        login_form.update({
+        login_data = {
             'username': username,
             'password': password,
-        })
+        }
+        redirect_res = post_login_form(
+            login_page, handle, 'Logging in as %s' % username, login_data)
 
-        redirect_url = handle.geturl()
+        if not redirect_res:
+            return
+        redirect_page, handle = redirect_res
 
-        post_url = self._search_regex(
-            r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
-            'post url', default=redirect_url, group='url')
-
-        if not post_url.startswith('http'):
-            post_url = compat_urlparse.urljoin(redirect_url, post_url)
-
-        headers = {'Referer': redirect_url}
-
-        try:
-            response = self._download_json(
-                post_url, None, 'Logging in as %s' % username,
-                data=urlencode_postdata(login_form),
-                headers=headers)
-        except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
-                response = self._parse_json(
-                    e.cause.read().decode('utf-8'), None)
-                fail(response['message'])
-            raise
-
-        if response.get('redirect'):
-            self._download_webpage(
-                response['redirect'], None, 'Downloading login redirect page',
-                headers=headers)
+        if re.search(r'(?i)<form[^>]+id="two-factor-submit"', redirect_page) is not None:
+            # TODO: Add mechanism to request an SMS or phone call
+            tfa_token = self._get_tfa_info('two-factor authentication token')
+            tfa_data = {
+                'authy_token': tfa_token,
+                'remember_2fa': 'true',
+            }
+            post_login_form(redirect_page, handle, 'Submitting TFA token', tfa_data)
 
     def _prefer_source(self, formats):
         try:

From 5316566edcbb1a2ac2e0559a1863b2204242b7d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Mar 2017 02:06:33 +0700
Subject: [PATCH 0435/1696] [twitch] Use better naming and simplify (closes
 #11974)

---
 youtube_dl/extractor/twitch.py | 31 ++++++++++++-------------------
 1 file changed, 12 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 096a2ac9d..ed36336bd 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -64,7 +64,7 @@ class TwitchBaseIE(InfoExtractor):
             raise ExtractorError(
                 'Unable to login. Twitch said: %s' % message, expected=True)
 
-        def post_login_form(page, urlh, note, data):
+        def login_step(page, urlh, note, data):
             form = self._hidden_inputs(page)
             form.update(data)
 
@@ -88,11 +88,10 @@ class TwitchBaseIE(InfoExtractor):
                     fail(response['message'])
                 raise
 
-            if response.get('redirect'):
-                redirect_url = urljoin(post_url, response['redirect'])
-                return self._download_webpage_handle(
-                    redirect_url, None, 'Downloading login redirect page',
-                    headers=headers)
+            redirect_url = urljoin(post_url, response['redirect'])
+            return self._download_webpage_handle(
+                redirect_url, None, 'Downloading login redirect page',
+                headers=headers)
 
         login_page, handle = self._download_webpage_handle(
             self._LOGIN_URL, None, 'Downloading login page')
@@ -101,25 +100,19 @@ class TwitchBaseIE(InfoExtractor):
         if 'blacklist_message' in login_page:
             fail(clean_html(login_page))
 
-        login_data = {
-            'username': username,
-            'password': password,
-        }
-        redirect_res = post_login_form(
-            login_page, handle, 'Logging in as %s' % username, login_data)
-
-        if not redirect_res:
-            return
-        redirect_page, handle = redirect_res
+        redirect_page, handle = login_step(
+            login_page, handle, 'Logging in as %s' % username, {
+                'username': username,
+                'password': password,
+        })
 
         if re.search(r'(?i)<form[^>]+id="two-factor-submit"', redirect_page) is not None:
             # TODO: Add mechanism to request an SMS or phone call
             tfa_token = self._get_tfa_info('two-factor authentication token')
-            tfa_data = {
+            login_step(redirect_page, handle, 'Submitting TFA token', {
                 'authy_token': tfa_token,
                 'remember_2fa': 'true',
-            }
-            post_login_form(redirect_page, handle, 'Submitting TFA token', tfa_data)
+            })
 
     def _prefer_source(self, formats):
         try:

From 75027364ba35d8852f393f67860be817fff05541 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Mar 2017 02:22:02 +0700
Subject: [PATCH 0436/1696] [ChangeLog] Actualize

---
 ChangeLog | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 13ccb0f8f..7e22db53e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,18 @@
 version <unreleased>
 
 Extractors
++ [twitch] Add basic support for two-factor authentication (#11974)
++ [vier] Add support for vijf.be (#12304)
++ [redbulltv] Add support for redbull.tv (#3919, #11948)
 * [douyutv] Switch to the PC API to escape the 5-min limitation (#12316)
++ [generic] Add support for rutube embeds
++ [rutube] Relax URL regular expression
++ [vrak] Add support for vrak.tv (#11452)
++ [brightcove:new] Add ability to smuggle geo_countries into URL
++ [brightcove:new] Raise GeoRestrictedError
+* [go] Relax URL regular expression (#12341)
+* [24video] Use original host for requests (#12339)
+* [ruutu] Disable DASH formats (#12322)
 
 
 version 2017.03.02

From afa4597618462df01b47febcd64c531f8ffdd63a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Mar 2017 02:23:08 +0700
Subject: [PATCH 0437/1696] release 2017.03.05

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 ++
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index fec4152e3..988d0d81b 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.03.02*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.03.02**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.03.05*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.03.05**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.03.02
+[debug] youtube-dl version 2017.03.05
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 7e22db53e..2f3ec1b47 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.03.05
 
 Extractors
 + [twitch] Add basic support for two-factor authentication (#11974)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index a08e00fce..f02c4dea0 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -626,6 +626,7 @@
  - **RaiTV**
  - **RBMARadio**
  - **RDS**: RDS.ca
+ - **RedBullTV**
  - **RedTube**
  - **RegioTV**
  - **RENTV**
@@ -916,6 +917,7 @@
  - **VoxMedia**
  - **Vporn**
  - **vpro**: npo.nl and ntr.nl
+ - **Vrak**
  - **VRT**
  - **vube**: Vube.com
  - **VuClip**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index f4c8d3d5f..215f9a3c1 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.03.02'
+__version__ = '2017.03.05'

From 6d0fe752bfeaa9a758099315f006bc15acd1ae76 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Sat, 4 Mar 2017 22:19:44 -0500
Subject: [PATCH 0438/1696] [external:ffmpeg] In test harness, limit to 10k
 download size

Otherwise, if you screw up a playlist test by including a playlist
dictionary key, you'll be there for eons while it downloads all the
files before erroring out.
---
 youtube_dl/downloader/external.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index bdd3545a2..127a92d20 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -270,6 +270,10 @@ class FFmpegFD(ExternalFD):
                 args += ['-rtmp_live', 'live']
 
         args += ['-i', url, '-c', 'copy']
+
+        if self.params.get('test', False):
+            args += ['-fs', compat_str(self._TEST_FILE_SIZE)] # -fs limit_size (output), expressed in bytes
+
         if protocol in ('m3u8', 'm3u8_native'):
             if self.params.get('hls_use_mpegts', False) or tmpfilename == '-':
                 args += ['-f', 'mpegts']

From a50862b7355544d3fd8473bf3ff35e9c6643d789 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Mar 2017 10:24:29 +0700
Subject: [PATCH 0439/1696] [downloader/external] Add missing import and PEP8

---
 youtube_dl/downloader/external.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 127a92d20..e13cf547d 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -6,7 +6,10 @@ import sys
 import re
 
 from .common import FileDownloader
-from ..compat import compat_setenv
+from ..compat import (
+    compat_setenv,
+    compat_str,
+)
 from ..postprocessor.ffmpeg import FFmpegPostProcessor, EXT_TO_OUT_FORMATS
 from ..utils import (
     cli_option,
@@ -272,7 +275,7 @@ class FFmpegFD(ExternalFD):
         args += ['-i', url, '-c', 'copy']
 
         if self.params.get('test', False):
-            args += ['-fs', compat_str(self._TEST_FILE_SIZE)] # -fs limit_size (output), expressed in bytes
+            args += ['-fs', compat_str(self._TEST_FILE_SIZE)]
 
         if protocol in ('m3u8', 'm3u8_native'):
             if self.params.get('hls_use_mpegts', False) or tmpfilename == '-':

From ed0cf9b38394b28bae5f05fd6b00c85a9c0e6755 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Mar 2017 23:22:27 +0700
Subject: [PATCH 0440/1696] [extractor/common] Move jwplayer formats extraction
 in separate method

---
 youtube_dl/extractor/common.py | 106 +++++++++++++++++----------------
 1 file changed, 56 insertions(+), 50 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index eb3c091aa..9b73a948c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2198,56 +2198,7 @@ class InfoExtractor(object):
 
             this_video_id = video_id or video_data['mediaid']
 
-            formats = []
-            for source in video_data['sources']:
-                source_url = self._proto_relative_url(source['file'])
-                if base_url:
-                    source_url = compat_urlparse.urljoin(base_url, source_url)
-                source_type = source.get('type') or ''
-                ext = mimetype2ext(source_type) or determine_ext(source_url)
-                if source_type == 'hls' or ext == 'm3u8':
-                    formats.extend(self._extract_m3u8_formats(
-                        source_url, this_video_id, 'mp4', 'm3u8_native', m3u8_id=m3u8_id, fatal=False))
-                elif ext == 'mpd':
-                    formats.extend(self._extract_mpd_formats(
-                        source_url, this_video_id, mpd_id=mpd_id, fatal=False))
-                # https://github.com/jwplayer/jwplayer/blob/master/src/js/providers/default.js#L67
-                elif source_type.startswith('audio') or ext in ('oga', 'aac', 'mp3', 'mpeg', 'vorbis'):
-                    formats.append({
-                        'url': source_url,
-                        'vcodec': 'none',
-                        'ext': ext,
-                    })
-                else:
-                    height = int_or_none(source.get('height'))
-                    if height is None:
-                        # Often no height is provided but there is a label in
-                        # format like 1080p.
-                        height = int_or_none(self._search_regex(
-                            r'^(\d{3,})[pP]$', source.get('label') or '',
-                            'height', default=None))
-                    a_format = {
-                        'url': source_url,
-                        'width': int_or_none(source.get('width')),
-                        'height': height,
-                        'ext': ext,
-                    }
-                    if source_url.startswith('rtmp'):
-                        a_format['ext'] = 'flv'
-
-                        # See com/longtailvideo/jwplayer/media/RTMPMediaProvider.as
-                        # of jwplayer.flash.swf
-                        rtmp_url_parts = re.split(
-                            r'((?:mp4|mp3|flv):)', source_url, 1)
-                        if len(rtmp_url_parts) == 3:
-                            rtmp_url, prefix, play_path = rtmp_url_parts
-                            a_format.update({
-                                'url': rtmp_url,
-                                'play_path': prefix + play_path,
-                            })
-                        if rtmp_params:
-                            a_format.update(rtmp_params)
-                    formats.append(a_format)
+            formats = self._parse_jwplayer_formats(video_data['sources'], this_video_id)
             self._sort_formats(formats)
 
             subtitles = {}
@@ -2278,6 +2229,61 @@ class InfoExtractor(object):
         else:
             return self.playlist_result(entries)
 
+    def _parse_jwplayer_formats(self, jwplayer_sources_data, video_id=None,
+                                m3u8_id=None, mpd_id=None, rtmp_params=None, base_url=None):
+        formats = []
+        for source in jwplayer_sources_data :
+            source_url = self._proto_relative_url(source['file'])
+            if base_url:
+                source_url = compat_urlparse.urljoin(base_url, source_url)
+            source_type = source.get('type') or ''
+            ext = mimetype2ext(source_type) or determine_ext(source_url)
+            if source_type == 'hls' or ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    source_url, video_id, 'mp4', 'm3u8_native', m3u8_id=m3u8_id, fatal=False))
+            elif ext == 'mpd':
+                formats.extend(self._extract_mpd_formats(
+                    source_url, video_id, mpd_id=mpd_id, fatal=False))
+            # https://github.com/jwplayer/jwplayer/blob/master/src/js/providers/default.js#L67
+            elif source_type.startswith('audio') or ext in ('oga', 'aac', 'mp3', 'mpeg', 'vorbis'):
+                formats.append({
+                    'url': source_url,
+                    'vcodec': 'none',
+                    'ext': ext,
+                })
+            else:
+                height = int_or_none(source.get('height'))
+                if height is None:
+                    # Often no height is provided but there is a label in
+                    # format like 1080p.
+                    height = int_or_none(self._search_regex(
+                        r'^(\d{3,})[pP]$', source.get('label') or '',
+                        'height', default=None))
+                a_format = {
+                    'url': source_url,
+                    'width': int_or_none(source.get('width')),
+                    'height': height,
+                    'ext': ext,
+                }
+                if source_url.startswith('rtmp'):
+                    a_format['ext'] = 'flv'
+
+                    # See com/longtailvideo/jwplayer/media/RTMPMediaProvider.as
+                    # of jwplayer.flash.swf
+                    rtmp_url_parts = re.split(
+                        r'((?:mp4|mp3|flv):)', source_url, 1)
+                    if len(rtmp_url_parts) == 3:
+                        rtmp_url, prefix, play_path = rtmp_url_parts
+                        a_format.update({
+                            'url': rtmp_url,
+                            'play_path': prefix + play_path,
+                        })
+                    if rtmp_params:
+                        a_format.update(rtmp_params)
+                formats.append(a_format)
+        return formats
+
+
     def _live_title(self, name):
         """ Generate the title for a live video """
         now = datetime.datetime.now()

From 0236cd0dfde1dda540c1067a9c5982d482005c47 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Mar 2017 23:25:03 +0700
Subject: [PATCH 0441/1696] [extractor/common] Improve height extraction and
 extract bitrate

---
 youtube_dl/extractor/common.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9b73a948c..2887db0c3 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2240,12 +2240,14 @@ class InfoExtractor(object):
             ext = mimetype2ext(source_type) or determine_ext(source_url)
             if source_type == 'hls' or ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
-                    source_url, video_id, 'mp4', 'm3u8_native', m3u8_id=m3u8_id, fatal=False))
+                    source_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id=m3u8_id, fatal=False))
             elif ext == 'mpd':
                 formats.extend(self._extract_mpd_formats(
                     source_url, video_id, mpd_id=mpd_id, fatal=False))
             # https://github.com/jwplayer/jwplayer/blob/master/src/js/providers/default.js#L67
-            elif source_type.startswith('audio') or ext in ('oga', 'aac', 'mp3', 'mpeg', 'vorbis'):
+            elif source_type.startswith('audio') or ext in (
+                    'oga', 'aac', 'mp3', 'mpeg', 'vorbis'):
                 formats.append({
                     'url': source_url,
                     'vcodec': 'none',
@@ -2255,19 +2257,19 @@ class InfoExtractor(object):
                 height = int_or_none(source.get('height'))
                 if height is None:
                     # Often no height is provided but there is a label in
-                    # format like 1080p.
+                    # format like "1080p", "720p SD", or 1080.
                     height = int_or_none(self._search_regex(
-                        r'^(\d{3,})[pP]$', source.get('label') or '',
+                        r'^(\d{3,4})[pP]?(?:\b|$)', compat_str(source.get('label') or ''),
                         'height', default=None))
                 a_format = {
                     'url': source_url,
                     'width': int_or_none(source.get('width')),
                     'height': height,
+                    'tbr': int_or_none(source.get('bitrate')),
                     'ext': ext,
                 }
                 if source_url.startswith('rtmp'):
                     a_format['ext'] = 'flv'
-
                     # See com/longtailvideo/jwplayer/media/RTMPMediaProvider.as
                     # of jwplayer.flash.swf
                     rtmp_url_parts = re.split(

From 1a2192cb904ff42a309ab2c2477fc226f8651f33 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Mar 2017 23:28:32 +0700
Subject: [PATCH 0442/1696] [extractor/common] Pass arguments to
 _parse_jwplayer_formats and PEP8

---
 youtube_dl/extractor/common.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 2887db0c3..78dc5be24 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2198,7 +2198,9 @@ class InfoExtractor(object):
 
             this_video_id = video_id or video_data['mediaid']
 
-            formats = self._parse_jwplayer_formats(video_data['sources'], this_video_id)
+            formats = self._parse_jwplayer_formats(
+                video_data['sources'], video_id=this_video_id, m3u8_id=m3u8_id,
+                mpd_id=mpd_id, rtmp_params=rtmp_params, base_url=base_url)
             self._sort_formats(formats)
 
             subtitles = {}
@@ -2232,7 +2234,7 @@ class InfoExtractor(object):
     def _parse_jwplayer_formats(self, jwplayer_sources_data, video_id=None,
                                 m3u8_id=None, mpd_id=None, rtmp_params=None, base_url=None):
         formats = []
-        for source in jwplayer_sources_data :
+        for source in jwplayer_sources_data:
             source_url = self._proto_relative_url(source['file'])
             if base_url:
                 source_url = compat_urlparse.urljoin(base_url, source_url)
@@ -2285,7 +2287,6 @@ class InfoExtractor(object):
                 formats.append(a_format)
         return formats
 
-
     def _live_title(self, name):
         """ Generate the title for a live video """
         now = datetime.datetime.now()

From 5dd376345b8c87e8c2130b80e73e690c5e721a28 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Mar 2017 23:31:38 +0700
Subject: [PATCH 0443/1696] [tunepk] Add extractor (closes #12197, closes
 #12243)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/tunepk.py     | 90 ++++++++++++++++++++++++++++++
 2 files changed, 91 insertions(+)
 create mode 100644 youtube_dl/extractor/tunepk.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index caf1dc766..b056dff53 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1000,6 +1000,7 @@ from .tunein import (
     TuneInTopicIE,
     TuneInShortenerIE,
 )
+from .tunepk import TunePkIE
 from .turbo import TurboIE
 from .tutv import TutvIE
 from .tv2 import (
diff --git a/youtube_dl/extractor/tunepk.py b/youtube_dl/extractor/tunepk.py
new file mode 100644
index 000000000..9d42651ce
--- /dev/null
+++ b/youtube_dl/extractor/tunepk.py
@@ -0,0 +1,90 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    int_or_none,
+    try_get,
+    unified_timestamp,
+)
+
+
+class TunePkIE(InfoExtractor):
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:
+                            (?:www\.)?tune\.pk/(?:video/|player/embed_player.php?.*?\bvid=)|
+                            embed\.tune\.pk/play/
+                        )
+                        (?P<id>\d+)
+                    '''
+    _TESTS = [{
+        'url': 'https://tune.pk/video/6919541/maudie-2017-international-trailer-1-ft-ethan-hawke-sally-hawkins',
+        'md5': '0c537163b7f6f97da3c5dd1e3ef6dd55',
+        'info_dict': {
+            'id': '6919541',
+            'ext': 'mp4',
+            'title': 'Maudie (2017) | International Trailer # 1 ft Ethan Hawke, Sally Hawkins',
+            'description': 'md5:eb5a04114fafef5cec90799a93a2d09c',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'timestamp': 1487327564,
+            'upload_date': '20170217',
+            'uploader': 'Movie Trailers',
+            'duration': 107,
+            'view_count': int,
+        }
+    }, {
+        'url': 'https://tune.pk/player/embed_player.php?vid=6919541&folder=2017/02/17/&width=600&height=350&autoplay=no',
+        'only_matching': True,
+    }, {
+        'url': 'https://embed.tune.pk/play/6919541?autoplay=no&ssl=yes&inline=true',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(
+            'https://tune.pk/video/%s' % video_id, video_id)
+
+        details = self._parse_json(
+            self._search_regex(
+                r'new\s+TunePlayer\(({.+?})\)\s*;\s*\n', webpage, 'tune player'),
+            video_id)['details']
+
+        video = details['video']
+        title = video.get('title') or self._og_search_title(
+            webpage, default=None) or self._html_search_meta(
+            'title', webpage, 'title', fatal=True)
+
+        formats = self._parse_jwplayer_formats(
+            details['player']['sources'], video_id)
+        self._sort_formats(formats)
+
+        description = self._og_search_description(
+            webpage, default=None) or self._html_search_meta(
+            'description', webpage, 'description')
+
+        thumbnail = video.get('thumb') or self._og_search_thumbnail(
+            webpage, default=None) or self._html_search_meta(
+            'thumbnail', webpage, 'thumbnail')
+
+        timestamp = unified_timestamp(video.get('date_added'))
+        uploader = try_get(
+            video, lambda x: x['uploader']['name'],
+            compat_str) or self._html_search_meta('author', webpage, 'author')
+
+        duration = int_or_none(video.get('duration'))
+        view_count = int_or_none(video.get('views'))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'uploader': uploader,
+            'duration': duration,
+            'view_count': view_count,
+            'formats': formats,
+        }

From d2b64e04b475b6ef7e5fb74d92ae9a35284fc35a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 6 Mar 2017 00:32:53 +0800
Subject: [PATCH 0444/1696] [addanime] Skip an invalid test

---
 youtube_dl/extractor/addanime.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/addanime.py b/youtube_dl/extractor/addanime.py
index 55a9322a7..9f8a71262 100644
--- a/youtube_dl/extractor/addanime.py
+++ b/youtube_dl/extractor/addanime.py
@@ -25,7 +25,8 @@ class AddAnimeIE(InfoExtractor):
             'ext': 'mp4',
             'description': 'One Piece 606',
             'title': 'One Piece 606',
-        }
+        },
+        'skip': 'Video is gone',
     }, {
         'url': 'http://add-anime.net/video/MDUGWYKNGBD8/One-Piece-687',
         'only_matching': True,

From fc11ad383311d721179483eea2e289e3a236e457 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 6 Mar 2017 03:21:03 +0700
Subject: [PATCH 0445/1696] [drtv:live] Bypass geo restriction

---
 youtube_dl/extractor/drtv.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index e966d7483..b879f2c2b 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -156,6 +156,7 @@ class DRTVIE(InfoExtractor):
 class DRTVLiveIE(InfoExtractor):
     IE_NAME = 'drtv:live'
     _VALID_URL = r'https?://(?:www\.)?dr\.dk/(?:tv|TV)/live/(?P<id>[\da-z-]+)'
+    _GEO_COUNTRIES = ['DK']
     _TEST = {
         'url': 'https://www.dr.dk/tv/live/dr1',
         'info_dict': {

From 96182695e4e37795a30ab143129c91dab18a9865 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 6 Mar 2017 03:23:01 +0700
Subject: [PATCH 0446/1696] [drtv] Add geo countries to GeoRestrictedError

---
 youtube_dl/extractor/drtv.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index b879f2c2b..e4917014a 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -15,6 +15,8 @@ from ..utils import (
 
 class DRTVIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?dr\.dk/(?:tv/se|nyheder|radio/ondemand)/(?:[^/]+/)*(?P<id>[\da-z-]+)(?:[/#?]|$)'
+    _GEO_BYPASS = False
+    _GEO_COUNTRIES = ['DK']
     IE_NAME = 'drtv'
     _TESTS = [{
         'url': 'https://www.dr.dk/tv/se/boern/ultra/klassen-ultra/klassen-darlig-taber-10',
@@ -137,7 +139,7 @@ class DRTVIE(InfoExtractor):
         if not formats and restricted_to_denmark:
             self.raise_geo_restricted(
                 'Unfortunately, DR is not allowed to show this program outside Denmark.',
-                expected=True)
+                countries=self._GEO_COUNTRIES)
 
         self._sort_formats(formats)
 

From 4b5de77bdb7765df4797bf068592926285ba709a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 6 Mar 2017 03:57:46 +0700
Subject: [PATCH 0447/1696] [utils] Process bytestrings in urljoin (closes
 #12369)

---
 test/test_utils.py  | 3 +++
 youtube_dl/utils.py | 7 ++++++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index aefd94518..173c49514 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -455,6 +455,9 @@ class TestUtil(unittest.TestCase):
 
     def test_urljoin(self):
         self.assertEqual(urljoin('http://foo.de/', '/a/b/c.txt'), 'http://foo.de/a/b/c.txt')
+        self.assertEqual(urljoin(b'http://foo.de/', '/a/b/c.txt'), 'http://foo.de/a/b/c.txt')
+        self.assertEqual(urljoin('http://foo.de/', b'/a/b/c.txt'), 'http://foo.de/a/b/c.txt')
+        self.assertEqual(urljoin(b'http://foo.de/', b'/a/b/c.txt'), 'http://foo.de/a/b/c.txt')
         self.assertEqual(urljoin('//foo.de/', '/a/b/c.txt'), '//foo.de/a/b/c.txt')
         self.assertEqual(urljoin('http://foo.de/', 'a/b/c.txt'), 'http://foo.de/a/b/c.txt')
         self.assertEqual(urljoin('http://foo.de', '/a/b/c.txt'), 'http://foo.de/a/b/c.txt')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 8738aa249..d293c7498 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1748,11 +1748,16 @@ def base_url(url):
 
 
 def urljoin(base, path):
+    if isinstance(path, bytes):
+        path = path.decode('utf-8')
     if not isinstance(path, compat_str) or not path:
         return None
     if re.match(r'^(?:https?:)?//', path):
         return path
-    if not isinstance(base, compat_str) or not re.match(r'^(?:https?:)?//', base):
+    if isinstance(base, bytes):
+        base = base.decode('utf-8')
+    if not isinstance(base, compat_str) or not re.match(
+            r'^(?:https?:)?//', base):
         return None
     return compat_urlparse.urljoin(base, path)
 

From 3f116b189bb990529a1a18ba7a3829b1592cfecd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 6 Mar 2017 04:01:21 +0700
Subject: [PATCH 0448/1696] [ChangeLog] Actualize

---
 ChangeLog | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 2f3ec1b47..c1f43a625 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,17 @@
+version <unreleased>
+
+Core
++ [utils] Process bytestrings in urljoin (#12369)
+* [extractor/common] Improve height extraction and extract bitrate
+* [extractor/common] Move jwplayer formats extraction in separate method
++ [external:ffmpeg] Limit test download size to 10KiB (#12362)
+
+Extractors
++ [drtv] Add geo countries to GeoRestrictedError
++ [drtv:live] Bypass geo restriction
++ [tunepk] Add extractor (#12197, #12243)
+
+
 version 2017.03.05
 
 Extractors

From 16647026264b35a40ecd56f5d9392d0643a2066c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 6 Mar 2017 04:04:39 +0700
Subject: [PATCH 0449/1696] release 2017.03.06

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 1 +
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 988d0d81b..a36d7322d 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.03.05*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.03.05**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.03.06*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.03.06**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.03.05
+[debug] youtube-dl version 2017.03.06
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index c1f43a625..648e04856 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.03.06
 
 Core
 + [utils] Process bytestrings in urljoin (#12369)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index f02c4dea0..85c59ca81 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -798,6 +798,7 @@
  - **tunein:program**
  - **tunein:station**
  - **tunein:topic**
+ - **TunePk**
  - **Turbo**
  - **Tutv**
  - **tv.dfb.de**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 215f9a3c1..71a984913 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.03.05'
+__version__ = '2017.03.06'

From da92da4b886a0e44fe28591ddf5b746fba1c9ade Mon Sep 17 00:00:00 2001
From: denneboomyo <denis_b555@freenet.de>
Date: Mon, 6 Mar 2017 11:00:17 +0100
Subject: [PATCH 0450/1696] Openload fix extraction (#12357)

* Fix extraction
---
 youtube_dl/extractor/openload.py | 43 +++++++++++++++++++++-----------
 1 file changed, 29 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index fc7ff43a6..25f6a9aca 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -75,22 +75,37 @@ class OpenloadIE(InfoExtractor):
             '<span[^>]+id="[^"]+"[^>]*>([0-9A-Za-z]+)</span>',
             webpage, 'openload ID')
 
-        first_char = int(ol_id[0])
-        urlcode = []
-        num = 1
+        video_url_chars = []
 
-        while num < len(ol_id):
-            i = ord(ol_id[num])
-            key = 0
-            if i <= 90:
-                key = i - 65
-            elif i >= 97:
-                key = 25 + i - 97
-            urlcode.append((key, compat_chr(int(ol_id[num + 2:num + 5]) // int(ol_id[num + 1]) - first_char)))
-            num += 5
+        first_char = ord(ol_id[0])
+        key = first_char - 55
+        maxKey = max(2, key)
+        key = min(maxKey, len(ol_id) - 14)
+        t = ol_id[key:key + 12]
 
-        video_url = 'https://openload.co/stream/' + ''.join(
-            [value for _, value in sorted(urlcode, key=lambda x: x[0])])
+        hashMap = {}
+        v = ol_id.replace(t, "")
+        h = 0
+
+        while h < len(t):
+            f = t[h:h + 2]
+            i = int(f, 16)
+            hashMap[h / 2] = i
+            h += 2
+
+        h = 0
+
+        while h < len(v):
+            B = v[h:h + 2]
+            i = int(B, 16)
+            index = (h / 2) % 6
+            A = hashMap[index]
+            i = i ^ A
+            video_url_chars.append(compat_chr(i))
+            h += 2
+
+        video_url = 'https://openload.co/stream/%s?mime=true'
+        video_url = video_url % (''.join(video_url_chars))
 
         title = self._og_search_title(webpage, default=None) or self._search_regex(
             r'<span[^>]+class=["\']title["\'][^>]*>([^<]+)', webpage,

From 92cb5763f440d0ca1627f8120a7ce29598eb9484 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 6 Mar 2017 18:04:19 +0800
Subject: [PATCH 0451/1696] [ChangeLog] Update after #12357

---
 ChangeLog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 648e04856..5fb4c20af 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [openload] Fix extraction (#10408, #12357)
+
+
 version 2017.03.06
 
 Core

From 54a3a8827baf71c553723b6766e676369e9c743c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 9 Jan 2017 02:36:39 +0800
Subject: [PATCH 0452/1696] [__init__] Metadata should be added after
 conversion

Fixes #5594
---
 ChangeLog              | 4 ++++
 youtube_dl/__init__.py | 8 +++++---
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 5fb4c20af..ad65505c5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,9 @@
 version <unreleased>
 
+Core
+* [__init__] Metadata are now added after conversion (#5594)
+
+
 Extractors
 * [openload] Fix extraction (#10408, #12357)
 
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 0c401baa6..ad5f13d2b 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -242,14 +242,11 @@ def _real_main(argv=None):
 
     # PostProcessors
     postprocessors = []
-    # Add the metadata pp first, the other pps will copy it
     if opts.metafromtitle:
         postprocessors.append({
             'key': 'MetadataFromTitle',
             'titleformat': opts.metafromtitle
         })
-    if opts.addmetadata:
-        postprocessors.append({'key': 'FFmpegMetadata'})
     if opts.extractaudio:
         postprocessors.append({
             'key': 'FFmpegExtractAudio',
@@ -279,6 +276,11 @@ def _real_main(argv=None):
         })
         if not already_have_thumbnail:
             opts.writethumbnail = True
+    # FFmpegMetadataPP should be run after FFmpegVideoConvertorPP and
+    # FFmpegExtractAudioPP as containers before conversion may not support
+    # metadata (3gp, webm, etc.)
+    if opts.addmetadata:
+        postprocessors.append({'key': 'FFmpegMetadata'})
     # XAttrMetadataPP should be run after post-processors that may change file
     # contents
     if opts.xattrs:

From e30ccf7047eb3b8dff8b778790f9b084e6d7f42e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 6 Mar 2017 23:05:38 +0700
Subject: [PATCH 0453/1696] [soundcloud] Update client id (closes #12376)

---
 youtube_dl/extractor/soundcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index b3aa4ce26..0ee4a8ff8 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -121,7 +121,7 @@ class SoundcloudIE(InfoExtractor):
         },
     ]
 
-    _CLIENT_ID = 'fDoItMDbsbZz8dY16ZzARCZmzgHBPotA'
+    _CLIENT_ID = '2t9loNQH90kzJcsFCODdigxfp325aq4z'
     _IPHONE_CLIENT_ID = '376f225bf427445fc4bfb6b99b72e0bf'
 
     @staticmethod

From 80146dcc6c27b46fb8340d3285d95f2f7674fb0b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 7 Mar 2017 03:57:54 +0700
Subject: [PATCH 0454/1696] [ChangeLog] Actualize

---
 ChangeLog | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index ad65505c5..622086440 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,10 +1,10 @@
 version <unreleased>
 
 Core
-* [__init__] Metadata are now added after conversion (#5594)
-
+* Metadata are now added after conversion (#5594)
 
 Extractors
+* [soundcloud] Update client id (#12376)
 * [openload] Fix extraction (#10408, #12357)
 
 

From dccd0ab35d1acc45e36241c505b5325d96ca501d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 7 Mar 2017 03:59:22 +0700
Subject: [PATCH 0455/1696] release 2017.03.07

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index a36d7322d..eb69696c8 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.03.06*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.03.06**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.03.07*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.03.07**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.03.06
+[debug] youtube-dl version 2017.03.07
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 622086440..601aad902 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.03.07
 
 Core
 * Metadata are now added after conversion (#5594)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 71a984913..bd451bf81 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.03.06'
+__version__ = '2017.03.07'

From d7d7f84c9565d682119d081324c26eb361e05aa1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 7 Mar 2017 04:03:52 +0700
Subject: [PATCH 0456/1696] Credit @benages for redbull.tv (#11948)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index fd1b8c9ea..74abda016 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -207,3 +207,4 @@ Marek Rusinowski
 Tobias Gruetzmacher
 Olivier Bilodeau
 Lars Vierbergen
+Juanjo Benages

From 9df53ea36ec84c6ab5a4a672d120f7850e2363ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 7 Mar 2017 04:04:49 +0700
Subject: [PATCH 0457/1696] Credit @puxlit for twitch 2fa (#11974)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 74abda016..273a6a034 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -208,3 +208,4 @@ Tobias Gruetzmacher
 Olivier Bilodeau
 Lars Vierbergen
 Juanjo Benages
+Xiao Di Guan

From fe646a2f106fef0651158c460dd766c236e2f5db Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 7 Mar 2017 15:34:06 +0800
Subject: [PATCH 0458/1696] [twitch] PEP8

---
 youtube_dl/extractor/twitch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index ed36336bd..2daf9dfac 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -104,7 +104,7 @@ class TwitchBaseIE(InfoExtractor):
             login_page, handle, 'Logging in as %s' % username, {
                 'username': username,
                 'password': password,
-        })
+            })
 
         if re.search(r'(?i)<form[^>]+id="two-factor-submit"', redirect_page) is not None:
             # TODO: Add mechanism to request an SMS or phone call

From 2e76bdc850ed5d5ffe95578b576b8fb66dcea8f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 7 Mar 2017 22:59:33 +0700
Subject: [PATCH 0459/1696] [brightcove:legacy] Relax videoPlayer validation
 check (closes #12381)

---
 youtube_dl/extractor/brightcove.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 66c8cb219..46ef8e605 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -193,7 +193,13 @@ class BrightcoveLegacyIE(InfoExtractor):
         if videoPlayer is not None:
             if isinstance(videoPlayer, list):
                 videoPlayer = videoPlayer[0]
-            if not (videoPlayer.isdigit() or videoPlayer.startswith('ref:')):
+            videoPlayer = videoPlayer.strip()
+            # UUID is also possible for videoPlayer (e.g.
+            # http://www.popcornflix.com/hoodies-vs-hooligans/7f2d2b87-bbf2-4623-acfb-ea942b4f01dd
+            # or http://www8.hp.com/cn/zh/home.html)
+            if not (re.match(
+                    r'^(?:\d+|[\da-fA-F]{8}-?[\da-fA-F]{4}-?[\da-fA-F]{4}-?[\da-fA-F]{4}-?[\da-fA-F]{12})$',
+                    videoPlayer) or videoPlayer.startswith('ref:')):
                 return None
             params['@videoPlayer'] = videoPlayer
         linkBase = find_param('linkBaseURL')

From b68a812ea839e44148516a34a15193189e58ba77 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 7 Mar 2017 23:00:21 +0700
Subject: [PATCH 0460/1696] [extractor/generic] Add test for brigthcove
 UUID-like videoPlayer

---
 youtube_dl/extractor/generic.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ebab9509d..bc22421ae 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -449,6 +449,23 @@ class GenericIE(InfoExtractor):
                 },
             }],
         },
+        {
+            # Brightcove with UUID in videoPlayer
+            'url': 'http://www8.hp.com/cn/zh/home.html',
+            'info_dict': {
+                'id': '5255815316001',
+                'ext': 'mp4',
+                'title': 'Sprocket Video - China',
+                'description': 'Sprocket Video - China',
+                'uploader': 'HP-Video Gallery',
+                'timestamp': 1482263210,
+                'upload_date': '20161220',
+                'uploader_id': '1107601872001',
+            },
+            'params': {
+                'skip_download': True,  # m3u8 download
+            },
+        },
         # ooyala video
         {
             'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',

From b08cc749d6e1a3078d807580c424437bca143b73 Mon Sep 17 00:00:00 2001
From: denneboomyo <denis_b555@freenet.de>
Date: Tue, 7 Mar 2017 23:01:27 +0100
Subject: [PATCH 0461/1696] [openload] Fix extraction

---
 youtube_dl/extractor/openload.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 25f6a9aca..5a5607357 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -78,10 +78,10 @@ class OpenloadIE(InfoExtractor):
         video_url_chars = []
 
         first_char = ord(ol_id[0])
-        key = first_char - 55
+        key = first_char - 50
         maxKey = max(2, key)
-        key = min(maxKey, len(ol_id) - 14)
-        t = ol_id[key:key + 12]
+        key = min(maxKey, len(ol_id) - 22)
+        t = ol_id[key:key + 20]
 
         hashMap = {}
         v = ol_id.replace(t, "")
@@ -98,8 +98,9 @@ class OpenloadIE(InfoExtractor):
         while h < len(v):
             B = v[h:h + 2]
             i = int(B, 16)
-            index = (h / 2) % 6
+            index = (h / 2) % 10
             A = hashMap[index]
+            i = i ^ 137
             i = i ^ A
             video_url_chars.append(compat_chr(i))
             h += 2

From d7344d33b11cc10d1a668c1ab9a65f2a34b4000f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 8 Mar 2017 18:25:04 +0700
Subject: [PATCH 0462/1696] [telequebec] Fix description extraction and update
 test (closes #12399)

---
 youtube_dl/extractor/telequebec.py | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/telequebec.py b/youtube_dl/extractor/telequebec.py
index 82d73c31d..fafaa826f 100644
--- a/youtube_dl/extractor/telequebec.py
+++ b/youtube_dl/extractor/telequebec.py
@@ -2,15 +2,17 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     int_or_none,
     smuggle_url,
+    try_get,
 )
 
 
 class TeleQuebecIE(InfoExtractor):
     _VALID_URL = r'https?://zonevideo\.telequebec\.tv/media/(?P<id>\d+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://zonevideo.telequebec.tv/media/20984/le-couronnement-de-new-york/couronnement-de-new-york',
         'md5': 'fe95a0957e5707b1b01f5013e725c90f',
         'info_dict': {
@@ -18,10 +20,14 @@ class TeleQuebecIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Le couronnement de New York',
             'description': 'md5:f5b3d27a689ec6c1486132b2d687d432',
-            'upload_date': '20160220',
-            'timestamp': 1455965438,
+            'upload_date': '20170201',
+            'timestamp': 1485972222,
         }
-    }
+    }, {
+        # no description
+        'url': 'http://zonevideo.telequebec.tv/media/30261',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         media_id = self._match_id(url)
@@ -31,9 +37,13 @@ class TeleQuebecIE(InfoExtractor):
         return {
             '_type': 'url_transparent',
             'id': media_id,
-            'url': smuggle_url('limelight:media:' + media_data['streamInfo']['sourceId'], {'geo_countries': ['CA']}),
+            'url': smuggle_url(
+                'limelight:media:' + media_data['streamInfo']['sourceId'],
+                {'geo_countries': ['CA']}),
             'title': media_data['title'],
-            'description': media_data.get('descriptions', [{'text': None}])[0].get('text'),
-            'duration': int_or_none(media_data.get('durationInMilliseconds'), 1000),
+            'description': try_get(
+                media_data, lambda x: x['descriptions'][0]['text'], compat_str),
+            'duration': int_or_none(
+                media_data.get('durationInMilliseconds'), 1000),
             'ie_key': 'LimelightMedia',
         }

From 0f6b87d067f33b5e9b8acf2f8a1f2afa9546439e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 8 Mar 2017 19:46:58 +0800
Subject: [PATCH 0463/1696] [miomio] Fix extraction

Closes #12291
Closes #12388
Closes #12402
---
 ChangeLog                      |  6 ++++++
 youtube_dl/extractor/miomio.py | 14 ++++++++++++--
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 601aad902..b000e2e94 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [miomio] Fix extraction (#12291, #12388, #12402)
+
+
 version 2017.03.07
 
 Core
diff --git a/youtube_dl/extractor/miomio.py b/youtube_dl/extractor/miomio.py
index ec1b4c4fe..40f72d66f 100644
--- a/youtube_dl/extractor/miomio.py
+++ b/youtube_dl/extractor/miomio.py
@@ -51,6 +51,7 @@ class MioMioIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'マツコの知らない世界【劇的進化SP！ビニール傘＆冷凍食品2016】 1_2 - 16 05 31',
         },
+        'skip': 'Unable to load videos',
     }]
 
     def _extract_mioplayer(self, webpage, video_id, title, http_headers):
@@ -94,9 +95,18 @@ class MioMioIE(InfoExtractor):
 
         return entries
 
+    def _download_chinese_webpage(self, *args, **kwargs):
+        # Requests with English locales return garbage
+        headers = {
+            'Accept-Language': 'zh-TW,en-US;q=0.7,en;q=0.3',
+        }
+        kwargs.setdefault('headers', {}).update(headers)
+        return self._download_webpage(*args, **kwargs)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_chinese_webpage(
+            url, video_id)
 
         title = self._html_search_meta(
             'description', webpage, 'title', fatal=True)
@@ -106,7 +116,7 @@ class MioMioIE(InfoExtractor):
 
         if '_h5' in mioplayer_path:
             player_url = compat_urlparse.urljoin(url, mioplayer_path)
-            player_webpage = self._download_webpage(
+            player_webpage = self._download_chinese_webpage(
                 player_url, video_id,
                 note='Downloading player webpage', headers={'Referer': url})
             entries = self._parse_html5_media_entries(player_url, player_webpage, video_id)

From 0cf2352e858982ed811122cf867fb5e25694d97a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 8 Mar 2017 21:19:08 +0700
Subject: [PATCH 0464/1696] [dplayit] Separate and rewrite extractor and bypass
 geo restriction (closes #12393)

---
 youtube_dl/extractor/dplay.py      | 117 +++++++++++++++++++++++------
 youtube_dl/extractor/extractors.py |   5 +-
 2 files changed, 100 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/dplay.py b/youtube_dl/extractor/dplay.py
index 32028bc3b..62e676389 100644
--- a/youtube_dl/extractor/dplay.py
+++ b/youtube_dl/extractor/dplay.py
@@ -6,37 +6,24 @@ import re
 import time
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
+from ..compat import (
+    compat_urlparse,
+    compat_HTTPError,
+)
 from ..utils import (
     USER_AGENTS,
+    ExtractorError,
     int_or_none,
+    unified_strdate,
+    remove_end,
     update_url_query,
 )
 
 
 class DPlayIE(InfoExtractor):
-    _VALID_URL = r'https?://(?P<domain>it\.dplay\.com|www\.dplay\.(?:dk|se|no))/[^/]+/(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?P<domain>www\.dplay\.(?:dk|se|no))/[^/]+/(?P<id>[^/?#]+)'
 
     _TESTS = [{
-        # geo restricted, via direct unsigned hls URL
-        'url': 'http://it.dplay.com/take-me-out/stagione-1-episodio-25/',
-        'info_dict': {
-            'id': '1255600',
-            'display_id': 'stagione-1-episodio-25',
-            'ext': 'mp4',
-            'title': 'Episodio 25',
-            'description': 'md5:cae5f40ad988811b197d2d27a53227eb',
-            'duration': 2761,
-            'timestamp': 1454701800,
-            'upload_date': '20160205',
-            'creator': 'RTIT',
-            'series': 'Take me out',
-            'season_number': 1,
-            'episode_number': 25,
-            'age_limit': 0,
-        },
-        'expected_warnings': ['Unable to download f4m manifest'],
-    }, {
         # non geo restricted, via secure api, unsigned download hls URL
         'url': 'http://www.dplay.se/nugammalt-77-handelser-som-format-sverige/season-1-svensken-lar-sig-njuta-av-livet/',
         'info_dict': {
@@ -168,3 +155,91 @@ class DPlayIE(InfoExtractor):
             'formats': formats,
             'subtitles': subtitles,
         }
+
+
+class DPlayItIE(InfoExtractor):
+    _VALID_URL = r'https?://it\.dplay\.com/[^/]+/[^/]+/(?P<id>[^/?#]+)'
+    _GEO_COUNTRIES = ['IT']
+    _TEST = {
+        'url': 'http://it.dplay.com/nove/biografie-imbarazzanti/luigi-di-maio-la-psicosi-di-stanislawskij/',
+        'md5': '2b808ffb00fc47b884a172ca5d13053c',
+        'info_dict': {
+            'id': '6918',
+            'display_id': 'luigi-di-maio-la-psicosi-di-stanislawskij',
+            'ext': 'mp4',
+            'title': 'Biografie imbarazzanti: Luigi Di Maio: la psicosi di Stanislawskij',
+            'description': 'md5:3c7a4303aef85868f867a26f5cc14813',
+            'thumbnail': r're:^https?://.*\.jpe?g',
+            'upload_date': '20160524',
+            'series': 'Biografie imbarazzanti',
+            'season_number': 1,
+            'episode': 'Luigi Di Maio: la psicosi di Stanislawskij',
+            'episode_number': 1,
+        },
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        video_id = self._search_regex(
+            r'url\s*:\s*["\']https://dplay-south-prod\.disco-api\.com/playback/videoPlaybackInfo/(\d+)',
+            webpage, 'video id')
+
+        title = remove_end(self._og_search_title(webpage), ' | Dplay')
+
+        try:
+            info = self._download_json(
+                'https://dplay-south-prod.disco-api.com/playback/videoPlaybackInfo/%s' % video_id,
+                display_id, headers={
+                    'Authorization': 'Bearer %s' % self._get_cookies(url).get(
+                        'dplayit_token').value,
+                    'Referer': url,
+                })
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code in (400, 403):
+                info = self._parse_json(e.cause.read().decode('utf-8'), display_id)
+                error = info['errors'][0]
+                if error.get('code') == 'access.denied.geoblocked':
+                    self.raise_geo_restricted(
+                        msg=error.get('detail'), countries=self._GEO_COUNTRIES)
+                raise ExtractorError(info['errors'][0]['detail'], expected=True)
+            raise
+
+        hls_url = info['data']['attributes']['streaming']['hls']['url']
+
+        formats = self._extract_m3u8_formats(
+            hls_url, display_id, ext='mp4', entry_protocol='m3u8_native',
+            m3u8_id='hls')
+
+        series = self._html_search_regex(
+            r'(?s)<h1[^>]+class=["\'].*?\bshow_title\b.*?["\'][^>]*>(.+?)</h1>',
+            webpage, 'series', fatal=False)
+        episode = self._search_regex(
+            r'<p[^>]+class=["\'].*?\bdesc_ep\b.*?["\'][^>]*>\s*<br/>\s*<b>([^<]+)',
+            webpage, 'episode', fatal=False)
+
+        mobj = re.search(
+            r'(?s)<span[^>]+class=["\']dates["\'][^>]*>.+?\bS\.(?P<season_number>\d+)\s+E\.(?P<episode_number>\d+)\s*-\s*(?P<upload_date>\d{2}/\d{2}/\d{4})',
+            webpage)
+        if mobj:
+            season_number = int(mobj.group('season_number'))
+            episode_number = int(mobj.group('episode_number'))
+            upload_date = unified_strdate(mobj.group('upload_date'))
+        else:
+            season_number = episode_number = upload_date = None
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': self._og_search_description(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'series': series,
+            'season_number': season_number,
+            'episode': episode,
+            'episode_number': episode_number,
+            'upload_date': upload_date,
+            'formats': formats,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index b056dff53..bd2762e47 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -246,7 +246,10 @@ from .dfb import DFBIE
 from .dhm import DHMIE
 from .dotsub import DotsubIE
 from .douyutv import DouyuTVIE
-from .dplay import DPlayIE
+from .dplay import (
+    DPlayIE,
+    DPlayItIE,
+)
 from .dramafever import (
     DramaFeverIE,
     DramaFeverSeriesIE,

From 0e7f9a9b48700efd40c4068b00364a7963dc9265 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 8 Mar 2017 21:30:30 +0700
Subject: [PATCH 0465/1696] [dplayit] Relax playback info URL extraction

---
 youtube_dl/extractor/dplay.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/dplay.py b/youtube_dl/extractor/dplay.py
index 62e676389..87c5dd63e 100644
--- a/youtube_dl/extractor/dplay.py
+++ b/youtube_dl/extractor/dplay.py
@@ -183,16 +183,15 @@ class DPlayItIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
-        video_id = self._search_regex(
-            r'url\s*:\s*["\']https://dplay-south-prod\.disco-api\.com/playback/videoPlaybackInfo/(\d+)',
+        info_url = self._search_regex(
+            r'url\s*:\s*["\']((?:https?:)?//[^/]+/playback/videoPlaybackInfo/\d+)',
             webpage, 'video id')
 
         title = remove_end(self._og_search_title(webpage), ' | Dplay')
 
         try:
             info = self._download_json(
-                'https://dplay-south-prod.disco-api.com/playback/videoPlaybackInfo/%s' % video_id,
-                display_id, headers={
+                info_url, display_id, headers={
                     'Authorization': 'Bearer %s' % self._get_cookies(url).get(
                         'dplayit_token').value,
                     'Referer': url,
@@ -231,7 +230,7 @@ class DPlayItIE(InfoExtractor):
             season_number = episode_number = upload_date = None
 
         return {
-            'id': video_id,
+            'id': info_url.rpartition('/')[-1],
             'display_id': display_id,
             'title': title,
             'description': self._og_search_description(webpage),

From 2913821723c826a0d2bfc16427592bf2b9d6d31c Mon Sep 17 00:00:00 2001
From: Thomas Christlieb <thomaschristlieb@hotmail.com>
Date: Thu, 9 Mar 2017 17:18:37 +0100
Subject: [PATCH 0466/1696] [prosiebensat1] Improve title extraction (closes
 #12318)

---
 youtube_dl/extractor/prosiebensat1.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 1245309a7..0cda992b4 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -369,7 +369,9 @@ class ProSiebenSat1IE(ProSiebenSat1BaseIE):
     def _extract_clip(self, url, webpage):
         clip_id = self._html_search_regex(
             self._CLIPID_REGEXES, webpage, 'clip id')
-        title = self._html_search_regex(self._TITLE_REGEXES, webpage, 'title')
+        title = self._html_search_regex(self._TITLE_REGEXES, webpage, 'title', default=None)
+        if title is None:
+            title = self._og_search_title(webpage)
         info = self._extract_video_info(url, clip_id)
         description = self._html_search_regex(
             self._DESCRIPTION_REGEXES, webpage, 'description', default=None)

From 76bee08fe72f154ad3754428eb80413ad0a3aa19 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Mar 2017 23:42:07 +0700
Subject: [PATCH 0467/1696] [prosiebensat1] Improve title extraction and add
 test

---
 youtube_dl/extractor/prosiebensat1.py | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 0cda992b4..d8a4bd244 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -300,6 +300,21 @@ class ProSiebenSat1IE(ProSiebenSat1BaseIE):
                 'skip_download': True,
             },
         },
+        {
+            # title in <h2 class="subtitle">
+            'url': 'http://www.prosieben.de/stars/oscar-award/videos/jetzt-erst-enthuellt-das-geheimnis-von-emma-stones-oscar-robe-clip',
+            'info_dict': {
+                'id': '4895826',
+                'ext': 'mp4',
+                'title': 'Jetzt erst enthüllt: Das Geheimnis von Emma Stones Oscar-Robe',
+                'description': 'md5:e5ace2bc43fadf7b63adc6187e9450b9',
+                'upload_date': '20170302',
+            },
+            'params': {
+                'skip_download': True,
+            },
+            'skip': 'geo restricted to Germany',
+        },
         {
             # geo restricted to Germany
             'url': 'http://www.kabeleinsdoku.de/tv/mayday-alarm-im-cockpit/video/102-notlandung-im-hudson-river-ganze-folge',
@@ -338,6 +353,7 @@ class ProSiebenSat1IE(ProSiebenSat1BaseIE):
         r'<header class="module_header">\s*<h2>([^<]+)</h2>\s*</header>',
         r'<h2 class="video-title" itemprop="name">\s*(.+?)</h2>',
         r'<div[^>]+id="veeseoTitle"[^>]*>(.+?)</div>',
+        r'<h2[^>]+class="subtitle"[^>]*>([^<]+)</h2>',
     ]
     _DESCRIPTION_REGEXES = [
         r'<p itemprop="description">\s*(.+?)</p>',
@@ -369,9 +385,9 @@ class ProSiebenSat1IE(ProSiebenSat1BaseIE):
     def _extract_clip(self, url, webpage):
         clip_id = self._html_search_regex(
             self._CLIPID_REGEXES, webpage, 'clip id')
-        title = self._html_search_regex(self._TITLE_REGEXES, webpage, 'title', default=None)
-        if title is None:
-            title = self._og_search_title(webpage)
+        title = self._html_search_regex(
+            self._TITLE_REGEXES, webpage, 'title',
+            default=None) or self._og_search_title(webpage)
         info = self._extract_video_info(url, clip_id)
         description = self._html_search_regex(
             self._DESCRIPTION_REGEXES, webpage, 'description', default=None)

From f802c4866017017afa642ca9ea1ea87edad081b4 Mon Sep 17 00:00:00 2001
From: runningbits <runningbits@users.noreply.github.com>
Date: Fri, 10 Mar 2017 16:59:32 +0100
Subject: [PATCH 0468/1696] [wdr:maus] Fix extraction and update tests

---
 youtube_dl/extractor/wdr.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index f7e6360a3..110999827 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -19,9 +19,9 @@ class WDRBaseIE(InfoExtractor):
     def _extract_wdr_video(self, webpage, display_id):
         # for wdr.de the data-extension is in a tag with the class "mediaLink"
         # for wdr.de radio players, in a tag with the class "wdrrPlayerPlayBtn"
-        # for wdrmaus its in a link to the page in a multiline "videoLink"-tag
+        # for wdrmaus it is in a link to the page in a multiline "videoLink"-tag
         json_metadata = self._html_search_regex(
-            r'class=(?:"(?:mediaLink|wdrrPlayerPlayBtn)\b[^"]*"[^>]+|"videoLink\b[^"]*"[\s]*>\n[^\n]*)data-extension="([^"]+)"',
+            r'class=(?:"(?:mediaLink|wdrrPlayerPlayBtn|videoButton)\b[^"]*"[^>]+|"videoLink\b[^"]*"[\s]*>\n[^\n]*)data-extension="([^"]+)"',
             webpage, 'media link', default=None, flags=re.MULTILINE)
 
         if not json_metadata:
@@ -161,23 +161,23 @@ class WDRIE(WDRBaseIE):
         {
             'url': 'http://www.wdrmaus.de/aktuelle-sendung/index.php5',
             'info_dict': {
-                'id': 'mdb-1096487',
-                'ext': 'flv',
+                'id': 'mdb-1323501',
+                'ext': 'mp4',
                 'upload_date': 're:^[0-9]{8}$',
                 'title': 're:^Die Sendung mit der Maus vom [0-9.]{10}$',
-                'description': '- Die Sendung mit der Maus -',
+                'description': 'Die Seite mit der Maus -',
             },
             'skip': 'The id changes from week to week because of the new episode'
         },
         {
-            'url': 'http://www.wdrmaus.de/sachgeschichten/sachgeschichten/achterbahn.php5',
+            'url': 'http://www.wdrmaus.de/filme/sachgeschichten/achterbahn.php5',
             'md5': '803138901f6368ee497b4d195bb164f2',
             'info_dict': {
                 'id': 'mdb-186083',
                 'ext': 'mp4',
                 'upload_date': '20130919',
                 'title': 'Sachgeschichte - Achterbahn ',
-                'description': '- Die Sendung mit der Maus -',
+                'description': 'Die Seite mit der Maus -',
             },
         },
         {
@@ -186,7 +186,7 @@ class WDRIE(WDRBaseIE):
             'info_dict': {
                 'id': 'mdb-869971',
                 'ext': 'flv',
-                'title': 'Funkhaus Europa Livestream',
+                'title': 'COSMO Livestream',
                 'description': 'md5:2309992a6716c347891c045be50992e4',
                 'upload_date': '20160101',
             },

From bd34c32bd754f30dd34b2d43604de73681b7148b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Mar 2017 23:07:36 +0700
Subject: [PATCH 0469/1696] [wdr] Actualize comment

---
 youtube_dl/extractor/wdr.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index 110999827..8bb7362bb 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -19,7 +19,8 @@ class WDRBaseIE(InfoExtractor):
     def _extract_wdr_video(self, webpage, display_id):
         # for wdr.de the data-extension is in a tag with the class "mediaLink"
         # for wdr.de radio players, in a tag with the class "wdrrPlayerPlayBtn"
-        # for wdrmaus it is in a link to the page in a multiline "videoLink"-tag
+        # for wdrmaus, in a tag with the class "videoButton" (previously a link
+        # to the page in a multiline "videoLink"-tag)
         json_metadata = self._html_search_regex(
             r'class=(?:"(?:mediaLink|wdrrPlayerPlayBtn|videoButton)\b[^"]*"[^>]+|"videoLink\b[^"]*"[\s]*>\n[^\n]*)data-extension="([^"]+)"',
             webpage, 'media link', default=None, flags=re.MULTILINE)
@@ -32,7 +33,7 @@ class WDRBaseIE(InfoExtractor):
         jsonp_url = media_link_obj['mediaObj']['url']
 
         metadata = self._download_json(
-            jsonp_url, 'metadata', transform_source=strip_jsonp)
+            jsonp_url, display_id, transform_source=strip_jsonp)
 
         metadata_tracker_data = metadata['trackerData']
         metadata_media_resource = metadata['mediaResource']

From c2ee861c6d9697d9b46fdad9719b087b87cfb481 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Mar 2017 23:16:53 +0700
Subject: [PATCH 0470/1696] [extractor/generic] Make title optional for
 jwplayer embeds (closes #12410)

---
 youtube_dl/extractor/generic.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index bc22421ae..ad47c3b6b 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2550,7 +2550,10 @@ class GenericIE(InfoExtractor):
             try:
                 jwplayer_data = self._parse_json(
                     jwplayer_data_str, video_id, transform_source=js_to_json)
-                return self._parse_jwplayer_data(jwplayer_data, video_id)
+                info = self._parse_jwplayer_data(
+                    jwplayer_data, video_id, require_title=False)
+                if not info.get('title'):
+                    info['title'] = video_title
             except ExtractorError:
                 pass
 

From f9e5c92c945bbc2c3c361f0e40f001d8f97e6b64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Mar 2017 23:23:24 +0700
Subject: [PATCH 0471/1696] [ChangeLog] Actualize

---
 ChangeLog | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index b000e2e94..e731e1998 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,14 @@
 version <unreleased>
 
 Extractors
+* [generic] Make title optional for jwplayer embeds (#12410)
+* [wdr:maus] Fix extraction (#12373)
+* [prosiebensat1] Improve title extraction (#12318, #12327)
+* [dplayit] Separate and rewrite extractor and bypass geo restriction (#12393)
 * [miomio] Fix extraction (#12291, #12388, #12402)
+* [telequebec] Fix description extraction (#12399)
+* [openload] Fix extraction (#12357)
+* [brightcove:legacy] Relax videoPlayer validation check (#12381)
 
 
 version 2017.03.07

From a8e687a4dac412ab66d63e7da93e032cfe440c05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Mar 2017 23:26:28 +0700
Subject: [PATCH 0472/1696] release 2017.03.10

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 1 +
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index eb69696c8..76e09c42a 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.03.07*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.03.07**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.03.10*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.03.10**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.03.07
+[debug] youtube-dl version 2017.03.10
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index e731e1998..848eba76b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.03.10
 
 Extractors
 * [generic] Make title optional for jwplayer embeds (#12410)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 85c59ca81..09dc830cb 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -212,6 +212,7 @@
  - **Dotsub**
  - **DouyuTV**: 斗鱼
  - **DPlay**
+ - **DPlayIt**
  - **dramafever**
  - **dramafever:series**
  - **DRBonanza**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index bd451bf81..d74046b37 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.03.07'
+__version__ = '2017.03.10'

From 4605c94d1a386a71f170dc46d491c4ef78828753 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Mar 2017 19:37:45 +0800
Subject: [PATCH 0473/1696] [__init__] Fix missing subtitles if --add-metadata
 is used (#12423)

The previous fix for #5594 is incorrect
---
 ChangeLog              |  6 ++++++
 youtube_dl/__init__.py | 15 ++++++++++-----
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 848eba76b..b1425e630 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Core
+* Fix missing subtitles if --add-metadata is used (#12423)
+
+
 version 2017.03.10
 
 Extractors
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index ad5f13d2b..c482f9375 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -259,6 +259,16 @@ def _real_main(argv=None):
             'key': 'FFmpegVideoConvertor',
             'preferedformat': opts.recodevideo,
         })
+    # FFmpegMetadataPP should be run after FFmpegVideoConvertorPP and
+    # FFmpegExtractAudioPP as containers before conversion may not support
+    # metadata (3gp, webm, etc.)
+    # And this post-processor should be placed before other metadata
+    # manipulating post-processors (FFmpegEmbedSubtitle) to prevent loss of
+    # extra metadata. By default ffmpeg preserves metadata applicable for both
+    # source and target containers. From this point the container won't change,
+    # so metadata can be added here.
+    if opts.addmetadata:
+        postprocessors.append({'key': 'FFmpegMetadata'})
     if opts.convertsubtitles:
         postprocessors.append({
             'key': 'FFmpegSubtitlesConvertor',
@@ -276,11 +286,6 @@ def _real_main(argv=None):
         })
         if not already_have_thumbnail:
             opts.writethumbnail = True
-    # FFmpegMetadataPP should be run after FFmpegVideoConvertorPP and
-    # FFmpegExtractAudioPP as containers before conversion may not support
-    # metadata (3gp, webm, etc.)
-    if opts.addmetadata:
-        postprocessors.append({'key': 'FFmpegMetadata'})
     # XAttrMetadataPP should be run after post-processors that may change file
     # contents
     if opts.xattrs:

From 70497994707b6bcbd6c0ecc3fb1073be4d75d970 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Mar 2017 23:16:51 +0700
Subject: [PATCH 0474/1696] [discoverygo:playlist] Add extractor (closes
 #12424)

---
 youtube_dl/extractor/discoverygo.py | 59 +++++++++++++++++++++++++++--
 youtube_dl/extractor/extractors.py  |  5 ++-
 2 files changed, 59 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/discoverygo.py b/youtube_dl/extractor/discoverygo.py
index 2042493a8..d14de8f00 100644
--- a/youtube_dl/extractor/discoverygo.py
+++ b/youtube_dl/extractor/discoverygo.py
@@ -1,17 +1,21 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
     extract_attributes,
+    ExtractorError,
     int_or_none,
     parse_age_limit,
-    ExtractorError,
+    remove_end,
+    unescapeHTML,
 )
 
 
-class DiscoveryGoIE(InfoExtractor):
-    _VALID_URL = r'''(?x)https?://(?:www\.)?(?:
+class DiscoveryGoBaseIE(InfoExtractor):
+    _VALID_URL_TEMPLATE = r'''(?x)https?://(?:www\.)?(?:
             discovery|
             investigationdiscovery|
             discoverylife|
@@ -21,7 +25,11 @@ class DiscoveryGoIE(InfoExtractor):
             sciencechannel|
             tlc|
             velocitychannel
-        )go\.com/(?:[^/]+/)*(?P<id>[^/?#&]+)'''
+        )go\.com/%s(?P<id>[^/?#&]+)'''
+
+
+class DiscoveryGoIE(DiscoveryGoBaseIE):
+    _VALID_URL = DiscoveryGoBaseIE._VALID_URL_TEMPLATE % r'(?:[^/]+/)+'
     _TEST = {
         'url': 'https://www.discoverygo.com/love-at-first-kiss/kiss-first-ask-questions-later/',
         'info_dict': {
@@ -113,3 +121,46 @@ class DiscoveryGoIE(InfoExtractor):
             'formats': formats,
             'subtitles': subtitles,
         }
+
+
+class DiscoveryGoPlaylistIE(DiscoveryGoBaseIE):
+    _VALID_URL = DiscoveryGoBaseIE._VALID_URL_TEMPLATE % ''
+    _TEST = {
+        'url': 'https://www.discoverygo.com/bering-sea-gold/',
+        'info_dict': {
+            'id': 'bering-sea-gold',
+            'title': 'Bering Sea Gold',
+            'description': 'md5:cc5c6489835949043c0cc3ad66c2fa0e',
+        },
+        'playlist_mincount': 6,
+    }
+
+    @classmethod
+    def suitable(cls, url):
+        return False if DiscoveryGoIE.suitable(url) else super(
+            DiscoveryGoPlaylistIE, cls).suitable(url)
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        entries = []
+        for mobj in re.finditer(r'data-json=(["\'])(?P<json>{.+?})\1', webpage):
+            data = self._parse_json(
+                mobj.group('json'), display_id,
+                transform_source=unescapeHTML, fatal=False)
+            if not isinstance(data, dict) or data.get('type') != 'episode':
+                continue
+            episode_url = data.get('socialUrl')
+            if not episode_url:
+                continue
+            entries.append(self.url_result(
+                episode_url, ie=DiscoveryGoIE.ie_key(),
+                video_id=data.get('id')))
+
+        return self.playlist_result(
+            entries, display_id,
+            remove_end(self._og_search_title(
+                webpage, fatal=False), ' | Discovery GO'),
+            self._og_search_description(webpage))
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index bd2762e47..24c478932 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -265,7 +265,10 @@ from .dvtv import DVTVIE
 from .dumpert import DumpertIE
 from .defense import DefenseGouvFrIE
 from .discovery import DiscoveryIE
-from .discoverygo import DiscoveryGoIE
+from .discoverygo import (
+    DiscoveryGoIE,
+    DiscoveryGoPlaylistIE,
+)
 from .disney import DisneyIE
 from .dispeak import DigitallySpeakingIE
 from .dropbox import DropboxIE

From a28f8d739683577ea632b1523eee418ed5177b73 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Mar 2017 23:18:42 +0700
Subject: [PATCH 0475/1696] [discoverygo] Bypass geo restriction

---
 youtube_dl/extractor/discoverygo.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/discoverygo.py b/youtube_dl/extractor/discoverygo.py
index d14de8f00..de94c4b09 100644
--- a/youtube_dl/extractor/discoverygo.py
+++ b/youtube_dl/extractor/discoverygo.py
@@ -30,6 +30,7 @@ class DiscoveryGoBaseIE(InfoExtractor):
 
 class DiscoveryGoIE(DiscoveryGoBaseIE):
     _VALID_URL = DiscoveryGoBaseIE._VALID_URL_TEMPLATE % r'(?:[^/]+/)+'
+    _GEO_COUNTRIES = ['US']
     _TEST = {
         'url': 'https://www.discoverygo.com/love-at-first-kiss/kiss-first-ask-questions-later/',
         'info_dict': {

From 57b0ddb35fd541f084f56e192fb46b4f61476664 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Mar 2017 23:21:08 +0700
Subject: [PATCH 0476/1696] [discoverygo] Actualize test

---
 youtube_dl/extractor/discoverygo.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/discoverygo.py b/youtube_dl/extractor/discoverygo.py
index de94c4b09..7cd5d4291 100644
--- a/youtube_dl/extractor/discoverygo.py
+++ b/youtube_dl/extractor/discoverygo.py
@@ -32,16 +32,16 @@ class DiscoveryGoIE(DiscoveryGoBaseIE):
     _VALID_URL = DiscoveryGoBaseIE._VALID_URL_TEMPLATE % r'(?:[^/]+/)+'
     _GEO_COUNTRIES = ['US']
     _TEST = {
-        'url': 'https://www.discoverygo.com/love-at-first-kiss/kiss-first-ask-questions-later/',
+        'url': 'https://www.discoverygo.com/bering-sea-gold/reaper-madness/',
         'info_dict': {
-            'id': '57a33c536b66d1cd0345eeb1',
+            'id': '58c167d86b66d12f2addeb01',
             'ext': 'mp4',
-            'title': 'Kiss First, Ask Questions Later!',
-            'description': 'md5:fe923ba34050eae468bffae10831cb22',
-            'duration': 2579,
-            'series': 'Love at First Kiss',
-            'season_number': 1,
-            'episode_number': 1,
+            'title': 'Reaper Madness',
+            'description': 'md5:09f2c625c99afb8946ed4fb7865f6e78',
+            'duration': 2519,
+            'series': 'Bering Sea Gold',
+            'season_number': 8,
+            'episode_number': 6,
             'age_limit': 14,
         },
     }

From 8c996232591848fe9c0c0585fd9958db4431c513 Mon Sep 17 00:00:00 2001
From: Starsam80 <samraskauskas@gmail.com>
Date: Sat, 11 Mar 2017 21:18:10 -0700
Subject: [PATCH 0477/1696] [crunchyroll] Extract season name

---
 youtube_dl/extractor/crunchyroll.py | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 9c6cf00ca..d15fd3744 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -177,6 +177,7 @@ class CrunchyrollIE(CrunchyrollBaseIE):
             'uploader': 'Kadokawa Pictures Inc.',
             'upload_date': '20170118',
             'series': "KONOSUBA -God's blessing on this wonderful world!",
+            'season': "KONOSUBA -God's blessing on this wonderful world! 2",
             'season_number': 2,
             'episode': 'Give Me Deliverance from this Judicial Injustice!',
             'episode_number': 1,
@@ -222,6 +223,23 @@ class CrunchyrollIE(CrunchyrollBaseIE):
             # just test metadata extraction
             'skip_download': True,
         },
+    }, {
+        # A video with a vastly different season name compared to the series name
+        'url': 'http://www.crunchyroll.com/nyarko-san-another-crawling-chaos/episode-1-test-590532',
+        'info_dict': {
+            'id': '590532',
+            'ext': 'mp4',
+            'title': 'Haiyoru! Nyaruani (ONA) Episode 1 – Test',
+            'description': 'Mahiro and Nyaruko talk about official certification.',
+            'uploader': 'TV TOKYO',
+            'upload_date': '20120305',
+            'series': 'Nyarko-san: Another Crawling Chaos',
+            'season': 'Haiyoru! Nyaruani (ONA)',
+        },
+        'params': {
+            # Just test metadata extraction
+            'skip_download': True,
+        },
     }]
 
     _FORMAT_IDS = {
@@ -491,7 +509,8 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         # webpage provide more accurate data than series_title from XML
         series = self._html_search_regex(
             r'id=["\']showmedia_about_episode_num[^>]+>\s*<a[^>]+>([^<]+)',
-            webpage, 'series', default=xpath_text(metadata, 'series_title'))
+            webpage, 'series', fatal=False)
+        season = xpath_text(metadata, 'series_title')
 
         episode = xpath_text(metadata, 'episode_title')
         episode_number = int_or_none(xpath_text(metadata, 'episode_number'))
@@ -508,6 +527,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             'uploader': video_uploader,
             'upload_date': video_upload_date,
             'series': series,
+            'season': season,
             'season_number': season_number,
             'episode': episode,
             'episode_number': episode_number,

From c1795ca6c8b1351a563a3e91023e46d18d59d52b Mon Sep 17 00:00:00 2001
From: Lucas M <lawlmart@users.noreply.github.com>
Date: Sun, 12 Mar 2017 11:51:59 -0700
Subject: [PATCH 0478/1696] [streamable] Update API URL

---
 youtube_dl/extractor/streamable.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/streamable.py b/youtube_dl/extractor/streamable.py
index e973c867c..9f5c237ef 100644
--- a/youtube_dl/extractor/streamable.py
+++ b/youtube_dl/extractor/streamable.py
@@ -65,7 +65,7 @@ class StreamableIE(InfoExtractor):
         # to return video info like the title properly sometimes, and doesn't
         # include info like the video duration
         video = self._download_json(
-            'https://streamable.com/ajax/videos/%s' % video_id, video_id)
+            'https://ajax.streamable.com/videos/%s' % video_id, video_id)
 
         # Format IDs:
         # 0 The video is being uploaded

From ff9d509d200577a0be962ee47894cd257c7ef818 Mon Sep 17 00:00:00 2001
From: Vijay Singh <sudovijay@users.noreply.github.com>
Date: Mon, 13 Mar 2017 01:52:35 +0530
Subject: [PATCH 0479/1696] [openload] Fix extraction

Just a minor fix for openload
---
 youtube_dl/extractor/openload.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 5a5607357..9a42ab895 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -100,7 +100,7 @@ class OpenloadIE(InfoExtractor):
             i = int(B, 16)
             index = (h / 2) % 10
             A = hashMap[index]
-            i = i ^ 137
+            i = i ^ 96
             i = i ^ A
             video_url_chars.append(compat_chr(i))
             h += 2

From e313d209c25bcf8adf3c888516624f5c9b2f2eaf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 13 Mar 2017 22:39:15 +0700
Subject: [PATCH 0480/1696] [mitele] Add support for ooyala videos (closes
 #12430)

---
 youtube_dl/extractor/mitele.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index 79e0b8ada..28b743cca 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import uuid
 
 from .common import InfoExtractor
+from .ooyala import OoyalaIE
 from ..compat import (
     compat_str,
     compat_urllib_parse_urlencode,
@@ -24,6 +25,9 @@ class MiTeleBaseIE(InfoExtractor):
             r'(?s)(<ms-video-player.+?</ms-video-player>)',
             webpage, 'ms video player'))
         video_id = player_data['data-media-id']
+        if player_data.get('data-cms-id') == 'ooyala':
+            return self.url_result(
+                'ooyala:%s' % video_id, ie=OoyalaIE.ie_key(), video_id=video_id)
         config_url = compat_urlparse.urljoin(url, player_data['data-config'])
         config = self._download_json(
             config_url, video_id, 'Downloading config JSON')

From 9d089630229e9d921da255fc6d3f671d307a0848 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 13 Mar 2017 22:41:28 +0700
Subject: [PATCH 0481/1696] [telecinco] Add test for #12430

---
 youtube_dl/extractor/telecinco.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
index d5abfc9e4..fdcc7d573 100644
--- a/youtube_dl/extractor/telecinco.py
+++ b/youtube_dl/extractor/telecinco.py
@@ -44,6 +44,10 @@ class TelecincoIE(MiTeleBaseIE):
     }, {
         'url': 'http://www.telecinco.es/espanasinirmaslejos/Espana-gran-destino-turistico_2_1240605043.html',
         'only_matching': True,
+    }, {
+        # ooyala video
+        'url': 'http://www.cuatro.com/chesterinlove/a-carta/chester-chester_in_love-chester_edu_2_2331030022.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 66bf351f8052fb71dce20c3a5ba1aa507532222e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Mar 2017 00:37:39 +0700
Subject: [PATCH 0482/1696] [facebook] Make title optional (closes #12443)

---
 youtube_dl/extractor/facebook.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 6315d40c5..b69c1ede0 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -196,6 +196,10 @@ class FacebookIE(InfoExtractor):
     }, {
         'url': 'https://www.facebookcorewwwi.onion/video.php?v=274175099429670',
         'only_matching': True,
+    }, {
+        # no title
+        'url': 'https://www.facebook.com/onlycleverentertainment/videos/1947995502095005/',
+        'only_matching': True,
     }]
 
     @staticmethod
@@ -353,15 +357,15 @@ class FacebookIE(InfoExtractor):
         self._sort_formats(formats)
 
         video_title = self._html_search_regex(
-            r'<h2\s+[^>]*class="uiHeaderTitle"[^>]*>([^<]*)</h2>', webpage, 'title',
-            default=None)
+            r'<h2\s+[^>]*class="uiHeaderTitle"[^>]*>([^<]*)</h2>', webpage,
+            'title', default=None)
         if not video_title:
             video_title = self._html_search_regex(
                 r'(?s)<span class="fbPhotosPhotoCaption".*?id="fbPhotoPageCaption"><span class="hasCaption">(.*?)</span>',
                 webpage, 'alternative title', default=None)
         if not video_title:
             video_title = self._html_search_meta(
-                'description', webpage, 'title')
+                'description', webpage, 'title', default=None)
         if video_title:
             video_title = limit_length(video_title, 80)
         else:

From 398887b4c09b3691379720314f4918bc094d1b7b Mon Sep 17 00:00:00 2001
From: Vijay Singh <sudovijay@users.noreply.github.com>
Date: Tue, 14 Mar 2017 05:19:18 +0530
Subject: [PATCH 0483/1696] [Openload] Fixed Extraction

They did changed it again.
---
 youtube_dl/extractor/openload.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 9a42ab895..5ea749f35 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -96,14 +96,16 @@ class OpenloadIE(InfoExtractor):
         h = 0
 
         while h < len(v):
-            B = v[h:h + 2]
+            B = v[h:h + 3]
             i = int(B, 16)
-            index = (h / 2) % 10
+            if (h / 3) % 3 == 0:
+                i = int(B, 8)
+            index = (h / 3) % 10
             A = hashMap[index]
-            i = i ^ 96
+            i = i ^ 47
             i = i ^ A
             video_url_chars.append(compat_chr(i))
-            h += 2
+            h += 3
 
         video_url = 'https://openload.co/stream/%s?mime=true'
         video_url = video_url % (''.join(video_url_chars))

From 2a751e137ffecf616f04f036bd89c87e967647bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 15 Mar 2017 01:58:59 +0700
Subject: [PATCH 0484/1696] [ChangeLog] Actualize

---
 ChangeLog | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index b1425e630..eaf1f7dbd 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -3,6 +3,15 @@ version <unreleased>
 Core
 * Fix missing subtitles if --add-metadata is used (#12423)
 
+Extractors
+* [facebook] Make title optional (#12443)
++ [mitele] Add support for ooyala videos (#12430)
+* [openload] Fix extraction (#12435, #12446)
+* [streamable] Update API URL (#12433)
++ [crunchyroll] Extract season name (#12428)
+* [discoverygo] Bypass geo restriction
++ [discoverygo:playlist] Add support for playlists (#12424)
+
 
 version 2017.03.10
 

From 5db83d79bfe192f8a7c80dd44fb2089f114a4189 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 15 Mar 2017 02:01:24 +0700
Subject: [PATCH 0485/1696] release 2017.03.15

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 1 +
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 76e09c42a..cd8592775 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.03.10*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.03.10**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.03.15*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.03.15**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.03.10
+[debug] youtube-dl version 2017.03.15
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index eaf1f7dbd..e10519792 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.03.15
 
 Core
 * Fix missing subtitles if --add-metadata is used (#12423)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 09dc830cb..cc0309f97 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -208,6 +208,7 @@
  - **Digiteka**
  - **Discovery**
  - **DiscoveryGo**
+ - **DiscoveryGoPlaylist**
  - **Disney**
  - **Dotsub**
  - **DouyuTV**: 斗鱼
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d74046b37..cbe686517 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.03.10'
+__version__ = '2017.03.15'

From ba448445b8baa66de92c65793f7ecba8927f0ce8 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 15 Mar 2017 01:40:54 +0100
Subject: [PATCH 0486/1696] [redbull] improve extraction

- extract 1080p quality
- correct ttml subtitle ext
- catch api errors
- reduce request size
---
 youtube_dl/extractor/redbulltv.py | 62 +++++++++++++++++++------------
 1 file changed, 39 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/redbulltv.py b/youtube_dl/extractor/redbulltv.py
index 5c73d5bca..afab62426 100644
--- a/youtube_dl/extractor/redbulltv.py
+++ b/youtube_dl/extractor/redbulltv.py
@@ -2,11 +2,13 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_HTTPError
 from ..utils import (
     float_or_none,
     int_or_none,
     try_get,
-    unified_timestamp,
+    # unified_timestamp,
+    ExtractorError,
 )
 
 
@@ -15,15 +17,15 @@ class RedBullTVIE(InfoExtractor):
     _TESTS = [{
         # film
         'url': 'https://www.redbull.tv/video/AP-1Q756YYX51W11/abc-of-wrc',
-        'md5': '78e860f631d7a846e712fab8c5fe2c38',
+        'md5': 'fb0445b98aa4394e504b413d98031d1f',
         'info_dict': {
             'id': 'AP-1Q756YYX51W11',
             'ext': 'mp4',
             'title': 'ABC of...WRC',
             'description': 'md5:5c7ed8f4015c8492ecf64b6ab31e7d31',
             'duration': 1582.04,
-            'timestamp': 1488405786,
-            'upload_date': '20170301',
+            # 'timestamp': 1488405786,
+            # 'upload_date': '20170301',
         },
     }, {
         # episode
@@ -34,8 +36,8 @@ class RedBullTVIE(InfoExtractor):
             'title': 'Grime - Hashtags S2 E4',
             'description': 'md5:334b741c8c1ce65be057eab6773c1cf5',
             'duration': 904.6,
-            'timestamp': 1487290093,
-            'upload_date': '20170217',
+            # 'timestamp': 1487290093,
+            # 'upload_date': '20170217',
             'series': 'Hashtags',
             'season_number': 2,
             'episode_number': 4,
@@ -48,29 +50,40 @@ class RedBullTVIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        access_token = self._download_json(
-            'https://api-v2.redbull.tv/start', video_id,
+        session = self._download_json(
+            'https://api-v2.redbull.tv/session', video_id,
             note='Downloading access token', query={
-                'build': '4.0.9',
-                'category': 'smartphone',
-                'os_version': 23,
-                'os_family': 'android',
-            })['auth']['access_token']
+                'build': '4.370.0',
+                'category': 'personal_computer',
+                'os_version': '1.0',
+                'os_family': 'http',
+            })
+        if session.get('code') == 'error':
+            raise ExtractorError('%s said: %s' % (
+                self.IE_NAME, session['message']))
+        auth = '%s %s' % (session.get('token_type', 'Bearer'), session['access_token'])
 
-        info = self._download_json(
-            'https://api-v2.redbull.tv/views/%s' % video_id,
-            video_id, note='Downloading video information',
-            headers={'Authorization': 'Bearer ' + access_token}
-        )['blocks'][0]['top'][0]
+        try:
+            info = self._download_json(
+                'https://api-v2.redbull.tv/content/%s' % video_id,
+                video_id, note='Downloading video information',
+                headers={'Authorization': auth}
+            )
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
+                error_message = self._parse_json(
+                    e.cause.read().decode(), video_id)['message']
+                raise ExtractorError('%s said: %s' % (
+                    self.IE_NAME, error_message), expected=True)
+            raise
 
         video = info['video_product']
 
         title = info['title'].strip()
-        m3u8_url = video['url']
 
         formats = self._extract_m3u8_formats(
-            m3u8_url, video_id, 'mp4', entry_protocol='m3u8_native',
-            m3u8_id='hls')
+            video['url'], video_id, 'mp4', 'm3u8_native')
+        self._sort_formats(formats)
 
         subtitles = {}
         for _, captions in (try_get(
@@ -82,9 +95,12 @@ class RedBullTVIE(InfoExtractor):
                 caption_url = caption.get('url')
                 if not caption_url:
                     continue
+                ext = caption.get('format')
+                if ext == 'xml':
+                    ext = 'ttml'
                 subtitles.setdefault(caption.get('lang') or 'en', []).append({
                     'url': caption_url,
-                    'ext': caption.get('format'),
+                    'ext': ext,
                 })
 
         subheading = info.get('subheading')
@@ -97,7 +113,7 @@ class RedBullTVIE(InfoExtractor):
             'description': info.get('long_description') or info.get(
                 'short_description'),
             'duration': float_or_none(video.get('duration'), scale=1000),
-            'timestamp': unified_timestamp(info.get('published')),
+            # 'timestamp': unified_timestamp(info.get('published')),
             'series': info.get('show_title'),
             'season_number': int_or_none(info.get('season_number')),
             'episode_number': int_or_none(info.get('episode_number')),

From a3096842856d0471b435fb0a85b295da7c4bcf7f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Mar 2017 03:28:01 +0700
Subject: [PATCH 0487/1696] [extractor/generic] Add forgotten return for
 jwplayer formats

---
 youtube_dl/extractor/generic.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ad47c3b6b..0fcb3fdac 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2554,6 +2554,7 @@ class GenericIE(InfoExtractor):
                     jwplayer_data, video_id, require_title=False)
                 if not info.get('title'):
                     info['title'] = video_title
+                return info
             except ExtractorError:
                 pass
 

From b51dc9db0e6ffc6a7725d92fa2c5de45a5b1be20 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Mar 2017 03:30:53 +0700
Subject: [PATCH 0488/1696] [extractor/common] Extract SMIL formats from
 jwplayer

---
 youtube_dl/extractor/common.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 78dc5be24..b51799bfa 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2247,6 +2247,9 @@ class InfoExtractor(object):
             elif ext == 'mpd':
                 formats.extend(self._extract_mpd_formats(
                     source_url, video_id, mpd_id=mpd_id, fatal=False))
+            elif ext == 'smil':
+                formats.extend(self._extract_smil_formats(
+                    source_url, video_id, fatal=False))
             # https://github.com/jwplayer/jwplayer/blob/master/src/js/providers/default.js#L67
             elif source_type.startswith('audio') or ext in (
                     'oga', 'aac', 'mp3', 'mpeg', 'vorbis'):

From 21bfcd3d6e41aed6113c874533fcfe41eb250d96 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 16 Mar 2017 12:50:45 +0100
Subject: [PATCH 0489/1696] [postprocessor/ffmpeg] Add support for flac

Requested at http://stackoverflow.com/q/42828041/35070
---
 youtube_dl/__init__.py             |  2 +-
 youtube_dl/postprocessor/ffmpeg.py | 34 +++++++++++++++++++-----------
 2 files changed, 23 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index c482f9375..2f640607f 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -196,7 +196,7 @@ def _real_main(argv=None):
     if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
         raise ValueError('Playlist end must be greater than playlist start')
     if opts.extractaudio:
-        if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
+        if opts.audioformat not in ['best', 'aac', 'flac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
             parser.error('invalid audio format specified')
     if opts.audioquality:
         opts.audioquality = opts.audioquality.strip('k').strip('K')
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 96ddb3b36..7c162d92a 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -26,15 +26,25 @@ from ..utils import (
 
 
 EXT_TO_OUT_FORMATS = {
-    "aac": "adts",
-    "m4a": "ipod",
-    "mka": "matroska",
-    "mkv": "matroska",
-    "mpg": "mpeg",
-    "ogv": "ogg",
-    "ts": "mpegts",
-    "wma": "asf",
-    "wmv": "asf",
+    'aac': 'adts',
+    'flac': 'flac',
+    'm4a': 'ipod',
+    'mka': 'matroska',
+    'mkv': 'matroska',
+    'mpg': 'mpeg',
+    'ogv': 'ogg',
+    'ts': 'mpegts',
+    'wma': 'asf',
+    'wmv': 'asf',
+}
+ACODECS = {
+    'mp3': 'libmp3lame',
+    'aac': 'aac',
+    'flac': 'flac',
+    'm4a': 'aac',
+    'opus': 'opus',
+    'vorbis': 'libvorbis',
+    'wav': None,
 }
 
 
@@ -237,7 +247,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
                 acodec = 'copy'
                 extension = 'm4a'
                 more_opts = ['-bsf:a', 'aac_adtstoasc']
-            elif filecodec in ['aac', 'mp3', 'vorbis', 'opus']:
+            elif filecodec in ['aac', 'flac', 'mp3', 'vorbis', 'opus']:
                 # Lossless if possible
                 acodec = 'copy'
                 extension = filecodec
@@ -256,8 +266,8 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
                     else:
                         more_opts += ['-b:a', self._preferredquality + 'k']
         else:
-            # We convert the audio (lossy)
-            acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'opus': 'opus', 'vorbis': 'libvorbis', 'wav': None}[self._preferredcodec]
+            # We convert the audio (lossy if codec is lossy)
+            acodec = ACODECS[self._preferredcodec]
             extension = self._preferredcodec
             more_opts = []
             if self._preferredquality is not None:

From 0efbc6b56d2b030e5dc98fa7f533a2e6cd41cf30 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 16 Mar 2017 12:54:47 +0100
Subject: [PATCH 0490/1696] [options] Mention flac support and sort
 alphabetically among the audio formats

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 8b51d3c6f..6b811535f 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -773,7 +773,7 @@ def parseOpts(overrideArguments=None):
         help='Convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
     postproc.add_option(
         '--audio-format', metavar='FORMAT', dest='audioformat', default='best',
-        help='Specify audio format: "best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; "%default" by default; No effect without -x')
+        help='Specify audio format: "best", "aac", "flac", "mp3", "m4a", "opus", "vorbis", or "wav"; "%default" by default; No effect without -x')
     postproc.add_option(
         '--audio-quality', metavar='QUALITY',
         dest='audioquality', default='5',

From 6ad476079db0dd806877cac1b73232d0ae16d50f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Mar 2017 22:39:48 +0700
Subject: [PATCH 0491/1696] [ChangeLog] Actualize

---
 ChangeLog | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index e10519792..da64f97ea 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,14 @@
+version <unreleased>
+
+Core
++ [postprocessor/ffmpeg] Add support for flac
++ [extractor/common] Extract SMIL formats from jwplayer
+
+Extractors
++ [generic] Add forgotten return for jwplayer formats
+* [redbulltv] Improve extraction
+
+
 version 2017.03.15
 
 Core

From 7d539ee10a8b0aeefb408ece19ce543f363006bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Mar 2017 22:42:12 +0700
Subject: [PATCH 0492/1696] release 2017.03.16

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 README.md                 | 5 +++--
 youtube_dl/version.py     | 2 +-
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index cd8592775..0e94b6cde 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.03.15*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.03.15**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.03.16*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.03.16**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.03.15
+[debug] youtube-dl version 2017.03.16
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index da64f97ea..75a8bd7a6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.03.16
 
 Core
 + [postprocessor/ffmpeg] Add support for flac
diff --git a/README.md b/README.md
index 0fc5984dc..86b44781c 100644
--- a/README.md
+++ b/README.md
@@ -375,8 +375,9 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
                                      (requires ffmpeg or avconv and ffprobe or
                                      avprobe)
     --audio-format FORMAT            Specify audio format: "best", "aac",
-                                     "vorbis", "mp3", "m4a", "opus", or "wav";
-                                     "best" by default; No effect without -x
+                                     "flac", "mp3", "m4a", "opus", "vorbis", or
+                                     "wav"; "best" by default; No effect without
+                                     -x
     --audio-quality QUALITY          Specify ffmpeg/avconv audio quality, insert
                                      a value between 0 (better) and 9 (worse)
                                      for VBR or a specific bitrate like 128K
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index cbe686517..f38f130bf 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.03.15'
+__version__ = '2017.03.16'

From 7f3590c43b8ae5cdba3c63e35e786083e3589485 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Mar 2017 00:00:01 +0700
Subject: [PATCH 0493/1696] [test_InfoExtractor] Add some realworld tests for
 _extract_jwplayer_data

---
 test/test_InfoExtractor.py | 93 +++++++++++++++++++++++++++++++++++++-
 1 file changed, 92 insertions(+), 1 deletion(-)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 437c7270e..881197afb 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -8,7 +8,7 @@ import sys
 import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from test.helper import FakeYDL
+from test.helper import FakeYDL, expect_dict
 from youtube_dl.extractor.common import InfoExtractor
 from youtube_dl.extractor import YoutubeIE, get_info_extractor
 from youtube_dl.utils import encode_data_uri, strip_jsonp, ExtractorError, RegexNotFoundError
@@ -84,6 +84,97 @@ class TestInfoExtractor(unittest.TestCase):
         self.assertRaises(ExtractorError, self.ie._download_json, uri, None)
         self.assertEqual(self.ie._download_json(uri, None, fatal=False), None)
 
+    def test_extract_jwplayer_data_realworld(self):
+        # from http://www.suffolk.edu/sjc/
+        expect_dict(
+            self,
+            self.ie._extract_jwplayer_data(r'''
+                <script type='text/javascript'>
+                    jwplayer('my-video').setup({
+                        file: 'rtmp://192.138.214.154/live/sjclive',
+                        fallback: 'true',
+                        width: '95%',
+                      aspectratio: '16:9',
+                      primary: 'flash',
+                      mediaid:'XEgvuql4'
+                    });
+                </script>
+                ''', None, require_title=False),
+            {
+                'id': 'XEgvuql4',
+                'formats': [{
+                    'url': 'rtmp://192.138.214.154/live/sjclive',
+                    'ext': 'flv'
+                }]
+            })
+
+        # from https://www.pornoxo.com/videos/7564/striptease-from-sexy-secretary/
+        expect_dict(
+            self,
+            self.ie._extract_jwplayer_data(r'''
+<script type="text/javascript">
+    jwplayer("mediaplayer").setup({
+        'videoid': "7564",
+        'width': "100%",
+        'aspectratio': "16:9",
+        'stretching': "exactfit",
+        'autostart': 'false',
+        'flashplayer': "https://t04.vipstreamservice.com/jwplayer/v5.10/player.swf",
+        'file': "https://cdn.pornoxo.com/key=MF+oEbaxqTKb50P-w9G3nA,end=1489689259,ip=104.199.146.27/ip=104.199.146.27/speed=6573765/buffer=3.0/2009-12/4b2157147afe5efa93ce1978e0265289c193874e02597.flv",
+        'image': "https://t03.vipstreamservice.com/thumbs/pxo-full/2009-12/14/a4b2157147afe5efa93ce1978e0265289c193874e02597.flv-full-13.jpg",
+        'filefallback': "https://cdn.pornoxo.com/key=9ZPsTR5EvPLQrBaak2MUGA,end=1489689259,ip=104.199.146.27/ip=104.199.146.27/speed=6573765/buffer=3.0/2009-12/m_4b2157147afe5efa93ce1978e0265289c193874e02597.mp4",
+        'logo.hide': true,
+        'skin': "https://t04.vipstreamservice.com/jwplayer/skin/modieus-blk.zip",
+        'plugins': "https://t04.vipstreamservice.com/jwplayer/dock/dockableskinnableplugin.swf",
+        'dockableskinnableplugin.piclink': "/index.php?key=ajax-videothumbsn&vid=7564&data=2009-12--14--4b2157147afe5efa93ce1978e0265289c193874e02597.flv--17370",
+        'controlbar': 'bottom',
+        'modes': [
+            {type: 'flash', src: 'https://t04.vipstreamservice.com/jwplayer/v5.10/player.swf'}
+        ],
+        'provider': 'http'
+    });
+    //noinspection JSAnnotator
+    invideo.setup({
+        adsUrl: "/banner-iframe/?zoneId=32",
+        adsUrl2: "",
+        autostart: false
+    });
+</script>
+            ''', 'dummy', require_title=False),
+            {
+                'thumbnail': 'https://t03.vipstreamservice.com/thumbs/pxo-full/2009-12/14/a4b2157147afe5efa93ce1978e0265289c193874e02597.flv-full-13.jpg',
+                'formats': [{
+                    'url': 'https://cdn.pornoxo.com/key=MF+oEbaxqTKb50P-w9G3nA,end=1489689259,ip=104.199.146.27/ip=104.199.146.27/speed=6573765/buffer=3.0/2009-12/4b2157147afe5efa93ce1978e0265289c193874e02597.flv',
+                    'ext': 'flv'
+                }]
+            })
+
+        # from http://www.indiedb.com/games/king-machine/videos
+        expect_dict(
+            self,
+            self.ie._extract_jwplayer_data(r'''
+<script>
+jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/\/www.indiedb.com\/","displaytitle":false,"autostart":false,"repeat":false,"title":"king machine trailer 1","sharing":{"link":"http:\/\/www.indiedb.com\/games\/king-machine\/videos\/king-machine-trailer-1","code":"<iframe width=\"560\" height=\"315\" src=\"http:\/\/www.indiedb.com\/media\/iframe\/1522983\" frameborder=\"0\" allowfullscreen><\/iframe><br><a href=\"http:\/\/www.indiedb.com\/games\/king-machine\/videos\/king-machine-trailer-1\">king machine trailer 1 - Indie DB<\/a>"},"related":{"file":"http:\/\/rss.indiedb.com\/media\/recommended\/1522983\/feed\/rss.xml","dimensions":"160x120","onclick":"link"},"sources":[{"file":"http:\/\/cdn.dbolical.com\/cache\/videos\/games\/1\/50\/49678\/encode_mp4\/king-machine-trailer.mp4","label":"360p SD","default":"true"},{"file":"http:\/\/cdn.dbolical.com\/cache\/videos\/games\/1\/50\/49678\/encode720p_mp4\/king-machine-trailer.mp4","label":"720p HD"}],"image":"http:\/\/media.indiedb.com\/cache\/images\/games\/1\/50\/49678\/thumb_620x2000\/king-machine-trailer.mp4.jpg","advertising":{"client":"vast","tag":"http:\/\/ads.intergi.com\/adrawdata\/3.0\/5205\/4251742\/0\/1013\/ADTECH;cors=yes;width=560;height=315;referring_url=http:\/\/www.indiedb.com\/games\/king-machine\/videos\/king-machine-trailer-1;content_url=http:\/\/www.indiedb.com\/games\/king-machine\/videos\/king-machine-trailer-1;media_id=1522983;title=king+machine+trailer+1;device=__DEVICE__;model=__MODEL__;os=Windows+OS;osversion=__OSVERSION__;ua=__UA__;ip=109.171.17.81;uniqueid=1522983;tags=__TAGS__;number=58cac25928151;time=1489683033"},"width":620,"height":349}).once("play", function(event) {
+            videoAnalytics("play");
+}).once("complete", function(event) {
+    videoAnalytics("completed");
+});
+</script>
+                ''', 'dummy'),
+            {
+                'title': 'king machine trailer 1',
+                'thumbnail': 'http://media.indiedb.com/cache/images/games/1/50/49678/thumb_620x2000/king-machine-trailer.mp4.jpg',
+                'formats': [{
+                    'url': 'http://cdn.dbolical.com/cache/videos/games/1/50/49678/encode_mp4/king-machine-trailer.mp4',
+                    'height': 360,
+                    'ext': 'mp4'
+                }, {
+                    'url': 'http://cdn.dbolical.com/cache/videos/games/1/50/49678/encode720p_mp4/king-machine-trailer.mp4',
+                    'height': 720,
+                    'ext': 'mp4'
+                }]
+            })
+
 
 if __name__ == '__main__':
     unittest.main()

From ea883a687c054692fcfe3cea15a22269044b64bb Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 17 Mar 2017 15:20:12 +0800
Subject: [PATCH 0494/1696] [openload] Fix extraction (closes #10408)

Thanks to @makgun02

Ref: http://pastebin.com/raw/JX9gHFUz
---
 ChangeLog                        |  6 +++++
 youtube_dl/extractor/openload.py | 43 ++++++++++++++++++++------------
 2 files changed, 33 insertions(+), 16 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 75a8bd7a6..eeb5813c5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [openload] Fix extraction (#10408)
+
+
 version 2017.03.16
 
 Core
diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 5ea749f35..fa876b127 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -78,34 +78,45 @@ class OpenloadIE(InfoExtractor):
         video_url_chars = []
 
         first_char = ord(ol_id[0])
-        key = first_char - 50
+        key = first_char - 55
         maxKey = max(2, key)
-        key = min(maxKey, len(ol_id) - 22)
-        t = ol_id[key:key + 20]
+        key = min(maxKey, len(ol_id) - 26)
+        t = ol_id[key:key + 24]
 
         hashMap = {}
-        v = ol_id.replace(t, "")
+        v = ol_id.replace(t, '')
         h = 0
 
         while h < len(t):
-            f = t[h:h + 2]
-            i = int(f, 16)
-            hashMap[h / 2] = i
-            h += 2
+            f = t[h:h + 3]
+            i = int(f, 8)
+            hashMap[h / 3] = i
+            h += 3
 
         h = 0
-
+        H = 0
         while h < len(v):
-            B = v[h:h + 3]
+            B = ''
+            C = ''
+            if len(v) >= h + 2:
+                B = v[h:h + 2]
+            if len(v) >= h + 3:
+                C = v[h:h + 3]
             i = int(B, 16)
-            if (h / 3) % 3 == 0:
-                i = int(B, 8)
-            index = (h / 3) % 10
+            h += 2
+            if H % 3 == 0:
+                i = int(C, 8)
+                h += 1
+            elif H % 2 == 0 and H != 0 and ord(v[H - 1]) < 60:
+                i = int(C, 10)
+                h += 1
+            index = H % 8
+
             A = hashMap[index]
-            i = i ^ 47
-            i = i ^ A
+            i ^= 213
+            i ^= A
             video_url_chars.append(compat_chr(i))
-            h += 3
+            H += 1
 
         video_url = 'https://openload.co/stream/%s?mime=true'
         video_url = video_url % (''.join(video_url_chars))

From 3e5856d860bb94b4dbe2fa38d9c50a6a92bb7401 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 17 Mar 2017 09:53:44 +0100
Subject: [PATCH 0495/1696] [discoverynetworks] add support for more domains
 and bypass geo restiction

---
 .../{tlc.py => discoverynetworks.py}          | 23 +++++++++++++------
 youtube_dl/extractor/extractors.py            |  2 +-
 2 files changed, 17 insertions(+), 8 deletions(-)
 rename youtube_dl/extractor/{tlc.py => discoverynetworks.py} (64%)

diff --git a/youtube_dl/extractor/tlc.py b/youtube_dl/extractor/discoverynetworks.py
similarity index 64%
rename from youtube_dl/extractor/tlc.py
rename to youtube_dl/extractor/discoverynetworks.py
index fd145ba42..b6653784c 100644
--- a/youtube_dl/extractor/tlc.py
+++ b/youtube_dl/extractor/discoverynetworks.py
@@ -9,13 +9,13 @@ from ..compat import (
     compat_parse_qs,
     compat_urlparse,
 )
+from ..utils import smuggle_url
 
 
-class TlcDeIE(InfoExtractor):
-    IE_NAME = 'tlc.de'
-    _VALID_URL = r'https?://(?:www\.)?tlc\.de/(?:[^/]+/)*videos/(?P<title>[^/?#]+)?(?:.*#(?P<id>\d+))?'
+class DiscoveryNetworksDeIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?(?:discovery|tlc|animalplanet|dmax)\.de/(?:.*#(?P<id>\d+)|(?:[^/]+/)*videos/(?P<title>[^/?#]+))'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.tlc.de/sendungen/breaking-amish/videos/#3235167922001',
         'info_dict': {
             'id': '3235167922001',
@@ -29,7 +29,13 @@ class TlcDeIE(InfoExtractor):
             'upload_date': '20140404',
             'uploader_id': '1659832546',
         },
-    }
+    }, {
+        'url': 'http://www.dmax.de/programme/storage-hunters-uk/videos/storage-hunters-uk-episode-6/',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.discovery.de/#5332316765001',
+        'only_matching': True,
+    }]
     BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/1659832546/default_default/index.html?videoId=%s'
 
     def _real_extract(self, url):
@@ -39,5 +45,8 @@ class TlcDeIE(InfoExtractor):
             title = mobj.group('title')
             webpage = self._download_webpage(url, title)
             brightcove_legacy_url = BrightcoveLegacyIE._extract_brightcove_url(webpage)
-            brightcove_id = compat_parse_qs(compat_urlparse.urlparse(brightcove_legacy_url).query)['@videoPlayer'][0]
-        return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)
+            brightcove_id = compat_parse_qs(compat_urlparse.urlparse(
+                brightcove_legacy_url).query)['@videoPlayer'][0]
+        return self.url_result(smuggle_url(
+            self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, {'geo_countries': ['DE']}),
+            'BrightcoveNew', brightcove_id)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 24c478932..79405b468 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -269,6 +269,7 @@ from .discoverygo import (
     DiscoveryGoIE,
     DiscoveryGoPlaylistIE,
 )
+from .discoverynetworks import DiscoveryNetworksDeIE
 from .disney import DisneyIE
 from .dispeak import DigitallySpeakingIE
 from .dropbox import DropboxIE
@@ -973,7 +974,6 @@ from .thisav import ThisAVIE
 from .thisoldhouse import ThisOldHouseIE
 from .threeqsdn import ThreeQSDNIE
 from .tinypic import TinyPicIE
-from .tlc import TlcDeIE
 from .tmz import (
     TMZIE,
     TMZArticleIE,

From e7a51a4c0235fafefc672d753017c770a306677a Mon Sep 17 00:00:00 2001
From: mrBliss <dewinant@gmail.com>
Date: Tue, 31 Jan 2017 13:59:18 +0100
Subject: [PATCH 0496/1696] [vtm] Add extractor (closes #9974)

Implementation of the approach described in #9974.
---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/vtm.py        | 136 +++++++++++++++++++++++++++++
 2 files changed, 137 insertions(+)
 create mode 100644 youtube_dl/extractor/vtm.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 79405b468..64316d4a8 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1174,6 +1174,7 @@ from .voxmedia import VoxMediaIE
 from .vporn import VpornIE
 from .vrt import VRTIE
 from .vrak import VrakIE
+from .vtm import VTMIE
 from .vube import VubeIE
 from .vuclip import VuClipIE
 from .vvvvid import VVVVIDIE
diff --git a/youtube_dl/extractor/vtm.py b/youtube_dl/extractor/vtm.py
new file mode 100644
index 000000000..f0a70040b
--- /dev/null
+++ b/youtube_dl/extractor/vtm.py
@@ -0,0 +1,136 @@
+from __future__ import unicode_literals
+
+import re
+
+from .generic import GenericIE
+from .common import InfoExtractor
+from ..utils import (
+    urlencode_postdata,
+    compat_urllib_parse_urlencode,
+    ExtractorError,
+    remove_end,
+)
+
+
+class VTMIE(InfoExtractor):
+    """Download full episodes that require an account from vtm.be or q2.be.
+
+    The generic extractor can be used to download clips that do no require an
+    account.
+    """
+    _VALID_URL = r'https?://(?:www\.)?(?P<site_id>vtm|q2)\.be/video[/?].+?'
+    _NETRC_MACHINE = 'vtm'
+    _APIKEY = '3_HZ0FtkMW_gOyKlqQzW5_0FHRC7Nd5XpXJZcDdXY4pk5eES2ZWmejRW5egwVm4ug-'
+    _TESTS = [
+        {
+            'url': 'http://vtm.be/video/volledige-afleveringen/id/vtm_20170219_VM0678361_vtmwatch',
+            'info_dict': {
+                'id': 'vtm_20170219_VM0678361_vtmwatch',
+                'ext': 'mp4',
+                'title': 'Allemaal Chris afl. 6',
+                'description': 'md5:4be86427521e7b07e0adb0c9c554ddb2',
+            },
+            'skip_download': True,
+        },
+        {
+            'url': 'http://vtm.be/video/volledige-afleveringen/id/257107153551000',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://vtm.be/video?aid=163157',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://www.q2.be/video/volledige-afleveringen/id/2be_20170301_VM0684442_q2',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://vtm.be/video?aid=168332',
+            'info_dict': {
+                'id': 'video?aid=168332',
+                'ext': 'mp4',
+                'title': 'Videozone',
+            },
+        },
+    ]
+
+    def _real_initialize(self):
+        self._logged_in = False
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None or password is None:
+            self.raise_login_required()
+
+        auth_data = {
+            'APIKey': self._APIKEY,
+            'sdk': 'js_6.1',
+            'format': 'json',
+            'loginID': username,
+            'password': password,
+        }
+
+        auth_info = self._download_json(
+            'https://accounts.eu1.gigya.com/accounts.login', None,
+            note='Logging in', errnote='Unable to log in',
+            data=urlencode_postdata(auth_data), fatal=True)
+
+        error_message = auth_info.get('errorDetails')
+        if error_message:
+            raise ExtractorError(
+                'Unable to login: %s' % error_message, expected=True)
+
+        self._uid = auth_info['UID']
+        self._uid_signature = auth_info['UIDSignature']
+        self._signature_timestamp = auth_info['signatureTimestamp']
+
+        self._logged_in = True
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        site_id = mobj.group('site_id')
+
+        webpage = self._download_webpage(url, None, "Downloading webpage")
+
+        # The URL sometimes contains the video id, but not always, e.g., test
+        # case 3. Fortunately, all webpages of videos requiring authentication
+        # contain the video id.
+        video_id = self._search_regex(
+            r'\\"vodId\\":\\"(.+?)\\"', webpage, 'video_id', default=None)
+
+        # It was most likely a video not requiring authentication.
+        if not video_id:
+            return self.url_result(url, 'Generic')
+
+        if not self._logged_in:
+            self._login()
+
+        title = self._html_search_regex(
+            r'\\"title\\":\\"(.+?)\\"', webpage, 'title', default=None)
+
+        description = self._html_search_regex(
+            r'<div[^>]+class="field-item\s+even">\s*<p>(.+?)</p>',
+            webpage, 'description', default=None)
+
+        data_url = 'http://vod.medialaan.io/api/1.0/item/%s/video' % video_id
+        m3u8_data = {
+            'app_id': 'vtm_watch' if site_id == 'vtm' else 'q2',
+            'user_network': 'vtm-sso',
+            'UID': self._uid,
+            'UIDSignature': self._uid_signature,
+            'signatureTimestamp': self._signature_timestamp,
+        }
+        data = self._download_json(data_url, video_id, query=m3u8_data)
+
+        formats = self._extract_m3u8_formats(
+            data['response']['uri'], video_id, entry_protocol='m3u8_native',
+            ext='mp4', m3u8_id='hls')
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'formats': formats,
+        }

From 2a721cdff2da0a9267c96ff2f4c19cda4ce0ab83 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Mar 2017 05:58:54 +0700
Subject: [PATCH 0497/1696] [medialaan] Fix and improve extraction (closes
 #11912)

---
 youtube_dl/extractor/extractors.py |   2 +-
 youtube_dl/extractor/medialaan.py  | 263 +++++++++++++++++++++++++++++
 youtube_dl/extractor/vtm.py        | 136 ---------------
 3 files changed, 264 insertions(+), 137 deletions(-)
 create mode 100644 youtube_dl/extractor/medialaan.py
 delete mode 100644 youtube_dl/extractor/vtm.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 64316d4a8..6b4742ed8 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1174,7 +1174,7 @@ from .voxmedia import VoxMediaIE
 from .vporn import VpornIE
 from .vrt import VRTIE
 from .vrak import VrakIE
-from .vtm import VTMIE
+from .medialaan import MedialaanIE
 from .vube import VubeIE
 from .vuclip import VuClipIE
 from .vvvvid import VVVVIDIE
diff --git a/youtube_dl/extractor/medialaan.py b/youtube_dl/extractor/medialaan.py
new file mode 100644
index 000000000..e70d4679d
--- /dev/null
+++ b/youtube_dl/extractor/medialaan.py
@@ -0,0 +1,263 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    parse_duration,
+    try_get,
+    unified_timestamp,
+    urlencode_postdata,
+)
+
+
+class MedialaanIE(InfoExtractor):
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:www\.)?
+                        (?:
+                            (?P<site_id>vtm|q2|vtmkzoom)\.be/
+                            (?:
+                                video(?:/[^/]+/id/|/?\?.*?\baid=)|
+                                (?:[^/]+/)*
+                            )
+                        )
+                        (?P<id>[^/?#&]+)
+                    '''
+    _NETRC_MACHINE = 'medialaan'
+    _APIKEY = '3_HZ0FtkMW_gOyKlqQzW5_0FHRC7Nd5XpXJZcDdXY4pk5eES2ZWmejRW5egwVm4ug-'
+    _SITE_TO_APP_ID = {
+        'vtm': 'vtm_watch',
+        'q2': 'q2',
+        'vtmkzoom': 'vtmkzoom',
+    }
+    _TESTS = [{
+        # vod
+        'url': 'http://vtm.be/video/volledige-afleveringen/id/vtm_20170219_VM0678361_vtmwatch',
+        'info_dict': {
+            'id': 'vtm_20170219_VM0678361_vtmwatch',
+            'ext': 'mp4',
+            'title': 'Allemaal Chris afl. 6',
+            'description': 'md5:4be86427521e7b07e0adb0c9c554ddb2',
+            'timestamp': 1487533280,
+            'upload_date': '20170219',
+            'duration': 2562,
+            'series': 'Allemaal Chris',
+            'season': 'Allemaal Chris',
+            'season_number': 1,
+            'season_id': '256936078124527',
+            'episode': 'Allemaal Chris afl. 6',
+            'episode_number': 6,
+            'episode_id': '256936078591527',
+        },
+        'params': {
+            'skip_download': True,
+        },
+        'skip': 'Requires account credentials',
+    }, {
+        # clip
+        'url': 'http://vtm.be/video?aid=168332',
+        'info_dict': {
+            'id': '168332',
+            'ext': 'mp4',
+            'title': '"Veronique liegt!"',
+            'description': 'md5:1385e2b743923afe54ba4adc38476155',
+            'timestamp': 1489002029,
+            'upload_date': '20170308',
+            'duration': 96,
+        },
+    }, {
+        # vod
+        'url': 'http://vtm.be/video/volledige-afleveringen/id/257107153551000',
+        'only_matching': True,
+    }, {
+        # vod
+        'url': 'http://vtm.be/video?aid=163157',
+        'only_matching': True,
+    }, {
+        # vod
+        'url': 'http://www.q2.be/video/volledige-afleveringen/id/2be_20170301_VM0684442_q2',
+        'only_matching': True,
+    }, {
+        # clip
+        'url': 'http://vitaya.be/de-jurk/precies-je-hebt-geen-borsten',
+        'only_matching': True,
+    }, {
+        # clip
+        'url': 'http://vtmkzoom.be/k3-dansstudio/een-nieuw-seizoen-van-k3-dansstudio',
+        'only_matching': True,
+    }]
+
+    def _real_initialize(self):
+        self._logged_in = False
+
+    def _login(self):
+        username, password = self._get_login_info()
+        if username is None:
+            self.raise_login_required()
+
+        auth_data = {
+            'APIKey': self._APIKEY,
+            'sdk': 'js_6.1',
+            'format': 'json',
+            'loginID': username,
+            'password': password,
+        }
+
+        auth_info = self._download_json(
+            'https://accounts.eu1.gigya.com/accounts.login', None,
+            note='Logging in', errnote='Unable to log in',
+            data=urlencode_postdata(auth_data))
+
+        error_message = auth_info.get('errorDetails') or auth_info.get('errorMessage')
+        if error_message:
+            raise ExtractorError(
+                'Unable to login: %s' % error_message, expected=True)
+
+        self._uid = auth_info['UID']
+        self._uid_signature = auth_info['UIDSignature']
+        self._signature_timestamp = auth_info['signatureTimestamp']
+
+        self._logged_in = True
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id, site_id = mobj.group('id', 'site_id')
+
+        webpage = self._download_webpage(url, video_id)
+
+        config = self._parse_json(
+            self._search_regex(
+                r'videoJSConfig\s*=\s*JSON\.parse\(\'({.+?})\'\);',
+                webpage, 'config', default='{}'), video_id,
+            transform_source=lambda s: s.replace(
+                '\\\\', '\\').replace(r'\"', '"').replace(r"\'", "'"))
+
+        vod_id = config.get('vodId') or self._search_regex(
+            (r'\\"vodId\\"\s*:\s*\\"(.+?)\\"',
+             r'<[^>]+id=["\']vod-(\d+)'),
+            webpage, 'video_id', default=None)
+
+        # clip, no authentication required
+        if not vod_id:
+            player = self._parse_json(
+                self._search_regex(
+                    r'vmmaplayer\(({.+?})\);', webpage, 'vmma player',
+                    default=''),
+                video_id, transform_source=lambda s: '[%s]' % s, fatal=False)
+            if player:
+                video = player[-1]
+                info = {
+                    'id': video_id,
+                    'url': video['videoUrl'],
+                    'title': video['title'],
+                    'thumbnail': video.get('imageUrl'),
+                    'timestamp': int_or_none(video.get('createdDate')),
+                    'duration': int_or_none(video.get('duration')),
+                }
+            else:
+                info = self._parse_html5_media_entries(
+                    url, webpage, video_id, m3u8_id='hls')[0]
+                info.update({
+                    'id': video_id,
+                    'title': self._html_search_meta('description', webpage),
+                    'duration': parse_duration(self._html_search_meta('duration', webpage)),
+                })
+        # vod, authentication required
+        else:
+            if not self._logged_in:
+                self._login()
+
+            settings = self._parse_json(
+                self._search_regex(
+                    r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);',
+                    webpage, 'drupal settings', default='{}'),
+                video_id)
+
+            def get(container, item):
+                return try_get(
+                    settings, lambda x: x[container][item],
+                    compat_str) or self._search_regex(
+                    r'"%s"\s*:\s*"([^"]+)' % item, webpage, item,
+                    default=None)
+
+            app_id = get('vod', 'app_id') or self._SITE_TO_APP_ID.get(site_id, 'vtm_watch')
+            sso = get('vod', 'gigyaDatabase') or 'vtm-sso'
+
+            data = self._download_json(
+                'http://vod.medialaan.io/api/1.0/item/%s/video' % vod_id,
+                video_id, query={
+                    'app_id': app_id,
+                    'user_network': sso,
+                    'UID': self._uid,
+                    'UIDSignature': self._uid_signature,
+                    'signatureTimestamp': self._signature_timestamp,
+                })
+
+            formats = self._extract_m3u8_formats(
+                data['response']['uri'], video_id, entry_protocol='m3u8_native',
+                ext='mp4', m3u8_id='hls')
+
+            self._sort_formats(formats)
+
+            info = {
+                'id': vod_id,
+                'formats': formats,
+            }
+
+            api_key = get('vod', 'apiKey')
+            channel = get('medialaanGigya', 'channel')
+
+            if api_key:
+                videos = self._download_json(
+                    'http://vod.medialaan.io/vod/v2/videos', video_id, fatal=False,
+                    query={
+                        'channels': channel,
+                        'ids': vod_id,
+                        'limit': 1,
+                        'apikey': api_key,
+                    })
+                if videos:
+                    video = try_get(
+                        videos, lambda x: x['response']['videos'][0], dict)
+                    if video:
+                        def get(container, item, expected_type=None):
+                            return try_get(
+                                video, lambda x: x[container][item], expected_type)
+
+                        def get_string(container, item):
+                            return get(container, item, compat_str)
+
+                        info.update({
+                            'series': get_string('program', 'title'),
+                            'season': get_string('season', 'title'),
+                            'season_number': int_or_none(get('season', 'number')),
+                            'season_id': get_string('season', 'id'),
+                            'episode': get_string('episode', 'title'),
+                            'episode_number': int_or_none(get('episode', 'number')),
+                            'episode_id': get_string('episode', 'id'),
+                            'duration': int_or_none(
+                                video.get('duration')) or int_or_none(
+                                video.get('durationMillis'), scale=1000),
+                            'title': get_string('episode', 'title'),
+                            'description': get_string('episode', 'text'),
+                            'timestamp': unified_timestamp(get_string(
+                                'publication', 'begin')),
+                        })
+
+            if not info.get('title'):
+                info['title'] = try_get(
+                    config, lambda x: x['videoConfig']['title'],
+                    compat_str) or self._html_search_regex(
+                    r'\\"title\\"\s*:\s*\\"(.+?)\\"', webpage, 'title',
+                    default=None) or self._og_search_title(webpage)
+
+        if not info.get('description'):
+            info['description'] = self._html_search_regex(
+                r'<div[^>]+class="field-item\s+even">\s*<p>(.+?)</p>',
+                webpage, 'description', default=None)
+
+        return info
diff --git a/youtube_dl/extractor/vtm.py b/youtube_dl/extractor/vtm.py
deleted file mode 100644
index f0a70040b..000000000
--- a/youtube_dl/extractor/vtm.py
+++ /dev/null
@@ -1,136 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .generic import GenericIE
-from .common import InfoExtractor
-from ..utils import (
-    urlencode_postdata,
-    compat_urllib_parse_urlencode,
-    ExtractorError,
-    remove_end,
-)
-
-
-class VTMIE(InfoExtractor):
-    """Download full episodes that require an account from vtm.be or q2.be.
-
-    The generic extractor can be used to download clips that do no require an
-    account.
-    """
-    _VALID_URL = r'https?://(?:www\.)?(?P<site_id>vtm|q2)\.be/video[/?].+?'
-    _NETRC_MACHINE = 'vtm'
-    _APIKEY = '3_HZ0FtkMW_gOyKlqQzW5_0FHRC7Nd5XpXJZcDdXY4pk5eES2ZWmejRW5egwVm4ug-'
-    _TESTS = [
-        {
-            'url': 'http://vtm.be/video/volledige-afleveringen/id/vtm_20170219_VM0678361_vtmwatch',
-            'info_dict': {
-                'id': 'vtm_20170219_VM0678361_vtmwatch',
-                'ext': 'mp4',
-                'title': 'Allemaal Chris afl. 6',
-                'description': 'md5:4be86427521e7b07e0adb0c9c554ddb2',
-            },
-            'skip_download': True,
-        },
-        {
-            'url': 'http://vtm.be/video/volledige-afleveringen/id/257107153551000',
-            'only_matching': True,
-        },
-        {
-            'url': 'http://vtm.be/video?aid=163157',
-            'only_matching': True,
-        },
-        {
-            'url': 'http://www.q2.be/video/volledige-afleveringen/id/2be_20170301_VM0684442_q2',
-            'only_matching': True,
-        },
-        {
-            'url': 'http://vtm.be/video?aid=168332',
-            'info_dict': {
-                'id': 'video?aid=168332',
-                'ext': 'mp4',
-                'title': 'Videozone',
-            },
-        },
-    ]
-
-    def _real_initialize(self):
-        self._logged_in = False
-
-    def _login(self):
-        (username, password) = self._get_login_info()
-        if username is None or password is None:
-            self.raise_login_required()
-
-        auth_data = {
-            'APIKey': self._APIKEY,
-            'sdk': 'js_6.1',
-            'format': 'json',
-            'loginID': username,
-            'password': password,
-        }
-
-        auth_info = self._download_json(
-            'https://accounts.eu1.gigya.com/accounts.login', None,
-            note='Logging in', errnote='Unable to log in',
-            data=urlencode_postdata(auth_data), fatal=True)
-
-        error_message = auth_info.get('errorDetails')
-        if error_message:
-            raise ExtractorError(
-                'Unable to login: %s' % error_message, expected=True)
-
-        self._uid = auth_info['UID']
-        self._uid_signature = auth_info['UIDSignature']
-        self._signature_timestamp = auth_info['signatureTimestamp']
-
-        self._logged_in = True
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        site_id = mobj.group('site_id')
-
-        webpage = self._download_webpage(url, None, "Downloading webpage")
-
-        # The URL sometimes contains the video id, but not always, e.g., test
-        # case 3. Fortunately, all webpages of videos requiring authentication
-        # contain the video id.
-        video_id = self._search_regex(
-            r'\\"vodId\\":\\"(.+?)\\"', webpage, 'video_id', default=None)
-
-        # It was most likely a video not requiring authentication.
-        if not video_id:
-            return self.url_result(url, 'Generic')
-
-        if not self._logged_in:
-            self._login()
-
-        title = self._html_search_regex(
-            r'\\"title\\":\\"(.+?)\\"', webpage, 'title', default=None)
-
-        description = self._html_search_regex(
-            r'<div[^>]+class="field-item\s+even">\s*<p>(.+?)</p>',
-            webpage, 'description', default=None)
-
-        data_url = 'http://vod.medialaan.io/api/1.0/item/%s/video' % video_id
-        m3u8_data = {
-            'app_id': 'vtm_watch' if site_id == 'vtm' else 'q2',
-            'user_network': 'vtm-sso',
-            'UID': self._uid,
-            'UIDSignature': self._uid_signature,
-            'signatureTimestamp': self._signature_timestamp,
-        }
-        data = self._download_json(data_url, video_id, query=m3u8_data)
-
-        formats = self._extract_m3u8_formats(
-            data['response']['uri'], video_id, entry_protocol='m3u8_native',
-            ext='mp4', m3u8_id='hls')
-
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'formats': formats,
-        }

From 5f0daab1ca60803f4f49b344ddb3757c418a2d8e Mon Sep 17 00:00:00 2001
From: Vijay Singh <sudovijay@users.noreply.github.com>
Date: Sat, 18 Mar 2017 04:32:55 +0530
Subject: [PATCH 0498/1696] [openload] Fix extraction

---
 youtube_dl/extractor/openload.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index fa876b127..435aec28e 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -80,8 +80,8 @@ class OpenloadIE(InfoExtractor):
         first_char = ord(ol_id[0])
         key = first_char - 55
         maxKey = max(2, key)
-        key = min(maxKey, len(ol_id) - 26)
-        t = ol_id[key:key + 24]
+        key = min(maxKey, len(ol_id) - 38)
+        t = ol_id[key:key + 36]
 
         hashMap = {}
         v = ol_id.replace(t, '')
@@ -110,7 +110,7 @@ class OpenloadIE(InfoExtractor):
             elif H % 2 == 0 and H != 0 and ord(v[H - 1]) < 60:
                 i = int(C, 10)
                 h += 1
-            index = H % 8
+            index = H % 12
 
             A = hashMap[index]
             i ^= 213

From febfe1e2626bab5dbb8d4e0bbe31aa225ce09d35 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Mar 2017 06:19:11 +0700
Subject: [PATCH 0499/1696] [adobepass] Detect and output error on authz token
 extraction (#12472)

---
 youtube_dl/extractor/adobepass.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
index d4816abf5..1b2d364ca 100644
--- a/youtube_dl/extractor/adobepass.py
+++ b/youtube_dl/extractor/adobepass.py
@@ -1458,6 +1458,8 @@ class AdobePassIE(InfoExtractor):
                     self._downloader.cache.store(self._MVPD_CACHE, requestor_id, {})
                     count += 1
                     continue
+                if '<error' in authorize:
+                    raise ExtractorError(xml_text(authorize, 'details'), expected=True)
                 authz_token = unescapeHTML(xml_text(authorize, 'authzToken'))
                 requestor_info[guid] = authz_token
                 self._downloader.cache.store(self._MVPD_CACHE, requestor_id, requestor_info)

From f68ef1e2abd876ffca65544fd3e42756f9c33be3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Mar 2017 23:23:47 +0700
Subject: [PATCH 0500/1696] [medialaan] Remove unrelated test

---
 youtube_dl/extractor/medialaan.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/youtube_dl/extractor/medialaan.py b/youtube_dl/extractor/medialaan.py
index e70d4679d..6e067474b 100644
--- a/youtube_dl/extractor/medialaan.py
+++ b/youtube_dl/extractor/medialaan.py
@@ -81,10 +81,6 @@ class MedialaanIE(InfoExtractor):
         # vod
         'url': 'http://www.q2.be/video/volledige-afleveringen/id/2be_20170301_VM0684442_q2',
         'only_matching': True,
-    }, {
-        # clip
-        'url': 'http://vitaya.be/de-jurk/precies-je-hebt-geen-borsten',
-        'only_matching': True,
     }, {
         # clip
         'url': 'http://vtmkzoom.be/k3-dansstudio/een-nieuw-seizoen-van-k3-dansstudio',

From 772b5ff57f702dd76986d1db17068da2116a2800 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 19 Mar 2017 00:45:04 +0100
Subject: [PATCH 0501/1696] [toongoggles] Add new extractor(closes #12171)

---
 youtube_dl/extractor/extractors.py  |  1 +
 youtube_dl/extractor/toongoggles.py | 81 +++++++++++++++++++++++++++++
 2 files changed, 82 insertions(+)
 create mode 100644 youtube_dl/extractor/toongoggles.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 6b4742ed8..97d68d9ca 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -986,6 +986,7 @@ from .tnaflix import (
 )
 from .toggle import ToggleIE
 from .tonline import TOnlineIE
+from .toongoggles import ToonGogglesIE
 from .toutv import TouTvIE
 from .toypics import ToypicsUserIE, ToypicsIE
 from .traileraddict import TrailerAddictIE
diff --git a/youtube_dl/extractor/toongoggles.py b/youtube_dl/extractor/toongoggles.py
new file mode 100644
index 000000000..b5ba1c01d
--- /dev/null
+++ b/youtube_dl/extractor/toongoggles.py
@@ -0,0 +1,81 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_duration,
+)
+
+
+class ToonGogglesIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?toongoggles\.com/shows/(?P<show_id>\d+)(?:/[^/]+/episodes/(?P<episode_id>\d+))?'
+    _TESTS = [{
+        'url': 'http://www.toongoggles.com/shows/217143/bernard-season-2/episodes/217147/football',
+        'md5': '18289fc2b951eff6b953a9d8f01e6831',
+        'info_dict': {
+            'id': '217147',
+            'ext': 'mp4',
+            'title': 'Football',
+            'uploader_id': '1',
+            'description': 'Bernard decides to play football in order to be better than Lloyd and tries to beat him no matter how, he even cheats.',
+            'upload_date': '20160718',
+            'timestamp': 1468879330,
+        }
+    }, {
+        'url': 'http://www.toongoggles.com/shows/227759/om-nom-stories-around-the-world',
+        'info_dict': {
+            'id': '227759',
+            'title': 'Om Nom Stories Around The World',
+        },
+        'playlist_mincount': 11,
+    }]
+
+    def _call_api(self, action, page_id, query):
+        query.update({
+            'for_ng': 1,
+            'for_web': 1,
+            'show_meta': 1,
+            'version': 7.0,
+        })
+        return self._download_json('http://api.toongoggles.com/' + action, page_id, query=query)
+
+    def _parse_episode_data(self, episode_data):
+        title = episode_data['episode_name']
+
+        return {
+            '_type': 'url_transparent',
+            'id': episode_data['episode_id'],
+            'title': title,
+            'url': 'kaltura:513551:' + episode_data['entry_id'],
+            'thumbnail': episode_data.get('thumbnail_url'),
+            'description': episode_data.get('description'),
+            'duration': parse_duration(episode_data.get('hms')),
+            'series': episode_data.get('show_name'),
+            'season_number': int_or_none(episode_data.get('season_num')),
+            'episode_id': episode_data.get('episode_id'),
+            'episode': title,
+            'episode_number': int_or_none(episode_data.get('episode_num')),
+            'categories': episode_data.get('categories'),
+            'ie_key': 'Kaltura',
+        }
+
+    def _real_extract(self, url):
+        show_id, episode_id = re.match(self._VALID_URL, url).groups()
+        if episode_id:
+            episode_data = self._call_api('search', episode_id, {
+                'filter': 'episode',
+                'id': episode_id,
+            })['objects'][0]
+            return self._parse_episode_data(episode_data)
+        else:
+            show_data = self._call_api('getepisodesbyshow', show_id, {
+                'max': 1000000000,
+                'showid': show_id,
+            })
+            entries = []
+            for episode_data in show_data.get('objects', []):
+                entries.append(self._parse_episode_data(episode_data))
+            return self.playlist_result(entries, show_id, show_data.get('show_name'))

From 46b18f2349670d395b9d84a57ee3d9b5d221ff4b Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Wed, 8 Mar 2017 18:13:54 -0500
Subject: [PATCH 0502/1696] [BostonGlobe] New. Nonstandard version of
 Brightcove.

Has a "data-brightcove-video-id" instead of a "data-video-id," otherwise
pretty much just Brightcove. Except the Globe isn't all Brightcove
videos, so fallback to Generic, too.

Also, abstract playlist_from_matches() from generic.py to common.py, and use
it here.

History of these changes can be found in
51170427d4b1143572a498dedaee61863a5b2c5b.
---
 youtube_dl/extractor/bostonglobe.py | 72 +++++++++++++++++++++++++++
 youtube_dl/extractor/common.py      | 28 +++++++----
 youtube_dl/extractor/extractors.py  |  1 +
 youtube_dl/extractor/generic.py     | 77 +++++++++++++----------------
 4 files changed, 126 insertions(+), 52 deletions(-)
 create mode 100644 youtube_dl/extractor/bostonglobe.py

diff --git a/youtube_dl/extractor/bostonglobe.py b/youtube_dl/extractor/bostonglobe.py
new file mode 100644
index 000000000..57882fbee
--- /dev/null
+++ b/youtube_dl/extractor/bostonglobe.py
@@ -0,0 +1,72 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+from ..utils import (
+    extract_attributes,
+)
+
+
+class BostonGlobeIE(InfoExtractor):
+    _VALID_URL = r'(?i)https?://(?:www\.)?bostonglobe\.com/.*/(?P<id>[^/]+)/\w+(?:\.html)?'
+    _TESTS = [
+        {
+            'url': 'http://www.bostonglobe.com/metro/2017/02/11/tree-finally-succumbs-disease-leaving-hole-neighborhood/h1b4lviqzMTIn9sVy8F3gP/story.html',
+            'md5': '0a62181079c85c2d2b618c9a738aedaf',
+            'info_dict': {
+                'title': 'A tree finally succumbs to disease, leaving a hole in a neighborhood',
+                'id': '5320421710001',
+                'ext': 'mp4',
+                'description': 'It arrived as a sapling when the Back Bay was in its infancy, a spindly American elm tamped down into a square of dirt cut into the brick sidewalk of 1880s Marlborough Street, no higher than the first bay window of the new brownstone behind it.',
+                'timestamp': 1486877593,
+                'upload_date': '20170212',
+                'uploader_id': '245991542',
+            },
+        },
+        {
+            # Embedded youtube video; we hand it off to the Generic extractor.
+            'url': 'https://www.bostonglobe.com/lifestyle/names/2017/02/17/does-ben-affleck-play-matt-damon-favorite-version-batman/ruqkc9VxKBYmh5txn1XhSI/story.html',
+            'md5': '582b40327089d5c0c949b3c54b13c24b',
+            'info_dict': {
+                'title': "Who Is Matt Damon's Favorite Batman?",
+                'id': 'ZW1QCnlA6Qc',
+                'ext': 'mp4',
+                'upload_date': '20170217',
+                'description': 'md5:3b3dccb9375867e0b4d527ed87d307cb',
+                'uploader': 'The Late Late Show with James Corden',
+                'uploader_id': 'TheLateLateShow',
+            },
+            'expected_warnings': ['404'],
+        },
+    ]
+
+    def _real_extract(self, url):
+        page_id = self._match_id(url)
+        webpage = self._download_webpage(url, page_id)
+
+        page_title = self._og_search_title(webpage, default=None)
+
+        # <video data-brightcove-video-id="5320421710001" data-account="245991542" data-player="SJWAiyYWg" data-embed="default" class="video-js" controls itemscope itemtype="http://schema.org/VideoObject">
+        entries = []
+        for video in re.findall(r'(?i)(<video[^>]+>)', webpage):
+            attrs = extract_attributes(video)
+
+            video_id = attrs.get('data-brightcove-video-id')
+            account_id = attrs.get('data-account')
+            player_id = attrs.get('data-player')
+            embed = attrs.get('data-embed')
+
+            if video_id and account_id and player_id and embed:
+                entries.append(
+                    'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
+                    % (account_id, player_id, embed, video_id))
+
+        if len(entries) == 0:
+            return self.url_result(url, 'Generic')
+        elif len(entries) == 1:
+            return self.url_result(entries[0], 'BrightcoveNew')
+        else:
+            return self.playlist_from_matches(entries, page_id, page_title, ie='BrightcoveNew')
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b51799bfa..0852b8e8c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -36,34 +36,35 @@ from ..utils import (
     clean_html,
     compiled_regex_type,
     determine_ext,
+    determine_protocol,
     error_to_compat_str,
     ExtractorError,
+    extract_attributes,
     fix_xml_ampersands,
     float_or_none,
     GeoRestrictedError,
     GeoUtils,
     int_or_none,
     js_to_json,
+    mimetype2ext,
+    orderedSet,
+    parse_codecs,
+    parse_duration,
     parse_iso8601,
+    parse_m3u8_attributes,
     RegexNotFoundError,
-    sanitize_filename,
     sanitized_Request,
+    sanitize_filename,
     unescapeHTML,
     unified_strdate,
     unified_timestamp,
+    update_Request,
+    update_url_query,
+    urljoin,
     url_basename,
     xpath_element,
     xpath_text,
     xpath_with_ns,
-    determine_protocol,
-    parse_duration,
-    mimetype2ext,
-    update_Request,
-    update_url_query,
-    parse_m3u8_attributes,
-    extract_attributes,
-    parse_codecs,
-    urljoin,
 )
 
 
@@ -714,6 +715,13 @@ class InfoExtractor(object):
             video_info['title'] = video_title
         return video_info
 
+    def playlist_from_matches(self, matches, video_id, video_title, getter=None, ie=None):
+        urlrs = orderedSet(
+            self.url_result(self._proto_relative_url(getter(m) if getter else m), ie)
+            for m in matches)
+        return self.playlist_result(
+            urlrs, playlist_id=video_id, playlist_title=video_title)
+
     @staticmethod
     def playlist_result(entries, playlist_id=None, playlist_title=None, playlist_description=None):
         """Returns a playlist"""
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 97d68d9ca..40a5c9842 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -117,6 +117,7 @@ from .bleacherreport import (
 from .blinkx import BlinkxIE
 from .bloomberg import BloombergIE
 from .bokecc import BokeCCIE
+from .bostonglobe import BostonGlobeIE
 from .bpb import BpbIE
 from .br import BRIE
 from .bravotv import BravoTVIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 0fcb3fdac..a71d6bac0 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1841,14 +1841,6 @@ class GenericIE(InfoExtractor):
         video_description = self._og_search_description(webpage, default=None)
         video_thumbnail = self._og_search_thumbnail(webpage, default=None)
 
-        # Helper method
-        def _playlist_from_matches(matches, getter=None, ie=None):
-            urlrs = orderedSet(
-                self.url_result(self._proto_relative_url(getter(m) if getter else m), ie)
-                for m in matches)
-            return self.playlist_result(
-                urlrs, playlist_id=video_id, playlist_title=video_title)
-
         # Look for Brightcove Legacy Studio embeds
         bc_urls = BrightcoveLegacyIE._extract_brightcove_urls(webpage)
         if bc_urls:
@@ -1869,28 +1861,28 @@ class GenericIE(InfoExtractor):
         # Look for Brightcove New Studio embeds
         bc_urls = BrightcoveNewIE._extract_urls(webpage)
         if bc_urls:
-            return _playlist_from_matches(bc_urls, ie='BrightcoveNew')
+            return self.playlist_from_matches(bc_urls, video_id, video_title, ie='BrightcoveNew')
 
         # Look for ThePlatform embeds
         tp_urls = ThePlatformIE._extract_urls(webpage)
         if tp_urls:
-            return _playlist_from_matches(tp_urls, ie='ThePlatform')
+            return self.playlist_from_matches(tp_urls, video_id, video_title, ie='ThePlatform')
 
         # Look for Vessel embeds
         vessel_urls = VesselIE._extract_urls(webpage)
         if vessel_urls:
-            return _playlist_from_matches(vessel_urls, ie=VesselIE.ie_key())
+            return self.playlist_from_matches(vessel_urls, video_id, video_title, ie=VesselIE.ie_key())
 
         # Look for embedded rtl.nl player
         matches = re.findall(
             r'<iframe[^>]+?src="((?:https?:)?//(?:www\.)?rtl\.nl/system/videoplayer/[^"]+(?:video_)?embed[^"]+)"',
             webpage)
         if matches:
-            return _playlist_from_matches(matches, ie='RtlNl')
+            return self.playlist_from_matches(matches, video_id, video_title, ie='RtlNl')
 
         vimeo_urls = VimeoIE._extract_urls(url, webpage)
         if vimeo_urls:
-            return _playlist_from_matches(vimeo_urls, ie=VimeoIE.ie_key())
+            return self.playlist_from_matches(vimeo_urls, video_id, video_title, ie=VimeoIE.ie_key())
 
         vid_me_embed_url = self._search_regex(
             r'src=[\'"](https?://vid\.me/[^\'"]+)[\'"]',
@@ -1912,25 +1904,25 @@ class GenericIE(InfoExtractor):
                 (?:embed|v|p)/.+?)
             \1''', webpage)
         if matches:
-            return _playlist_from_matches(
-                matches, lambda m: unescapeHTML(m[1]))
+            return self.playlist_from_matches(
+                matches, video_id, video_title, lambda m: unescapeHTML(m[1]))
 
         # Look for lazyYT YouTube embed
         matches = re.findall(
             r'class="lazyYT" data-youtube-id="([^"]+)"', webpage)
         if matches:
-            return _playlist_from_matches(matches, lambda m: unescapeHTML(m))
+            return self.playlist_from_matches(matches, video_id, video_title, lambda m: unescapeHTML(m))
 
         # Look for Wordpress "YouTube Video Importer" plugin
         matches = re.findall(r'''(?x)<div[^>]+
             class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
             data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
         if matches:
-            return _playlist_from_matches(matches, lambda m: m[-1])
+            return self.playlist_from_matches(matches, video_id, video_title, lambda m: m[-1])
 
         matches = DailymotionIE._extract_urls(webpage)
         if matches:
-            return _playlist_from_matches(matches)
+            return self.playlist_from_matches(matches, video_id, video_title)
 
         # Look for embedded Dailymotion playlist player (#3822)
         m = re.search(
@@ -1939,8 +1931,8 @@ class GenericIE(InfoExtractor):
             playlists = re.findall(
                 r'list\[\]=/playlist/([^/]+)/', unescapeHTML(m.group('url')))
             if playlists:
-                return _playlist_from_matches(
-                    playlists, lambda p: '//dailymotion.com/playlist/%s' % p)
+                return self.playlist_from_matches(
+                    playlists, video_id, video_title, lambda p: '//dailymotion.com/playlist/%s' % p)
 
         # Look for embedded Wistia player
         match = re.search(
@@ -2047,8 +2039,9 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             embeds = self._parse_json(mobj.group(1), video_id, fatal=False)
             if embeds:
-                return _playlist_from_matches(
-                    embeds, getter=lambda v: OoyalaIE._url_for_embed_code(smuggle_url(v['provider_video_id'], {'domain': url})), ie='Ooyala')
+                return self.playlist_from_matches(
+                    embeds, video_id, video_title,
+                    getter=lambda v: OoyalaIE._url_for_embed_code(smuggle_url(v['provider_video_id'], {'domain': url})), ie='Ooyala')
 
         # Look for Aparat videos
         mobj = re.search(r'<iframe .*?src="(http://www\.aparat\.com/video/[^"]+)"', webpage)
@@ -2110,13 +2103,13 @@ class GenericIE(InfoExtractor):
         # Look for funnyordie embed
         matches = re.findall(r'<iframe[^>]+?src="(https?://(?:www\.)?funnyordie\.com/embed/[^"]+)"', webpage)
         if matches:
-            return _playlist_from_matches(
-                matches, getter=unescapeHTML, ie='FunnyOrDie')
+            return self.playlist_from_matches(
+                matches, video_id, video_title, getter=unescapeHTML, ie='FunnyOrDie')
 
         # Look for BBC iPlayer embed
         matches = re.findall(r'setPlaylist\("(https?://www\.bbc\.co\.uk/iplayer/[^/]+/[\da-z]{8})"\)', webpage)
         if matches:
-            return _playlist_from_matches(matches, ie='BBCCoUk')
+            return self.playlist_from_matches(matches, video_id, video_title, ie='BBCCoUk')
 
         # Look for embedded RUTV player
         rutv_url = RUTVIE._extract_url(webpage)
@@ -2131,32 +2124,32 @@ class GenericIE(InfoExtractor):
         # Look for embedded SportBox player
         sportbox_urls = SportBoxEmbedIE._extract_urls(webpage)
         if sportbox_urls:
-            return _playlist_from_matches(sportbox_urls, ie='SportBoxEmbed')
+            return self.playlist_from_matches(sportbox_urls, video_id, video_title, ie='SportBoxEmbed')
 
         # Look for embedded XHamster player
         xhamster_urls = XHamsterEmbedIE._extract_urls(webpage)
         if xhamster_urls:
-            return _playlist_from_matches(xhamster_urls, ie='XHamsterEmbed')
+            return self.playlist_from_matches(xhamster_urls, video_id, video_title, ie='XHamsterEmbed')
 
         # Look for embedded TNAFlixNetwork player
         tnaflix_urls = TNAFlixNetworkEmbedIE._extract_urls(webpage)
         if tnaflix_urls:
-            return _playlist_from_matches(tnaflix_urls, ie=TNAFlixNetworkEmbedIE.ie_key())
+            return self.playlist_from_matches(tnaflix_urls, video_id, video_title, ie=TNAFlixNetworkEmbedIE.ie_key())
 
         # Look for embedded PornHub player
         pornhub_urls = PornHubIE._extract_urls(webpage)
         if pornhub_urls:
-            return _playlist_from_matches(pornhub_urls, ie=PornHubIE.ie_key())
+            return self.playlist_from_matches(pornhub_urls, video_id, video_title, ie=PornHubIE.ie_key())
 
         # Look for embedded DrTuber player
         drtuber_urls = DrTuberIE._extract_urls(webpage)
         if drtuber_urls:
-            return _playlist_from_matches(drtuber_urls, ie=DrTuberIE.ie_key())
+            return self.playlist_from_matches(drtuber_urls, video_id, video_title, ie=DrTuberIE.ie_key())
 
         # Look for embedded RedTube player
         redtube_urls = RedTubeIE._extract_urls(webpage)
         if redtube_urls:
-            return _playlist_from_matches(redtube_urls, ie=RedTubeIE.ie_key())
+            return self.playlist_from_matches(redtube_urls, video_id, video_title, ie=RedTubeIE.ie_key())
 
         # Look for embedded Tvigle player
         mobj = re.search(
@@ -2202,12 +2195,12 @@ class GenericIE(InfoExtractor):
         # Look for embedded soundcloud player
         soundcloud_urls = SoundcloudIE._extract_urls(webpage)
         if soundcloud_urls:
-            return _playlist_from_matches(soundcloud_urls, getter=unescapeHTML, ie=SoundcloudIE.ie_key())
+            return self.playlist_from_matches(soundcloud_urls, video_id, video_title, getter=unescapeHTML, ie=SoundcloudIE.ie_key())
 
         # Look for tunein player
         tunein_urls = TuneInBaseIE._extract_urls(webpage)
         if tunein_urls:
-            return _playlist_from_matches(tunein_urls)
+            return self.playlist_from_matches(tunein_urls, video_id, video_title)
 
         # Look for embedded mtvservices player
         mtvservices_url = MTVServicesEmbeddedIE._extract_url(webpage)
@@ -2490,35 +2483,35 @@ class GenericIE(InfoExtractor):
         # Look for DBTV embeds
         dbtv_urls = DBTVIE._extract_urls(webpage)
         if dbtv_urls:
-            return _playlist_from_matches(dbtv_urls, ie=DBTVIE.ie_key())
+            return self.playlist_from_matches(dbtv_urls, video_id, video_title, ie=DBTVIE.ie_key())
 
         # Look for Videa embeds
         videa_urls = VideaIE._extract_urls(webpage)
         if videa_urls:
-            return _playlist_from_matches(videa_urls, ie=VideaIE.ie_key())
+            return self.playlist_from_matches(videa_urls, video_id, video_title, ie=VideaIE.ie_key())
 
         # Look for 20 minuten embeds
         twentymin_urls = TwentyMinutenIE._extract_urls(webpage)
         if twentymin_urls:
-            return _playlist_from_matches(
-                twentymin_urls, ie=TwentyMinutenIE.ie_key())
+            return self.playlist_from_matches(
+                twentymin_urls, video_id, video_title, ie=TwentyMinutenIE.ie_key())
 
         # Look for Openload embeds
         openload_urls = OpenloadIE._extract_urls(webpage)
         if openload_urls:
-            return _playlist_from_matches(
-                openload_urls, ie=OpenloadIE.ie_key())
+            return self.playlist_from_matches(
+                openload_urls, video_id, video_title, ie=OpenloadIE.ie_key())
 
         # Look for VideoPress embeds
         videopress_urls = VideoPressIE._extract_urls(webpage)
         if videopress_urls:
-            return _playlist_from_matches(
-                videopress_urls, ie=VideoPressIE.ie_key())
+            return self.playlist_from_matches(
+                videopress_urls, video_id, video_title, ie=VideoPressIE.ie_key())
 
         # Look for Rutube embeds
         rutube_urls = RutubeIE._extract_urls(webpage)
         if rutube_urls:
-            return _playlist_from_matches(
+            return self.playlist_from_matches(
                 rutube_urls, ie=RutubeIE.ie_key())
 
         # Looking for http://schema.org/VideoObject

From 68220649fa0b1c06c16a80ce51cc21f8d3264a4c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 19 Mar 2017 20:42:17 +0800
Subject: [PATCH 0503/1696] [ChangeLog] Update after #12099

---
 ChangeLog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChangeLog b/ChangeLog
index eeb5813c5..d70637b69 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
++ [bostonglobe] Add extractor for bostonglobe.com (#12099)
 * [openload] Fix extraction (#10408)
 
 

From 45e6ad21b4f024c1721dc3dd2b53f15d7efa8aa6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Mar 2017 23:48:02 +0700
Subject: [PATCH 0504/1696] Credit @mrBliss for vtm (#11912)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 273a6a034..2d676b210 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -209,3 +209,4 @@ Olivier Bilodeau
 Lars Vierbergen
 Juanjo Benages
 Xiao Di Guan
+Thomas Winant

From 9487ce03e998337cbc69db250009e11b52c3b255 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Mar 2017 23:59:40 +0700
Subject: [PATCH 0505/1696] [YoutubeDL] Allow multiple input URLs to be used
 with stdout as output template

---
 youtube_dl/YoutubeDL.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 13a3a909e..cb502c26f 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1872,6 +1872,7 @@ class YoutubeDL(object):
         """Download a given list of URLs."""
         outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
         if (len(url_list) > 1 and
+                outtmpl != '-' and
                 '%' not in outtmpl and
                 self.params.get('max_downloads') != 1):
             raise SameFileError(outtmpl)

From 0ecdd3adbd104786c901944a316b87f58056bcdf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Mar 2017 00:03:58 +0700
Subject: [PATCH 0506/1696] [ChangeLog] Actualize

---
 ChangeLog | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index d70637b69..5a316f0ac 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,15 @@
 version <unreleased>
 
+Core
++ [YoutubeDL] Allow multiple input URLs to be used with stdout (-) as
+  output template
++ [adobepass] Detect and output error on authz token extraction (#12472)
+
 Extractors
 + [bostonglobe] Add extractor for bostonglobe.com (#12099)
++ [toongoggles] Add support for toongoggles.com (#12171)
++ [medialaan] Add support for Medialaan sites (#9974, #11912)
++ [discoverynetworks] Add support for more domains and bypass geo restiction
 * [openload] Fix extraction (#10408)
 
 

From 0e9a73e6120965fc2c2a1a2a1a30f7d38af4c73a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Mar 2017 00:07:57 +0700
Subject: [PATCH 0507/1696] release 2017.03.20

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 5 ++++-
 youtube_dl/version.py     | 2 +-
 4 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 0e94b6cde..4273fedbf 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.03.16*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.03.16**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.03.20*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.03.20**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.03.16
+[debug] youtube-dl version 2017.03.20
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 5a316f0ac..bbbf3c34d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.03.20
 
 Core
 + [YoutubeDL] Allow multiple input URLs to be used with stdout (-) as
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index cc0309f97..6a7be28cb 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -108,6 +108,7 @@
  - **blinkx**
  - **Bloomberg**
  - **BokeCC**
+ - **BostonGlobe**
  - **Bpb**: Bundeszentrale für politische Bildung
  - **BR**: Bayerischer Rundfunk Mediathek
  - **BravoTV**
@@ -209,6 +210,7 @@
  - **Discovery**
  - **DiscoveryGo**
  - **DiscoveryGoPlaylist**
+ - **DiscoveryNetworksDe**
  - **Disney**
  - **Dotsub**
  - **DouyuTV**: 斗鱼
@@ -425,6 +427,7 @@
  - **MatchTV**
  - **MDR**: MDR.DE and KiKA
  - **media.ccc.de**
+ - **Medialaan**
  - **Meipai**: 美拍
  - **MelonVOD**
  - **META**
@@ -777,12 +780,12 @@
  - **ThisAV**
  - **ThisOldHouse**
  - **tinypic**: tinypic.com videos
- - **tlc.de**
  - **TMZ**
  - **TMZArticle**
  - **TNAFlix**
  - **TNAFlixNetworkEmbed**
  - **toggle**
+ - **ToonGoggles**
  - **Tosh**: Tosh.0
  - **tou.tv**
  - **Toypics**: Toypics user profile
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index f38f130bf..a65f2e741 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.03.16'
+__version__ = '2017.03.20'

From 957f453429d584615ac4d2277caeb0d75d0fe1d9 Mon Sep 17 00:00:00 2001
From: Vijay Singh <sudovijay@users.noreply.github.com>
Date: Mon, 20 Mar 2017 09:22:32 +0530
Subject: [PATCH 0508/1696] [Openload.co] Fixed Extraction

They did it again. just a minor change though. here's quick fix
---
 youtube_dl/extractor/openload.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 435aec28e..58ffde541 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -110,7 +110,7 @@ class OpenloadIE(InfoExtractor):
             elif H % 2 == 0 and H != 0 and ord(v[H - 1]) < 60:
                 i = int(C, 10)
                 h += 1
-            index = H % 12
+            index = H % 7
 
             A = hashMap[index]
             i ^= 213

From 8a8cc339b6b5189b3c1fdb15ef7224c035b21a6b Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Mon, 20 Mar 2017 11:35:13 -0400
Subject: [PATCH 0509/1696] [senateisvp] Allow https URL scheme for embeds

---
 youtube_dl/extractor/senateisvp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/senateisvp.py b/youtube_dl/extractor/senateisvp.py
index 387a4f7f6..db5ef8b57 100644
--- a/youtube_dl/extractor/senateisvp.py
+++ b/youtube_dl/extractor/senateisvp.py
@@ -89,7 +89,7 @@ class SenateISVPIE(InfoExtractor):
     @staticmethod
     def _search_iframe_url(webpage):
         mobj = re.search(
-            r"<iframe[^>]+src=['\"](?P<url>http://www\.senate\.gov/isvp/?\?[^'\"]+)['\"]",
+            r"<iframe[^>]+src=['\"](?P<url>https?://www\.senate\.gov/isvp/?\?[^'\"]+)['\"]",
             webpage)
         if mobj:
             return mobj.group('url')

From 97952bdb78854bf09c688eb535dc7b67265934c1 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Tue, 21 Mar 2017 13:12:14 -0400
Subject: [PATCH 0510/1696] [generic] Add test for Senate ISVP iframe embed

---
 youtube_dl/extractor/generic.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a71d6bac0..cb6308d29 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1542,6 +1542,17 @@ class GenericIE(InfoExtractor):
             'url': 'http://www.golfchannel.com/topics/shows/golftalkcentral.htm',
             'only_matching': True,
         },
+        {
+            # Senate ISVP iframe https
+            'url': 'https://www.hsgac.senate.gov/hearings/canadas-fast-track-refugee-plan-unanswered-questions-and-implications-for-us-national-security',
+            'md5': 'fb8c70b0b515e5037981a2492099aab8',
+            'info_dict': {
+                'id': 'govtaff020316',
+                'ext': 'mp4',
+                'title': 'Integrated Senate Video Player',
+            },
+            'add_ie': [SenateISVPIE.ie_key()],
+        },
         # {
         #     # TODO: find another test
         #     # http://schema.org/VideoObject

From 21fbf0f955f584ad2d02608850a69a2fd74b65a6 Mon Sep 17 00:00:00 2001
From: Throaway <Throaway@null.com>
Date: Mon, 20 Mar 2017 16:29:39 -0700
Subject: [PATCH 0511/1696] [pornhub] Decode obfuscated video URL (closes
 #12470)

---
 youtube_dl/extractor/pornhub.py | 37 ++++++++++++++++++++++++++++++---
 1 file changed, 34 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 9b413590a..eb316ad14 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -1,7 +1,9 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import functools
 import itertools
+import operator
 # import os
 import re
 
@@ -129,9 +131,38 @@ class PornHubIE(InfoExtractor):
 
         tv_webpage = dl_webpage('tv')
 
-        video_url = self._search_regex(
-            r'<video[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//.+?)\1', tv_webpage,
-            'video url', group='url')
+        encoded_url = self._search_regex(r'(var.*mediastring.*)</script>',
+            tv_webpage, 'encoded url')
+        assignments = encoded_url.split(";")
+        js_vars = {}
+
+        def parse_js_value(inp):
+            inp = re.sub(r'/\*[^*]*\*/', "", inp)
+
+            if "+" in inp:
+                inps = inp.split("+")
+                return functools.reduce(operator.concat, map(parse_js_value, inps))
+
+            inp = inp.strip()
+            if inp in js_vars:
+                return js_vars[inp]
+
+            # Hope it's a string!
+            assert inp.startswith('"') and inp.endswith('"')
+            return inp[1:-1]
+
+        for assn in assignments:
+            assn = assn.strip()
+            if len(assn) == 0:
+                continue
+
+            assert assn.startswith("var ")
+            assn = assn[4:]
+            vname, value = assn.split("=", 1)
+
+            js_vars[vname] = parse_js_value(value)
+
+        video_url = js_vars["mediastring"]
 
         title = self._search_regex(
             r'<h1>([^>]+)</h1>', tv_webpage, 'title', default=None)

From e1e35d1ac66ab99202e8265ac811906de2aa87dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Mar 2017 01:59:27 +0700
Subject: [PATCH 0512/1696] [pornhub] Improve extraction and style (closes
 #12515)

---
 youtube_dl/extractor/pornhub.py | 35 ++++++++++++++-------------------
 1 file changed, 15 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index eb316ad14..b25f1f193 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -20,6 +20,7 @@ from ..utils import (
     js_to_json,
     orderedSet,
     # sanitized_Request,
+    remove_quotes,
     str_to_int,
 )
 # from ..aes import (
@@ -131,38 +132,32 @@ class PornHubIE(InfoExtractor):
 
         tv_webpage = dl_webpage('tv')
 
-        encoded_url = self._search_regex(r'(var.*mediastring.*)</script>',
-            tv_webpage, 'encoded url')
-        assignments = encoded_url.split(";")
+        assignments = self._search_regex(
+            r'(var.+?mediastring.+?)</script>', tv_webpage,
+            'encoded url').split(';')
+
         js_vars = {}
 
         def parse_js_value(inp):
-            inp = re.sub(r'/\*[^*]*\*/', "", inp)
-
-            if "+" in inp:
-                inps = inp.split("+")
-                return functools.reduce(operator.concat, map(parse_js_value, inps))
-
+            inp = re.sub(r'/\*(?:(?!\*/).)*?\*/', '', inp)
+            if '+' in inp:
+                inps = inp.split('+')
+                return functools.reduce(
+                    operator.concat, map(parse_js_value, inps))
             inp = inp.strip()
             if inp in js_vars:
                 return js_vars[inp]
-
-            # Hope it's a string!
-            assert inp.startswith('"') and inp.endswith('"')
-            return inp[1:-1]
+            return remove_quotes(inp)
 
         for assn in assignments:
             assn = assn.strip()
-            if len(assn) == 0:
+            if not assn:
                 continue
-
-            assert assn.startswith("var ")
-            assn = assn[4:]
-            vname, value = assn.split("=", 1)
-
+            assn = re.sub(r'var\s+', '', assn)
+            vname, value = assn.split('=', 1)
             js_vars[vname] = parse_js_value(value)
 
-        video_url = js_vars["mediastring"]
+        video_url = js_vars['mediastring']
 
         title = self._search_regex(
             r'<h1>([^>]+)</h1>', tv_webpage, 'title', default=None)

From 8e5a7c5e67a8fad446d22a7619cd6a09823a05e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Mar 2017 02:28:04 +0700
Subject: [PATCH 0513/1696] [pluralsight] Omit module title from video title
 (closes #12506)

---
 youtube_dl/extractor/pluralsight.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index e0cbd045e..0c6e036ca 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -40,7 +40,7 @@ class PluralsightIE(PluralsightBaseIE):
         'info_dict': {
             'id': 'hosting-sql-server-windows-azure-iaas-m7-mgmt-04',
             'ext': 'mp4',
-            'title': 'Management of SQL Server - Demo Monitoring',
+            'title': 'Demo Monitoring',
             'duration': 338,
         },
         'skip': 'Requires pluralsight account credentials',
@@ -187,7 +187,7 @@ class PluralsightIE(PluralsightBaseIE):
         if not clip:
             raise ExtractorError('Unable to resolve clip')
 
-        title = '%s - %s' % (module['title'], clip['title'])
+        title = clip['title']
 
         QUALITIES = {
             'low': {'width': 640, 'height': 480},

From e8686e51d77607347802f82c57278e7d675d022c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Mar 2017 02:35:09 +0700
Subject: [PATCH 0514/1696] [ChangeLog] Actualize

---
 ChangeLog | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index bbbf3c34d..82e82b588 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,11 @@
+version <unreleased>
+
+Extractors
+- [pluralsight] Omit module title from video title (#12506)
+* [pornhub] Decode obfuscated video URL (#12470, #12515)
+* [senateisvp] Allow https URL scheme for embeds (#12512)
+
+
 version 2017.03.20
 
 Core

From 093dad9e256e3237dfad3c57a19ae10ddadcbf9b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Mar 2017 02:36:50 +0700
Subject: [PATCH 0515/1696] release 2017.03.22

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 4273fedbf..31ba1de3d 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.03.20*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.03.20**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.03.22*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.03.22**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.03.20
+[debug] youtube-dl version 2017.03.22
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 82e82b588..dc5acbca9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.03.22
 
 Extractors
 - [pluralsight] Omit module title from video title (#12506)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a65f2e741..4d722873d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.03.20'
+__version__ = '2017.03.22'

From c183e14f89078593ab47f06e5076f00bee3c9dd9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Mar 2017 22:26:59 +0700
Subject: [PATCH 0516/1696] [viu] Relax _VALID_URL (closes #12529)

---
 youtube_dl/extractor/viu.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/viu.py b/youtube_dl/extractor/viu.py
index 3fd889c8e..db6a65d2e 100644
--- a/youtube_dl/extractor/viu.py
+++ b/youtube_dl/extractor/viu.py
@@ -44,7 +44,7 @@ class ViuBaseIE(InfoExtractor):
 
 
 class ViuIE(ViuBaseIE):
-    _VALID_URL = r'(?:viu:|https?://www\.viu\.com/[a-z]{2}/media/)(?P<id>\d+)'
+    _VALID_URL = r'(?:viu:|https?://[^/]+\.viu\.com/[a-z]{2}/media/)(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://www.viu.com/en/media/1116705532?containerId=playlist-22168059',
         'info_dict': {
@@ -69,6 +69,9 @@ class ViuIE(ViuBaseIE):
             'skip_download': 'm3u8 download',
         },
         'skip': 'Geo-restricted to Indonesia',
+    }, {
+        'url': 'https://india.viu.com/en/media/1126286865',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 391d076d7cf037b1d7849ea7cbbdd04950c46f96 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Mar 2017 23:22:14 +0700
Subject: [PATCH 0517/1696] [condenast] Fix extraction and style (closes
 #12526)

---
 youtube_dl/extractor/condenast.py | 43 +++++++++++++++++++++----------
 1 file changed, 30 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/condenast.py b/youtube_dl/extractor/condenast.py
index 8d8f60598..d3463b874 100644
--- a/youtube_dl/extractor/condenast.py
+++ b/youtube_dl/extractor/condenast.py
@@ -9,13 +9,14 @@ from ..compat import (
     compat_urlparse,
 )
 from ..utils import (
-    orderedSet,
-    remove_end,
-    extract_attributes,
-    mimetype2ext,
     determine_ext,
+    extract_attributes,
     int_or_none,
+    js_to_json,
+    mimetype2ext,
+    orderedSet,
     parse_iso8601,
+    remove_end,
 )
 
 
@@ -66,6 +67,16 @@ class CondeNastIE(InfoExtractor):
             'upload_date': '20130314',
             'timestamp': 1363219200,
         }
+    }, {
+        'url': 'http://video.gq.com/watch/the-closer-with-keith-olbermann-the-only-true-surprise-trump-s-an-idiot?c=series',
+        'info_dict': {
+            'id': '58d1865bfd2e6126e2000015',
+            'ext': 'mp4',
+            'title': 'The Only True Surprise? Trump’s an Idiot',
+            'uploader': 'gq',
+            'upload_date': '20170321',
+            'timestamp': 1490126427,
+        },
     }, {
         # JS embed
         'url': 'http://player.cnevids.com/embedjs/55f9cf8b61646d1acf00000c/5511d76261646d5566020000.js',
@@ -114,26 +125,33 @@ class CondeNastIE(InfoExtractor):
             })
         video_id = query['videoId']
         video_info = None
-        info_page = self._download_webpage(
+        info_page = self._download_json(
             'http://player.cnevids.com/player/video.js',
-            video_id, 'Downloading video info', query=query, fatal=False)
+            video_id, 'Downloading video info', fatal=False, query=query)
         if info_page:
-            video_info = self._parse_json(self._search_regex(
-                r'loadCallback\(({.+})\)', info_page, 'video info'), video_id)['video']
-        else:
+            video_info = info_page.get('video')
+        if not video_info:
             info_page = self._download_webpage(
                 'http://player.cnevids.com/player/loader.js',
                 video_id, 'Downloading loader info', query=query)
-            video_info = self._parse_json(self._search_regex(
-                r'var\s+video\s*=\s*({.+?});', info_page, 'video info'), video_id)
+            video_info = self._parse_json(
+                self._search_regex(
+                    r'(?s)var\s+config\s*=\s*({.+?});', info_page, 'config'),
+                video_id, transform_source=js_to_json)['video']
+
         title = video_info['title']
 
         formats = []
-        for fdata in video_info.get('sources', [{}])[0]:
+        for fdata in video_info['sources']:
             src = fdata.get('src')
             if not src:
                 continue
             ext = mimetype2ext(fdata.get('type')) or determine_ext(src)
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    src, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls', fatal=False))
+                continue
             quality = fdata.get('quality')
             formats.append({
                 'format_id': ext + ('-%s' % quality if quality else ''),
@@ -169,7 +187,6 @@ class CondeNastIE(InfoExtractor):
                 path=remove_end(parsed_url.path, '.js').replace('/embedjs/', '/embed/')))
             url_type = 'embed'
 
-        self.to_screen('Extracting from %s with the Condé Nast extractor' % self._SITES[site])
         webpage = self._download_webpage(url, item_id)
 
         if url_type == 'series':

From ca5ed022e962ecd6992c145ac7bc00b5963e5d69 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 22 Mar 2017 17:28:24 +0100
Subject: [PATCH 0518/1696] [hbo] add support for free episode urls and new
 formats extraction(closes #12519)

---
 youtube_dl/extractor/hbo.py | 45 ++++++++++++++++++++++++++++++-------
 1 file changed, 37 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/hbo.py b/youtube_dl/extractor/hbo.py
index 8116ad9bd..931f71a5a 100644
--- a/youtube_dl/extractor/hbo.py
+++ b/youtube_dl/extractor/hbo.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     xpath_text,
     xpath_element,
@@ -14,14 +15,26 @@ from ..utils import (
 
 class HBOBaseIE(InfoExtractor):
     _FORMATS_INFO = {
+        'pro7': {
+            'width': 1280,
+            'height': 720,
+        },
         '1920': {
             'width': 1280,
             'height': 720,
         },
+        'pro6': {
+            'width': 768,
+            'height': 432,
+        },
         '640': {
             'width': 768,
             'height': 432,
         },
+        'pro5': {
+            'width': 640,
+            'height': 360,
+        },
         'highwifi': {
             'width': 640,
             'height': 360,
@@ -78,6 +91,17 @@ class HBOBaseIE(InfoExtractor):
                     formats.extend(self._extract_m3u8_formats(
                         video_url.replace('.tar', '/base_index_w8.m3u8'),
                         video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+                elif source.tag == 'hls':
+                    # #EXT-X-BYTERANGE is not supported by native hls downloader
+                    # and ffmpeg (#10955)
+                    # formats.extend(self._extract_m3u8_formats(
+                    #     video_url.replace('.tar', '/base_index.m3u8'),
+                    #     video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+                    continue
+                elif source.tag == 'dash':
+                    formats.extend(self._extract_mpd_formats(
+                        video_url.replace('.tar', '/manifest.mpd'),
+                        video_id, mpd_id='dash', fatal=False))
                 else:
                     format_info = self._FORMATS_INFO.get(source.tag, {})
                     formats.append({
@@ -112,10 +136,11 @@ class HBOBaseIE(InfoExtractor):
 
 
 class HBOIE(HBOBaseIE):
+    IE_NAME = 'hbo'
     _VALID_URL = r'https?://(?:www\.)?hbo\.com/video/video\.html\?.*vid=(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.hbo.com/video/video.html?autoplay=true&g=u&vid=1437839',
-        'md5': '1c33253f0c7782142c993c0ba62a8753',
+        'md5': '2c6a6bc1222c7e91cb3334dad1746e5a',
         'info_dict': {
             'id': '1437839',
             'ext': 'mp4',
@@ -131,11 +156,12 @@ class HBOIE(HBOBaseIE):
 
 
 class HBOEpisodeIE(HBOBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?hbo\.com/(?!video)([^/]+/)+video/(?P<id>[0-9a-z-]+)\.html'
+    IE_NAME = 'hbo:episode'
+    _VALID_URL = r'https?://(?:www\.)?hbo\.com/(?P<path>(?!video)(?:(?:[^/]+/)+video|watch-free-episodes)/(?P<id>[0-9a-z-]+))(?:\.html)?'
 
     _TESTS = [{
         'url': 'http://www.hbo.com/girls/episodes/5/52-i-love-you-baby/video/ep-52-inside-the-episode.html?autoplay=true',
-        'md5': '689132b253cc0ab7434237fc3a293210',
+        'md5': '61ead79b9c0dfa8d3d4b07ef4ac556fb',
         'info_dict': {
             'id': '1439518',
             'display_id': 'ep-52-inside-the-episode',
@@ -147,16 +173,19 @@ class HBOEpisodeIE(HBOBaseIE):
     }, {
         'url': 'http://www.hbo.com/game-of-thrones/about/video/season-5-invitation-to-the-set.html?autoplay=true',
         'only_matching': True,
+    }, {
+        'url': 'http://www.hbo.com/watch-free-episodes/last-week-tonight-with-john-oliver',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
+        path, display_id = re.match(self._VALID_URL, url).groups()
 
-        webpage = self._download_webpage(url, display_id)
+        content = self._download_json(
+            'http://www.hbo.com/api/content/' + path, display_id)['content']
 
-        video_id = self._search_regex(
-            r'(?P<q1>[\'"])videoId(?P=q1)\s*:\s*(?P<q2>[\'"])(?P<video_id>\d+)(?P=q2)',
-            webpage, 'video ID', group='video_id')
+        video_id = compat_str((content.get('parsed', {}).get(
+            'common:FullBleedVideo', {}) or content['selectedEpisode'])['videoId'])
 
         info_dict = self._extract_from_id(video_id)
         info_dict['display_id'] = display_id

From 579c99a284481243f30e80151c90a753f613778d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Mar 2017 23:48:06 +0700
Subject: [PATCH 0519/1696] [cloudy] Fix extraction (closes #12525)

---
 youtube_dl/extractor/cloudy.py | 113 +++++++++++----------------------
 1 file changed, 36 insertions(+), 77 deletions(-)

diff --git a/youtube_dl/extractor/cloudy.py b/youtube_dl/extractor/cloudy.py
index ae5ba0015..9bc8dbea4 100644
--- a/youtube_dl/extractor/cloudy.py
+++ b/youtube_dl/extractor/cloudy.py
@@ -1,97 +1,56 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..compat import (
-    compat_parse_qs,
-    compat_HTTPError,
-)
 from ..utils import (
-    ExtractorError,
-    HEADRequest,
-    remove_end,
+    str_to_int,
+    unified_strdate,
 )
 
 
 class CloudyIE(InfoExtractor):
     _IE_DESC = 'cloudy.ec'
-    _VALID_URL = r'''(?x)
-        https?://(?:www\.)?cloudy\.ec/
-        (?:v/|embed\.php\?id=)
-        (?P<id>[A-Za-z0-9]+)
-        '''
-    _EMBED_URL = 'http://www.cloudy.ec/embed.php?id=%s'
-    _API_URL = 'http://www.cloudy.ec/api/player.api.php'
-    _MAX_TRIES = 2
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?cloudy\.ec/(?:v/|embed\.php\?.*?\bid=)(?P<id>[A-Za-z0-9]+)'
+    _TESTS = [{
         'url': 'https://www.cloudy.ec/v/af511e2527aac',
-        'md5': '5cb253ace826a42f35b4740539bedf07',
+        'md5': '29832b05028ead1b58be86bf319397ca',
         'info_dict': {
             'id': 'af511e2527aac',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Funny Cats and Animals Compilation june 2013',
+            'upload_date': '20130913',
+            'view_count': int,
         }
-    }
-
-    def _extract_video(self, video_id, file_key, error_url=None, try_num=0):
-
-        if try_num > self._MAX_TRIES - 1:
-            raise ExtractorError('Unable to extract video URL', expected=True)
-
-        form = {
-            'file': video_id,
-            'key': file_key,
-        }
-
-        if error_url:
-            form.update({
-                'numOfErrors': try_num,
-                'errorCode': '404',
-                'errorUrl': error_url,
-            })
-
-        player_data = self._download_webpage(
-            self._API_URL, video_id, 'Downloading player data', query=form)
-        data = compat_parse_qs(player_data)
-
-        try_num += 1
-
-        if 'error' in data:
-            raise ExtractorError(
-                '%s error: %s' % (self.IE_NAME, ' '.join(data['error_msg'])),
-                expected=True)
-
-        title = data.get('title', [None])[0]
-        if title:
-            title = remove_end(title, '&asdasdas').strip()
-
-        video_url = data.get('url', [None])[0]
-
-        if video_url:
-            try:
-                self._request_webpage(HEADRequest(video_url), video_id, 'Checking video URL')
-            except ExtractorError as e:
-                if isinstance(e.cause, compat_HTTPError) and e.cause.code in [404, 410]:
-                    self.report_warning('Invalid video URL, requesting another', video_id)
-                    return self._extract_video(video_id, file_key, video_url, try_num)
-
-        return {
-            'id': video_id,
-            'url': video_url,
-            'title': title,
-        }
+    }, {
+        'url': 'http://www.cloudy.ec/embed.php?autoplay=1&id=af511e2527aac',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
-        url = self._EMBED_URL % video_id
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(
+            'http://www.cloudy.ec/embed.php?id=%s' % video_id, video_id)
 
-        file_key = self._search_regex(
-            [r'key\s*:\s*"([^"]+)"', r'filekey\s*=\s*"([^"]+)"'],
-            webpage, 'file_key')
+        info = self._parse_html5_media_entries(url, webpage, video_id)[0]
 
-        return self._extract_video(video_id, file_key)
+        webpage = self._download_webpage(
+            'https://www.cloudy.ec/v/%s' % video_id, video_id, fatal=False)
+
+        if webpage:
+            info.update({
+                'title': self._search_regex(
+                    r'<h\d[^>]*>([^<]+)<', webpage, 'title'),
+                'upload_date': unified_strdate(self._search_regex(
+                    r'>Published at (\d{4}-\d{1,2}-\d{1,2})', webpage,
+                    'upload date', fatal=False)),
+                'view_count': str_to_int(self._search_regex(
+                    r'([\d,.]+) views<', webpage, 'view count', fatal=False)),
+            })
+
+        if not info.get('title'):
+            info['title'] = video_id
+
+        info['id'] = video_id
+
+        return info

From b0f7f21cb92ca3af1795f68737ffa25196968dc6 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 23 Mar 2017 09:22:17 +0100
Subject: [PATCH 0520/1696] [channel9] fix extraction(closes #11323)

---
 youtube_dl/extractor/channel9.py | 345 ++++++++++++-------------------
 1 file changed, 127 insertions(+), 218 deletions(-)

diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py
index 865dbcaba..b1cb58530 100644
--- a/youtube_dl/extractor/channel9.py
+++ b/youtube_dl/extractor/channel9.py
@@ -5,8 +5,10 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
-    parse_filesize,
-    qualities,
+    unescapeHTML,
+    int_or_none,
+    parse_iso8601,
+    clean_html,
 )
 
 
@@ -20,46 +22,50 @@ class Channel9IE(InfoExtractor):
     '''
     IE_DESC = 'Channel 9'
     IE_NAME = 'channel9'
-    _VALID_URL = r'https?://(?:www\.)?channel9\.msdn\.com/(?P<contentpath>.+?)(?P<rss>/RSS)?/?(?:[?#&]|$)'
+    _VALID_URL = r'https?://(?:www\.)?(?:channel9\.msdn\.com|s\.ch9\.ms)/(?P<contentpath>.+?)(?P<rss>/RSS)?/?(?:[?#&]|$)'
 
     _TESTS = [{
         'url': 'http://channel9.msdn.com/Events/TechEd/Australia/2013/KOS002',
-        'md5': 'bbd75296ba47916b754e73c3a4bbdf10',
+        'md5': '32083d4eaf1946db6d454313f44510ca',
         'info_dict': {
-            'id': 'Events/TechEd/Australia/2013/KOS002',
-            'ext': 'mp4',
+            'id': '6c413323-383a-49dc-88f9-a22800cab024',
+            'ext': 'wmv',
             'title': 'Developer Kick-Off Session: Stuff We Love',
-            'description': 'md5:c08d72240b7c87fcecafe2692f80e35f',
+            'description': 'md5:b80bf9355a503c193aff7ec6cd5a7731',
             'duration': 4576,
-            'thumbnail': r're:http://.*\.jpg',
+            'thumbnail': r're:https?://.*\.jpg',
+            'timestamp': 1377717420,
+            'upload_date': '20130828',
             'session_code': 'KOS002',
-            'session_day': 'Day 1',
             'session_room': 'Arena 1A',
-            'session_speakers': ['Ed Blankenship', 'Andrew Coates', 'Brady Gaster', 'Patrick Klug',
-                                 'Mads Kristensen'],
+            'session_speakers': ['Andrew Coates', 'Brady Gaster', 'Mads Kristensen', 'Ed Blankenship', 'Patrick Klug'],
         },
     }, {
         'url': 'http://channel9.msdn.com/posts/Self-service-BI-with-Power-BI-nuclear-testing',
-        'md5': 'b43ee4529d111bc37ba7ee4f34813e68',
+        'md5': 'dcf983ee6acd2088e7188c3cf79b46bc',
         'info_dict': {
-            'id': 'posts/Self-service-BI-with-Power-BI-nuclear-testing',
-            'ext': 'mp4',
+            'id': 'fe8e435f-bb93-4e01-8e97-a28c01887024',
+            'ext': 'wmv',
             'title': 'Self-service BI with Power BI - nuclear testing',
-            'description': 'md5:d1e6ecaafa7fb52a2cacdf9599829f5b',
+            'description': 'md5:2d17fec927fc91e9e17783b3ecc88f54',
             'duration': 1540,
-            'thumbnail': r're:http://.*\.jpg',
+            'thumbnail': r're:https?://.*\.jpg',
+            'timestamp': 1386381991,
+            'upload_date': '20131207',
             'authors': ['Mike Wilmot'],
         },
     }, {
         # low quality mp4 is best
         'url': 'https://channel9.msdn.com/Events/CPP/CppCon-2015/Ranges-for-the-Standard-Library',
         'info_dict': {
-            'id': 'Events/CPP/CppCon-2015/Ranges-for-the-Standard-Library',
+            'id': '33ad69d2-6a4e-4172-83a1-a523013dec76',
             'ext': 'mp4',
             'title': 'Ranges for the Standard Library',
-            'description': 'md5:2e6b4917677af3728c5f6d63784c4c5d',
+            'description': 'md5:9895e0a9fd80822d2f01c454b8f4a372',
             'duration': 5646,
-            'thumbnail': r're:http://.*\.jpg',
+            'thumbnail': r're:https?://.*\.jpg',
+            'upload_date': '20150930',
+            'timestamp': 1443640735,
         },
         'params': {
             'skip_download': True,
@@ -70,7 +76,7 @@ class Channel9IE(InfoExtractor):
             'id': 'Niners/Splendid22/Queue/76acff796e8f411184b008028e0d492b',
             'title': 'Channel 9',
         },
-        'playlist_count': 2,
+        'playlist_mincount': 100,
     }, {
         'url': 'https://channel9.msdn.com/Events/DEVintersection/DEVintersection-2016/RSS',
         'only_matching': True,
@@ -81,189 +87,6 @@ class Channel9IE(InfoExtractor):
 
     _RSS_URL = 'http://channel9.msdn.com/%s/RSS'
 
-    def _formats_from_html(self, html):
-        FORMAT_REGEX = r'''
-            (?x)
-            <a\s+href="(?P<url>[^"]+)">(?P<quality>[^<]+)</a>\s*
-            <span\s+class="usage">\((?P<note>[^\)]+)\)</span>\s*
-            (?:<div\s+class="popup\s+rounded">\s*
-            <h3>File\s+size</h3>\s*(?P<filesize>.*?)\s*
-            </div>)?                                                # File size part may be missing
-        '''
-        quality = qualities((
-            'MP3', 'MP4',
-            'Low Quality WMV', 'Low Quality MP4',
-            'Mid Quality WMV', 'Mid Quality MP4',
-            'High Quality WMV', 'High Quality MP4'))
-        formats = [{
-            'url': x.group('url'),
-            'format_id': x.group('quality'),
-            'format_note': x.group('note'),
-            'format': '%s (%s)' % (x.group('quality'), x.group('note')),
-            'filesize_approx': parse_filesize(x.group('filesize')),
-            'quality': quality(x.group('quality')),
-            'vcodec': 'none' if x.group('note') == 'Audio only' else None,
-        } for x in list(re.finditer(FORMAT_REGEX, html))]
-
-        self._sort_formats(formats)
-
-        return formats
-
-    def _extract_title(self, html):
-        title = self._html_search_meta('title', html, 'title')
-        if title is None:
-            title = self._og_search_title(html)
-            TITLE_SUFFIX = ' (Channel 9)'
-            if title is not None and title.endswith(TITLE_SUFFIX):
-                title = title[:-len(TITLE_SUFFIX)]
-        return title
-
-    def _extract_description(self, html):
-        DESCRIPTION_REGEX = r'''(?sx)
-            <div\s+class="entry-content">\s*
-            <div\s+id="entry-body">\s*
-            (?P<description>.+?)\s*
-            </div>\s*
-            </div>
-        '''
-        m = re.search(DESCRIPTION_REGEX, html)
-        if m is not None:
-            return m.group('description')
-        return self._html_search_meta('description', html, 'description')
-
-    def _extract_duration(self, html):
-        m = re.search(r'"length": *"(?P<hours>\d{2}):(?P<minutes>\d{2}):(?P<seconds>\d{2})"', html)
-        return ((int(m.group('hours')) * 60 * 60) + (int(m.group('minutes')) * 60) + int(m.group('seconds'))) if m else None
-
-    def _extract_slides(self, html):
-        m = re.search(r'<a href="(?P<slidesurl>[^"]+)" class="slides">Slides</a>', html)
-        return m.group('slidesurl') if m is not None else None
-
-    def _extract_zip(self, html):
-        m = re.search(r'<a href="(?P<zipurl>[^"]+)" class="zip">Zip</a>', html)
-        return m.group('zipurl') if m is not None else None
-
-    def _extract_avg_rating(self, html):
-        m = re.search(r'<p class="avg-rating">Avg Rating: <span>(?P<avgrating>[^<]+)</span></p>', html)
-        return float(m.group('avgrating')) if m is not None else 0
-
-    def _extract_rating_count(self, html):
-        m = re.search(r'<div class="rating-count">\((?P<ratingcount>[^<]+)\)</div>', html)
-        return int(self._fix_count(m.group('ratingcount'))) if m is not None else 0
-
-    def _extract_view_count(self, html):
-        m = re.search(r'<li class="views">\s*<span class="count">(?P<viewcount>[^<]+)</span> Views\s*</li>', html)
-        return int(self._fix_count(m.group('viewcount'))) if m is not None else 0
-
-    def _extract_comment_count(self, html):
-        m = re.search(r'<li class="comments">\s*<a href="#comments">\s*<span class="count">(?P<commentcount>[^<]+)</span> Comments\s*</a>\s*</li>', html)
-        return int(self._fix_count(m.group('commentcount'))) if m is not None else 0
-
-    def _fix_count(self, count):
-        return int(str(count).replace(',', '')) if count is not None else None
-
-    def _extract_authors(self, html):
-        m = re.search(r'(?s)<li class="author">(.*?)</li>', html)
-        if m is None:
-            return None
-        return re.findall(r'<a href="/Niners/[^"]+">([^<]+)</a>', m.group(1))
-
-    def _extract_session_code(self, html):
-        m = re.search(r'<li class="code">\s*(?P<code>.+?)\s*</li>', html)
-        return m.group('code') if m is not None else None
-
-    def _extract_session_day(self, html):
-        m = re.search(r'<li class="day">\s*<a href="/Events/[^"]+">(?P<day>[^<]+)</a>\s*</li>', html)
-        return m.group('day').strip() if m is not None else None
-
-    def _extract_session_room(self, html):
-        m = re.search(r'<li class="room">\s*(?P<room>.+?)\s*</li>', html)
-        return m.group('room') if m is not None else None
-
-    def _extract_session_speakers(self, html):
-        return re.findall(r'<a href="/Events/Speakers/[^"]+">([^<]+)</a>', html)
-
-    def _extract_content(self, html, content_path):
-        # Look for downloadable content
-        formats = self._formats_from_html(html)
-        slides = self._extract_slides(html)
-        zip_ = self._extract_zip(html)
-
-        # Nothing to download
-        if len(formats) == 0 and slides is None and zip_ is None:
-            self._downloader.report_warning('None of recording, slides or zip are available for %s' % content_path)
-            return
-
-        # Extract meta
-        title = self._extract_title(html)
-        description = self._extract_description(html)
-        thumbnail = self._og_search_thumbnail(html)
-        duration = self._extract_duration(html)
-        avg_rating = self._extract_avg_rating(html)
-        rating_count = self._extract_rating_count(html)
-        view_count = self._extract_view_count(html)
-        comment_count = self._extract_comment_count(html)
-
-        common = {
-            '_type': 'video',
-            'id': content_path,
-            'description': description,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'avg_rating': avg_rating,
-            'rating_count': rating_count,
-            'view_count': view_count,
-            'comment_count': comment_count,
-        }
-
-        result = []
-
-        if slides is not None:
-            d = common.copy()
-            d.update({'title': title + '-Slides', 'url': slides})
-            result.append(d)
-
-        if zip_ is not None:
-            d = common.copy()
-            d.update({'title': title + '-Zip', 'url': zip_})
-            result.append(d)
-
-        if len(formats) > 0:
-            d = common.copy()
-            d.update({'title': title, 'formats': formats})
-            result.append(d)
-
-        return result
-
-    def _extract_entry_item(self, html, content_path):
-        contents = self._extract_content(html, content_path)
-        if contents is None:
-            return contents
-
-        if len(contents) > 1:
-            raise ExtractorError('Got more than one entry')
-        result = contents[0]
-        result['authors'] = self._extract_authors(html)
-
-        return result
-
-    def _extract_session(self, html, content_path):
-        contents = self._extract_content(html, content_path)
-        if contents is None:
-            return contents
-
-        session_meta = {
-            'session_code': self._extract_session_code(html),
-            'session_day': self._extract_session_day(html),
-            'session_room': self._extract_session_room(html),
-            'session_speakers': self._extract_session_speakers(html),
-        }
-
-        for content in contents:
-            content.update(session_meta)
-
-        return self.playlist_result(contents)
-
     def _extract_list(self, video_id, rss_url=None):
         if not rss_url:
             rss_url = self._RSS_URL % video_id
@@ -274,9 +97,7 @@ class Channel9IE(InfoExtractor):
         return self.playlist_result(entries, video_id, title_text)
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        content_path = mobj.group('contentpath')
-        rss = mobj.group('rss')
+        content_path, rss = re.match(self._VALID_URL, url).groups()
 
         if rss:
             return self._extract_list(content_path, url)
@@ -284,17 +105,105 @@ class Channel9IE(InfoExtractor):
         webpage = self._download_webpage(
             url, content_path, 'Downloading web page')
 
-        page_type = self._search_regex(
-            r'<meta[^>]+name=(["\'])WT\.entryid\1[^>]+content=(["\'])(?P<pagetype>[^:]+).+?\2',
-            webpage, 'page type', default=None, group='pagetype')
-        if page_type:
-            if page_type == 'Entry':      # Any 'item'-like page, may contain downloadable content
-                return self._extract_entry_item(webpage, content_path)
-            elif page_type == 'Session':  # Event session page, may contain downloadable content
-                return self._extract_session(webpage, content_path)
-            elif page_type == 'Event':
-                return self._extract_list(content_path)
+        episode_data = self._search_regex(
+            r"data-episode='([^']+)'", webpage, 'episode data', default=None)
+        if episode_data:
+            episode_data = self._parse_json(unescapeHTML(
+                episode_data), content_path)
+            content_id = episode_data['contentId']
+            is_session = '/Sessions(' in episode_data['api']
+            content_url = 'https://channel9.msdn.com/odata' + episode_data['api']
+            if is_session:
+                content_url += '?$expand=Speakers'
             else:
-                raise ExtractorError('Unexpected WT.entryid %s' % page_type, expected=True)
-        else:  # Assuming list
+                content_url += '?$expand=Authors'
+            content_data = self._download_json(content_url, content_id)
+            title = content_data['Title']
+
+            formats = []
+            qualities = [
+                'VideoMP4Low',
+                'VideoWMV',
+                'VideoMP4Medium',
+                'VideoMP4High',
+                'VideoWMVHQ',
+            ]
+            for q in qualities:
+                q_url = content_data.get(q)
+                if not q_url:
+                    continue
+                formats.append({
+                    'format_id': q,
+                    'url': q_url,
+                })
+            slides = content_data.get('Slides')
+            zip_file = content_data.get('ZipFile')
+
+            if not formats and not slides and not zip_file:
+                raise ExtractorError(
+                    'None of recording, slides or zip are available for %s' % content_path)
+
+            subtitles = {}
+            for caption in content_data.get('Captions', []):
+                caption_url = caption.get('Url')
+                if not caption_url:
+                    continue
+                subtitles.setdefault(caption.get('Language', 'en'), []).append({
+                    'url': caption_url,
+                    'ext': 'vtt',
+                })
+
+            common = {
+                'id': content_id,
+                'title': title,
+                'description': clean_html(content_data.get('Description') or content_data.get('Body')),
+                'thumbnail': content_data.get('Thumbnail') or content_data.get('VideoPlayerPreviewImage'),
+                'duration': int_or_none(content_data.get('MediaLengthInSeconds')),
+                'timestamp': parse_iso8601(content_data.get('PublishedDate')),
+                'avg_rating': int_or_none(content_data.get('Rating')),
+                'rating_count': int_or_none(content_data.get('RatingCount')),
+                'view_count': int_or_none(content_data.get('Views')),
+                'comment_count': int_or_none(content_data.get('CommentCount')),
+                'subtitles': subtitles,
+            }
+            if is_session:
+                speakers = []
+                for s in content_data.get('Speakers', []):
+                    speaker_name = s.get('FullName')
+                    if not speaker_name:
+                        continue
+                    speakers.append(speaker_name)
+
+                common.update({
+                    'session_code': content_data.get('Code'),
+                    'session_room': content_data.get('Room'),
+                    'session_speakers': speakers,
+                })
+            else:
+                authors = []
+                for a in content_data.get('Authors', []):
+                    author_name = a.get('DisplayName')
+                    if not author_name:
+                        continue
+                    authors.append(author_name)
+                common['authors'] = authors
+
+            contents = []
+
+            if slides:
+                d = common.copy()
+                d.update({'title': title + '-Slides', 'url': slides})
+                contents.append(d)
+
+            if zip_file:
+                d = common.copy()
+                d.update({'title': title + '-Zip', 'url': zip_file})
+                contents.append(d)
+
+            if formats:
+                d = common.copy()
+                d.update({'title': title, 'formats': formats})
+                contents.append(d)
+            return self.playlist_result(contents)
+        else:
             return self._extract_list(content_path)

From 52d5ecabd518db46fc02b8624b2ad04ba7cf2114 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 23 Mar 2017 13:48:32 +0100
Subject: [PATCH 0521/1696] [bellmedia] add support for etalk.ca(closes #12447)

---
 youtube_dl/extractor/bellmedia.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bellmedia.py b/youtube_dl/extractor/bellmedia.py
index 1f5b6ed92..8820a3914 100644
--- a/youtube_dl/extractor/bellmedia.py
+++ b/youtube_dl/extractor/bellmedia.py
@@ -21,10 +21,11 @@ class BellMediaIE(InfoExtractor):
                 animalplanet|
                 bravo|
                 mtv|
-                space
+                space|
+                etalk
             )\.ca|
             much\.com
-        )/.*?(?:\bvid=|-vid|~|%7E|/(?:episode)?)(?P<id>[0-9]{6,})'''
+        )/.*?(?:\bvid(?:eoid)?=|-vid|~|%7E|/(?:episode)?)(?P<id>[0-9]{6,})'''
     _TESTS = [{
         'url': 'http://www.ctv.ca/video/player?vid=706966',
         'md5': 'ff2ebbeae0aa2dcc32a830c3fd69b7b0',
@@ -58,6 +59,9 @@ class BellMediaIE(InfoExtractor):
     }, {
         'url': 'http://www.ctv.ca/DCs-Legends-of-Tomorrow/Video/S2E11-Turncoat-vid1051430',
         'only_matching': True,
+    }, {
+        'url': 'http://www.etalk.ca/video?videoid=663455',
+        'only_matching': True,
     }]
     _DOMAINS = {
         'thecomedynetwork': 'comedy',
@@ -65,6 +69,7 @@ class BellMediaIE(InfoExtractor):
         'sciencechannel': 'discsci',
         'investigationdiscovery': 'invdisc',
         'animalplanet': 'aniplan',
+        'etalk': 'ctv',
     }
 
     def _real_extract(self, url):

From d0572557c2a88e34d85715af4271e8b5decbdfdb Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 23 Mar 2017 13:52:07 +0100
Subject: [PATCH 0522/1696] [ninecninemedia] remove mp4 url extraction request

---
 youtube_dl/extractor/ninecninemedia.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/youtube_dl/extractor/ninecninemedia.py b/youtube_dl/extractor/ninecninemedia.py
index d9943fc2c..8961309fd 100644
--- a/youtube_dl/extractor/ninecninemedia.py
+++ b/youtube_dl/extractor/ninecninemedia.py
@@ -34,12 +34,6 @@ class NineCNineMediaStackIE(NineCNineMediaBaseIE):
         formats.extend(self._extract_f4m_formats(
             stack_base_url + 'f4m', stack_id,
             f4m_id='hds', fatal=False))
-        mp4_url = self._download_webpage(stack_base_url + 'pd', stack_id, fatal=False)
-        if mp4_url:
-            formats.append({
-                'url': mp4_url,
-                'format_id': 'mp4',
-            })
         self._sort_formats(formats)
 
         return {

From a5d783f525a8d4b62777434607c7f1efc5c34ece Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Mar 2017 23:47:43 +0700
Subject: [PATCH 0523/1696] [channel9] Extract more formats

---
 youtube_dl/extractor/channel9.py | 76 +++++++++++++++++++++++++++-----
 1 file changed, 65 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py
index b1cb58530..717e4eb3b 100644
--- a/youtube_dl/extractor/channel9.py
+++ b/youtube_dl/extractor/channel9.py
@@ -9,6 +9,7 @@ from ..utils import (
     int_or_none,
     parse_iso8601,
     clean_html,
+    qualities,
 )
 
 
@@ -120,22 +121,75 @@ class Channel9IE(InfoExtractor):
             content_data = self._download_json(content_url, content_id)
             title = content_data['Title']
 
+            QUALITIES = (
+                'mp3',
+                'wmv', 'mp4',
+                'wmv-low', 'mp4-low',
+                'wmv-mid', 'mp4-mid',
+                'wmv-high', 'mp4-high',
+            )
+
+            quality_key = qualities(QUALITIES)
+
+            def quality(quality_id, format_url):
+                return (len(QUALITIES) if '_Source.' in format_url
+                        else quality_key(quality_id))
+
             formats = []
-            qualities = [
-                'VideoMP4Low',
-                'VideoWMV',
-                'VideoMP4Medium',
-                'VideoMP4High',
-                'VideoWMVHQ',
-            ]
-            for q in qualities:
-                q_url = content_data.get(q)
-                if not q_url:
+            urls = set()
+
+            SITE_QUALITIES = {
+                'MP3': 'mp3',
+                'MP4': 'mp4',
+                'Low Quality WMV': 'wmv-low',
+                'Low Quality MP4': 'mp4-low',
+                'Mid Quality WMV': 'wmv-mid',
+                'Mid Quality MP4': 'mp4-mid',
+                'High Quality WMV': 'wmv-high',
+                'High Quality MP4': 'mp4-high',
+            }
+
+            formats_select = self._search_regex(
+                r'(?s)<select[^>]+name=["\']format[^>]+>(.+?)</select', webpage,
+                'formats select', default=None)
+            if formats_select:
+                for mobj in re.finditer(
+                        r'<option\b[^>]+\bvalue=(["\'])(?P<url>(?:(?!\1).)+)\1[^>]*>\s*(?P<format>[^<]+?)\s*<',
+                        formats_select):
+                    format_url = mobj.group('url')
+                    if format_url in urls:
+                        continue
+                    urls.add(format_url)
+                    format_id = mobj.group('format')
+                    quality_id = SITE_QUALITIES.get(format_id, format_id)
+                    formats.append({
+                        'url': format_url,
+                        'format_id': quality_id,
+                        'quality': quality(quality_id, format_url),
+                        'vcodec': 'none' if quality_id == 'mp3' else None,
+                    })
+
+            API_QUALITIES = {
+                'VideoMP4Low': 'mp4-low',
+                'VideoWMV': 'wmv-mid',
+                'VideoMP4Medium': 'mp4-mid',
+                'VideoMP4High': 'mp4-high',
+                'VideoWMVHQ': 'wmv-hq',
+            }
+
+            for format_id, q in API_QUALITIES.items():
+                q_url = content_data.get(format_id)
+                if not q_url or q_url in urls:
                     continue
+                urls.add(q_url)
                 formats.append({
-                    'format_id': q,
                     'url': q_url,
+                    'format_id': q,
+                    'quality': quality(q, q_url),
                 })
+
+            self._sort_formats(formats)
+
             slides = content_data.get('Slides')
             zip_file = content_data.get('ZipFile')
 

From bea7af694748f3d731ab4340539251f2daf5cc10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Mar 2017 23:58:12 +0700
Subject: [PATCH 0524/1696] [channel9] Remove expired comment and sort imports

---
 youtube_dl/extractor/channel9.py | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py
index 717e4eb3b..e92894246 100644
--- a/youtube_dl/extractor/channel9.py
+++ b/youtube_dl/extractor/channel9.py
@@ -4,23 +4,16 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    clean_html,
     ExtractorError,
-    unescapeHTML,
     int_or_none,
     parse_iso8601,
-    clean_html,
     qualities,
+    unescapeHTML,
 )
 
 
 class Channel9IE(InfoExtractor):
-    '''
-    Common extractor for channel9.msdn.com.
-
-    The type of provided URL (video or playlist) is determined according to
-    meta Search.PageType from web page HTML rather than URL itself, as it is
-    not always possible to do.
-    '''
     IE_DESC = 'Channel 9'
     IE_NAME = 'channel9'
     _VALID_URL = r'https?://(?:www\.)?(?:channel9\.msdn\.com|s\.ch9\.ms)/(?P<contentpath>.+?)(?P<rss>/RSS)?/?(?:[?#&]|$)'

From 7963b6cba8d020d5553bb98aee1d098870f78f42 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Mar 2017 00:19:58 +0700
Subject: [PATCH 0525/1696] [ChangeLog] Actualize

---
 ChangeLog | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index dc5acbca9..2df64ea73 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,15 @@
+version <unreleased>
+
+Extractors
+- [9c9media] Remove mp4 URL extraction request
++ [bellmedia] Add support for etalk.ca and space.ca (#12447)
+* [channel9] Fix extraction (#11323)
+* [cloudy] Fix extraction (#12525)
++ [hbo] Add support for free episode URLs and new formats extraction (#12519)
+* [condenast] Fix extraction and style (#12526)
+* [viu] Relax URL regular expression (#12529)
+
+
 version 2017.03.22
 
 Extractors

From a3ccd6bd11454b9760ef2c5f09f02f3afdb11af5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Mar 2017 00:24:23 +0700
Subject: [PATCH 0526/1696] release 2017.03.24

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 4 ++--
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 31ba1de3d..dfff41d2d 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.03.22*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.03.22**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.03.24*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.03.24**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.03.22
+[debug] youtube-dl version 2017.03.24
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 2df64ea73..78377dcb4 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.03.24
 
 Extractors
 - [9c9media] Remove mp4 URL extraction request
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 6a7be28cb..7c99ba3c2 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -312,8 +312,8 @@
  - **GPUTechConf**
  - **Groupon**
  - **Hark**
- - **HBO**
- - **HBOEpisode**
+ - **hbo**
+ - **hbo:episode**
  - **HearThisAt**
  - **Heise**
  - **HellPorno**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4d722873d..13904c724 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.03.22'
+__version__ = '2017.03.24'

From 54b960f340ed5398136ef0206d17cafba2575678 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Mar 2017 00:45:24 +0700
Subject: [PATCH 0527/1696] [generic] Do not follow redirects to the same URL

---
 youtube_dl/extractor/generic.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index cb6308d29..da9d04efc 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2635,11 +2635,14 @@ class GenericIE(InfoExtractor):
                     found = re.search(REDIRECT_REGEX, refresh_header)
             if found:
                 new_url = compat_urlparse.urljoin(url, unescapeHTML(found.group(1)))
-                self.report_following_redirect(new_url)
-                return {
-                    '_type': 'url',
-                    'url': new_url,
-                }
+                if new_url != url:
+                    self.report_following_redirect(new_url)
+                    return {
+                        '_type': 'url',
+                        'url': new_url,
+                    }
+                else:
+                    found = None
 
         if not found:
             # twitter:player is a https URL to iframe player that may or may not

From d0ba55871e6754fdc8a6a28543581989ba3c50fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Mar 2017 01:17:17 +0700
Subject: [PATCH 0528/1696] [youtube] Improve _VALID_URLs (closes #12538)

---
 youtube_dl/extractor/youtube.py | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index caa048249..ca40de522 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -59,6 +59,8 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
     # If True it will raise an error if no login info is provided
     _LOGIN_REQUIRED = False
 
+    _PLAYLIST_ID_RE = r'(?:PL|LL|EC|UU|FL|RD|UL|TL)[0-9A-Za-z-_]{10,}'
+
     def _set_language(self):
         self._set_cookie(
             '.youtube.com', 'PREF', 'f1=50000000&hl=en',
@@ -265,9 +267,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                          )
                      )?                                                       # all until now is optional -> you can pass the naked ID
                      ([0-9A-Za-z_-]{11})                                      # here is it! the YouTube video ID
-                     (?!.*?\blist=)                                            # combined list/video URLs are handled by the playlist IE
+                     (?!.*?\blist=
+                        (?:
+                            %(playlist_id)s|                                  # combined list/video URLs are handled by the playlist IE
+                            WL                                                # WL are handled by the watch later IE
+                        )
+                     )
                      (?(1).+)?                                                # if we found the ID, everything can follow
-                     $"""
+                     $""" % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
     _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
     _formats = {
         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
@@ -924,6 +931,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'url': 'sJL6WA-aGkQ',
             'only_matching': True,
         },
+        {
+            'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
+            'only_matching': True,
+        },
     ]
 
     def __init__(self, *args, **kwargs):
@@ -1864,8 +1875,8 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
                         )
                         .*
                      |
-                        ((?:PL|LL|EC|UU|FL|RD|UL|TL)[0-9A-Za-z-_]{10,})
-                     )"""
+                        (%(playlist_id)s)
+                     )""" % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
     _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s&disable_polymer=true'
     _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})&amp;[^"]*?index=(?P<index>\d+)(?:[^>]+>(?P<title>[^<]+))?'
     IE_NAME = 'youtube:playlist'

From 31a1214076ff41efbaa7f7243565da830d1e2c7e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 25 Mar 2017 07:03:13 +0100
Subject: [PATCH 0529/1696] [franceculture] fix extraction(closes #12547)

---
 youtube_dl/extractor/franceculture.py | 31 ++++++++++++++++-----------
 1 file changed, 19 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/franceculture.py b/youtube_dl/extractor/franceculture.py
index b98da692c..df3d757f3 100644
--- a/youtube_dl/extractor/franceculture.py
+++ b/youtube_dl/extractor/franceculture.py
@@ -4,6 +4,8 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..utils import (
     determine_ext,
+    extract_attributes,
+    int_or_none,
     unified_strdate,
 )
 
@@ -19,6 +21,7 @@ class FranceCultureIE(InfoExtractor):
             'title': 'Rendez-vous au pays des geeks',
             'thumbnail': r're:^https?://.*\.jpg$',
             'upload_date': '20140301',
+            'timestamp': 1393642916,
             'vcodec': 'none',
         }
     }
@@ -28,30 +31,34 @@ class FranceCultureIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
-        video_url = self._search_regex(
-            r'(?s)<div[^>]+class="[^"]*?title-zone-diffusion[^"]*?"[^>]*>.*?<button[^>]+data-asset-source="([^"]+)"',
-            webpage, 'video path')
+        video_data = extract_attributes(self._search_regex(
+            r'(?s)<div[^>]+class="[^"]*?(?:title-zone-diffusion|heading-zone-(?:wrapper|player-button))[^"]*?"[^>]*>.*?(<button[^>]+data-asset-source="[^"]+"[^>]+>)',
+            webpage, 'video data'))
 
-        title = self._og_search_title(webpage)
+        video_url = video_data['data-asset-source']
+        title = video_data.get('data-asset-title') or self._og_search_title(webpage)
 
-        upload_date = unified_strdate(self._search_regex(
-            '(?s)<div[^>]+class="date"[^>]*>.*?<span[^>]+class="inner"[^>]*>([^<]+)<',
-            webpage, 'upload date', fatal=False))
+        description = self._html_search_regex(
+            r'(?s)<div[^>]+class="intro"[^>]*>.*?<h2>(.+?)</h2>',
+            webpage, 'description', default=None)
         thumbnail = self._search_regex(
-            r'(?s)<figure[^>]+itemtype="https://schema.org/ImageObject"[^>]*>.*?<img[^>]+data-dejavu-src="([^"]+)"',
+            r'(?s)<figure[^>]+itemtype="https://schema.org/ImageObject"[^>]*>.*?<img[^>]+(?:data-dejavu-)?src="([^"]+)"',
             webpage, 'thumbnail', fatal=False)
         uploader = self._html_search_regex(
-            r'(?s)<div id="emission".*?<span class="author">(.*?)</span>',
+            r'(?s)<span class="author">(.*?)</span>',
             webpage, 'uploader', default=None)
-        vcodec = 'none' if determine_ext(video_url.lower()) == 'mp3' else None
+        ext = determine_ext(video_url.lower())
 
         return {
             'id': display_id,
             'display_id': display_id,
             'url': video_url,
             'title': title,
+            'description': description,
             'thumbnail': thumbnail,
-            'vcodec': vcodec,
+            'ext': ext,
+            'vcodec': 'none' if ext == 'mp3' else None,
             'uploader': uploader,
-            'upload_date': upload_date,
+            'timestamp': int_or_none(video_data.get('data-asset-created-date')),
+            'duration': int_or_none(video_data.get('data-duration')),
         }

From 1088d76da6cbc83d64faca5a1a987944af04b0ce Mon Sep 17 00:00:00 2001
From: zurfyx <zurfyx@gmail.com>
Date: Sat, 25 Mar 2017 00:45:32 +0100
Subject: [PATCH 0530/1696] [atresplayer] Fix login error detection

---
 youtube_dl/extractor/atresplayer.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index e3c669830..23a536ff2 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -90,7 +90,8 @@ class AtresPlayerIE(InfoExtractor):
             request, None, 'Logging in as %s' % username)
 
         error = self._html_search_regex(
-            r'(?s)<ul class="list_error">(.+?)</ul>', response, 'error', default=None)
+            r'(?s)<ul[^>]+class="[^"]*\blist_error\b[^"]*">(.+?)</ul>',
+            response, 'error', default=None)
         if error:
             raise ExtractorError(
                 'Unable to login: %s' % error, expected=True)

From 048086920bdb92cc5d63847e9e1d2fd645910363 Mon Sep 17 00:00:00 2001
From: zurfyx <zurfyx@gmail.com>
Date: Sat, 25 Mar 2017 01:08:47 +0100
Subject: [PATCH 0531/1696] [atresplayer] Extract HD manifest

---
 youtube_dl/extractor/atresplayer.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 23a536ff2..940c548f4 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -161,7 +161,8 @@ class AtresPlayerIE(InfoExtractor):
                 # this videos are protected by DRM, the f4m downloader doesn't support them
                 continue
             else:
-                f4m_url = video_url[:-9] + '/manifest.f4m'
+                video_url_hd = video_url.replace('free_es', 'es')
+                f4m_url = video_url_hd[:-9] + '/manifest.f4m'
             formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False))
         self._sort_formats(formats)
 

From c7301e677bddb5d676ebf207a3ac485fce330057 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Mar 2017 18:03:46 +0700
Subject: [PATCH 0532/1696] [atresplayer] Extract DASH and ISM formats

---
 youtube_dl/extractor/atresplayer.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 940c548f4..ffac9df0e 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -160,10 +160,15 @@ class AtresPlayerIE(InfoExtractor):
                 f4m_url = 'http://drg.antena3.com/{0}hds/es/sd.f4m'.format(f4m_path)
                 # this videos are protected by DRM, the f4m downloader doesn't support them
                 continue
-            else:
-                video_url_hd = video_url.replace('free_es', 'es')
-                f4m_url = video_url_hd[:-9] + '/manifest.f4m'
-            formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False))
+            video_url_hd = video_url.replace('free_es', 'es')
+            formats.extend(self._extract_f4m_formats(
+                video_url_hd[:-9] + '/manifest.f4m', video_id, f4m_id='hds',
+                fatal=False))
+            formats.extend(self._extract_mpd_formats(
+                video_url_hd[:-9] + '/manifest.mpd', video_id, mpd_id='dash',
+                fatal=False))
+            formats.extend(self._extract_ism_formats(
+                video_url_hd, video_id, ism_id='mss', fatal=False))
         self._sort_formats(formats)
 
         path_data = player.get('pathData')

From e8e4cc5a6a3ad8bf94d9ff9e5bb2d72712e14c34 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Sun, 19 Mar 2017 20:52:25 -0400
Subject: [PATCH 0533/1696] [generic] Replace LazyYT test with skiplagged

discourse.ubuntu.com has gone away, repalce with skiplagged.com.
Be nice to have a non-frontpage URL that might be more stable,
though I don't have one. Maybe this should move to html
in test/test_InfoExtractor.py?
---
 youtube_dl/extractor/generic.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index da9d04efc..4fff93efe 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -902,12 +902,13 @@ class GenericIE(InfoExtractor):
         },
         # LazyYT
         {
-            'url': 'http://discourse.ubuntu.com/t/unity-8-desktop-mode-windows-on-mir/1986',
+            'url': 'https://skiplagged.com/',
             'info_dict': {
-                'id': '1986',
-                'title': 'Unity 8 desktop-mode windows on Mir! - Ubuntu Discourse',
+                'id': 'skiplagged',
+                'title': 'Skiplagged: The smart way to find cheap flights',
             },
-            'playlist_mincount': 2,
+            'playlist_mincount': 1,
+            'add_ie': ['Youtube'],
         },
         # Cinchcast embed
         {

From 7aa0ee321b4095da7a2430f383bea773115e1491 Mon Sep 17 00:00:00 2001
From: gkoelln <gkoelln7@gmail.com>
Date: Sat, 25 Mar 2017 08:12:25 -0500
Subject: [PATCH 0534/1696] [fox] Add metadata extraction

Add series, season number, episode number and episode.
---
 youtube_dl/extractor/fox.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/fox.py b/youtube_dl/extractor/fox.py
index 9f2e5d065..cc5d62ebc 100644
--- a/youtube_dl/extractor/fox.py
+++ b/youtube_dl/extractor/fox.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 from .adobepass import AdobePassIE
 from ..utils import (
+    int_or_none,
     smuggle_url,
     update_url_query,
 )
@@ -47,9 +48,12 @@ class FOXIE(AdobePassIE):
             resource = self._get_mvpd_resource('fbc-fox', None, ap_p['videoGUID'], rating)
             query['auth'] = self._extract_mvpd_auth(url, video_id, 'fbc-fox', resource)
 
-        return {
+        info = self._search_json_ld(webpage, video_id, fatal=False)
+        info.update({
             '_type': 'url_transparent',
             'ie_key': 'ThePlatform',
             'url': smuggle_url(update_url_query(release_url, query), {'force_smil_url': True}),
             'id': video_id,
-        }
+        })
+
+        return info

From d97729c83a747f48d83f4aba9b85d2a14a58b8b7 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 25 Mar 2017 14:28:53 +0100
Subject: [PATCH 0535/1696] [fox] remove unused import

---
 youtube_dl/extractor/fox.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/fox.py b/youtube_dl/extractor/fox.py
index cc5d62ebc..159fdf9c4 100644
--- a/youtube_dl/extractor/fox.py
+++ b/youtube_dl/extractor/fox.py
@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 
 from .adobepass import AdobePassIE
 from ..utils import (
-    int_or_none,
     smuggle_url,
     update_url_query,
 )

From c6c22e984d0d35172d8e39d2136d2059494d22b2 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Sat, 25 Mar 2017 10:36:40 -0400
Subject: [PATCH 0536/1696] [test_download] Print additional IEs in summary
 output

---
 test/test_download.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/test/test_download.py b/test/test_download.py
index 30034f978..01a8bcb89 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -71,6 +71,18 @@ class TestDownload(unittest.TestCase):
 
     maxDiff = None
 
+    def __str__(self):
+        """Identify each test with the `add_ie` attribute, if available."""
+
+        def strclass(cls):
+            """From 2.7's unittest; 2.6 had _strclass so we can't import it."""
+            return '%s.%s' % (cls.__module__, cls.__name__)
+
+        add_ie = getattr(self, self._testMethodName).add_ie
+        return '%s (%s)%s:' % (self._testMethodName,
+                               strclass(self.__class__),
+                               ' [%s]' % add_ie if add_ie else '')
+
     def setUp(self):
         self.defs = defs
 
@@ -233,6 +245,8 @@ for n, test_case in enumerate(defs):
         i += 1
     test_method = generator(test_case, tname)
     test_method.__name__ = str(tname)
+    ie_list = test_case.get('add_ie')
+    test_method.add_ie = ie_list and ','.join(ie_list)
     setattr(TestDownload, test_method.__name__, test_method)
     del test_method
 

From 610a6d10538d8ecab8e51dc083f02adbd09f706f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Mar 2017 21:40:28 +0700
Subject: [PATCH 0537/1696] [atresplayer] Do not extract ISM formats As per
 @remitamine: the ISM downloader does not support videos served from wowza
 servers(it will produce broken files)

---
 youtube_dl/extractor/atresplayer.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index ffac9df0e..bfda1e24e 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -167,8 +167,6 @@ class AtresPlayerIE(InfoExtractor):
             formats.extend(self._extract_mpd_formats(
                 video_url_hd[:-9] + '/manifest.mpd', video_id, mpd_id='dash',
                 fatal=False))
-            formats.extend(self._extract_ism_formats(
-                video_url_hd, video_id, ism_id='mss', fatal=False))
         self._sort_formats(formats)
 
         path_data = player.get('pathData')

From d66d43c5547daf4fc1a269824a8432477fbb099d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 25 Mar 2017 18:13:13 +0100
Subject: [PATCH 0538/1696] [atvat] Add new extractor(closes #5325)

---
 youtube_dl/extractor/atvat.py      | 73 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  1 +
 2 files changed, 74 insertions(+)
 create mode 100644 youtube_dl/extractor/atvat.py

diff --git a/youtube_dl/extractor/atvat.py b/youtube_dl/extractor/atvat.py
new file mode 100644
index 000000000..1584d53fc
--- /dev/null
+++ b/youtube_dl/extractor/atvat.py
@@ -0,0 +1,73 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    int_or_none,
+    unescapeHTML,
+)
+
+
+class ATVAtIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?atv\.at/(?:[^/]+/){2}(?P<id>[dv]\d+)'
+    _TESTS = [{
+        'url': 'http://atv.at/aktuell/di-210317-2005-uhr/v1698449/',
+        'md5': 'c3b6b975fb3150fc628572939df205f2',
+        'info_dict': {
+            'id': '1698447',
+            'ext': 'mp4',
+            'title': 'DI, 21.03.17 | 20:05 Uhr 1/1',
+        }
+    }, {
+        'url': 'http://atv.at/aktuell/meinrad-knapp/d8416/',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        video_data = self._parse_json(unescapeHTML(self._search_regex(
+            r'class="[^"]*jsb_video/FlashPlayer[^"]*"[^>]+data-jsb="([^"]+)"',
+            webpage, 'player data')), display_id)['config']['initial_video']
+
+        video_id = video_data['id']
+        video_title = video_data['title']
+
+        parts = []
+        for part in video_data.get('parts', []):
+            part_id = part['id']
+            part_title = part['title']
+
+            formats = []
+            for source in part.get('sources', []):
+                source_url = source.get('src')
+                if not source_url:
+                    continue
+                ext = determine_ext(source_url)
+                if ext == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        source_url, part_id, 'mp4', 'm3u8_native',
+                        m3u8_id='hls', fatal=False))
+                else:
+                    formats.append({
+                        'format_id': source.get('delivery'),
+                        'url': source_url,
+                    })
+            self._sort_formats(formats)
+
+            parts.append({
+                'id': part_id,
+                'title': part_title,
+                'thumbnail': part.get('preview_image_url'),
+                'duration': int_or_none(part.get('duration')),
+                'is_live': part.get('is_livestream'),
+                'formats': formats,
+            })
+
+        return {
+            '_type': 'multi_video',
+            'id': video_id,
+            'title': video_title,
+            'entries': parts,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 40a5c9842..6a7028a4d 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -71,6 +71,7 @@ from .arte import (
 )
 from .atresplayer import AtresPlayerIE
 from .atttechchannel import ATTTechChannelIE
+from .atvat import ATVAtIE
 from .audimedia import AudiMediaIE
 from .audioboom import AudioBoomIE
 from .audiomack import AudiomackIE, AudiomackAlbumIE

From 51ef4919dfd51b5bd562f39f865a117f9a5cd304 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 6 Mar 2017 00:31:44 +0800
Subject: [PATCH 0539/1696] [afreecatv] Fix extraction (closes #12179)

---
 ChangeLog                         |  6 +++
 youtube_dl/extractor/afreecatv.py | 72 +++++++++++++------------------
 2 files changed, 36 insertions(+), 42 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 78377dcb4..45d6f244d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [afreecatv] Fix extraction (#12179)
+
+
 version 2017.03.24
 
 Extractors
diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py
index e0a0f7c57..b774d6db8 100644
--- a/youtube_dl/extractor/afreecatv.py
+++ b/youtube_dl/extractor/afreecatv.py
@@ -4,15 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse_urlparse,
-    compat_urlparse,
-)
+from ..compat import compat_xpath
 from ..utils import (
     ExtractorError,
     int_or_none,
-    update_url_query,
-    xpath_element,
     xpath_text,
 )
 
@@ -43,7 +38,8 @@ class AfreecaTVIE(InfoExtractor):
             'uploader': 'dailyapril',
             'uploader_id': 'dailyapril',
             'upload_date': '20160503',
-        }
+        },
+        'skip': 'Video is gone',
     }, {
         'url': 'http://afbbs.afreecatv.com:8080/app/read_ucc_bbs.cgi?nStationNo=16711924&nTitleNo=36153164&szBjId=dailyapril&nBbsNo=18605867',
         'info_dict': {
@@ -71,6 +67,19 @@ class AfreecaTVIE(InfoExtractor):
                 'upload_date': '20160502',
             },
         }],
+        'skip': 'Video is gone',
+    }, {
+        'url': 'http://vod.afreecatv.com/PLAYER/STATION/18650793',
+        'info_dict': {
+            'id': '18650793',
+            'ext': 'flv',
+            'uploader': '윈아디',
+            'uploader_id': 'badkids',
+            'title': '오늘은 다르다! 쏘님의 우월한 위아래~ 댄스리액션!',
+        },
+        'params': {
+            'skip_download': True,  # requires rtmpdump
+        },
     }, {
         'url': 'http://www.afreecatv.com/player/Player.swf?szType=szBjId=djleegoon&nStationNo=11273158&nBbsNo=13161095&nTitleNo=36327652',
         'only_matching': True,
@@ -90,40 +99,33 @@ class AfreecaTVIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        parsed_url = compat_urllib_parse_urlparse(url)
-        info_url = compat_urlparse.urlunparse(parsed_url._replace(
-            netloc='afbbs.afreecatv.com:8080',
-            path='/api/video/get_video_info.php'))
 
         video_xml = self._download_xml(
-            update_url_query(info_url, {'nTitleNo': video_id}), video_id)
+            'http://afbbs.afreecatv.com:8080/api/video/get_video_info.php',
+            video_id, query={'nTitleNo': video_id})
 
-        if xpath_element(video_xml, './track/video/file') is None:
+        video_element = video_xml.findall(compat_xpath('./track/video'))[1]
+        if video_element is None or video_element.text is None:
             raise ExtractorError('Specified AfreecaTV video does not exist',
                                  expected=True)
 
-        title = xpath_text(video_xml, './track/title', 'title')
+        video_url_raw = video_element.text
+
+        app, playpath = video_url_raw.split('mp4:')
+
+        title = xpath_text(video_xml, './track/title', 'title', fatal=True)
         uploader = xpath_text(video_xml, './track/nickname', 'uploader')
         uploader_id = xpath_text(video_xml, './track/bj_id', 'uploader id')
         duration = int_or_none(xpath_text(video_xml, './track/duration',
                                           'duration'))
         thumbnail = xpath_text(video_xml, './track/titleImage', 'thumbnail')
 
-        entries = []
-        for i, video_file in enumerate(video_xml.findall('./track/video/file')):
-            video_key = self.parse_video_key(video_file.get('key', ''))
-            if not video_key:
-                continue
-            entries.append({
-                'id': '%s_%s' % (video_id, video_key.get('part', i + 1)),
-                'title': title,
-                'upload_date': video_key.get('upload_date'),
-                'duration': int_or_none(video_file.get('duration')),
-                'url': video_file.text,
-            })
-
-        info = {
+        return {
             'id': video_id,
+            'url': app,
+            'ext': 'flv',
+            'play_path': 'mp4:' + playpath,
+            'rtmp_live': True,  # downloading won't end without this
             'title': title,
             'uploader': uploader,
             'uploader_id': uploader_id,
@@ -131,20 +133,6 @@ class AfreecaTVIE(InfoExtractor):
             'thumbnail': thumbnail,
         }
 
-        if len(entries) > 1:
-            info['_type'] = 'multi_video'
-            info['entries'] = entries
-        elif len(entries) == 1:
-            info['url'] = entries[0]['url']
-            info['upload_date'] = entries[0].get('upload_date')
-        else:
-            raise ExtractorError(
-                'No files found for the specified AfreecaTV video, either'
-                ' the URL is incorrect or the video has been made private.',
-                expected=True)
-
-        return info
-
 
 class AfreecaTVGlobalIE(AfreecaTVIE):
     IE_NAME = 'afreecatv:global'

From 03486dbb0133e42074c272f60e24f18c856fdf0d Mon Sep 17 00:00:00 2001
From: Random User <rndusr@posteo.de>
Date: Sat, 25 Mar 2017 19:37:45 +0100
Subject: [PATCH 0540/1696] Add test for JWPlayer where config is passed as
 variable

---
 youtube_dl/extractor/generic.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 9868ca6d0..c8c103ae3 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -972,6 +972,20 @@ class GenericIE(InfoExtractor):
                 'thumbnail': r're:^https?://.*\.jpg$',
             },
         },
+        {
+            # JWPlayer config passed as variable
+            'url': 'http://www.txxx.com/videos/3326530/ariele/',
+            'info_dict': {
+                'id': '3326530_hq',
+                'ext': 'mp4',
+                'title': 'ARIELE | Tube Cup',
+                'uploader': 'www.txxx.com',
+                'age_limit': 18,
+            },
+            'params': {
+                'skip_download': True,
+            }
+        },
         # rtl.nl embed
         {
             'url': 'http://www.rtlnieuws.nl/nieuws/buitenland/aanslagen-kopenhagen',

From fb4fc44928d042a33287fd3e8e18b721c29ff8e8 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 25 Mar 2017 19:37:54 +0100
Subject: [PATCH 0541/1696] [downloader/hls] immediately delegate downloading
 to ffmpeg in case live stream

---
 youtube_dl/downloader/hls.py          | 21 +++++++++++++--------
 youtube_dl/extractor/arkena.py        |  3 +--
 youtube_dl/extractor/ceskatelevize.py |  3 +--
 youtube_dl/extractor/eyedotv.py       |  2 +-
 youtube_dl/extractor/freshlive.py     |  5 ++---
 youtube_dl/extractor/livestream.py    | 15 ++++++++-------
 youtube_dl/extractor/vk.py            |  3 +--
 7 files changed, 27 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 4989abce1..7534e4da5 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -30,6 +30,15 @@ class HlsFD(FragmentFD):
 
     FD_NAME = 'hlsnative'
 
+    def _delegate_to_ffmpeg(self, filename, info_dict):
+        self.report_warning(
+            'hlsnative has detected features it does not support, '
+            'extraction will be delegated to ffmpeg')
+        fd = FFmpegFD(self.ydl, self.params)
+        for ph in self._progress_hooks:
+            fd.add_progress_hook(ph)
+        return fd.real_download(filename, info_dict)
+
     @staticmethod
     def can_download(manifest, info_dict):
         UNSUPPORTED_FEATURES = (
@@ -53,10 +62,12 @@ class HlsFD(FragmentFD):
         )
         check_results = [not re.search(feature, manifest) for feature in UNSUPPORTED_FEATURES]
         check_results.append(can_decrypt_frag or '#EXT-X-KEY:METHOD=AES-128' not in manifest)
-        check_results.append(not info_dict.get('is_live'))
         return all(check_results)
 
     def real_download(self, filename, info_dict):
+        if info_dict.get('is_live'):
+            return self._delegate_to_ffmpeg(filename, info_dict)
+
         man_url = info_dict['url']
         self.to_screen('[%s] Downloading m3u8 manifest' % self.FD_NAME)
 
@@ -68,13 +79,7 @@ class HlsFD(FragmentFD):
             if info_dict.get('extra_param_to_segment_url'):
                 self.report_error('pycrypto not found. Please install it.')
                 return False
-            self.report_warning(
-                'hlsnative has detected features it does not support, '
-                'extraction will be delegated to ffmpeg')
-            fd = FFmpegFD(self.ydl, self.params)
-            for ph in self._progress_hooks:
-                fd.add_progress_hook(ph)
-            return fd.real_download(filename, info_dict)
+            return self._delegate_to_ffmpeg(filename, info_dict)
 
         total_frags = 0
         for line in s.splitlines():
diff --git a/youtube_dl/extractor/arkena.py b/youtube_dl/extractor/arkena.py
index 50ffb442d..4495ddbb0 100644
--- a/youtube_dl/extractor/arkena.py
+++ b/youtube_dl/extractor/arkena.py
@@ -93,8 +93,7 @@ class ArkenaIE(InfoExtractor):
                 exts = (mimetype2ext(f.get('Type')), determine_ext(f_url, None))
                 if kind == 'm3u8' or 'm3u8' in exts:
                     formats.extend(self._extract_m3u8_formats(
-                        f_url, video_id, 'mp4',
-                        entry_protocol='m3u8' if is_live else 'm3u8_native',
+                        f_url, video_id, 'mp4', 'm3u8_native',
                         m3u8_id=kind, fatal=False, live=is_live))
                 elif kind == 'flash' or 'f4m' in exts:
                     formats.extend(self._extract_f4m_formats(
diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index b1dfacf80..dd2529a6d 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -160,8 +160,7 @@ class CeskaTelevizeIE(InfoExtractor):
                 for format_id, stream_url in item.get('streamUrls', {}).items():
                     if 'playerType=flash' in stream_url:
                         stream_formats = self._extract_m3u8_formats(
-                            stream_url, playlist_id, 'mp4',
-                            entry_protocol='m3u8' if is_live else 'm3u8_native',
+                            stream_url, playlist_id, 'mp4', 'm3u8_native',
                             m3u8_id='hls-%s' % format_id, fatal=False)
                     else:
                         stream_formats = self._extract_mpd_formats(
diff --git a/youtube_dl/extractor/eyedotv.py b/youtube_dl/extractor/eyedotv.py
index 2f3035147..f62ddebae 100644
--- a/youtube_dl/extractor/eyedotv.py
+++ b/youtube_dl/extractor/eyedotv.py
@@ -54,7 +54,7 @@ class EyedoTVIE(InfoExtractor):
             'id': video_id,
             'title': title,
             'formats': self._extract_m3u8_formats(
-                m3u8_url, video_id, 'mp4', 'm3u8' if is_live else 'm3u8_native'),
+                m3u8_url, video_id, 'mp4', 'm3u8_native'),
             'description': xpath_text(video_data, _add_ns('Description')),
             'duration': parse_duration(xpath_text(video_data, _add_ns('Duration'))),
             'uploader': xpath_text(video_data, _add_ns('Createur')),
diff --git a/youtube_dl/extractor/freshlive.py b/youtube_dl/extractor/freshlive.py
index a90f9156c..72a845945 100644
--- a/youtube_dl/extractor/freshlive.py
+++ b/youtube_dl/extractor/freshlive.py
@@ -56,9 +56,8 @@ class FreshLiveIE(InfoExtractor):
         is_live = info.get('liveStreamUrl') is not None
 
         formats = self._extract_m3u8_formats(
-            stream_url, video_id, ext='mp4',
-            entry_protocol='m3u8' if is_live else 'm3u8_native',
-            m3u8_id='hls')
+            stream_url, video_id, 'mp4',
+            'm3u8_native', m3u8_id='hls')
 
         if is_live:
             title = self._live_title(title)
diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index c863413bf..7f946c6ed 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -119,7 +119,8 @@ class LivestreamIE(InfoExtractor):
         m3u8_url = video_data.get('m3u8_url')
         if m3u8_url:
             formats.extend(self._extract_m3u8_formats(
-                m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+                m3u8_url, video_id, 'mp4', 'm3u8_native',
+                m3u8_id='hls', fatal=False))
 
         f4m_url = video_data.get('f4m_url')
         if f4m_url:
@@ -158,11 +159,11 @@ class LivestreamIE(InfoExtractor):
         if smil_url:
             formats.extend(self._extract_smil_formats(smil_url, broadcast_id))
 
-        entry_protocol = 'm3u8' if is_live else 'm3u8_native'
         m3u8_url = stream_info.get('m3u8_url')
         if m3u8_url:
             formats.extend(self._extract_m3u8_formats(
-                m3u8_url, broadcast_id, 'mp4', entry_protocol, m3u8_id='hls', fatal=False))
+                m3u8_url, broadcast_id, 'mp4', 'm3u8_native',
+                m3u8_id='hls', fatal=False))
 
         rtsp_url = stream_info.get('rtsp_url')
         if rtsp_url:
@@ -276,7 +277,7 @@ class LivestreamOriginalIE(InfoExtractor):
             'view_count': view_count,
         }
 
-    def _extract_video_formats(self, video_data, video_id, entry_protocol):
+    def _extract_video_formats(self, video_data, video_id):
         formats = []
 
         progressive_url = video_data.get('progressiveUrl')
@@ -289,7 +290,8 @@ class LivestreamOriginalIE(InfoExtractor):
         m3u8_url = video_data.get('httpUrl')
         if m3u8_url:
             formats.extend(self._extract_m3u8_formats(
-                m3u8_url, video_id, 'mp4', entry_protocol, m3u8_id='hls', fatal=False))
+                m3u8_url, video_id, 'mp4', 'm3u8_native',
+                m3u8_id='hls', fatal=False))
 
         rtsp_url = video_data.get('rtspUrl')
         if rtsp_url:
@@ -340,11 +342,10 @@ class LivestreamOriginalIE(InfoExtractor):
                 }
             video_data = self._download_json(stream_url, content_id)
             is_live = video_data.get('isLive')
-            entry_protocol = 'm3u8' if is_live else 'm3u8_native'
             info.update({
                 'id': content_id,
                 'title': self._live_title(info['title']) if is_live else info['title'],
-                'formats': self._extract_video_formats(video_data, content_id, entry_protocol),
+                'formats': self._extract_video_formats(video_data, content_id),
                 'is_live': is_live,
             })
             return info
diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 7c42a4f54..dc2719cf9 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -432,8 +432,7 @@ class VKIE(VKBaseIE):
                 })
             elif format_id == 'hls':
                 formats.extend(self._extract_m3u8_formats(
-                    format_url, video_id, 'mp4',
-                    entry_protocol='m3u8' if is_live else 'm3u8_native',
+                    format_url, video_id, 'mp4', 'm3u8_native',
                     m3u8_id=format_id, fatal=False, live=is_live))
             elif format_id == 'rtmp':
                 formats.append({

From c73e330e7adc9c0c15ac51aeea8fbb7dad95351a Mon Sep 17 00:00:00 2001
From: Random User <rndusr@posteo.de>
Date: Sat, 25 Mar 2017 19:38:30 +0100
Subject: [PATCH 0542/1696] _find_jwplayer_data() returns dict or None

This simplifies code for callers of `_find_jwplayer_data()` which no longer have
to run `_parse_json()` on the return value.

It also makes sure that `_find_jwplayer_data()` returns either a `dict` or
`None` and nothing else.
---
 youtube_dl/extractor/common.py  | 18 ++++++++++++------
 youtube_dl/extractor/generic.py | 12 ++++--------
 youtube_dl/extractor/tvnoe.py   |  5 ++---
 3 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index eb3c091aa..c2ca73ee1 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2161,18 +2161,24 @@ class InfoExtractor(object):
                     })
         return formats
 
-    @staticmethod
-    def _find_jwplayer_data(webpage):
+    def _find_jwplayer_data(self, webpage, video_id=None, transform_source=js_to_json):
         mobj = re.search(
             r'jwplayer\((?P<quote>[\'"])[^\'" ]+(?P=quote)\)\.setup\s*\((?P<options>[^)]+)\)',
             webpage)
         if mobj:
-            return mobj.group('options')
+            try:
+                jwplayer_data = self._parse_json(mobj.group('options'),
+                                                 video_id=video_id,
+                                                 transform_source=transform_source)
+            except ExtractorError:
+                pass
+            else:
+                if isinstance(jwplayer_data, dict):
+                    return jwplayer_data
 
     def _extract_jwplayer_data(self, webpage, video_id, *args, **kwargs):
-        jwplayer_data = self._parse_json(
-            self._find_jwplayer_data(webpage), video_id,
-            transform_source=js_to_json)
+        jwplayer_data = self._find_jwplayer_data(
+            webpage, video_id, transform_source=js_to_json)
         return self._parse_jwplayer_data(
             jwplayer_data, video_id, *args, **kwargs)
 
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c8c103ae3..3fe0237b6 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2518,14 +2518,10 @@ class GenericIE(InfoExtractor):
                 self._sort_formats(entry['formats'])
             return self.playlist_result(entries)
 
-        jwplayer_data_str = self._find_jwplayer_data(webpage)
-        if jwplayer_data_str:
-            try:
-                jwplayer_data = self._parse_json(
-                    jwplayer_data_str, video_id, transform_source=js_to_json)
-                return self._parse_jwplayer_data(jwplayer_data, video_id)
-            except ExtractorError:
-                pass
+        jwplayer_data = self._find_jwplayer_data(
+            webpage, video_id, transform_source=js_to_json)
+        if jwplayer_data:
+            return self._parse_jwplayer_data(jwplayer_data, video_id)
 
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
diff --git a/youtube_dl/extractor/tvnoe.py b/youtube_dl/extractor/tvnoe.py
index 1a5b76bf2..26a5aeae4 100644
--- a/youtube_dl/extractor/tvnoe.py
+++ b/youtube_dl/extractor/tvnoe.py
@@ -31,9 +31,8 @@ class TVNoeIE(InfoExtractor):
             r'<iframe[^>]+src="([^"]+)"', webpage, 'iframe URL')
 
         ifs_page = self._download_webpage(iframe_url, video_id)
-        jwplayer_data = self._parse_json(
-            self._find_jwplayer_data(ifs_page),
-            video_id, transform_source=js_to_json)
+        jwplayer_data = self._find_jwplayer_data(
+            ifs_page, video_id, transform_source=js_to_json)
         info_dict = self._parse_jwplayer_data(
             jwplayer_data, video_id, require_title=False, base_url=iframe_url)
 

From 51098426b83a8ebce4b0c08e869ce023232089fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Mar 2017 02:30:10 +0700
Subject: [PATCH 0543/1696] [utils] Introduce expand_path

---
 test/test_utils.py  | 10 ++++++++++
 youtube_dl/utils.py |  6 ++++++
 2 files changed, 16 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 173c49514..8c50b46e8 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -56,6 +56,7 @@ from youtube_dl.utils import (
     read_batch_urls,
     sanitize_filename,
     sanitize_path,
+    expand_path,
     prepend_extension,
     replace_extension,
     remove_start,
@@ -95,6 +96,8 @@ from youtube_dl.utils import (
 from youtube_dl.compat import (
     compat_chr,
     compat_etree_fromstring,
+    compat_getenv,
+    compat_setenv,
     compat_urlparse,
     compat_parse_qs,
 )
@@ -214,6 +217,13 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(sanitize_path('./abc'), 'abc')
         self.assertEqual(sanitize_path('./../abc'), '..\\abc')
 
+    def test_expand_path(self):
+        compat_setenv('YOUTUBE-DL-EXPATH-PATH', 'expanded')
+        self.assertEqual(expand_path('%YOUTUBE-DL-EXPATH-PATH%'), 'expanded')
+        self.assertEqual(expand_path('%HOMEPATH%'), compat_getenv('HOMEPATH'))
+        self.assertEqual(expand_path('~'), compat_getenv('HOME'))
+        self.assertEqual(expand_path('~/%YOUTUBE-DL-EXPATH-PATH%'), '%s/expanded' % compat_getenv('HOME'))
+
     def test_prepend_extension(self):
         self.assertEqual(prepend_extension('abc.ext', 'temp'), 'abc.temp.ext')
         self.assertEqual(prepend_extension('abc.ext', 'temp', 'ext'), 'abc.temp.ext')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d293c7498..2340bc306 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -39,6 +39,7 @@ from .compat import (
     compat_basestring,
     compat_chr,
     compat_etree_fromstring,
+    compat_expanduser,
     compat_html_entities,
     compat_html_entities_html5,
     compat_http_client,
@@ -539,6 +540,11 @@ def sanitized_Request(url, *args, **kwargs):
     return compat_urllib_request.Request(sanitize_url(url), *args, **kwargs)
 
 
+def expand_path(s):
+    """Expand shell variables and ~"""
+    return os.path.expandvars(compat_expanduser(s))
+
+
 def orderedSet(iterable):
     """ Remove all duplicates from the input iterable """
     res = []

From 590bc6f6a1cb513852a22f6db0ee36e9bd138f64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Mar 2017 02:31:16 +0700
Subject: [PATCH 0544/1696] Use expand_path where appropriate (closes #12556)

---
 youtube_dl/YoutubeDL.py | 6 +++---
 youtube_dl/__init__.py  | 8 ++++----
 youtube_dl/cache.py     | 9 ++++++---
 3 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index cb502c26f..21586f0f4 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -29,7 +29,6 @@ import random
 from .compat import (
     compat_basestring,
     compat_cookiejar,
-    compat_expanduser,
     compat_get_terminal_size,
     compat_http_client,
     compat_kwargs,
@@ -54,6 +53,7 @@ from .utils import (
     encode_compat_str,
     encodeFilename,
     error_to_compat_str,
+    expand_path,
     ExtractorError,
     format_bytes,
     formatSeconds,
@@ -672,7 +672,7 @@ class YoutubeDL(object):
                         FORMAT_RE.format(numeric_field),
                         r'%({0})s'.format(numeric_field), outtmpl)
 
-            tmpl = compat_expanduser(outtmpl)
+            tmpl = expand_path(outtmpl)
             filename = tmpl % template_dict
             # Temporary fix for #4787
             # 'Treat' all problem characters by passing filename through preferredencoding
@@ -2170,7 +2170,7 @@ class YoutubeDL(object):
         if opts_cookiefile is None:
             self.cookiejar = compat_cookiejar.CookieJar()
         else:
-            opts_cookiefile = compat_expanduser(opts_cookiefile)
+            opts_cookiefile = expand_path(opts_cookiefile)
             self.cookiejar = compat_cookiejar.MozillaCookieJar(
                 opts_cookiefile)
             if os.access(opts_cookiefile, os.R_OK):
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 2f640607f..f15606568 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -16,7 +16,6 @@ from .options import (
     parseOpts,
 )
 from .compat import (
-    compat_expanduser,
     compat_getpass,
     compat_shlex_split,
     workaround_optparse_bug9161,
@@ -26,6 +25,7 @@ from .utils import (
     decodeOption,
     DEFAULT_OUTTMPL,
     DownloadError,
+    expand_path,
     match_filter_func,
     MaxDownloadsReached,
     preferredencoding,
@@ -88,7 +88,7 @@ def _real_main(argv=None):
                 batchfd = sys.stdin
             else:
                 batchfd = io.open(
-                    compat_expanduser(opts.batchfile),
+                    expand_path(opts.batchfile),
                     'r', encoding='utf-8', errors='ignore')
             batch_urls = read_batch_urls(batchfd)
             if opts.verbose:
@@ -238,7 +238,7 @@ def _real_main(argv=None):
 
     any_getting = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson or opts.dump_single_json
     any_printing = opts.print_json
-    download_archive_fn = compat_expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
+    download_archive_fn = expand_path(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 
     # PostProcessors
     postprocessors = []
@@ -449,7 +449,7 @@ def _real_main(argv=None):
 
         try:
             if opts.load_info_filename is not None:
-                retcode = ydl.download_with_info_file(compat_expanduser(opts.load_info_filename))
+                retcode = ydl.download_with_info_file(expand_path(opts.load_info_filename))
             else:
                 retcode = ydl.download(all_urls)
         except MaxDownloadsReached:
diff --git a/youtube_dl/cache.py b/youtube_dl/cache.py
index 5fe839eb1..7bdade1bd 100644
--- a/youtube_dl/cache.py
+++ b/youtube_dl/cache.py
@@ -8,8 +8,11 @@ import re
 import shutil
 import traceback
 
-from .compat import compat_expanduser, compat_getenv
-from .utils import write_json_file
+from .compat import compat_getenv
+from .utils import (
+    expand_path,
+    write_json_file,
+)
 
 
 class Cache(object):
@@ -21,7 +24,7 @@ class Cache(object):
         if res is None:
             cache_root = compat_getenv('XDG_CACHE_HOME', '~/.cache')
             res = os.path.join(cache_root, 'youtube-dl')
-        return compat_expanduser(res)
+        return expand_path(res)
 
     def _get_cache_fn(self, section, key, dtype):
         assert re.match(r'^[a-zA-Z0-9_.-]+$', section), \

From 5b7cc56b05ff4e3936da7a7c0bec5f8d5c9f27c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Mar 2017 02:32:14 +0700
Subject: [PATCH 0545/1696] [atresplayer] PEP 8

---
 youtube_dl/extractor/atresplayer.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index bfda1e24e..99af6dc5a 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -156,8 +156,8 @@ class AtresPlayerIE(InfoExtractor):
             if format_id == 'token' or not video_url.startswith('http'):
                 continue
             if 'geodeswowsmpra3player' in video_url:
-                f4m_path = video_url.split('smil:', 1)[-1].split('free_', 1)[0]
-                f4m_url = 'http://drg.antena3.com/{0}hds/es/sd.f4m'.format(f4m_path)
+                # f4m_path = video_url.split('smil:', 1)[-1].split('free_', 1)[0]
+                # f4m_url = 'http://drg.antena3.com/{0}hds/es/sd.f4m'.format(f4m_path)
                 # this videos are protected by DRM, the f4m downloader doesn't support them
                 continue
             video_url_hd = video_url.replace('free_es', 'es')

From 15495cf3e5f8dbab5559936006df77e3ac0a370b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Mar 2017 02:32:46 +0700
Subject: [PATCH 0546/1696] [franceculture] PEP 8

---
 youtube_dl/extractor/franceculture.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/franceculture.py b/youtube_dl/extractor/franceculture.py
index df3d757f3..b8fa17588 100644
--- a/youtube_dl/extractor/franceculture.py
+++ b/youtube_dl/extractor/franceculture.py
@@ -6,7 +6,6 @@ from ..utils import (
     determine_ext,
     extract_attributes,
     int_or_none,
-    unified_strdate,
 )
 
 

From d212c93d16bbb6aeb8645f1dbb15a78f3d9414a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Mar 2017 02:34:25 +0700
Subject: [PATCH 0547/1696] [pluralsight] PEP 8

---
 youtube_dl/extractor/pluralsight.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 0c6e036ca..e45d9fe55 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -169,11 +169,10 @@ class PluralsightIE(PluralsightBaseIE):
 
         collection = course['modules']
 
-        module, clip = None, None
+        clip = None
 
         for module_ in collection:
             if name in (module_.get('moduleName'), module_.get('name')):
-                module = module_
                 for clip_ in module_.get('clips', []):
                     clip_index = clip_.get('clipIndex')
                     if clip_index is None:

From 41c5e60dd57c0df10f4aa05dee95af2bbc1dc8dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Mar 2017 03:07:56 +0700
Subject: [PATCH 0548/1696] [test_utils] Fix expand_path tests

---
 test/test_utils.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 8c50b46e8..b9a02666d 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -218,11 +218,16 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(sanitize_path('./../abc'), '..\\abc')
 
     def test_expand_path(self):
+        def env(var):
+            return '%{0}%'.format(var) if sys.platform == 'win32' else '${0}'.format(var)
+
         compat_setenv('YOUTUBE-DL-EXPATH-PATH', 'expanded')
-        self.assertEqual(expand_path('%YOUTUBE-DL-EXPATH-PATH%'), 'expanded')
-        self.assertEqual(expand_path('%HOMEPATH%'), compat_getenv('HOMEPATH'))
+        self.assertEqual(expand_path(env('YOUTUBE-DL-EXPATH-PATH')), 'expanded')
+        self.assertEqual(expand_path(env('HOMEPATH')), compat_getenv('HOMEPATH'))
         self.assertEqual(expand_path('~'), compat_getenv('HOME'))
-        self.assertEqual(expand_path('~/%YOUTUBE-DL-EXPATH-PATH%'), '%s/expanded' % compat_getenv('HOME'))
+        self.assertEqual(
+            expand_path('~/%s' % env('YOUTUBE-DL-EXPATH-PATH')),
+            '%s/expanded' % compat_getenv('HOME'))
 
     def test_prepend_extension(self):
         self.assertEqual(prepend_extension('abc.ext', 'temp'), 'abc.temp.ext')

From a426ef6d783038e570db252a2e9e72800ffcb381 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Mar 2017 03:22:48 +0700
Subject: [PATCH 0549/1696] [test_utils] Do not use dash in env variables'
 names

---
 test/test_utils.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index b9a02666d..aa4569b81 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -221,12 +221,12 @@ class TestUtil(unittest.TestCase):
         def env(var):
             return '%{0}%'.format(var) if sys.platform == 'win32' else '${0}'.format(var)
 
-        compat_setenv('YOUTUBE-DL-EXPATH-PATH', 'expanded')
-        self.assertEqual(expand_path(env('YOUTUBE-DL-EXPATH-PATH')), 'expanded')
-        self.assertEqual(expand_path(env('HOMEPATH')), compat_getenv('HOMEPATH'))
+        compat_setenv('YOUTUBE_DL_EXPATH_PATH', 'expanded')
+        self.assertEqual(expand_path(env('YOUTUBE_DL_EXPATH_PATH')), 'expanded')
+        self.assertEqual(expand_path(env('HOME')), compat_getenv('HOME'))
         self.assertEqual(expand_path('~'), compat_getenv('HOME'))
         self.assertEqual(
-            expand_path('~/%s' % env('YOUTUBE-DL-EXPATH-PATH')),
+            expand_path('~/%s' % env('YOUTUBE_DL_EXPATH_PATH')),
             '%s/expanded' % compat_getenv('HOME'))
 
     def test_prepend_extension(self):

From 942b44a0525f677924c660bcb00902d705d91fc2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Mar 2017 03:24:25 +0700
Subject: [PATCH 0550/1696] [test_compat] Do not use dash in env variables'
 names

---
 test/test_compat.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index b57424948..d6c54e135 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -27,11 +27,11 @@ from youtube_dl.compat import (
 class TestCompat(unittest.TestCase):
     def test_compat_getenv(self):
         test_str = 'тест'
-        compat_setenv('YOUTUBE-DL-TEST', test_str)
-        self.assertEqual(compat_getenv('YOUTUBE-DL-TEST'), test_str)
+        compat_setenv('YOUTUBE_DL_COMPAT_GETENV', test_str)
+        self.assertEqual(compat_getenv('YOUTUBE_DL_COMPAT_GETENV'), test_str)
 
     def test_compat_setenv(self):
-        test_var = 'YOUTUBE-DL-TEST'
+        test_var = 'YOUTUBE_DL_COMPAT_SETENV'
         test_str = 'тест'
         compat_setenv(test_var, test_str)
         compat_getenv(test_var)

From 2bfaf89b6cc6dd07ed6ca32086c72a98b67c20ba Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 25 Mar 2017 23:06:33 +0100
Subject: [PATCH 0551/1696] [downloader/hls] move check for m3u8 live streams
 to get_suitable_downloader

---
 youtube_dl/downloader/__init__.py |  3 +++
 youtube_dl/downloader/hls.py      | 21 ++++++++-------------
 2 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index 16952e359..2e485df9d 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -43,6 +43,9 @@ def get_suitable_downloader(info_dict, params={}):
         if ed.can_download(info_dict):
             return ed
 
+    if protocol.startswith('m3u8') and info_dict.get('is_live'):
+        return FFmpegFD
+
     if protocol == 'm3u8' and params.get('hls_prefer_native') is True:
         return HlsFD
 
diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 7534e4da5..4989abce1 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -30,15 +30,6 @@ class HlsFD(FragmentFD):
 
     FD_NAME = 'hlsnative'
 
-    def _delegate_to_ffmpeg(self, filename, info_dict):
-        self.report_warning(
-            'hlsnative has detected features it does not support, '
-            'extraction will be delegated to ffmpeg')
-        fd = FFmpegFD(self.ydl, self.params)
-        for ph in self._progress_hooks:
-            fd.add_progress_hook(ph)
-        return fd.real_download(filename, info_dict)
-
     @staticmethod
     def can_download(manifest, info_dict):
         UNSUPPORTED_FEATURES = (
@@ -62,12 +53,10 @@ class HlsFD(FragmentFD):
         )
         check_results = [not re.search(feature, manifest) for feature in UNSUPPORTED_FEATURES]
         check_results.append(can_decrypt_frag or '#EXT-X-KEY:METHOD=AES-128' not in manifest)
+        check_results.append(not info_dict.get('is_live'))
         return all(check_results)
 
     def real_download(self, filename, info_dict):
-        if info_dict.get('is_live'):
-            return self._delegate_to_ffmpeg(filename, info_dict)
-
         man_url = info_dict['url']
         self.to_screen('[%s] Downloading m3u8 manifest' % self.FD_NAME)
 
@@ -79,7 +68,13 @@ class HlsFD(FragmentFD):
             if info_dict.get('extra_param_to_segment_url'):
                 self.report_error('pycrypto not found. Please install it.')
                 return False
-            return self._delegate_to_ffmpeg(filename, info_dict)
+            self.report_warning(
+                'hlsnative has detected features it does not support, '
+                'extraction will be delegated to ffmpeg')
+            fd = FFmpegFD(self.ydl, self.params)
+            for ph in self._progress_hooks:
+                fd.add_progress_hook(ph)
+            return fd.real_download(filename, info_dict)
 
         total_frags = 0
         for line in s.splitlines():

From f7923a4c399e0ce8e6cd230db92aefbfcff297c3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 26 Mar 2017 22:07:12 +0800
Subject: [PATCH 0552/1696] [ChangeLog] Update after #12307

---
 ChangeLog | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 45d6f244d..adc64053b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,10 @@
 version <unreleased>
 
+Core
+* Don't raise an error if JWPlayer config data is not a Javascript object
+  literal. _find_jwplayer_data() now returns a dict rather than an str.
+  (#12307)
+
 Extractors
 * [afreecatv] Fix extraction (#12179)
 

From 82eefd0be00b7557782ae75602b463e226dd964f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Mar 2017 23:39:12 +0700
Subject: [PATCH 0553/1696] [ChangeLog] Actualize

---
 ChangeLog | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index adc64053b..e79067cff 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -2,11 +2,21 @@ version <unreleased>
 
 Core
 * Don't raise an error if JWPlayer config data is not a Javascript object
-  literal. _find_jwplayer_data() now returns a dict rather than an str.
-  (#12307)
+  literal. _find_jwplayer_data now returns a dict rather than an str. (#12307)
+* Expand environment variables for options representing paths (#12556)
++ [utils] Introduce expand_path
+* [downloader/hls] Delegate downloading to ffmpeg immediately for live streams
 
 Extractors
 * [afreecatv] Fix extraction (#12179)
++ [atvat] Add support for atv.at (#5325)
++ [fox] Add metadata extraction (#12391)
++ [atresplayer] Extract DASH formats
++ [atresplayer] Extract HD manifest (#12548)
+* [atresplayer] Fix login error detection (#12548)
+* [franceculture] Fix extraction (#12547)
+* [youtube] Improve URL regular expression (#12538)
+* [generic] Do not follow redirects to the same URL
 
 
 version 2017.03.24

From 9e691da06791a0a617ed69ef21e272536e247ed1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Mar 2017 08:11:40 +0700
Subject: [PATCH 0554/1696] release 2017.03.26

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 1 +
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index dfff41d2d..2f717926c 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.03.24*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.03.24**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.03.26*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.03.26**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.03.24
+[debug] youtube-dl version 2017.03.26
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index e79067cff..07725b12a 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.03.26
 
 Core
 * Don't raise an error if JWPlayer config data is not a Javascript object
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 7c99ba3c2..e9dbc021b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -67,6 +67,7 @@
  - **arte.tv:playlist**
  - **AtresPlayer**
  - **ATTTechChannel**
+ - **ATVAt**
  - **AudiMedia**
  - **AudioBoom**
  - **audiomack**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 13904c724..94e8198ec 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.03.24'
+__version__ = '2017.03.26'

From aea1dccbd07b073ef36b325a9a21eb3f642322d9 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Tue, 28 Mar 2017 15:42:03 +0200
Subject: [PATCH 0555/1696] [openload] fix extractor

---
 youtube_dl/extractor/openload.py | 73 +++++++++++++-------------------
 1 file changed, 30 insertions(+), 43 deletions(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 58ffde541..d8036b54a 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -75,51 +75,38 @@ class OpenloadIE(InfoExtractor):
             '<span[^>]+id="[^"]+"[^>]*>([0-9A-Za-z]+)</span>',
             webpage, 'openload ID')
 
-        video_url_chars = []
-
-        first_char = ord(ol_id[0])
-        key = first_char - 55
-        maxKey = max(2, key)
-        key = min(maxKey, len(ol_id) - 38)
-        t = ol_id[key:key + 36]
-
-        hashMap = {}
-        v = ol_id.replace(t, '')
-        h = 0
-
-        while h < len(t):
-            f = t[h:h + 3]
-            i = int(f, 8)
-            hashMap[h / 3] = i
-            h += 3
-
-        h = 0
-        H = 0
-        while h < len(v):
-            B = ''
-            C = ''
-            if len(v) >= h + 2:
-                B = v[h:h + 2]
-            if len(v) >= h + 3:
-                C = v[h:h + 3]
-            i = int(B, 16)
-            h += 2
-            if H % 3 == 0:
-                i = int(C, 8)
-                h += 1
-            elif H % 2 == 0 and H != 0 and ord(v[H - 1]) < 60:
-                i = int(C, 10)
-                h += 1
-            index = H % 7
-
-            A = hashMap[index]
-            i ^= 213
-            i ^= A
-            video_url_chars.append(compat_chr(i))
-            H += 1
+        decoded = ''
+        a = ol_id[0:24]
+        b = []
+        for i in range(0, len(a), 8):
+            b.append(int(a[i:i + 8] or '0', 16))
+        ol_id = ol_id[24:]
+        j = 0
+        k = 0
+        while j < len(ol_id):
+            c = 128
+            d = 0
+            e = 0
+            f = 0
+            _more = True
+            while _more:
+                if j + 1 >= len(ol_id):
+                    c = 143
+                f = int(ol_id[j:j + 2] or '0', 16)
+                j += 2
+                d += (f & 127) << e
+                e += 7
+                _more = f >= c
+            g = d ^ b[k % 3]
+            for i in range(4):
+                char_dec = (g >> 8 * i) & (c + 127)
+                char = compat_chr(char_dec)
+                if char != '#':
+                    decoded += char
+            k += 1
 
         video_url = 'https://openload.co/stream/%s?mime=true'
-        video_url = video_url % (''.join(video_url_chars))
+        video_url = video_url % decoded
 
         title = self._og_search_title(webpage, default=None) or self._search_regex(
             r'<span[^>]+class=["\']title["\'][^>]*>([^<]+)', webpage,

From 12ee65ea0d09c6ac42ad06b3d561b4a26db00cfe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 28 Mar 2017 23:35:48 +0700
Subject: [PATCH 0556/1696] [options] Mention ISM for --fragment-retries and
 --skip-unavailable-fragments

---
 youtube_dl/options.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 6b811535f..2d2f5e47b 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -459,11 +459,11 @@ def parseOpts(overrideArguments=None):
     downloader.add_option(
         '--fragment-retries',
         dest='fragment_retries', metavar='RETRIES', default=10,
-        help='Number of retries for a fragment (default is %default), or "infinite" (DASH and hlsnative only)')
+        help='Number of retries for a fragment (default is %default), or "infinite" (DASH, hlsnative and ISM)')
     downloader.add_option(
         '--skip-unavailable-fragments',
         action='store_true', dest='skip_unavailable_fragments', default=True,
-        help='Skip unavailable fragments (DASH and hlsnative only)')
+        help='Skip unavailable fragments (DASH, hlsnative and ISM)')
     downloader.add_option(
         '--abort-on-unavailable-fragment',
         action='store_false', dest='skip_unavailable_fragments',

From 128244657b92582f7f4793c2d1be86b04032ac7f Mon Sep 17 00:00:00 2001
From: plroman <dev@plr.pm>
Date: Tue, 28 Mar 2017 23:23:20 +0200
Subject: [PATCH 0557/1696] [allocine] Fix extraction

---
 youtube_dl/extractor/allocine.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/allocine.py b/youtube_dl/extractor/allocine.py
index 90f11d39f..0463a070b 100644
--- a/youtube_dl/extractor/allocine.py
+++ b/youtube_dl/extractor/allocine.py
@@ -70,7 +70,7 @@ class AllocineIE(InfoExtractor):
         if model:
             model_data = self._parse_json(model, display_id)
 
-            for video_url in model_data['sources'].values():
+            for video_url in model_data['videos'][0]['sources'].values():
                 video_id, format_id = url_basename(video_url).split('_')[:2]
                 formats.append({
                     'format_id': format_id,
@@ -78,7 +78,7 @@ class AllocineIE(InfoExtractor):
                     'url': video_url,
                 })
 
-            title = model_data['title']
+            title = model_data['videos'][0]['title']
         else:
             video_id = display_id
             media_data = self._download_json(

From 639e5b2a848c0a73e8525472dd8bb4b14a8c4746 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 29 Mar 2017 04:43:12 +0700
Subject: [PATCH 0558/1696] [allocine] Extract more metadata

---
 youtube_dl/extractor/allocine.py | 46 +++++++++++++++++++++++---------
 1 file changed, 34 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/allocine.py b/youtube_dl/extractor/allocine.py
index 0463a070b..cd533acfc 100644
--- a/youtube_dl/extractor/allocine.py
+++ b/youtube_dl/extractor/allocine.py
@@ -2,9 +2,13 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
-    remove_end,
+    int_or_none,
     qualities,
+    remove_end,
+    try_get,
+    unified_timestamp,
     url_basename,
 )
 
@@ -22,6 +26,10 @@ class AllocineIE(InfoExtractor):
             'title': 'Astérix - Le Domaine des Dieux Teaser VF',
             'description': 'md5:4a754271d9c6f16c72629a8a993ee884',
             'thumbnail': r're:http://.*\.jpg',
+            'duration': 39,
+            'timestamp': 1404273600,
+            'upload_date': '20140702',
+            'view_count': int,
         },
     }, {
         'url': 'http://www.allocine.fr/video/player_gen_cmedia=19540403&cfilm=222257.html',
@@ -33,6 +41,10 @@ class AllocineIE(InfoExtractor):
             'title': 'Planes 2 Bande-annonce VF',
             'description': 'Regardez la bande annonce du film Planes 2 (Planes 2 Bande-annonce VF). Planes 2, un film de Roberts Gannaway',
             'thumbnail': r're:http://.*\.jpg',
+            'duration': 69,
+            'timestamp': 1385659800,
+            'upload_date': '20131128',
+            'view_count': int,
         },
     }, {
         'url': 'http://www.allocine.fr/video/player_gen_cmedia=19544709&cfilm=181290.html',
@@ -44,6 +56,10 @@ class AllocineIE(InfoExtractor):
             'title': 'Dragons 2 - Bande annonce finale VF',
             'description': 'md5:6cdd2d7c2687d4c6aafe80a35e17267a',
             'thumbnail': r're:http://.*\.jpg',
+            'duration': 144,
+            'timestamp': 1397589900,
+            'upload_date': '20140415',
+            'view_count': int,
         },
     }, {
         'url': 'http://www.allocine.fr/video/video-19550147/',
@@ -69,34 +85,37 @@ class AllocineIE(InfoExtractor):
             r'data-model="([^"]+)"', webpage, 'data model', default=None)
         if model:
             model_data = self._parse_json(model, display_id)
-
-            for video_url in model_data['videos'][0]['sources'].values():
+            video = model_data['videos'][0]
+            title = video['title']
+            for video_url in video['sources'].values():
                 video_id, format_id = url_basename(video_url).split('_')[:2]
                 formats.append({
                     'format_id': format_id,
                     'quality': quality(format_id),
                     'url': video_url,
                 })
-
-            title = model_data['videos'][0]['title']
+            duration = int_or_none(video.get('duration'))
+            view_count = int_or_none(video.get('view_count'))
+            timestamp = unified_timestamp(try_get(
+                video, lambda x: x['added_at']['date'], compat_str))
         else:
             video_id = display_id
             media_data = self._download_json(
                 'http://www.allocine.fr/ws/AcVisiondataV5.ashx?media=%s' % video_id, display_id)
+            title = remove_end(
+                self._html_search_regex(
+                    r'(?s)<title>(.+?)</title>', webpage, 'title').strip(),
+                ' - AlloCiné')
             for key, value in media_data['video'].items():
                 if not key.endswith('Path'):
                     continue
-
                 format_id = key[:-len('Path')]
                 formats.append({
                     'format_id': format_id,
                     'quality': quality(format_id),
                     'url': value,
                 })
-
-            title = remove_end(self._html_search_regex(
-                r'(?s)<title>(.+?)</title>', webpage, 'title'
-            ).strip(), ' - AlloCiné')
+            duration, view_count, timestamp = [None] * 3
 
         self._sort_formats(formats)
 
@@ -104,7 +123,10 @@ class AllocineIE(InfoExtractor):
             'id': video_id,
             'display_id': display_id,
             'title': title,
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'formats': formats,
             'description': self._og_search_description(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'duration': duration,
+            'timestamp': timestamp,
+            'view_count': view_count,
+            'formats': formats,
         }

From 82be732b174ea8e9984e7b0582c69e41b266d1da Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 31 Mar 2017 12:24:23 +0100
Subject: [PATCH 0559/1696] [adn] Add new extractor

---
 youtube_dl/extractor/adn.py        | 136 +++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |   1 +
 2 files changed, 137 insertions(+)
 create mode 100644 youtube_dl/extractor/adn.py

diff --git a/youtube_dl/extractor/adn.py b/youtube_dl/extractor/adn.py
new file mode 100644
index 000000000..e44caa00b
--- /dev/null
+++ b/youtube_dl/extractor/adn.py
@@ -0,0 +1,136 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import base64
+import json
+import os
+
+from .common import InfoExtractor
+from ..aes import aes_cbc_decrypt
+from ..compat import compat_ord
+from ..utils import (
+    bytes_to_intlist,
+    ExtractorError,
+    float_or_none,
+    intlist_to_bytes,
+    srt_subtitles_timecode,
+    strip_or_none,
+)
+
+
+class ADNIE(InfoExtractor):
+    IE_DESC = 'Anime Digital Network'
+    _VALID_URL = r'https?://(?:www\.)?animedigitalnetwork\.fr/video/[^/]+/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://animedigitalnetwork.fr/video/blue-exorcist-kyoto-saga/7778-episode-1-debut-des-hostilites',
+        'md5': 'e497370d847fd79d9d4c74be55575c7a',
+        'info_dict': {
+            'id': '7778',
+            'ext': 'mp4',
+            'title': 'Blue Exorcist - Kyôto Saga - Épisode 1',
+            'description': 'md5:2f7b5aa76edbc1a7a92cedcda8a528d5',
+        }
+    }
+
+    def _get_subtitles(self, sub_path, video_id):
+        if not sub_path:
+            return None
+
+        enc_subtitles = self._download_webpage(
+            'http://animedigitalnetwork.fr/' + sub_path,
+            video_id, fatal=False)
+        if not enc_subtitles:
+            return None
+
+        # http://animedigitalnetwork.fr/components/com_vodvideo/videojs/adn-vjs.min.js
+        dec_subtitles = intlist_to_bytes(aes_cbc_decrypt(
+            bytes_to_intlist(base64.b64decode(enc_subtitles[24:])),
+            bytes_to_intlist(b'\xb5@\xcfq\xa3\x98"N\xe4\xf3\x12\x98}}\x16\xd8'),
+            bytes_to_intlist(base64.b64decode(enc_subtitles[:24]))
+        ))
+        subtitles_json = self._parse_json(
+            dec_subtitles[:-compat_ord(dec_subtitles[-1])],
+            None, fatal=False)
+        if not subtitles_json:
+            return None
+
+        subtitles = {}
+        for sub_lang, sub in subtitles_json.items():
+            srt = ''
+            for num, current in enumerate(sub):
+                start, end, text = (
+                    float_or_none(current.get('startTime')),
+                    float_or_none(current.get('endTime')),
+                    current.get('text'))
+                if start is None or end is None or text is None:
+                    continue
+                srt += os.linesep.join(
+                    (
+                        '%d' % num,
+                        '%s --> %s' % (
+                            srt_subtitles_timecode(start),
+                            srt_subtitles_timecode(end)),
+                        text,
+                        os.linesep,
+                    ))
+
+            if sub_lang == 'vostf':
+                sub_lang = 'fr'
+            subtitles.setdefault(sub_lang, []).extend([{
+                'ext': 'json',
+                'data': json.dumps(sub),
+            }, {
+                'ext': 'srt',
+                'data': srt,
+            }])
+        return subtitles
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        player_config = self._parse_json(self._search_regex(
+            r'playerConfig\s*=\s*({.+});', webpage, 'player config'), video_id)
+
+        video_info = {}
+        video_info_str = self._search_regex(
+            r'videoInfo\s*=\s*({.+});', webpage,
+            'video info', fatal=False)
+        if video_info_str:
+            video_info = self._parse_json(
+                video_info_str, video_id, fatal=False) or {}
+
+        options = player_config.get('options') or {}
+        metas = options.get('metas') or {}
+        title = metas.get('title') or video_info['title']
+        links = player_config.get('links') or {}
+
+        formats = []
+        for format_id, qualities in links.items():
+            for load_balancer_url in qualities.values():
+                load_balancer_data = self._download_json(
+                    load_balancer_url, video_id, fatal=False) or {}
+                m3u8_url = load_balancer_data.get('location')
+                if not m3u8_url:
+                    continue
+                m3u8_formats = self._extract_m3u8_formats(
+                    m3u8_url, video_id, 'mp4', 'm3u8_native',
+                    m3u8_id=format_id, fatal=False)
+                if format_id == 'vf':
+                    for f in m3u8_formats:
+                        f['language'] = 'fr'
+                formats.extend(m3u8_formats)
+        error = options.get('error')
+        if not formats and error:
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, error), expected=True)
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': strip_or_none(metas.get('summary') or video_info.get('resume')),
+            'thumbnail': video_info.get('image'),
+            'formats': formats,
+            'subtitles': self.extract_subtitles(player_config.get('subtitles'), video_id),
+            'episode': metas.get('subtitle') or video_info.get('videoTitle'),
+            'series': video_info.get('playlistTitle'),
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 6a7028a4d..43933ad5b 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -19,6 +19,7 @@ from .acast import (
     ACastChannelIE,
 )
 from .addanime import AddAnimeIE
+from .adn import ADNIE
 from .adobetv import (
     AdobeTVIE,
     AdobeTVShowIE,

From 3e943cfe09eda6ef9b0fa419fdd22155fbaa047f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 31 Mar 2017 14:54:06 +0100
Subject: [PATCH 0560/1696] [generic] pass base_url to _parse_jwplayer_data

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 274f81738..73911940c 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2568,7 +2568,7 @@ class GenericIE(InfoExtractor):
             webpage, video_id, transform_source=js_to_json)
         if jwplayer_data:
             info = self._parse_jwplayer_data(
-                jwplayer_data, video_id, require_title=False)
+                jwplayer_data, video_id, require_title=False, base_url=url)
             if not info.get('title'):
                 info['title'] = video_title
             return info

From 1640eb096166c81918125a0a7462eb2edb063167 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 31 Mar 2017 23:57:35 +0700
Subject: [PATCH 0561/1696] [YoutubeDL] Return early when extraction of
 url_transparent fails

---
 youtube_dl/YoutubeDL.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 21586f0f4..54bc8b06d 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -837,6 +837,12 @@ class YoutubeDL(object):
                 ie_result['url'], ie_key=ie_result.get('ie_key'),
                 extra_info=extra_info, download=False, process=False)
 
+            # extract_info may return None when ignoreerrors is enabled and
+            # extraction failed with an error, don't crash and return early
+            # in this case
+            if not info:
+                return info
+
             force_properties = dict(
                 (k, v) for k, v in ie_result.items() if v is not None)
             for f in ('_type', 'url', 'ie_key'):

From 7453999580f2809153a84420d3ca72b24186c02b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 Apr 2017 00:25:27 +0700
Subject: [PATCH 0562/1696] [packtpub] Add extractor (closes #12610)

---
 youtube_dl/extractor/extractors.py |   4 +
 youtube_dl/extractor/packtpub.py   | 138 +++++++++++++++++++++++++++++
 2 files changed, 142 insertions(+)
 create mode 100644 youtube_dl/extractor/packtpub.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 43933ad5b..6ad7444fe 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -729,6 +729,10 @@ from .orf import (
     ORFFM4IE,
     ORFIPTVIE,
 )
+from .packtpub import (
+    PacktPubIE,
+    PacktPubCourseIE,
+)
 from .pandatv import PandaTVIE
 from .pandoratv import PandoraTVIE
 from .parliamentliveuk import ParliamentLiveUKIE
diff --git a/youtube_dl/extractor/packtpub.py b/youtube_dl/extractor/packtpub.py
new file mode 100644
index 000000000..881f3bcc7
--- /dev/null
+++ b/youtube_dl/extractor/packtpub.py
@@ -0,0 +1,138 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    clean_html,
+    ExtractorError,
+    remove_end,
+    strip_or_none,
+    unified_timestamp,
+    urljoin,
+)
+
+
+class PacktPubBaseIE(InfoExtractor):
+    _PACKT_BASE = 'https://www.packtpub.com'
+    _MAPT_REST = '%s/mapt-rest' % _PACKT_BASE
+
+
+class PacktPubIE(PacktPubBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?packtpub\.com/mapt/video/[^/]+/(?P<course_id>\d+)/(?P<chapter_id>\d+)/(?P<id>\d+)'
+
+    _TEST = {
+        'url': 'https://www.packtpub.com/mapt/video/web-development/9781787122215/20528/20530/Project+Intro',
+        'md5': '1e74bd6cfd45d7d07666f4684ef58f70',
+        'info_dict': {
+            'id': '20530',
+            'ext': 'mp4',
+            'title': 'Project Intro',
+            'thumbnail': r're:(?i)^https?://.*\.jpg',
+            'timestamp': 1490918400,
+            'upload_date': '20170331',
+        },
+    }
+
+    def _handle_error(self, response):
+        if response.get('status') != 'success':
+            raise ExtractorError(
+                '% said: %s' % (self.IE_NAME, response['message']),
+                expected=True)
+
+    def _download_json(self, *args, **kwargs):
+        response = super(PacktPubIE, self)._download_json(*args, **kwargs)
+        self._handle_error(response)
+        return response
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        course_id, chapter_id, video_id = mobj.group(
+            'course_id', 'chapter_id', 'id')
+
+        video = self._download_json(
+            '%s/users/me/products/%s/chapters/%s/sections/%s'
+            % (self._MAPT_REST, course_id, chapter_id, video_id), video_id,
+            'Downloading JSON video')['data']
+
+        content = video.get('content')
+        if not content:
+            raise ExtractorError('This video is locked', expected=True)
+
+        video_url = content['file']
+
+        metadata = self._download_json(
+            '%s/products/%s/chapters/%s/sections/%s/metadata'
+            % (self._MAPT_REST, course_id, chapter_id, video_id),
+            video_id)['data']
+
+        title = metadata['pageTitle']
+        course_title = metadata.get('title')
+        if course_title:
+            title = remove_end(title, ' - %s' % course_title)
+        timestamp = unified_timestamp(metadata.get('publicationDate'))
+        thumbnail = urljoin(self._PACKT_BASE, metadata.get('filepath'))
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+        }
+
+
+class PacktPubCourseIE(PacktPubBaseIE):
+    _VALID_URL = r'(?P<url>https?://(?:www\.)?packtpub\.com/mapt/video/[^/]+/(?P<id>\d+))'
+    _TEST = {
+        'url': 'https://www.packtpub.com/mapt/video/web-development/9781787122215',
+        'info_dict': {
+            'id': '9781787122215',
+            'title': 'Learn Nodejs by building 12 projects [Video]',
+        },
+        'playlist_count': 90,
+    }
+
+    @classmethod
+    def suitable(cls, url):
+        return False if PacktPubIE.suitable(url) else super(
+            PacktPubCourseIE, cls).suitable(url)
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        url, course_id = mobj.group('url', 'id')
+
+        course = self._download_json(
+            '%s/products/%s/metadata' % (self._MAPT_REST, course_id),
+            course_id)['data']
+
+        entries = []
+        for chapter_num, chapter in enumerate(course['tableOfContents'], 1):
+            if chapter.get('type') != 'chapter':
+                continue
+            children = chapter.get('children')
+            if not isinstance(children, list):
+                continue
+            chapter_info = {
+                'chapter': chapter.get('title'),
+                'chapter_number': chapter_num,
+                'chapter_id': chapter.get('id'),
+            }
+            for section in children:
+                if section.get('type') != 'section':
+                    continue
+                section_url = section.get('seoUrl')
+                if not isinstance(section_url, compat_str):
+                    continue
+                entry = {
+                    '_type': 'url_transparent',
+                    'url': urljoin(url + '/', section_url),
+                    'title': strip_or_none(section.get('title')),
+                    'description': clean_html(section.get('summary')),
+                    'ie_key': PacktPubIE.ie_key(),
+                }
+                entry.update(chapter_info)
+                entries.append(entry)
+
+        return self.playlist_result(entries, course_id, course.get('title'))

From 77c8ebe6318055cc34eaedca63f4866c4c47437a Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 31 Mar 2017 23:28:24 +0100
Subject: [PATCH 0563/1696] [vrv] Add new extractor

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/vrv.py        | 151 +++++++++++++++++++++++++++++
 2 files changed, 152 insertions(+)
 create mode 100644 youtube_dl/extractor/vrv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 6ad7444fe..1b427e256 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1182,6 +1182,7 @@ from .voxmedia import VoxMediaIE
 from .vporn import VpornIE
 from .vrt import VRTIE
 from .vrak import VrakIE
+from .vrv import VRVIE
 from .medialaan import MedialaanIE
 from .vube import VubeIE
 from .vuclip import VuClipIE
diff --git a/youtube_dl/extractor/vrv.py b/youtube_dl/extractor/vrv.py
new file mode 100644
index 000000000..33618c951
--- /dev/null
+++ b/youtube_dl/extractor/vrv.py
@@ -0,0 +1,151 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import base64
+import json
+import hashlib
+import hmac
+import random
+import string
+import time
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_parse_urlencode,
+    compat_urlparse,
+)
+from ..utils import (
+    float_or_none,
+    int_or_none,
+)
+
+
+class VRVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?vrv\.co/watch/(?P<id>[A-Z0-9]+)'
+    _TEST = {
+        'url': 'https://vrv.co/watch/GR9PNZ396/Hidden-America-with-Jonah-Ray:BOSTON-WHERE-THE-PAST-IS-THE-PRESENT',
+        'info_dict': {
+            'id': 'GR9PNZ396',
+            'ext': 'mp4',
+            'title': 'BOSTON: WHERE THE PAST IS THE PRESENT',
+            'description': 'md5:4ec8844ac262ca2df9e67c0983c6b83f',
+            'uploader_id': 'seeso',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
+    _API_DOMAIN = None
+    _API_PARAMS = {}
+    _CMS_SIGNING = {}
+
+    def _call_api(self, path, video_id, note, data=None):
+        base_url = self._API_DOMAIN + '/core/' + path
+        encoded_query = compat_urllib_parse_urlencode({
+            'oauth_consumer_key': self._API_PARAMS['oAuthKey'],
+            'oauth_nonce': ''.join([random.choice(string.ascii_letters) for _ in range(32)]),
+            'oauth_signature_method': 'HMAC-SHA1',
+            'oauth_timestamp': int(time.time()),
+            'oauth_version': '1.0',
+        })
+        headers = self.geo_verification_headers()
+        if data:
+            data = json.dumps(data).encode()
+            headers['Content-Type'] = 'application/json'
+        method = 'POST' if data else 'GET'
+        base_string = '&'.join([method, compat_urlparse.quote(base_url, ''), compat_urlparse.quote(encoded_query, '')])
+        oauth_signature = base64.b64encode(hmac.new(
+            (self._API_PARAMS['oAuthSecret'] + '&').encode('ascii'),
+            base_string.encode(), hashlib.sha1).digest()).decode()
+        encoded_query += '&oauth_signature=' + compat_urlparse.quote(oauth_signature, '')
+        return self._download_json(
+            '?'.join([base_url, encoded_query]), video_id,
+            note='Downloading %s JSON metadata' % note, headers=headers, data=data)
+
+    def _call_cms(self, path, video_id, note):
+        return self._download_json(
+            self._API_DOMAIN + path, video_id, query=self._CMS_SIGNING,
+            note='Downloading %s JSON metadata' % note, headers=self.geo_verification_headers())
+
+    def _set_api_params(self, webpage, video_id):
+        if not self._API_PARAMS:
+            self._API_PARAMS = self._parse_json(self._search_regex(
+                r'window\.__APP_CONFIG__\s*=\s*({.+?})</script>',
+                webpage, 'api config'), video_id)['cxApiParams']
+            self._API_DOMAIN = self._API_PARAMS.get('apiDomain', 'https://api.vrv.co')
+
+    def _set_cms_signing(self, video_id):
+        if not self._CMS_SIGNING:
+            self._CMS_SIGNING = self._call_api('index', video_id, 'CMS Signing')['cms_signing']
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(
+            url, video_id,
+            headers=self.geo_verification_headers())
+        media_resource = self._parse_json(self._search_regex(
+            r'window\.__INITIAL_STATE__\s*=\s*({.+?})</script>',
+            webpage, 'inital state'), video_id).get('watch', {}).get('mediaResource') or {}
+
+        video_data = media_resource.get('json')
+        if not video_data:
+            self._set_api_params(webpage, video_id)
+            episode_path = self._call_api('cms_resource', video_id, 'episode resource path', data={
+                'resource_key': 'cms:/episodes/' + video_id,
+            })['__links__']['cms_resource']['href']
+            self._set_cms_signing(video_id)
+            video_data = self._call_cms(episode_path, video_id, 'video')
+        title = video_data['title']
+
+        streams_json = media_resource.get('streams', {}).get('json', {})
+        if not streams_json:
+            self._set_api_params(webpage, video_id)
+            streams_path = video_data['__links__']['streams']['href']
+            self._set_cms_signing(video_id)
+            streams_json = self._call_cms(streams_path, video_id, 'streams')
+
+        audio_locale = streams_json.get('audio_locale')
+        formats = []
+        for stream_id, stream in streams_json.get('streams', {}).get('adaptive_hls', {}).items():
+            stream_url = stream.get('url')
+            if not stream_url:
+                continue
+            stream_id = stream_id or audio_locale
+            m3u8_formats = self._extract_m3u8_formats(
+                stream_url, video_id, 'mp4', m3u8_id=stream_id,
+                note='Downloading %s m3u8 information' % stream_id,
+                fatal=False)
+            if audio_locale:
+                for f in m3u8_formats:
+                    f['language'] = audio_locale
+            formats.extend(m3u8_formats)
+        self._sort_formats(formats)
+
+        thumbnails = []
+        for thumbnail in video_data.get('images', {}).get('thumbnails', []):
+            thumbnail_url = thumbnail.get('source')
+            if not thumbnail_url:
+                continue
+            thumbnails.append({
+                'url': thumbnail_url,
+                'width': int_or_none(thumbnail.get('width')),
+                'height': int_or_none(thumbnail.get('height')),
+            })
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnails': thumbnails,
+            'description': video_data.get('description'),
+            'duration': float_or_none(video_data.get('duration_ms'), 1000),
+            'uploader_id': video_data.get('channel_id'),
+            'series': video_data.get('series_title'),
+            'season': video_data.get('season_title'),
+            'season_number': int_or_none(video_data.get('season_number')),
+            'season_id': video_data.get('season_id'),
+            'episode': title,
+            'episode_number': int_or_none(video_data.get('episode_number')),
+            'episode_id': video_data.get('production_episode_id'),
+        }

From be61efdf1754d026f270f6d87446040231d56954 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 1 Apr 2017 07:26:40 +0100
Subject: [PATCH 0564/1696] [tvplay] Bypass geo restriction

---
 youtube_dl/extractor/tvplay.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index 3eda0a399..99ff82a5d 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -225,7 +225,11 @@ class TVPlayIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
+        geo_country = self._search_regex(
+            r'https?://[^/]+\.([a-z]{2})', url,
+            'geo country', default=None)
+        if geo_country:
+            self._initialize_geo_bypass([geo_country.upper()])
         video = self._download_json(
             'http://playapi.mtgx.tv/v3/videos/%s' % video_id, video_id, 'Downloading video JSON')
 

From e97fc8d6b837921ea8429727f026238b857e1b31 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 1 Apr 2017 07:50:24 +0100
Subject: [PATCH 0565/1696] [cwtv] extract ISM formats

---
 youtube_dl/extractor/cwtv.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/cwtv.py b/youtube_dl/extractor/cwtv.py
index 1ab9333b2..f4cf0f1c5 100644
--- a/youtube_dl/extractor/cwtv.py
+++ b/youtube_dl/extractor/cwtv.py
@@ -82,6 +82,11 @@ class CWTVIE(InfoExtractor):
                             'url': quality_url,
                             'tbr': tbr,
                         })
+        video_metadata = video_data['assetFields']
+        ism_url = video_metadata.get('smoothStreamingUrl')
+        if ism_url:
+            formats.extend(self._extract_ism_formats(
+                ism_url, video_id, ism_id='mss', fatal=False))
         self._sort_formats(formats)
 
         thumbnails = [{
@@ -90,8 +95,6 @@ class CWTVIE(InfoExtractor):
             'height': image.get('height'),
         } for image_id, image in video_data['images'].items() if image.get('uri')] if video_data.get('images') else None
 
-        video_metadata = video_data['assetFields']
-
         subtitles = {
             'en': [{
                 'url': video_metadata['UnicornCcUrl'],

From ca77b92f94010bdf2d44de44cb23e32075b7dcaa Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 1 Apr 2017 09:33:23 +0100
Subject: [PATCH 0566/1696] [crunchyroll] pass geo verifcation proxy

---
 youtube_dl/extractor/crunchyroll.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index d15fd3744..2ed8b30bb 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -390,7 +390,9 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         else:
             webpage_url = 'http://www.' + mobj.group('url')
 
-        webpage = self._download_webpage(self._add_skip_wall(webpage_url), video_id, 'Downloading webpage')
+        webpage = self._download_webpage(
+            self._add_skip_wall(webpage_url), video_id,
+            headers=self.geo_verification_headers())
         note_m = self._html_search_regex(
             r'<div class="showmedia-trailer-notice">(.+?)</div>',
             webpage, 'trailer-notice', default='')
@@ -565,7 +567,9 @@ class CrunchyrollShowPlaylistIE(CrunchyrollBaseIE):
     def _real_extract(self, url):
         show_id = self._match_id(url)
 
-        webpage = self._download_webpage(self._add_skip_wall(url), show_id)
+        webpage = self._download_webpage(
+            self._add_skip_wall(url), show_id,
+            headers=self.geo_verification_headers())
         title = self._html_search_regex(
             r'(?s)<h1[^>]*>\s*<span itemprop="name">(.*?)</span>',
             webpage, 'title')

From 2cd668ee591df4f271ed4394ba9b38262ae3c40e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 Apr 2017 18:55:48 +0700
Subject: [PATCH 0567/1696] [xfileshare] Improve extraction and extract hls
 formats

---
 youtube_dl/extractor/xfileshare.py | 57 +++++++++++++++++++++---------
 1 file changed, 41 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index e616adce3..6de5b26d7 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -6,6 +6,7 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     decode_packed_codes,
+    determine_ext,
     ExtractorError,
     int_or_none,
     NO_DEFAULT,
@@ -95,6 +96,16 @@ class XFileShareIE(InfoExtractor):
         # removed by administrator
         'url': 'http://xvidstage.com/amfy7atlkx25',
         'only_matching': True,
+    }, {
+        'url': 'http://vidabc.com/i8ybqscrphfv',
+        'info_dict': {
+            'id': 'i8ybqscrphfv',
+            'ext': 'mp4',
+            'title': 're:Beauty and the Beast 2017',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
@@ -133,31 +144,45 @@ class XFileShareIE(InfoExtractor):
             webpage, 'title', default=None) or self._og_search_title(
             webpage, default=None) or video_id).strip()
 
-        def extract_video_url(default=NO_DEFAULT):
-            return self._search_regex(
-                (r'file\s*:\s*(["\'])(?P<url>http.+?)\1,',
-                 r'file_link\s*=\s*(["\'])(?P<url>http.+?)\1',
-                 r'addVariable\((\\?["\'])file\1\s*,\s*(\\?["\'])(?P<url>http.+?)\2\)',
-                 r'<embed[^>]+src=(["\'])(?P<url>http.+?)\1'),
-                webpage, 'file url', default=default, group='url')
+        def extract_formats(default=NO_DEFAULT):
+            urls = []
+            for regex in (
+                    r'file\s*:\s*(["\'])(?P<url>http(?:(?!\1).)+\.(?:m3u8|mp4|flv)(?:(?!\1).)*)\1',
+                    r'file_link\s*=\s*(["\'])(?P<url>http(?:(?!\1).)+)\1',
+                    r'addVariable\((\\?["\'])file\1\s*,\s*(\\?["\'])(?P<url>http(?:(?!\2).)+)\2\)',
+                    r'<embed[^>]+src=(["\'])(?P<url>http(?:(?!\1).)+\.(?:m3u8|mp4|flv)(?:(?!\1).)*)\1'):
+                for mobj in re.finditer(regex, webpage):
+                    video_url = mobj.group('url')
+                    if video_url not in urls:
+                        urls.append(video_url)
+            formats = []
+            for video_url in urls:
+                if determine_ext(video_url) == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        video_url, video_id, 'mp4',
+                        entry_protocol='m3u8_native', m3u8_id='hls',
+                        fatal=False))
+                else:
+                    formats.append({
+                        'url': video_url,
+                        'format_id': 'sd',
+                    })
+            if not formats and default is not NO_DEFAULT:
+                return default
+            self._sort_formats(formats)
+            return formats
 
-        video_url = extract_video_url(default=None)
+        formats = extract_formats(default=None)
 
-        if not video_url:
+        if not formats:
             webpage = decode_packed_codes(self._search_regex(
                 r"(}\('(.+)',(\d+),(\d+),'[^']*\b(?:file|embed)\b[^']*'\.split\('\|'\))",
                 webpage, 'packed code'))
-            video_url = extract_video_url()
+            formats = extract_formats()
 
         thumbnail = self._search_regex(
             r'image\s*:\s*["\'](http[^"\']+)["\'],', webpage, 'thumbnail', default=None)
 
-        formats = [{
-            'format_id': 'sd',
-            'url': video_url,
-            'quality': 1,
-        }]
-
         return {
             'id': video_id,
             'title': title,

From eecea00d36f29f3b22e5936ed48fa91456ab066a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 Apr 2017 18:56:35 +0700
Subject: [PATCH 0568/1696] [xfileshare] Add support for vidabc.com (closes
 #12589)

---
 youtube_dl/extractor/xfileshare.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index 6de5b26d7..6856fb3bf 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -27,6 +27,7 @@ class XFileShareIE(InfoExtractor):
         ('vidto.me', 'Vidto'),
         ('streamin.to', 'Streamin.To'),
         ('xvidstage.com', 'XVIDSTAGE'),
+        ('vidabc.com', 'Vid ABC'),
     )
 
     IE_DESC = 'XFileShare based sites: %s' % ', '.join(list(zip(*_SITES))[1])

From 91399b2fcc95e72f052ee9eab8e12b68d1815c9e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 1 Apr 2017 13:32:38 +0100
Subject: [PATCH 0569/1696] [funimation] fix extraction(closes #10696)(#11773)

---
 youtube_dl/extractor/funimation.py | 211 ++++++++++-------------------
 1 file changed, 73 insertions(+), 138 deletions(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index eba00cd5a..e44a2a87f 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -7,9 +7,9 @@ from ..compat import (
     compat_urllib_parse_unquote_plus,
 )
 from ..utils import (
-    clean_html,
     determine_ext,
     int_or_none,
+    js_to_json,
     sanitized_Request,
     ExtractorError,
     urlencode_postdata
@@ -17,34 +17,26 @@ from ..utils import (
 
 
 class FunimationIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?funimation\.com/shows/[^/]+/videos/(?:official|promotional)/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?funimation(?:\.com|now\.uk)/shows/[^/]+/(?P<id>[^/?#&]+)'
 
     _NETRC_MACHINE = 'funimation'
 
     _TESTS = [{
-        'url': 'http://www.funimation.com/shows/air/videos/official/breeze',
+        'url': 'https://www.funimation.com/shows/hacksign/role-play/',
         'info_dict': {
-            'id': '658',
-            'display_id': 'breeze',
-            'ext': 'mp4',
-            'title': 'Air - 1 - Breeze',
-            'description': 'md5:1769f43cd5fc130ace8fd87232207892',
-            'thumbnail': r're:https?://.*\.jpg',
-        },
-        'skip': 'Access without user interaction is forbidden by CloudFlare, and video removed',
-    }, {
-        'url': 'http://www.funimation.com/shows/hacksign/videos/official/role-play',
-        'info_dict': {
-            'id': '31128',
+            'id': '91144',
             'display_id': 'role-play',
             'ext': 'mp4',
-            'title': '.hack//SIGN - 1 - Role Play',
+            'title': '.hack//SIGN - Role Play',
             'description': 'md5:b602bdc15eef4c9bbb201bb6e6a4a2dd',
             'thumbnail': r're:https?://.*\.jpg',
         },
-        'skip': 'Access without user interaction is forbidden by CloudFlare',
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
-        'url': 'http://www.funimation.com/shows/attack-on-titan-junior-high/videos/promotional/broadcast-dub-preview',
+        'url': 'https://www.funimation.com/shows/attack-on-titan-junior-high/broadcast-dub-preview/',
         'info_dict': {
             'id': '9635',
             'display_id': 'broadcast-dub-preview',
@@ -54,25 +46,13 @@ class FunimationIE(InfoExtractor):
             'thumbnail': r're:https?://.*\.(?:jpg|png)',
         },
         'skip': 'Access without user interaction is forbidden by CloudFlare',
+    }, {
+        'url': 'https://www.funimationnow.uk/shows/puzzle-dragons-x/drop-impact/simulcast/',
+        'only_matching': True,
     }]
 
     _LOGIN_URL = 'http://www.funimation.com/login'
 
-    def _download_webpage(self, *args, **kwargs):
-        try:
-            return super(FunimationIE, self)._download_webpage(*args, **kwargs)
-        except ExtractorError as ee:
-            if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
-                response = ee.cause.read()
-                if b'>Please complete the security check to access<' in response:
-                    raise ExtractorError(
-                        'Access to funimation.com is blocked by CloudFlare. '
-                        'Please browse to http://www.funimation.com/, solve '
-                        'the reCAPTCHA, export browser cookies to a text file,'
-                        ' and then try again with --cookies YOUR_COOKIE_FILE.',
-                        expected=True)
-            raise
-
     def _extract_cloudflare_session_ua(self, url):
         ci_session_cookie = self._get_cookies(url).get('ci_session')
         if ci_session_cookie:
@@ -114,119 +94,74 @@ class FunimationIE(InfoExtractor):
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        def _search_kane(name):
+            return self._search_regex(
+                r"KANE_customdimensions\.%s\s*=\s*'([^']+)';" % name,
+                webpage, name, default=None)
+
+        title_data = self._parse_json(self._search_regex(
+            r'TITLE_DATA\s*=\s*({[^}]+})',
+            webpage, 'title data', default=''),
+            display_id, js_to_json, fatal=False) or {}
+
+        video_id = title_data.get('id') or self._search_regex([
+            r"KANE_customdimensions.videoID\s*=\s*'(\d+)';",
+            r'<iframe[^>]+src="/player/(\d+)"',
+        ], webpage, 'video_id', default=None)
+        if not video_id:
+            player_url = self._html_search_meta([
+                'al:web:url',
+                'og:video:url',
+                'og:video:secure_url',
+            ], webpage, fatal=True)
+            video_id = self._search_regex(r'/player/(\d+)', player_url, 'video id')
+
+        title = episode = title_data.get('title') or _search_kane('videoTitle') or self._og_search_title(webpage)
+        series = _search_kane('showName')
+        if series:
+            title = '%s - %s' % (series, title)
+        description = self._html_search_meta(['description', 'og:description'], webpage, fatal=True)
+
+        try:
+            sources = self._download_json(
+                'https://prod-api-funimationnow.dadcdigital.com/api/source/catalog/video/%s/signed/' % video_id,
+                video_id)['items']
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+                error = self._parse_json(e.cause.read(), video_id)['errors'][0]
+                raise ExtractorError('%s said: %s' % (
+                    self.IE_NAME, error.get('detail') or error.get('title')), expected=True)
+            raise
 
-        errors = []
         formats = []
-
-        ERRORS_MAP = {
-            'ERROR_MATURE_CONTENT_LOGGED_IN': 'matureContentLoggedIn',
-            'ERROR_MATURE_CONTENT_LOGGED_OUT': 'matureContentLoggedOut',
-            'ERROR_SUBSCRIPTION_LOGGED_OUT': 'subscriptionLoggedOut',
-            'ERROR_VIDEO_EXPIRED': 'videoExpired',
-            'ERROR_TERRITORY_UNAVAILABLE': 'territoryUnavailable',
-            'SVODBASIC_SUBSCRIPTION_IN_PLAYER': 'basicSubscription',
-            'SVODNON_SUBSCRIPTION_IN_PLAYER': 'nonSubscription',
-            'ERROR_PLAYER_NOT_RESPONDING': 'playerNotResponding',
-            'ERROR_UNABLE_TO_CONNECT_TO_CDN': 'unableToConnectToCDN',
-            'ERROR_STREAM_NOT_FOUND': 'streamNotFound',
-        }
-
-        USER_AGENTS = (
-            # PC UA is served with m3u8 that provides some bonus lower quality formats
-            ('pc', 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0'),
-            # Mobile UA allows to extract direct links and also does not fail when
-            # PC UA fails with hulu error (e.g.
-            # http://www.funimation.com/shows/hacksign/videos/official/role-play)
-            ('mobile', 'Mozilla/5.0 (Linux; Android 4.4.2; Nexus 4 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.114 Mobile Safari/537.36'),
-        )
-
-        user_agent = self._extract_cloudflare_session_ua(url)
-        if user_agent:
-            USER_AGENTS = ((None, user_agent),)
-
-        for kind, user_agent in USER_AGENTS:
-            request = sanitized_Request(url)
-            request.add_header('User-Agent', user_agent)
-            webpage = self._download_webpage(
-                request, display_id,
-                'Downloading %s webpage' % kind if kind else 'Downloading webpage')
-
-            playlist = self._parse_json(
-                self._search_regex(
-                    r'var\s+playersData\s*=\s*(\[.+?\]);\n',
-                    webpage, 'players data'),
-                display_id)[0]['playlist']
-
-            items = next(item['items'] for item in playlist if item.get('items'))
-            item = next(item for item in items if item.get('itemAK') == display_id)
-
-            error_messages = {}
-            video_error_messages = self._search_regex(
-                r'var\s+videoErrorMessages\s*=\s*({.+?});\n',
-                webpage, 'error messages', default=None)
-            if video_error_messages:
-                error_messages_json = self._parse_json(video_error_messages, display_id, fatal=False)
-                if error_messages_json:
-                    for _, error in error_messages_json.items():
-                        type_ = error.get('type')
-                        description = error.get('description')
-                        content = error.get('content')
-                        if type_ == 'text' and description and content:
-                            error_message = ERRORS_MAP.get(description)
-                            if error_message:
-                                error_messages[error_message] = content
-
-            for video in item.get('videoSet', []):
-                auth_token = video.get('authToken')
-                if not auth_token:
-                    continue
-                funimation_id = video.get('FUNImationID') or video.get('videoId')
-                preference = 1 if video.get('languageMode') == 'dub' else 0
-                if not auth_token.startswith('?'):
-                    auth_token = '?%s' % auth_token
-                for quality, height in (('sd', 480), ('hd', 720), ('hd1080', 1080)):
-                    format_url = video.get('%sUrl' % quality)
-                    if not format_url:
-                        continue
-                    if not format_url.startswith(('http', '//')):
-                        errors.append(format_url)
-                        continue
-                    if determine_ext(format_url) == 'm3u8':
-                        formats.extend(self._extract_m3u8_formats(
-                            format_url + auth_token, display_id, 'mp4', entry_protocol='m3u8_native',
-                            preference=preference, m3u8_id='%s-hls' % funimation_id, fatal=False))
-                    else:
-                        tbr = int_or_none(self._search_regex(
-                            r'-(\d+)[Kk]', format_url, 'tbr', default=None))
-                        formats.append({
-                            'url': format_url + auth_token,
-                            'format_id': '%s-http-%dp' % (funimation_id, height),
-                            'height': height,
-                            'tbr': tbr,
-                            'preference': preference,
-                        })
-
-        if not formats and errors:
-            raise ExtractorError(
-                '%s returned error: %s'
-                % (self.IE_NAME, clean_html(error_messages.get(errors[0], errors[0]))),
-                expected=True)
-
+        for source in sources:
+            source_url = source.get('src')
+            if not source_url:
+                continue
+            source_type = source.get('videoType') or determine_ext(source_url)
+            if source_type == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    source_url, video_id, 'mp4',
+                    m3u8_id='hls', fatal=False))
+            else:
+                formats.append({
+                    'format_id': source_type,
+                    'url': source_url,
+                })
         self._sort_formats(formats)
 
-        title = item['title']
-        artist = item.get('artist')
-        if artist:
-            title = '%s - %s' % (artist, title)
-        description = self._og_search_description(webpage) or item.get('description')
-        thumbnail = self._og_search_thumbnail(webpage) or item.get('posterUrl')
-        video_id = item.get('itemId') or display_id
-
         return {
             'id': video_id,
             'display_id': display_id,
             'title': title,
             'description': description,
-            'thumbnail': thumbnail,
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'series': series,
+            'season_number': int_or_none(title_data.get('seasonNum') or _search_kane('season')),
+            'episode_number': int_or_none(title_data.get('episodeNum')),
+            'episode': episode,
+            'season_id': title_data.get('seriesId'),
             'formats': formats,
         }

From a6f3a162f35cc05ac5a34773b438dd4c5f0d164a Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 1 Apr 2017 15:35:39 +0100
Subject: [PATCH 0570/1696] [limelight] improve extraction for audio only
 formats

---
 youtube_dl/extractor/limelight.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index 422be2528..f52c2e169 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -62,13 +62,21 @@ class LimelightBaseIE(InfoExtractor):
                 fmt = {
                     'url': stream_url,
                     'abr': float_or_none(stream.get('audioBitRate')),
-                    'vbr': float_or_none(stream.get('videoBitRate')),
                     'fps': float_or_none(stream.get('videoFrameRate')),
-                    'width': int_or_none(stream.get('videoWidthInPixels')),
-                    'height': int_or_none(stream.get('videoHeightInPixels')),
                     'ext': ext,
                 }
-                rtmp = re.search(r'^(?P<url>rtmpe?://(?P<host>[^/]+)/(?P<app>.+))/(?P<playpath>mp4:.+)$', stream_url)
+                width = int_or_none(stream.get('videoWidthInPixels'))
+                height = int_or_none(stream.get('videoHeightInPixels'))
+                vbr = float_or_none(stream.get('videoBitRate'))
+                if width or height or vbr:
+                    fmt.update({
+                        'width': width,
+                        'height': height,
+                        'vbr': vbr,
+                    })
+                else:
+                    fmt['vcodec'] = 'none'
+                rtmp = re.search(r'^(?P<url>rtmpe?://(?P<host>[^/]+)/(?P<app>.+))/(?P<playpath>mp[34]:.+)$', stream_url)
                 if rtmp:
                     format_id = 'rtmp'
                     if stream.get('videoBitRate'):

From 48ab554feb9c6d3e0f13e1357e04f4c89089e2d3 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 1 Apr 2017 18:09:36 +0100
Subject: [PATCH 0571/1696] [vrv] add support for series pages

---
 youtube_dl/extractor/extractors.py |  5 +-
 youtube_dl/extractor/vrv.py        | 88 ++++++++++++++++++++++--------
 2 files changed, 68 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 1b427e256..980333a11 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1182,7 +1182,10 @@ from .voxmedia import VoxMediaIE
 from .vporn import VpornIE
 from .vrt import VRTIE
 from .vrak import VrakIE
-from .vrv import VRVIE
+from .vrv import (
+    VRVIE,
+    VRVSeriesIE,
+)
 from .medialaan import MedialaanIE
 from .vube import VubeIE
 from .vuclip import VuClipIE
diff --git a/youtube_dl/extractor/vrv.py b/youtube_dl/extractor/vrv.py
index 33618c951..487047fd7 100644
--- a/youtube_dl/extractor/vrv.py
+++ b/youtube_dl/extractor/vrv.py
@@ -20,22 +20,7 @@ from ..utils import (
 )
 
 
-class VRVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?vrv\.co/watch/(?P<id>[A-Z0-9]+)'
-    _TEST = {
-        'url': 'https://vrv.co/watch/GR9PNZ396/Hidden-America-with-Jonah-Ray:BOSTON-WHERE-THE-PAST-IS-THE-PRESENT',
-        'info_dict': {
-            'id': 'GR9PNZ396',
-            'ext': 'mp4',
-            'title': 'BOSTON: WHERE THE PAST IS THE PRESENT',
-            'description': 'md5:4ec8844ac262ca2df9e67c0983c6b83f',
-            'uploader_id': 'seeso',
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }
+class VRVBaseIE(InfoExtractor):
     _API_DOMAIN = None
     _API_PARAMS = {}
     _CMS_SIGNING = {}
@@ -64,6 +49,8 @@ class VRVIE(InfoExtractor):
             note='Downloading %s JSON metadata' % note, headers=headers, data=data)
 
     def _call_cms(self, path, video_id, note):
+        if not self._CMS_SIGNING:
+            self._CMS_SIGNING = self._call_api('index', video_id, 'CMS Signing')['cms_signing']
         return self._download_json(
             self._API_DOMAIN + path, video_id, query=self._CMS_SIGNING,
             note='Downloading %s JSON metadata' % note, headers=self.geo_verification_headers())
@@ -75,9 +62,30 @@ class VRVIE(InfoExtractor):
                 webpage, 'api config'), video_id)['cxApiParams']
             self._API_DOMAIN = self._API_PARAMS.get('apiDomain', 'https://api.vrv.co')
 
-    def _set_cms_signing(self, video_id):
-        if not self._CMS_SIGNING:
-            self._CMS_SIGNING = self._call_api('index', video_id, 'CMS Signing')['cms_signing']
+    def _get_cms_resource(self, resource_key, video_id):
+        return self._call_api(
+            'cms_resource', video_id, 'resource path', data={
+                'resource_key': resource_key,
+            })['__links__']['cms_resource']['href']
+
+
+class VRVIE(VRVBaseIE):
+    IE_NAME = 'vrv'
+    _VALID_URL = r'https?://(?:www\.)?vrv\.co/watch/(?P<id>[A-Z0-9]+)'
+    _TEST = {
+        'url': 'https://vrv.co/watch/GR9PNZ396/Hidden-America-with-Jonah-Ray:BOSTON-WHERE-THE-PAST-IS-THE-PRESENT',
+        'info_dict': {
+            'id': 'GR9PNZ396',
+            'ext': 'mp4',
+            'title': 'BOSTON: WHERE THE PAST IS THE PRESENT',
+            'description': 'md5:4ec8844ac262ca2df9e67c0983c6b83f',
+            'uploader_id': 'seeso',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -91,10 +99,8 @@ class VRVIE(InfoExtractor):
         video_data = media_resource.get('json')
         if not video_data:
             self._set_api_params(webpage, video_id)
-            episode_path = self._call_api('cms_resource', video_id, 'episode resource path', data={
-                'resource_key': 'cms:/episodes/' + video_id,
-            })['__links__']['cms_resource']['href']
-            self._set_cms_signing(video_id)
+            episode_path = self._get_cms_resource(
+                'cms:/episodes/' + video_id, video_id)
             video_data = self._call_cms(episode_path, video_id, 'video')
         title = video_data['title']
 
@@ -102,7 +108,6 @@ class VRVIE(InfoExtractor):
         if not streams_json:
             self._set_api_params(webpage, video_id)
             streams_path = video_data['__links__']['streams']['href']
-            self._set_cms_signing(video_id)
             streams_json = self._call_cms(streams_path, video_id, 'streams')
 
         audio_locale = streams_json.get('audio_locale')
@@ -149,3 +154,38 @@ class VRVIE(InfoExtractor):
             'episode_number': int_or_none(video_data.get('episode_number')),
             'episode_id': video_data.get('production_episode_id'),
         }
+
+
+class VRVSeriesIE(VRVBaseIE):
+    IE_NAME = 'vrv:series'
+    _VALID_URL = r'https?://(?:www\.)?vrv\.co/series/(?P<id>[A-Z0-9]+)'
+    _TEST = {
+        'url': 'https://vrv.co/series/G68VXG3G6/The-Perfect-Insider',
+        'info_dict': {
+            'id': 'G68VXG3G6',
+        },
+        'playlist_mincount': 11,
+    }
+
+    def _real_extract(self, url):
+        series_id = self._match_id(url)
+        webpage = self._download_webpage(
+            url, series_id,
+            headers=self.geo_verification_headers())
+
+        self._set_api_params(webpage, series_id)
+        seasons_path = self._get_cms_resource(
+            'cms:/seasons?series_id=' + series_id, series_id)
+        seasons_data = self._call_cms(seasons_path, series_id, 'seasons')
+
+        entries = []
+        for season in seasons_data.get('items', []):
+            episodes_path = season['__links__']['season/episodes']['href']
+            episodes = self._call_cms(episodes_path, series_id, 'episodes')
+            for episode in episodes.get('items', []):
+                episode_id = episode['id']
+                entries.append(self.url_result(
+                    'https://vrv.co/watch/' + episode_id,
+                    'VRV', episode_id, episode.get('title')))
+
+        return self.playlist_result(entries, series_id)

From 51342717cddafde83dbf39f2212be40a196a577a Mon Sep 17 00:00:00 2001
From: Timendum <timedum@gmail.com>
Date: Tue, 14 Mar 2017 16:11:09 +0100
Subject: [PATCH 0572/1696] [rai] Fix extraction

---
 youtube_dl/extractor/extractors.py |   2 +-
 youtube_dl/extractor/rai.py        | 359 ++++++++++++++++-------------
 2 files changed, 197 insertions(+), 164 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 980333a11..d9e8d53ac 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -802,7 +802,7 @@ from .radiojavan import RadioJavanIE
 from .radiobremen import RadioBremenIE
 from .radiofrance import RadioFranceIE
 from .rai import (
-    RaiTVIE,
+    RaiPlayIE,
     RaiIE,
 )
 from .rbmaradio import RBMARadioIE
diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 41afbd9af..b67e94f88 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -3,8 +3,8 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..compat import compat_urlparse
 from ..utils import (
-    determine_ext,
     ExtractorError,
+    determine_ext,
     find_xpath_attr,
     fix_xml_ampersands,
     int_or_none,
@@ -55,7 +55,200 @@ class RaiBaseIE(InfoExtractor):
 
         return formats
 
-    def _extract_from_content_id(self, content_id, base_url):
+
+class RaiPlayIE(RaiBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?raiplay\.it/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})\.html'
+    _TESTS = [{
+        'url': 'http://www.raiplay.it/video/2016/10/La-Casa-Bianca-e06118bb-59a9-4636-b914-498e4cfd2c66.html?source=twitter',
+        'md5': '340aa3b7afb54bfd14a8c11786450d76',
+        'info_dict': {
+            'id': 'e06118bb-59a9-4636-b914-498e4cfd2c66',
+            'ext': 'mp4',
+            'title': 'La Casa Bianca',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'uploader': r're:^Rai.+',
+            'description': 're:^[A-Za-z]+'
+        }
+    }, {
+        'url': 'http://www.raiplay.it/video/2016/11/gazebotraindesi-efebe701-969c-4593-92f3-285f0d1ce750.html?',
+        'md5': 'ed4da3d70ccf8129a33ab16b34d20ab8',
+        'info_dict': {
+            'id': 'efebe701-969c-4593-92f3-285f0d1ce750',
+            'ext': 'mp4',
+            'title': 'Gazebo - #gazebotraindesi',
+            'thumbnail': r're:^https?://.*\.png$',
+            'uploader': r're:^Rai.+',
+            'description': r're:^[A-Za-z]+'
+        }
+    }, {
+        'url': 'http://www.raiplay.it/video/2014/04/Report-del-07042014-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
+        'md5': '8970abf8caf8aef4696e7b1f2adfc696',
+        'info_dict': {
+            'id': 'cb27157f-9dd0-4aee-b788-b1f67643a391',
+            'ext': 'mp4',
+            'title': 'Report - Report del 07/04/2014',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'uploader': r're:^Rai.+',
+            'description': r're:^[A-Za-z]+'
+        }
+    }]
+    _RESOLUTION = '600x400'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        # remove query and fragment part from url
+        canonical_url = compat_urlparse.urljoin(url, compat_urlparse.urlparse(url).path)
+        webpage = self._download_webpage(canonical_url, video_id)
+
+        media = self._download_json('%s?json' % canonical_url,
+                                    video_id, 'Downloading video JSON')
+
+        thumbnails = []
+        if 'images' in media:
+            for _, value in media.get('images').items():
+                if value:
+                    thumbnails.append({
+                        'url': value.replace('[RESOLUTION]', self._RESOLUTION)
+                    })
+
+        if 'video' not in media:
+            raise ExtractorError('No video found')
+
+        video = media.get('video')
+        duration = parse_duration(video.get('duration')),
+        formats = self._extract_relinker_formats(video.get('contentUrl'), video_id)
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage).replace(' - video - RaiPlay', ''),
+            'description': self._og_search_description(webpage),
+            'uploader': media.get('channel'),
+            'duration': duration,
+            'thumbnails': thumbnails,
+            'formats': formats
+        }
+
+
+class RaiIE(RaiBaseIE):
+    _VALID_URL = r'https?://.+\.(?:rai|rainews)\.it/dl/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
+    _TESTS = [{
+        # subdomain test case
+        'url': 'http://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html',
+        'info_dict': {
+            'id': '04a9f4bd-b563-40cf-82a6-aad3529cb4a9',
+            'ext': 'mp4',
+            'title': 'TG PRIMO TEMPO',
+            'upload_date': '20140612',
+            'duration': 1758,
+            'thumbnail': r're:^https?://.*\.jpg$'
+        }
+    }, {
+        # rainews test case
+        'url': 'http://www.rainews.it/dl/rainews/media/Weekend-al-cinema-da-Hollywood-arriva-il-thriller-di-Tate-Taylor-La-ragazza-del-treno-1632c009-c843-4836-bb65-80c33084a64b.html',
+        'info_dict': {
+            'id': '1632c009-c843-4836-bb65-80c33084a64b',
+            'ext': 'mp4',
+            'title': 'Weekend al cinema, da Hollywood arriva il thriller di Tate Taylor \"La ragazza del treno\" ',
+            'upload_date': '20161103',
+            'thumbnail': r're:^https?://.*\.png$',
+            'description': r're:^[A-Za-z]+'
+        }
+    }, {
+        # with media information
+        'url': 'http://www.rai.it/dl/RaiTV/programmi/media/ContentItem-efb17665-691c-45d5-a60c-5301333cbb0c.html',
+        'md5': '11959b4e44fa74de47011b5799490adf',
+        'info_dict': {
+            'id': 'efb17665-691c-45d5-a60c-5301333cbb0c',
+            'ext': 'mp4',
+            'title': 'TG1 ore 20:00 del 03/11/2016',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'upload_date': '20161103',
+            'description': r're:^[A-Za-z]+'
+        }
+    }, {
+        # drawMediaRaiTV test case
+        'url': 'http://www.report.rai.it/dl/Report/puntata/ContentItem-0c7a664b-d0f4-4b2c-8835-3f82e46f433e.html',
+        'md5': '2dd727e61114e1ee9c47f0da6914e178',
+        'info_dict': {
+            'id': '59d69d28-6bb6-409d-a4b5-ed44096560af',
+            'ext': 'mp4',
+            'title': 'Il pacco',
+            'description': 'md5:4b1afae1364115ce5d78ed83cd2e5b3a',
+            'upload_date': '20141221',
+        },
+    }, {
+        # Direct relinker URL
+        'url': 'http://www.rai.tv/dl/RaiTV/dirette/PublishingBlock-1912dbbf-3f96-44c3-b4cf-523681fbacbc.html?channel=EuroNews',
+        # HDS live stream, MD5 is unstable
+        'info_dict': {
+            'id': '1912dbbf-3f96-44c3-b4cf-523681fbacbc',
+            'ext': 'flv',
+            'title': 'EuroNews',
+        },
+    }, {
+        # Embedded content item ID
+        'url': 'http://www.tg1.rai.it/dl/tg1/2010/edizioni/ContentSet-9b6e0cba-4bef-4aef-8cf0-9f7f665b7dfb-tg1.html?item=undefined',
+        'info_dict': {
+            'id': 'd80d4b70-3812-4501-a888-92edec729f00',
+            'ext': 'mp4',
+            'title': r're:TG1 ore \d{2}:\d{2} del \d{2}/\d{2}/\d{4}',
+            'upload_date': r're:\d{8}',
+            'description': r're:.+',
+        },
+    }, {
+        'url': 'http://www.rainews.it/dl/rainews/live/ContentItem-3156f2f2-dc70-4953-8e2f-70d7489d4ce9.html',
+        # HDS live stream, MD5 is unstable
+        'info_dict': {
+            'id': '3156f2f2-dc70-4953-8e2f-70d7489d4ce9',
+            'ext': 'mp4',
+            'title': 'La diretta di Rainews24',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        iframe_url = self._search_regex(
+            [r'<iframe[^>]+src="([^"]*/dl/[^"]+\?iframe\b[^"]*)"',
+             r'drawMediaRaiTV\(["\'](.+?)["\']'],
+            webpage, 'iframe', default=None)
+        if iframe_url:
+            if not iframe_url.startswith('http'):
+                iframe_url = compat_urlparse.urljoin(url, iframe_url)
+            return self.url_result(iframe_url)
+
+        content_item_id = self._search_regex(
+            r'initEdizione\((?P<q1>[\'"])ContentItem-(?P<content_id>[^\'"]+)(?P=q1)',
+            webpage, 'content item ID', group='content_id', default=None)
+        if content_item_id:
+            return self._extract_from_content_id(content_item_id, url)
+
+        try:
+            return self._extract_from_content_id(video_id, url)
+        except ExtractorError:
+            # no media data, only direct relinker
+            pass
+
+        relinker_url = compat_urlparse.urljoin(url, self._search_regex(
+            r'(?:var\s+videoURL|mediaInfo\.mediaUri)\s*=\s*(?P<q1>[\'"])(?P<url>(https?:)?//mediapolis\.rai\.it/relinker/relinkerServlet\.htm\?cont=\d+)(?P=q1)',
+            webpage, 'relinker URL', group='url'))
+        formats = self._extract_relinker_formats(relinker_url, video_id)
+        self._sort_formats(formats)
+
+        title = self._search_regex(
+            r'var\s+videoTitolo\s*=\s*([\'"])(?P<title>[^\'"]+)\1',
+            webpage, 'title', group='title', default=None) or self._og_search_title(webpage)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+        }
+
+    def _extract_from_content_id(self, content_id, url):
         media = self._download_json(
             'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-%s.html?json' % content_id,
             content_id, 'Downloading video JSON')
@@ -65,7 +258,7 @@ class RaiBaseIE(InfoExtractor):
             thumbnail_url = media.get(image_type)
             if thumbnail_url:
                 thumbnails.append({
-                    'url': compat_urlparse.urljoin(base_url, thumbnail_url),
+                    'url': compat_urlparse.urljoin(url, thumbnail_url),
                 })
 
         formats = []
@@ -105,163 +298,3 @@ class RaiBaseIE(InfoExtractor):
             'formats': formats,
             'subtitles': subtitles,
         }
-
-
-class RaiTVIE(RaiBaseIE):
-    _VALID_URL = r'https?://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/(?:[^/]+/)+(?:media|ondemand)/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
-    _TESTS = [
-        {
-            'url': 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
-            'md5': '8970abf8caf8aef4696e7b1f2adfc696',
-            'info_dict': {
-                'id': 'cb27157f-9dd0-4aee-b788-b1f67643a391',
-                'ext': 'mp4',
-                'title': 'Report del 07/04/2014',
-                'description': 'md5:f27c544694cacb46a078db84ec35d2d9',
-                'upload_date': '20140407',
-                'duration': 6160,
-                'thumbnail': r're:^https?://.*\.jpg$',
-            }
-        },
-        {
-            # no m3u8 stream
-            'url': 'http://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html',
-            # HDS download, MD5 is unstable
-            'info_dict': {
-                'id': '04a9f4bd-b563-40cf-82a6-aad3529cb4a9',
-                'ext': 'flv',
-                'title': 'TG PRIMO TEMPO',
-                'upload_date': '20140612',
-                'duration': 1758,
-                'thumbnail': r're:^https?://.*\.jpg$',
-            },
-            'skip': 'Geo-restricted to Italy',
-        },
-        {
-            'url': 'http://www.rainews.it/dl/rainews/media/state-of-the-net-Antonella-La-Carpia-regole-virali-7aafdea9-0e5d-49d5-88a6-7e65da67ae13.html',
-            'md5': '35cf7c229f22eeef43e48b5cf923bef0',
-            'info_dict': {
-                'id': '7aafdea9-0e5d-49d5-88a6-7e65da67ae13',
-                'ext': 'mp4',
-                'title': 'State of the Net, Antonella La Carpia: regole virali',
-                'description': 'md5:b0ba04a324126903e3da7763272ae63c',
-                'upload_date': '20140613',
-            },
-            'skip': 'Error 404',
-        },
-        {
-            'url': 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-b4a49761-e0cc-4b14-8736-2729f6f73132-tg2.html',
-            'info_dict': {
-                'id': 'b4a49761-e0cc-4b14-8736-2729f6f73132',
-                'ext': 'mp4',
-                'title': 'Alluvione in Sardegna e dissesto idrogeologico',
-                'description': 'Edizione delle ore 20:30 ',
-            },
-            'skip': 'invalid urls',
-        },
-        {
-            'url': 'http://www.ilcandidato.rai.it/dl/ray/media/Il-Candidato---Primo-episodio-Le-Primarie-28e5525a-b495-45e8-a7c3-bc48ba45d2b6.html',
-            'md5': 'e57493e1cb8bc7c564663f363b171847',
-            'info_dict': {
-                'id': '28e5525a-b495-45e8-a7c3-bc48ba45d2b6',
-                'ext': 'mp4',
-                'title': 'Il Candidato - Primo episodio: "Le Primarie"',
-                'description': 'md5:364b604f7db50594678f483353164fb8',
-                'upload_date': '20140923',
-                'duration': 386,
-                'thumbnail': r're:^https?://.*\.jpg$',
-            }
-        },
-    ]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        return self._extract_from_content_id(video_id, url)
-
-
-class RaiIE(RaiBaseIE):
-    _VALID_URL = r'https?://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
-    _TESTS = [
-        {
-            'url': 'http://www.report.rai.it/dl/Report/puntata/ContentItem-0c7a664b-d0f4-4b2c-8835-3f82e46f433e.html',
-            'md5': '2dd727e61114e1ee9c47f0da6914e178',
-            'info_dict': {
-                'id': '59d69d28-6bb6-409d-a4b5-ed44096560af',
-                'ext': 'mp4',
-                'title': 'Il pacco',
-                'description': 'md5:4b1afae1364115ce5d78ed83cd2e5b3a',
-                'upload_date': '20141221',
-            },
-        },
-        {
-            # Direct relinker URL
-            'url': 'http://www.rai.tv/dl/RaiTV/dirette/PublishingBlock-1912dbbf-3f96-44c3-b4cf-523681fbacbc.html?channel=EuroNews',
-            # HDS live stream, MD5 is unstable
-            'info_dict': {
-                'id': '1912dbbf-3f96-44c3-b4cf-523681fbacbc',
-                'ext': 'flv',
-                'title': 'EuroNews',
-            },
-            'skip': 'Geo-restricted to Italy',
-        },
-        {
-            # Embedded content item ID
-            'url': 'http://www.tg1.rai.it/dl/tg1/2010/edizioni/ContentSet-9b6e0cba-4bef-4aef-8cf0-9f7f665b7dfb-tg1.html?item=undefined',
-            'md5': '84c1135ce960e8822ae63cec34441d63',
-            'info_dict': {
-                'id': '0960e765-62c8-474a-ac4b-7eb3e2be39c8',
-                'ext': 'mp4',
-                'title': 'TG1 ore 20:00 del 02/07/2016',
-                'upload_date': '20160702',
-            },
-        },
-        {
-            'url': 'http://www.rainews.it/dl/rainews/live/ContentItem-3156f2f2-dc70-4953-8e2f-70d7489d4ce9.html',
-            # HDS live stream, MD5 is unstable
-            'info_dict': {
-                'id': '3156f2f2-dc70-4953-8e2f-70d7489d4ce9',
-                'ext': 'flv',
-                'title': 'La diretta di Rainews24',
-            },
-        },
-    ]
-
-    @classmethod
-    def suitable(cls, url):
-        return False if RaiTVIE.suitable(url) else super(RaiIE, cls).suitable(url)
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        iframe_url = self._search_regex(
-            [r'<iframe[^>]+src="([^"]*/dl/[^"]+\?iframe\b[^"]*)"',
-             r'drawMediaRaiTV\(["\'](.+?)["\']'],
-            webpage, 'iframe', default=None)
-        if iframe_url:
-            if not iframe_url.startswith('http'):
-                iframe_url = compat_urlparse.urljoin(url, iframe_url)
-            return self.url_result(iframe_url)
-
-        content_item_id = self._search_regex(
-            r'initEdizione\((?P<q1>[\'"])ContentItem-(?P<content_id>[^\'"]+)(?P=q1)',
-            webpage, 'content item ID', group='content_id', default=None)
-        if content_item_id:
-            return self._extract_from_content_id(content_item_id, url)
-
-        relinker_url = compat_urlparse.urljoin(url, self._search_regex(
-            r'(?:var\s+videoURL|mediaInfo\.mediaUri)\s*=\s*(?P<q1>[\'"])(?P<url>(https?:)?//mediapolis\.rai\.it/relinker/relinkerServlet\.htm\?cont=\d+)(?P=q1)',
-            webpage, 'relinker URL', group='url'))
-        formats = self._extract_relinker_formats(relinker_url, video_id)
-        self._sort_formats(formats)
-
-        title = self._search_regex(
-            r'var\s+videoTitolo\s*=\s*([\'"])(?P<title>[^\'"]+)\1',
-            webpage, 'title', group='title', default=None) or self._og_search_title(webpage)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'formats': formats,
-        }

From b8d8cced9b55c57f3b09e83972be9d6318a459ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Apr 2017 02:14:42 +0700
Subject: [PATCH 0573/1696] [rai] Improve extraction (closes #11790) * Fix
 georestriction detection * Detect live streams + Extract relinker metadata *
 Improve ContentItem detection + Extract series metadata * Fix tests

---
 youtube_dl/extractor/rai.py | 359 +++++++++++++++++++++++-------------
 1 file changed, 233 insertions(+), 126 deletions(-)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index b67e94f88..b77b0a08e 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -1,23 +1,40 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
-from ..compat import compat_urlparse
+from ..compat import (
+    compat_urlparse,
+    compat_str,
+)
 from ..utils import (
     ExtractorError,
     determine_ext,
     find_xpath_attr,
     fix_xml_ampersands,
+    GeoRestrictedError,
     int_or_none,
     parse_duration,
+    strip_or_none,
+    try_get,
     unified_strdate,
+    unified_timestamp,
     update_url_query,
+    urljoin,
     xpath_text,
 )
 
 
 class RaiBaseIE(InfoExtractor):
-    def _extract_relinker_formats(self, relinker_url, video_id):
+    _UUID_RE = r'[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}'
+    _GEO_COUNTRIES = ['IT']
+    _GEO_BYPASS = False
+
+    def _extract_relinker_info(self, relinker_url, video_id):
         formats = []
+        geoprotection = None
+        is_live = None
+        duration = None
 
         for platform in ('mon', 'flash', 'native'):
             relinker = self._download_xml(
@@ -27,9 +44,27 @@ class RaiBaseIE(InfoExtractor):
                 query={'output': 45, 'pl': platform},
                 headers=self.geo_verification_headers())
 
-            media_url = find_xpath_attr(relinker, './url', 'type', 'content').text
+            if not geoprotection:
+                geoprotection = xpath_text(
+                    relinker, './geoprotection', default=None) == 'Y'
+
+            if not is_live:
+                is_live = xpath_text(
+                    relinker, './is_live', default=None) == 'Y'
+            if not duration:
+                duration = parse_duration(xpath_text(
+                    relinker, './duration', default=None))
+
+            url_elem = find_xpath_attr(relinker, './url', 'type', 'content')
+            if url_elem is None:
+                continue
+
+            media_url = url_elem.text
+
+            # This does not imply geo restriction (e.g.
+            # http://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html)
             if media_url == 'http://download.rai.it/video_no_available.mp4':
-                self.raise_geo_restricted()
+                continue
 
             ext = determine_ext(media_url)
             if (ext == 'm3u8' and platform != 'mon') or (ext == 'f4m' and platform != 'flash'):
@@ -53,11 +88,18 @@ class RaiBaseIE(InfoExtractor):
                     'format_id': 'http-%d' % bitrate if bitrate > 0 else 'http',
                 })
 
-        return formats
+        if not formats and geoprotection is True:
+            self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
+
+        return dict((k, v) for k, v in {
+            'is_live': is_live,
+            'duration': duration,
+            'formats': formats,
+        }.items() if v is not None)
 
 
 class RaiPlayIE(RaiBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?raiplay\.it/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})\.html'
+    _VALID_URL = r'(?P<url>https?://(?:www\.)?raiplay\.it/.+?-(?P<id>%s)\.html)' % RaiBaseIE._UUID_RE
     _TESTS = [{
         'url': 'http://www.raiplay.it/video/2016/10/La-Casa-Bianca-e06118bb-59a9-4636-b914-498e4cfd2c66.html?source=twitter',
         'md5': '340aa3b7afb54bfd14a8c11786450d76',
@@ -65,110 +107,130 @@ class RaiPlayIE(RaiBaseIE):
             'id': 'e06118bb-59a9-4636-b914-498e4cfd2c66',
             'ext': 'mp4',
             'title': 'La Casa Bianca',
+            'alt_title': 'S2016 - Puntata del 23/10/2016',
+            'description': 'md5:a09d45890850458077d1f68bb036e0a5',
             'thumbnail': r're:^https?://.*\.jpg$',
-            'uploader': r're:^Rai.+',
-            'description': 're:^[A-Za-z]+'
-        }
-    }, {
-        'url': 'http://www.raiplay.it/video/2016/11/gazebotraindesi-efebe701-969c-4593-92f3-285f0d1ce750.html?',
-        'md5': 'ed4da3d70ccf8129a33ab16b34d20ab8',
-        'info_dict': {
-            'id': 'efebe701-969c-4593-92f3-285f0d1ce750',
-            'ext': 'mp4',
-            'title': 'Gazebo - #gazebotraindesi',
-            'thumbnail': r're:^https?://.*\.png$',
-            'uploader': r're:^Rai.+',
-            'description': r're:^[A-Za-z]+'
-        }
+            'uploader': 'Rai 3',
+            'creator': 'Rai 3',
+            'duration': 3278,
+            'timestamp': 1477764300,
+            'upload_date': '20161029',
+            'series': 'La Casa Bianca',
+            'season': '2016',
+        },
     }, {
         'url': 'http://www.raiplay.it/video/2014/04/Report-del-07042014-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
         'md5': '8970abf8caf8aef4696e7b1f2adfc696',
         'info_dict': {
             'id': 'cb27157f-9dd0-4aee-b788-b1f67643a391',
             'ext': 'mp4',
-            'title': 'Report - Report del 07/04/2014',
+            'title': 'Report del 07/04/2014',
+            'alt_title': 'S2013/14 - Puntata del 07/04/2014',
+            'description': 'md5:f27c544694cacb46a078db84ec35d2d9',
             'thumbnail': r're:^https?://.*\.jpg$',
-            'uploader': r're:^Rai.+',
-            'description': r're:^[A-Za-z]+'
-        }
+            'uploader': 'Rai 5',
+            'creator': 'Rai 5',
+            'duration': 6160,
+            'series': 'Report',
+            'season_number': 5,
+            'season': '2013/14',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://www.raiplay.it/video/2016/11/gazebotraindesi-efebe701-969c-4593-92f3-285f0d1ce750.html?',
+        'only_matching': True,
     }]
-    _RESOLUTION = '600x400'
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        url, video_id = mobj.group('url', 'id')
 
-        # remove query and fragment part from url
-        canonical_url = compat_urlparse.urljoin(url, compat_urlparse.urlparse(url).path)
-        webpage = self._download_webpage(canonical_url, video_id)
+        media = self._download_json(
+            '%s?json' % url, video_id, 'Downloading video JSON')
 
-        media = self._download_json('%s?json' % canonical_url,
-                                    video_id, 'Downloading video JSON')
+        title = media['name']
+
+        video = media['video']
+
+        relinker_info = self._extract_relinker_info(video['contentUrl'], video_id)
+        self._sort_formats(relinker_info['formats'])
 
         thumbnails = []
         if 'images' in media:
             for _, value in media.get('images').items():
                 if value:
                     thumbnails.append({
-                        'url': value.replace('[RESOLUTION]', self._RESOLUTION)
+                        'url': value.replace('[RESOLUTION]', '600x400')
                     })
 
-        if 'video' not in media:
-            raise ExtractorError('No video found')
+        timestamp = unified_timestamp(try_get(
+            media, lambda x: x['availabilities'][0]['start'], compat_str))
 
-        video = media.get('video')
-        duration = parse_duration(video.get('duration')),
-        formats = self._extract_relinker_formats(video.get('contentUrl'), video_id)
-        self._sort_formats(formats)
-
-        return {
+        info = {
             'id': video_id,
-            'title': self._og_search_title(webpage).replace(' - video - RaiPlay', ''),
-            'description': self._og_search_description(webpage),
+            'title': title,
+            'alt_title': media.get('subtitle'),
+            'description': media.get('description'),
             'uploader': media.get('channel'),
-            'duration': duration,
+            'creator': media.get('editor'),
+            'duration': parse_duration(video.get('duration')),
+            'timestamp': timestamp,
             'thumbnails': thumbnails,
-            'formats': formats
+            'series': try_get(
+                media, lambda x: x['isPartOf']['name'], compat_str),
+            'season_number': int_or_none(try_get(
+                media, lambda x: x['isPartOf']['numeroStagioni'])),
+            'season': media.get('stagione') or None,
         }
 
+        info.update(relinker_info)
+
+        return info
+
 
 class RaiIE(RaiBaseIE):
-    _VALID_URL = r'https?://.+\.(?:rai|rainews)\.it/dl/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
+    _VALID_URL = r'https?://[^/]+\.(?:rai\.(?:it|tv)|rainews\.it)/dl/.+?-(?P<id>%s)(?:-.+?)?\.html' % RaiBaseIE._UUID_RE
     _TESTS = [{
-        # subdomain test case
+        # var uniquename = "ContentItem-..."
+        # data-id="ContentItem-..."
         'url': 'http://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html',
         'info_dict': {
             'id': '04a9f4bd-b563-40cf-82a6-aad3529cb4a9',
             'ext': 'mp4',
             'title': 'TG PRIMO TEMPO',
-            'upload_date': '20140612',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 1758,
-            'thumbnail': r're:^https?://.*\.jpg$'
+            'upload_date': '20140612',
         }
     }, {
-        # rainews test case
+        # with ContentItem in many metas
         'url': 'http://www.rainews.it/dl/rainews/media/Weekend-al-cinema-da-Hollywood-arriva-il-thriller-di-Tate-Taylor-La-ragazza-del-treno-1632c009-c843-4836-bb65-80c33084a64b.html',
         'info_dict': {
             'id': '1632c009-c843-4836-bb65-80c33084a64b',
             'ext': 'mp4',
-            'title': 'Weekend al cinema, da Hollywood arriva il thriller di Tate Taylor \"La ragazza del treno\" ',
-            'upload_date': '20161103',
+            'title': 'Weekend al cinema, da Hollywood arriva il thriller di Tate Taylor "La ragazza del treno"',
+            'description': 'I film in uscita questa settimana.',
             'thumbnail': r're:^https?://.*\.png$',
-            'description': r're:^[A-Za-z]+'
+            'duration': 833,
+            'upload_date': '20161103',
         }
     }, {
-        # with media information
+        # with ContentItem in og:url
         'url': 'http://www.rai.it/dl/RaiTV/programmi/media/ContentItem-efb17665-691c-45d5-a60c-5301333cbb0c.html',
         'md5': '11959b4e44fa74de47011b5799490adf',
         'info_dict': {
             'id': 'efb17665-691c-45d5-a60c-5301333cbb0c',
             'ext': 'mp4',
             'title': 'TG1 ore 20:00 del 03/11/2016',
+            'description': 'TG1 edizione integrale ore 20:00 del giorno 03/11/2016',
             'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 2214,
             'upload_date': '20161103',
-            'description': r're:^[A-Za-z]+'
         }
     }, {
-        # drawMediaRaiTV test case
+        # drawMediaRaiTV(...)
         'url': 'http://www.report.rai.it/dl/Report/puntata/ContentItem-0c7a664b-d0f4-4b2c-8835-3f82e46f433e.html',
         'md5': '2dd727e61114e1ee9c47f0da6914e178',
         'info_dict': {
@@ -176,83 +238,67 @@ class RaiIE(RaiBaseIE):
             'ext': 'mp4',
             'title': 'Il pacco',
             'description': 'md5:4b1afae1364115ce5d78ed83cd2e5b3a',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'upload_date': '20141221',
         },
     }, {
-        # Direct relinker URL
+        # initEdizione('ContentItem-...'
+        'url': 'http://www.tg1.rai.it/dl/tg1/2010/edizioni/ContentSet-9b6e0cba-4bef-4aef-8cf0-9f7f665b7dfb-tg1.html?item=undefined',
+        'info_dict': {
+            'id': 'c2187016-8484-4e3a-8ac8-35e475b07303',
+            'ext': 'mp4',
+            'title': r're:TG1 ore \d{2}:\d{2} del \d{2}/\d{2}/\d{4}',
+            'duration': 2274,
+            'upload_date': '20170401',
+        },
+        'skip': 'Changes daily',
+    }, {
+        # HDS live stream with only relinker URL
         'url': 'http://www.rai.tv/dl/RaiTV/dirette/PublishingBlock-1912dbbf-3f96-44c3-b4cf-523681fbacbc.html?channel=EuroNews',
-        # HDS live stream, MD5 is unstable
         'info_dict': {
             'id': '1912dbbf-3f96-44c3-b4cf-523681fbacbc',
             'ext': 'flv',
             'title': 'EuroNews',
         },
-    }, {
-        # Embedded content item ID
-        'url': 'http://www.tg1.rai.it/dl/tg1/2010/edizioni/ContentSet-9b6e0cba-4bef-4aef-8cf0-9f7f665b7dfb-tg1.html?item=undefined',
-        'info_dict': {
-            'id': 'd80d4b70-3812-4501-a888-92edec729f00',
-            'ext': 'mp4',
-            'title': r're:TG1 ore \d{2}:\d{2} del \d{2}/\d{2}/\d{4}',
-            'upload_date': r're:\d{8}',
-            'description': r're:.+',
+        'params': {
+            'skip_download': True,
         },
     }, {
+        # HLS live stream with ContentItem in og:url
         'url': 'http://www.rainews.it/dl/rainews/live/ContentItem-3156f2f2-dc70-4953-8e2f-70d7489d4ce9.html',
-        # HDS live stream, MD5 is unstable
         'info_dict': {
             'id': '3156f2f2-dc70-4953-8e2f-70d7489d4ce9',
             'ext': 'mp4',
             'title': 'La diretta di Rainews24',
         },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        iframe_url = self._search_regex(
-            [r'<iframe[^>]+src="([^"]*/dl/[^"]+\?iframe\b[^"]*)"',
-             r'drawMediaRaiTV\(["\'](.+?)["\']'],
-            webpage, 'iframe', default=None)
-        if iframe_url:
-            if not iframe_url.startswith('http'):
-                iframe_url = compat_urlparse.urljoin(url, iframe_url)
-            return self.url_result(iframe_url)
-
-        content_item_id = self._search_regex(
-            r'initEdizione\((?P<q1>[\'"])ContentItem-(?P<content_id>[^\'"]+)(?P=q1)',
-            webpage, 'content item ID', group='content_id', default=None)
-        if content_item_id:
-            return self._extract_from_content_id(content_item_id, url)
-
-        try:
-            return self._extract_from_content_id(video_id, url)
-        except ExtractorError:
-            # no media data, only direct relinker
-            pass
-
-        relinker_url = compat_urlparse.urljoin(url, self._search_regex(
-            r'(?:var\s+videoURL|mediaInfo\.mediaUri)\s*=\s*(?P<q1>[\'"])(?P<url>(https?:)?//mediapolis\.rai\.it/relinker/relinkerServlet\.htm\?cont=\d+)(?P=q1)',
-            webpage, 'relinker URL', group='url'))
-        formats = self._extract_relinker_formats(relinker_url, video_id)
-        self._sort_formats(formats)
-
-        title = self._search_regex(
-            r'var\s+videoTitolo\s*=\s*([\'"])(?P<title>[^\'"]+)\1',
-            webpage, 'title', group='title', default=None) or self._og_search_title(webpage)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'formats': formats,
-        }
-
     def _extract_from_content_id(self, content_id, url):
         media = self._download_json(
             'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-%s.html?json' % content_id,
             content_id, 'Downloading video JSON')
 
+        title = media['name'].strip()
+
+        media_type = media['type']
+        if 'Audio' in media_type:
+            relinker_info = {
+                'formats': {
+                    'format_id': media.get('formatoAudio'),
+                    'url': media['audioUrl'],
+                    'ext': media.get('formatoAudio'),
+                }
+            }
+        elif 'Video' in media_type:
+            relinker_info = self._extract_relinker_info(media['mediaUri'], content_id)
+        else:
+            raise ExtractorError('not a media file')
+
+        self._sort_formats(relinker_info['formats'])
+
         thumbnails = []
         for image_type in ('image', 'image_medium', 'image_300'):
             thumbnail_url = media.get(image_type)
@@ -261,20 +307,6 @@ class RaiIE(RaiBaseIE):
                     'url': compat_urlparse.urljoin(url, thumbnail_url),
                 })
 
-        formats = []
-        media_type = media['type']
-        if 'Audio' in media_type:
-            formats.append({
-                'format_id': media.get('formatoAudio'),
-                'url': media['audioUrl'],
-                'ext': media.get('formatoAudio'),
-            })
-        elif 'Video' in media_type:
-            formats.extend(self._extract_relinker_formats(media['mediaUri'], content_id))
-            self._sort_formats(formats)
-        else:
-            raise ExtractorError('not a media file')
-
         subtitles = {}
         captions = media.get('subtitlesUrl')
         if captions:
@@ -287,14 +319,89 @@ class RaiIE(RaiBaseIE):
                 'url': captions,
             }]
 
-        return {
+        info = {
             'id': content_id,
-            'title': media['name'],
-            'description': media.get('desc'),
+            'title': title,
+            'description': strip_or_none(media.get('desc')),
             'thumbnails': thumbnails,
             'uploader': media.get('author'),
             'upload_date': unified_strdate(media.get('date')),
             'duration': parse_duration(media.get('length')),
-            'formats': formats,
             'subtitles': subtitles,
         }
+
+        info.update(relinker_info)
+
+        return info
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        content_item_id = None
+
+        content_item_url = self._html_search_meta(
+            ('og:url', 'og:video', 'og:video:secure_url', 'twitter:url',
+             'twitter:player', 'jsonlink'), webpage, default=None)
+        if content_item_url:
+            content_item_id = self._search_regex(
+                r'ContentItem-(%s)' % self._UUID_RE, content_item_url,
+                'content item id', default=None)
+
+        if not content_item_id:
+            content_item_id = self._search_regex(
+                r'''(?x)
+                    (?:
+                        (?:initEdizione|drawMediaRaiTV)\(|
+                        <(?:[^>]+\bdata-id|var\s+uniquename)=
+                    )
+                    (["\'])
+                    (?:(?!\1).)*\bContentItem-(?P<id>%s)
+                ''' % self._UUID_RE,
+                webpage, 'content item id', default=None, group='id')
+
+        content_item_ids = set()
+        content_item_ids.add(content_item_id)
+        if video_id not in content_item_ids:
+            content_item_ids.add(video_id)
+
+        for content_item_id in content_item_ids:
+            try:
+                return self._extract_from_content_id(content_item_id, url)
+            except GeoRestrictedError:
+                raise
+            except ExtractorError:
+                pass
+
+        relinker_url = self._search_regex(
+            r'''(?x)
+                (?:
+                    var\s+videoURL|
+                    mediaInfo\.mediaUri
+                )\s*=\s*
+                ([\'"])
+                (?P<url>
+                    (?:https?:)?
+                    //mediapolis(?:vod)?\.rai\.it/relinker/relinkerServlet\.htm\?
+                    (?:(?!\1).)*\bcont=(?:(?!\1).)+)\1
+            ''',
+            webpage, 'relinker URL', group='url')
+
+        relinker_info = self._extract_relinker_info(
+            urljoin(url, relinker_url), video_id)
+        self._sort_formats(relinker_info['formats'])
+
+        title = self._search_regex(
+            r'var\s+videoTitolo\s*=\s*([\'"])(?P<title>[^\'"]+)\1',
+            webpage, 'title', group='title',
+            default=None) or self._og_search_title(webpage)
+
+        info = {
+            'id': video_id,
+            'title': title,
+        }
+
+        info.update(relinker_info)
+
+        return info

From 361f293ab85c29ab62cb91577d2be34814d5c552 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Apr 2017 02:24:13 +0700
Subject: [PATCH 0574/1696] [rai] Skip not found content item id

---
 youtube_dl/extractor/rai.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index b77b0a08e..077546a73 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -362,7 +362,8 @@ class RaiIE(RaiBaseIE):
                 webpage, 'content item id', default=None, group='id')
 
         content_item_ids = set()
-        content_item_ids.add(content_item_id)
+        if content_item_id:
+            content_item_ids.add(content_item_id)
         if video_id not in content_item_ids:
             content_item_ids.add(video_id)
 

From a76c25146a93052f367a0fb8cdd9a08ba9cef491 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Apr 2017 02:37:18 +0700
Subject: [PATCH 0575/1696] [ChangeLog] Actualize

---
 ChangeLog | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 07725b12a..3ffc647f1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,27 @@
+version <unreleased>
+
+Core
+[YoutubeDL] Return early when extraction of url_transparent fails
+
+Extractors
+* [rai] Fix and improve extraction (#11790)
++ [vrv] Add support for series pages
+* [limelight] Improve extraction for audio only formats
+* [funimation] Fix extraction (#10696, #11773)
++ [xfileshare] Add support for vidabc.com (#12589)
++ [xfileshare] Improve extraction and extract hls formats
++ [crunchyroll] Pass geo verifcation proxy
++ [cwtv] Extract ISM formats
++ [tvplay] Bypass geo restriction
++ [vrv] Add support for vrv.co
++ [packtpub] Add support for packtpub.com (#12610)
++ [generic] Pass base_url to _parse_jwplayer_data
++ [adn] Add support for animedigitalnetwork.fr (#4866)
++ [allocine] Extract more metadata
+* [allocine] Fix extraction (#12592)
+* [openload] Fix extraction
+
+
 version 2017.03.26
 
 Core

From b56e41a701d73072b7d62a151b7aafd87955dfe8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Apr 2017 02:39:15 +0700
Subject: [PATCH 0576/1696] release 2017.04.02

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 README.md                 | 8 ++++----
 docs/supportedsites.md    | 9 +++++++--
 youtube_dl/version.py     | 2 +-
 5 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 2f717926c..c1b737619 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.03.26*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.03.26**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.02*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.02**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.03.26
+[debug] youtube-dl version 2017.04.02
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 3ffc647f1..0199bdf1f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.04.02
 
 Core
 [YoutubeDL] Return early when extraction of url_transparent fails
diff --git a/README.md b/README.md
index 86b44781c..41f647aaa 100644
--- a/README.md
+++ b/README.md
@@ -181,10 +181,10 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
     -R, --retries RETRIES            Number of retries (default is 10), or
                                      "infinite".
     --fragment-retries RETRIES       Number of retries for a fragment (default
-                                     is 10), or "infinite" (DASH and hlsnative
-                                     only)
-    --skip-unavailable-fragments     Skip unavailable fragments (DASH and
-                                     hlsnative only)
+                                     is 10), or "infinite" (DASH, hlsnative and
+                                     ISM)
+    --skip-unavailable-fragments     Skip unavailable fragments (DASH, hlsnative
+                                     and ISM)
     --abort-on-unavailable-fragment  Abort downloading when some fragment is not
                                      available
     --buffer-size SIZE               Size of download buffer (e.g. 1024 or 16K)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index e9dbc021b..5c1855111 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -28,6 +28,7 @@
  - **acast**
  - **acast:channel**
  - **AddAnime**
+ - **ADN**: Anime Digital Network
  - **AdobeTV**
  - **AdobeTVChannel**
  - **AdobeTVShow**
@@ -572,6 +573,8 @@
  - **orf:iptv**: iptv.ORF.at
  - **orf:oe1**: Radio Österreich 1
  - **orf:tvthek**: ORF TVthek
+ - **PacktPub**
+ - **PacktPubCourse**
  - **PandaTV**: 熊猫TV
  - **pandora.tv**: 판도라TV
  - **parliamentlive.tv**: UK parliament videos
@@ -629,7 +632,7 @@
  - **radiofrance**
  - **RadioJavan**
  - **Rai**
- - **RaiTV**
+ - **RaiPlay**
  - **RBMARadio**
  - **RDS**: RDS.ca
  - **RedBullTV**
@@ -926,6 +929,8 @@
  - **vpro**: npo.nl and ntr.nl
  - **Vrak**
  - **VRT**
+ - **vrv**
+ - **vrv:series**
  - **vube**: Vube.com
  - **VuClip**
  - **VVVVID**
@@ -953,7 +958,7 @@
  - **WSJ**: Wall Street Journal
  - **XBef**
  - **XboxClips**
- - **XFileShare**: XFileShare based sites: DaClips, FileHoot, GorillaVid, MovPod, PowerWatch, Rapidvideo.ws, TheVideoBee, Vidto, Streamin.To, XVIDSTAGE
+ - **XFileShare**: XFileShare based sites: DaClips, FileHoot, GorillaVid, MovPod, PowerWatch, Rapidvideo.ws, TheVideoBee, Vidto, Streamin.To, XVIDSTAGE, Vid ABC
  - **XHamster**
  - **XHamsterEmbed**
  - **xiami:album**: 虾米音乐 - 专辑
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 94e8198ec..f612d03ca 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.03.26'
+__version__ = '2017.04.02'

From b3633fa0ce0f98801582f8e4e348436b0f361eb7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Apr 2017 03:20:28 +0700
Subject: [PATCH 0577/1696] [pericope] Add support for pscp.tv URLs

---
 youtube_dl/extractor/periscope.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 0e3623024..1add6b840 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -20,7 +20,7 @@ class PeriscopeBaseIE(InfoExtractor):
 class PeriscopeIE(PeriscopeBaseIE):
     IE_DESC = 'Periscope'
     IE_NAME = 'periscope'
-    _VALID_URL = r'https?://(?:www\.)?periscope\.tv/[^/]+/(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:periscope|pscp)\.tv/[^/]+/(?P<id>[^/?#]+)'
     # Alive example URLs can be found here http://onperiscope.com/
     _TESTS = [{
         'url': 'https://www.periscope.tv/w/aJUQnjY3MjA3ODF8NTYxMDIyMDl2zCg2pECBgwTqRpQuQD352EMPTKQjT4uqlM3cgWFA-g==',
@@ -41,6 +41,9 @@ class PeriscopeIE(PeriscopeBaseIE):
     }, {
         'url': 'https://www.periscope.tv/bastaakanoggano/1OdKrlkZZjOJX',
         'only_matching': True,
+    }, {
+        'url': 'https://www.periscope.tv/w/1ZkKzPbMVggJv',
+        'only_matching': True,
     }]
 
     @staticmethod
@@ -103,7 +106,7 @@ class PeriscopeIE(PeriscopeBaseIE):
 
 
 class PeriscopeUserIE(PeriscopeBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?periscope\.tv/(?P<id>[^/]+)/?$'
+    _VALID_URL = r'https?://(?:www\.)?(?:periscope|pscp)\.tv/(?P<id>[^/]+)/?$'
     IE_DESC = 'Periscope user videos'
     IE_NAME = 'periscope:user'
 

From 4457823dda410c5406f5ab5474b9b1f9325fa7ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Apr 2017 03:56:49 +0700
Subject: [PATCH 0578/1696] [extractor/common] Move censorship checks to a
 separate method and add check for just another ISP

---
 youtube_dl/extractor/common.py | 48 ++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 6c3c095f7..cdfa7000b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -547,6 +547,34 @@ class InfoExtractor(object):
 
         return encoding
 
+    def __check_blocked(self, content):
+        first_block = content[:512]
+        if ('<title>Access to this site is blocked</title>' in content and
+                'Websense' in first_block):
+            msg = 'Access to this webpage has been blocked by Websense filtering software in your network.'
+            blocked_iframe = self._html_search_regex(
+                r'<iframe src="([^"]+)"', content,
+                'Websense information URL', default=None)
+            if blocked_iframe:
+                msg += ' Visit %s for more details' % blocked_iframe
+            raise ExtractorError(msg, expected=True)
+        if '<title>The URL you requested has been blocked</title>' in first_block:
+            msg = (
+                'Access to this webpage has been blocked by Indian censorship. '
+                'Use a VPN or proxy server (with --proxy) to route around it.')
+            block_msg = self._html_search_regex(
+                r'</h1><p>(.*?)</p>',
+                content, 'block message', default=None)
+            if block_msg:
+                msg += ' (Message: "%s")' % block_msg.replace('\n', ' ')
+            raise ExtractorError(msg, expected=True)
+        if ('<title>TTK :: Доступ к ресурсу ограничен</title>' in content and
+                'blocklist.rkn.gov.ru' in content):
+            raise ExtractorError(
+                'Access to this webpage has been blocked by decision of the Russian government. '
+                'Visit http://blocklist.rkn.gov.ru/ for a block reason.',
+                expected=True)
+
     def _webpage_read_content(self, urlh, url_or_request, video_id, note=None, errnote=None, fatal=True, prefix=None, encoding=None):
         content_type = urlh.headers.get('Content-Type', '')
         webpage_bytes = urlh.read()
@@ -588,25 +616,7 @@ class InfoExtractor(object):
         except LookupError:
             content = webpage_bytes.decode('utf-8', 'replace')
 
-        if ('<title>Access to this site is blocked</title>' in content and
-                'Websense' in content[:512]):
-            msg = 'Access to this webpage has been blocked by Websense filtering software in your network.'
-            blocked_iframe = self._html_search_regex(
-                r'<iframe src="([^"]+)"', content,
-                'Websense information URL', default=None)
-            if blocked_iframe:
-                msg += ' Visit %s for more details' % blocked_iframe
-            raise ExtractorError(msg, expected=True)
-        if '<title>The URL you requested has been blocked</title>' in content[:512]:
-            msg = (
-                'Access to this webpage has been blocked by Indian censorship. '
-                'Use a VPN or proxy server (with --proxy) to route around it.')
-            block_msg = self._html_search_regex(
-                r'</h1><p>(.*?)</p>',
-                content, 'block message', default=None)
-            if block_msg:
-                msg += ' (Message: "%s")' % block_msg.replace('\n', ' ')
-            raise ExtractorError(msg, expected=True)
+        self.__check_blocked(content)
 
         return content
 

From fd47550885fc0abcdddbfdec70e00bc0d6a9865e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Apr 2017 04:42:10 +0700
Subject: [PATCH 0579/1696] [extractor/common] Add coding cookie

---
 youtube_dl/extractor/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index cdfa7000b..ae8af61de 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import base64

From 61e2331ad83569b3256e32f415c22cc439c000ed Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 1 Apr 2017 23:47:35 +0100
Subject: [PATCH 0580/1696] [tv5mondeplus] Add new extractor(closes #11386)

---
 youtube_dl/extractor/extractors.py   |  1 +
 youtube_dl/extractor/tv5mondeplus.py | 78 ++++++++++++++++++++++++++++
 2 files changed, 79 insertions(+)
 create mode 100644 youtube_dl/extractor/tv5mondeplus.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d9e8d53ac..46e5e3a1d 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1023,6 +1023,7 @@ from .tv2 import (
 )
 from .tv3 import TV3IE
 from .tv4 import TV4IE
+from .tv5mondeplus import TV5MondePlusIE
 from .tva import TVAIE
 from .tvanouvelles import (
     TVANouvellesIE,
diff --git a/youtube_dl/extractor/tv5mondeplus.py b/youtube_dl/extractor/tv5mondeplus.py
new file mode 100644
index 000000000..8b2d7812a
--- /dev/null
+++ b/youtube_dl/extractor/tv5mondeplus.py
@@ -0,0 +1,78 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    extract_attributes,
+    get_element_by_class,
+    int_or_none,
+    parse_duration,
+    parse_iso8601,
+)
+
+
+class TV5MondePlusIE(InfoExtractor):
+    IE_DESC = 'TV5MONDE+'
+    _VALID_URL = r'https?://(?:www\.)?tv5mondeplus\.com/toutes-les-videos/[^/]+/(?P<id>[^/?#]+)'
+    _TEST = {
+        'url': 'http://www.tv5mondeplus.com/toutes-les-videos/documentaire/tdah-mon-amour-tele-quebec-tdah-mon-amour-ep001-enfants',
+        'md5': '12130fc199f020673138a83466542ec6',
+        'info_dict': {
+            'id': '0a774110-dc60-4037-f769-996439514f1f',
+            'ext': 'mp4',
+            'title': 'Tdah, mon amour - Enfants',
+            'description': 'md5:b65f0cc50e46947e62e5d352e9916cc4',
+            'upload_date': '20170401',
+            'timestamp': 1491022860,
+        }
+    }
+    _GEO_BYPASS = False
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        if ">Ce programme n'est malheureusement pas disponible pour votre zone géographique.<" in webpage:
+            self.raise_geo_restricted(countries=['FR'])
+
+        series = get_element_by_class('video-detail__title', webpage)
+        title = episode = get_element_by_class(
+            'video-detail__subtitle', webpage) or series
+        if series and series != title:
+            title = '%s - %s' % (series, title)
+        vpl_data = extract_attributes(self._search_regex(
+            r'(<[^>]+class="video_player_loader"[^>]+>)',
+            webpage, 'video player loader'))
+
+        video_files = self._parse_json(
+            vpl_data['data-broadcast'], display_id).get('files', [])
+        formats = []
+        for video_file in video_files:
+            v_url = video_file.get('url')
+            if not v_url:
+                continue
+            video_format = video_file.get('format') or determine_ext(v_url)
+            if video_format == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    v_url, display_id, 'mp4', 'm3u8_native',
+                    m3u8_id='hls', fatal=False))
+            else:
+                formats.append({
+                    'url': v_url,
+                    'format_id': video_format,
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': vpl_data.get('data-guid') or display_id,
+            'display_id': display_id,
+            'title': title,
+            'description': get_element_by_class('video-detail__description', webpage),
+            'thumbnail': vpl_data.get('data-image'),
+            'duration': int_or_none(vpl_data.get('data-duration')) or parse_duration(self._html_search_meta('duration', webpage)),
+            'timestamp': parse_iso8601(self._html_search_meta('uploadDate', webpage)),
+            'formats': formats,
+            'episode': episode,
+            'series': series,
+        }

From dbf70c489f1bcf36d44afb8384d7abdb8a34d960 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 2 Apr 2017 00:26:48 +0100
Subject: [PATCH 0581/1696] [tv5mondeplus] clean description and use stable id

---
 youtube_dl/extractor/tv5mondeplus.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/tv5mondeplus.py b/youtube_dl/extractor/tv5mondeplus.py
index 8b2d7812a..88b6baa31 100644
--- a/youtube_dl/extractor/tv5mondeplus.py
+++ b/youtube_dl/extractor/tv5mondeplus.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
+    clean_html,
     determine_ext,
     extract_attributes,
     get_element_by_class,
@@ -19,10 +20,10 @@ class TV5MondePlusIE(InfoExtractor):
         'url': 'http://www.tv5mondeplus.com/toutes-les-videos/documentaire/tdah-mon-amour-tele-quebec-tdah-mon-amour-ep001-enfants',
         'md5': '12130fc199f020673138a83466542ec6',
         'info_dict': {
-            'id': '0a774110-dc60-4037-f769-996439514f1f',
+            'id': 'tdah-mon-amour-tele-quebec-tdah-mon-amour-ep001-enfants',
             'ext': 'mp4',
             'title': 'Tdah, mon amour - Enfants',
-            'description': 'md5:b65f0cc50e46947e62e5d352e9916cc4',
+            'description': 'md5:230e3aca23115afcf8006d1bece6df74',
             'upload_date': '20170401',
             'timestamp': 1491022860,
         }
@@ -65,10 +66,10 @@ class TV5MondePlusIE(InfoExtractor):
         self._sort_formats(formats)
 
         return {
-            'id': vpl_data.get('data-guid') or display_id,
+            'id': display_id,
             'display_id': display_id,
             'title': title,
-            'description': get_element_by_class('video-detail__description', webpage),
+            'description': clean_html(get_element_by_class('video-detail__description', webpage)),
             'thumbnail': vpl_data.get('data-image'),
             'duration': int_or_none(vpl_data.get('data-duration')) or parse_duration(self._html_search_meta('duration', webpage)),
             'timestamp': parse_iso8601(self._html_search_meta('uploadDate', webpage)),

From a9bb61a4259972378dec5ef45d181fec150146b8 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 2 Apr 2017 09:22:09 +0100
Subject: [PATCH 0582/1696] [discoveryvr] Add new extractor(closes #12578)

---
 youtube_dl/extractor/discoveryvr.py | 59 +++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py  |  1 +
 2 files changed, 60 insertions(+)
 create mode 100644 youtube_dl/extractor/discoveryvr.py

diff --git a/youtube_dl/extractor/discoveryvr.py b/youtube_dl/extractor/discoveryvr.py
new file mode 100644
index 000000000..cb63c2649
--- /dev/null
+++ b/youtube_dl/extractor/discoveryvr.py
@@ -0,0 +1,59 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import parse_duration
+
+
+class DiscoveryVRIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?discoveryvr\.com/watch/(?P<id>[^/?#]+)'
+    _TEST = {
+        'url': 'http://www.discoveryvr.com/watch/discovery-vr-an-introduction',
+        'md5': '32b1929798c464a54356378b7912eca4',
+        'info_dict': {
+            'id': 'discovery-vr-an-introduction',
+            'ext': 'mp4',
+            'title': 'Discovery VR - An Introduction',
+            'description': 'md5:80d418a10efb8899d9403e61d8790f06',
+        }
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        bootstrap_data = self._search_regex(
+            r'root\.DVR\.bootstrapData\s+=\s+"({.+?})";',
+            webpage, 'bootstrap data')
+        bootstrap_data = self._parse_json(
+            bootstrap_data.encode('utf-8').decode('unicode_escape'),
+            display_id)
+        videos = self._parse_json(bootstrap_data['videos'], display_id)['allVideos']
+        video_data = next(video for video in videos if video.get('slug') == display_id)
+
+        series = video_data.get('showTitle')
+        title = episode = video_data.get('title') or series
+        if series and series != title:
+            title = '%s - %s' % (series, title)
+
+        formats = []
+        for f, format_id in (('cdnUriM3U8', 'mobi'), ('webVideoUrlSd', 'sd'), ('webVideoUrlHd', 'hd')):
+            f_url = video_data.get(f)
+            if not f_url:
+                continue
+            formats.append({
+                'format_id': format_id,
+                'url': f_url,
+            })
+
+        return {
+            'id': display_id,
+            'display_id': display_id,
+            'title': title,
+            'description': video_data.get('description'),
+            'thumbnail': video_data.get('thumbnail'),
+            'duration': parse_duration(video_data.get('runTime')),
+            'formats': formats,
+            'episode': episode,
+            'series': series,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 46e5e3a1d..13ca1d2cd 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -273,6 +273,7 @@ from .discoverygo import (
     DiscoveryGoPlaylistIE,
 )
 from .discoverynetworks import DiscoveryNetworksDeIE
+from .discoveryvr import DiscoveryVRIE
 from .disney import DisneyIE
 from .dispeak import DigitallySpeakingIE
 from .dropbox import DropboxIE

From e2435ba5f3c03f737bb3d2f209c3b47ef87fd41e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 3 Apr 2017 03:52:44 +0700
Subject: [PATCH 0583/1696] [ChangeLog] Actualize

---
 ChangeLog | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 0199bdf1f..cb58b1687 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,19 @@
+version <unreleased>
+
+Core
++ [extractor/common] Add censorship check for TransTelekom ISP
+* [extractor/common] Move censorship checks to a separate method
+
+Extractors
++ [discoveryvr] Add support for discoveryvr.com (#12578)
++ [tv5mondeplus] Add support for tv5mondeplus.com (#11386)
++ [periscope] Add support for pscp.tv URLs (#12618, #12625)
+
+
 version 2017.04.02
 
 Core
-[YoutubeDL] Return early when extraction of url_transparent fails
+* [YoutubeDL] Return early when extraction of url_transparent fails
 
 Extractors
 * [rai] Fix and improve extraction (#11790)

From b022f4f600a2d73c65e569a9f0a19eed47272938 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 3 Apr 2017 03:53:55 +0700
Subject: [PATCH 0584/1696] release 2017.04.03

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 ++
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index c1b737619..8461a6173 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.02*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.02**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.03*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.03**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.04.02
+[debug] youtube-dl version 2017.04.03
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index cb58b1687..06a03bb9a 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.04.03
 
 Core
 + [extractor/common] Add censorship check for TransTelekom ISP
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 5c1855111..a075c7012 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -213,6 +213,7 @@
  - **DiscoveryGo**
  - **DiscoveryGoPlaylist**
  - **DiscoveryNetworksDe**
+ - **DiscoveryVR**
  - **Disney**
  - **Dotsub**
  - **DouyuTV**: 斗鱼
@@ -815,6 +816,7 @@
  - **TV2Article**
  - **TV3**
  - **TV4**: tv4.se and tv4play.se
+ - **TV5MondePlus**: TV5MONDE+
  - **TVA**
  - **TVANouvelles**
  - **TVANouvellesArticle**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index f612d03ca..1b260064b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.04.02'
+__version__ = '2017.04.03'

From 2ab0bfcd810b521bcf4de6dd555c0a47e8a69f70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 4 Apr 2017 03:05:18 +0700
Subject: [PATCH 0585/1696] [vshare] Add extractor (closes #12278)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/vshare.py     | 38 ++++++++++++++++++++++++++++++
 2 files changed, 39 insertions(+)
 create mode 100644 youtube_dl/extractor/vshare.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 13ca1d2cd..7398dceef 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1188,6 +1188,7 @@ from .vrv import (
     VRVIE,
     VRVSeriesIE,
 )
+from .vshare import VShareIE
 from .medialaan import MedialaanIE
 from .vube import VubeIE
 from .vuclip import VuClipIE
diff --git a/youtube_dl/extractor/vshare.py b/youtube_dl/extractor/vshare.py
new file mode 100644
index 000000000..5addbc280
--- /dev/null
+++ b/youtube_dl/extractor/vshare.py
@@ -0,0 +1,38 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class VShareIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?vshare\.io/[dv]/(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'https://vshare.io/d/0f64ce6',
+        'md5': '16d7b8fef58846db47419199ff1ab3e7',
+        'info_dict': {
+            'id': '0f64ce6',
+            'title': 'vl14062007715967',
+            'ext': 'mp4',
+        }
+    }, {
+        'url': 'https://vshare.io/v/0f64ce6/width-650/height-430/1',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(
+            'https://vshare.io/d/%s' % video_id, video_id)
+
+        title = self._html_search_regex(
+            r'(?s)<div id="root-container">(.+?)<br/>', webpage, 'title')
+        video_url = self._search_regex(
+            r'<a[^>]+href=(["\'])(?P<url>(?:https?:)?//.+?)\1[^>]*>[Cc]lick\s+here',
+            webpage, 'video url', group='url')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': video_url,
+        }

From b68e00b08ae60e60d9f5c2654884f3c58595075f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 3 Apr 2017 21:35:54 +0100
Subject: [PATCH 0586/1696] [rtl2] add support for you.rtl2.de(closes #10257)

---
 youtube_dl/extractor/extractors.py |   6 +-
 youtube_dl/extractor/rtl2.py       | 110 ++++++++++++++++++++++++++++-
 2 files changed, 114 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 7398dceef..2904dd4d1 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -834,7 +834,11 @@ from .rozhlas import RozhlasIE
 from .rtbf import RTBFIE
 from .rte import RteIE, RteRadioIE
 from .rtlnl import RtlNlIE
-from .rtl2 import RTL2IE
+from .rtl2 import (
+    RTL2IE,
+    RTL2YouIE,
+    RTL2YouSeriesIE,
+)
 from .rtp import RTPIE
 from .rts import RTSIE
 from .rtve import RTVEALaCartaIE, RTVELiveIE, RTVEInfantilIE, RTVELiveIE, RTVETelevisionIE
diff --git a/youtube_dl/extractor/rtl2.py b/youtube_dl/extractor/rtl2.py
index 721ee733c..666e90e90 100644
--- a/youtube_dl/extractor/rtl2.py
+++ b/youtube_dl/extractor/rtl2.py
@@ -1,13 +1,26 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import base64
 import re
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..aes import aes_cbc_decrypt
+from ..compat import (
+    compat_ord,
+    compat_str,
+)
+from ..utils import (
+    bytes_to_intlist,
+    ExtractorError,
+    intlist_to_bytes,
+    int_or_none,
+    strip_or_none,
+)
 
 
 class RTL2IE(InfoExtractor):
+    IE_NAME = 'rtl2'
     _VALID_URL = r'http?://(?:www\.)?rtl2\.de/[^?#]*?/(?P<id>[^?#/]*?)(?:$|/(?:$|[?#]))'
     _TESTS = [{
         'url': 'http://www.rtl2.de/sendung/grip-das-motormagazin/folge/folge-203-0',
@@ -98,3 +111,98 @@ class RTL2IE(InfoExtractor):
             'duration': int_or_none(video_info.get('duration')),
             'formats': formats,
         }
+
+
+class RTL2YouBaseIE(InfoExtractor):
+    _BACKWERK_BASE_URL = 'https://p-you-backwerk.rtl2apps.de/'
+
+
+class RTL2YouIE(RTL2YouBaseIE):
+    IE_NAME = 'rtl2:you'
+    _VALID_URL = r'http?://you\.rtl2\.de/(?:video/\d+/|youplayer/index\.html\?.*?\bvid=)(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://you.rtl2.de/video/3002/15740/MJUNIK%20%E2%80%93%20Home%20of%20YOU/307-hirn-wo-bist-du',
+        'info_dict': {
+            'id': '15740',
+            'ext': 'mp4',
+            'title': 'MJUNIK – Home of YOU - #307 Hirn, wo bist du?!',
+            'description': 'md5:ddaa95c61b372b12b66e115b2772fe01',
+            'age_limit': 12,
+        },
+    }, {
+        'url': 'http://you.rtl2.de/youplayer/index.html?vid=15712',
+        'only_matching': True,
+    }]
+    _AES_KEY = b'\xe9W\xe4.<*\xb8\x1a\xd2\xb6\x92\xf3C\xd3\xefL\x1b\x03*\xbbbH\xc0\x03\xffo\xc2\xf2(\xaa\xaa!'
+    _GEO_COUNTRIES = ['DE']
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        stream_data = self._download_json(
+            self._BACKWERK_BASE_URL + 'stream/video/' + video_id, video_id)
+
+        data, iv = base64.b64decode(stream_data['streamUrl']).decode().split(':')
+        stream_url = intlist_to_bytes(aes_cbc_decrypt(
+            bytes_to_intlist(base64.b64decode(data)),
+            bytes_to_intlist(self._AES_KEY),
+            bytes_to_intlist(base64.b64decode(iv))
+        ))
+        if b'rtl2_you_video_not_found' in stream_url:
+            raise ExtractorError('video not found', expected=True)
+
+        formats = self._extract_m3u8_formats(
+            stream_url[:-compat_ord(stream_url[-1])].decode(),
+            video_id, 'mp4', 'm3u8_native')
+        self._sort_formats(formats)
+
+        video_data = self._download_json(
+            self._BACKWERK_BASE_URL + 'video/' + video_id, video_id)
+
+        series = video_data.get('formatTitle')
+        title = episode = video_data.get('title') or series
+        if series and series != title:
+            title = '%s - %s' % (series, title)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'description': strip_or_none(video_data.get('description')),
+            'thumbnail': video_data.get('image'),
+            'duration': int_or_none(stream_data.get('duration') or video_data.get('duration'), 1000),
+            'series': series,
+            'episode': episode,
+            'age_limit': int_or_none(video_data.get('minimumAge')),
+        }
+
+
+class RTL2YouSeriesIE(RTL2YouBaseIE):
+    IE_NAME = 'rtl2:you:series'
+    _VALID_URL = r'http?://you\.rtl2\.de/videos/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://you.rtl2.de/videos/115/dragon-ball',
+        'info_dict': {
+            'id': '115',
+        },
+        'playlist_mincount': 5,
+    }
+
+    def _real_extract(self, url):
+        series_id = self._match_id(url)
+        stream_data = self._download_json(
+            self._BACKWERK_BASE_URL + 'videos',
+            series_id, query={
+                'formatId': series_id,
+                'limit': 1000000000,
+            })
+
+        entries = []
+        for video in stream_data.get('videos', []):
+            video_id = compat_str(video['videoId'])
+            if not video_id:
+                continue
+            entries.append(self.url_result(
+                'http://you.rtl2.de/video/%s/%s' % (series_id, video_id),
+                'RTL2You', video_id))
+        return self.playlist_result(entries, series_id)

From fac39cccd47555af5e9ba2c9f5f36f8ff3b628ad Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 3 Apr 2017 23:39:56 +0100
Subject: [PATCH 0587/1696] [odnoklassniki] fix format sorting for 1080p
 quality

---
 youtube_dl/extractor/odnoklassniki.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 986708e75..0ee56a45b 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -176,7 +176,7 @@ class OdnoklassnikiIE(InfoExtractor):
             })
             return info
 
-        quality = qualities(('mobile', 'lowest', 'low', 'sd', 'hd'))
+        quality = qualities(('mobile', 'lowest', 'low', 'sd', 'hd', 'full'))
 
         formats = [{
             'url': f['url'],

From 7f04386b89ced3235ea53c6c041dbfff3cbe667b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 6 Apr 2017 02:28:09 +0700
Subject: [PATCH 0588/1696] [aenetworks] Add more video URL regexes (closes
 #12657)

---
 youtube_dl/extractor/aenetworks.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index dd96a47ce..9a5074037 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -107,7 +107,10 @@ class AENetworksIE(AENetworksBaseIE):
         }
         video_id = self._html_search_meta('aetn:VideoID', webpage)
         media_url = self._search_regex(
-            r"media_url\s*=\s*'([^']+)'", webpage, 'video url')
+            [r"media_url\s*=\s*'(?P<url>[^']+)'",
+             r'data-media-url=(?P<url>(?:https?:)?//[^\s>]+)',
+             r'data-media-url=(["\'])(?P<url>(?:(?!\1).)+?)\1'],
+            webpage, 'video url', group='url')
         theplatform_metadata = self._download_theplatform_metadata(self._search_regex(
             r'https?://link.theplatform.com/s/([^?]+)', media_url, 'theplatform_path'), video_id)
         info = self._parse_theplatform_metadata(theplatform_metadata)

From 0378b8b91713a3ff7960ad92391e5994648f18fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 6 Apr 2017 23:38:16 +0700
Subject: [PATCH 0589/1696] [xfileshare] Add support for vidbom.com (closes
 #12661)

---
 youtube_dl/extractor/xfileshare.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index 6856fb3bf..a2fcb14c9 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -28,6 +28,7 @@ class XFileShareIE(InfoExtractor):
         ('streamin.to', 'Streamin.To'),
         ('xvidstage.com', 'XVIDSTAGE'),
         ('vidabc.com', 'Vid ABC'),
+        ('vidbom.com', 'VidBom'),
     )
 
     IE_DESC = 'XFileShare based sites: %s' % ', '.join(list(zip(*_SITES))[1])

From 3461f5db0615fa46c2caa393f648f1ba538d1b66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 6 Apr 2017 23:39:09 +0700
Subject: [PATCH 0590/1696] [xfileshare] Add support for vidlo.us (closes
 #12660)

---
 youtube_dl/extractor/xfileshare.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index a2fcb14c9..1b5cd122a 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -29,6 +29,7 @@ class XFileShareIE(InfoExtractor):
         ('xvidstage.com', 'XVIDSTAGE'),
         ('vidabc.com', 'Vid ABC'),
         ('vidbom.com', 'VidBom'),
+        ('vidlo.us', 'vidlo'),
     )
 
     IE_DESC = 'XFileShare based sites: %s' % ', '.join(list(zip(*_SITES))[1])

From a84da06f49c04203b9dadd0b4d4091228ca5afcd Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Sat, 8 Apr 2017 00:21:11 -0400
Subject: [PATCH 0591/1696] [test_download] Improve diagnostic on wrong 'id'

---
 test/test_download.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/test_download.py b/test/test_download.py
index 01a8bcb89..bd9dd86e8 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -11,6 +11,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from test.helper import (
     assertGreaterEqual,
     expect_warnings,
+    expect_value,
     get_params,
     gettestcases,
     expect_info_dict,
@@ -200,6 +201,7 @@ def generator(test_case, tname):
                     test_case['playlist_duration_sum'], got_duration)
 
             for tc in test_cases:
+                expect_value(self, res_dict['id'], tc['info_dict']['id'], 'id')
                 tc_filename = get_tc_filename(tc)
                 if not test_case.get('params', {}).get('skip_download', False):
                     self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename)

From 3bef10a50ca61e4d6e06b0990b8ccf5f739c0601 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Sat, 8 Apr 2017 00:27:52 -0400
Subject: [PATCH 0592/1696] [test_download] typo in comment

---
 test/test_download.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_download.py b/test/test_download.py
index bd9dd86e8..61015562d 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -152,7 +152,7 @@ def generator(test_case, tname):
             try_num = 1
             while True:
                 try:
-                    # We're not using .download here sine that is just a shim
+                    # We're not using .download here since that is just a shim
                     # for outside error handling, and returns the exit code
                     # instead of the result dict.
                     res_dict = ydl.extract_info(

From 80b2fdf9ac99ebcbd6eec87670aa6d646da7dac4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 14:10:12 +0700
Subject: [PATCH 0593/1696] [test_download] Match info dicts against tests
 before matching info file

---
 test/test_download.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/test/test_download.py b/test/test_download.py
index 61015562d..5c5824d65 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -200,8 +200,9 @@ def generator(test_case, tname):
                 self.assertEqual(
                     test_case['playlist_duration_sum'], got_duration)
 
-            for tc in test_cases:
-                expect_value(self, res_dict['id'], tc['info_dict']['id'], 'id')
+            for tc_num, tc in enumerate(test_cases):
+                tc_res_dict = res_dict['entries'][tc_num] if is_playlist else res_dict
+                expect_info_dict(self, tc_res_dict, tc.get('info_dict', {}))
                 tc_filename = get_tc_filename(tc)
                 if not test_case.get('params', {}).get('skip_download', False):
                     self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename)

From 1b3feca0a700c4d4d7c4d8b01fc5f033cf48f421 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 14:11:03 +0700
Subject: [PATCH 0594/1696] [raiplay] Extract subtitles

---
 youtube_dl/extractor/rai.py | 34 +++++++++++++++++++++++-----------
 1 file changed, 23 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 077546a73..81eb9db85 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -97,6 +97,25 @@ class RaiBaseIE(InfoExtractor):
             'formats': formats,
         }.items() if v is not None)
 
+    @staticmethod
+    def _extract_subtitles(url, subtitle_url):
+        subtitles = {}
+        if subtitle_url and isinstance(subtitle_url, compat_str):
+            subtitle_url = urljoin(url, subtitle_url)
+            STL_EXT = '.stl'
+            SRT_EXT = '.srt'
+            subtitles['it'] = [{
+                'ext': 'stl',
+                'url': subtitle_url,
+            }]
+            if subtitle_url.endswith(STL_EXT):
+                srt_url = subtitle_url[:-len(STL_EXT)] + SRT_EXT
+                subtitles['it'].append({
+                    'ext': 'srt',
+                    'url': srt_url,
+                })
+        return subtitles
+
 
 class RaiPlayIE(RaiBaseIE):
     _VALID_URL = r'(?P<url>https?://(?:www\.)?raiplay\.it/.+?-(?P<id>%s)\.html)' % RaiBaseIE._UUID_RE
@@ -168,6 +187,8 @@ class RaiPlayIE(RaiBaseIE):
         timestamp = unified_timestamp(try_get(
             media, lambda x: x['availabilities'][0]['start'], compat_str))
 
+        subtitles = self._extract_subtitles(url, video.get('subtitles'))
+
         info = {
             'id': video_id,
             'title': title,
@@ -183,6 +204,7 @@ class RaiPlayIE(RaiBaseIE):
             'season_number': int_or_none(try_get(
                 media, lambda x: x['isPartOf']['numeroStagioni'])),
             'season': media.get('stagione') or None,
+            'subtitles': subtitles,
         }
 
         info.update(relinker_info)
@@ -307,17 +329,7 @@ class RaiIE(RaiBaseIE):
                     'url': compat_urlparse.urljoin(url, thumbnail_url),
                 })
 
-        subtitles = {}
-        captions = media.get('subtitlesUrl')
-        if captions:
-            STL_EXT = '.stl'
-            SRT_EXT = '.srt'
-            if captions.endswith(STL_EXT):
-                captions = captions[:-len(STL_EXT)] + SRT_EXT
-            subtitles['it'] = [{
-                'ext': 'srt',
-                'url': captions,
-            }]
+        subtitles = self._extract_subtitles(url, media.get('subtitlesUrl'))
 
         info = {
             'id': content_id,

From c4d6fc6d659ba74aab3f59aca02ff0e2e0763eb4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 14:11:20 +0700
Subject: [PATCH 0595/1696] [test_subtitles] Fix raiplay test

---
 test/test_subtitles.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 27e763edd..1b8de822a 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -21,7 +21,7 @@ from youtube_dl.extractor import (
     NPOIE,
     ComedyCentralIE,
     NRKTVIE,
-    RaiTVIE,
+    RaiPlayIE,
     VikiIE,
     ThePlatformIE,
     ThePlatformFeedIE,
@@ -258,9 +258,9 @@ class TestNRKSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['no']), '544fa917d3197fcbee64634559221cc2')
 
 
-class TestRaiSubtitles(BaseTestSubtitles):
-    url = 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-cb27157f-9dd0-4aee-b788-b1f67643a391.html'
-    IE = RaiTVIE
+class TestRaiPlaySubtitles(BaseTestSubtitles):
+    url = 'http://www.raiplay.it/video/2014/04/Report-del-07042014-cb27157f-9dd0-4aee-b788-b1f67643a391.html'
+    IE = RaiPlayIE
 
     def test_allsubtitles(self):
         self.DL.params['writesubtitles'] = True

From 60e5016199320333ad3f3584b5b0c0147cfaeba3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 14:19:01 +0700
Subject: [PATCH 0596/1696] [test_download] Remove unused import

---
 test/test_download.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/test_download.py b/test/test_download.py
index 5c5824d65..fd7d2e8be 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -11,7 +11,6 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from test.helper import (
     assertGreaterEqual,
     expect_warnings,
-    expect_value,
     get_params,
     gettestcases,
     expect_info_dict,

From 4c0397329688144b8c0ec1d34227772fdfd03527 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 8 Apr 2017 15:39:58 +0800
Subject: [PATCH 0597/1696] [airmozilla] Fix extraction (closes #12670)

---
 ChangeLog                          |  6 ++++++
 youtube_dl/extractor/airmozilla.py | 28 ++++++++++------------------
 2 files changed, 16 insertions(+), 18 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 06a03bb9a..47804428f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [airmozilla] Fix extraction (#12670)
+
+
 version 2017.04.03
 
 Core
diff --git a/youtube_dl/extractor/airmozilla.py b/youtube_dl/extractor/airmozilla.py
index 0e0691879..9e38136b4 100644
--- a/youtube_dl/extractor/airmozilla.py
+++ b/youtube_dl/extractor/airmozilla.py
@@ -15,12 +15,12 @@ class AirMozillaIE(InfoExtractor):
     _VALID_URL = r'https?://air\.mozilla\.org/(?P<id>[0-9a-z-]+)/?'
     _TEST = {
         'url': 'https://air.mozilla.org/privacy-lab-a-meetup-for-privacy-minded-people-in-san-francisco/',
-        'md5': '2e3e7486ba5d180e829d453875b9b8bf',
+        'md5': '8d02f53ee39cf006009180e21df1f3ba',
         'info_dict': {
             'id': '6x4q2w',
             'ext': 'mp4',
             'title': 'Privacy Lab - a meetup for privacy minded people in San Francisco',
-            'thumbnail': r're:https?://vid\.ly/(?P<id>[0-9a-z-]+)/poster',
+            'thumbnail': r're:https?://.*/poster\.jpg',
             'description': 'Brings together privacy professionals and others interested in privacy at for-profits, non-profits, and NGOs in an effort to contribute to the state of the ecosystem...',
             'timestamp': 1422487800,
             'upload_date': '20150128',
@@ -34,21 +34,13 @@ class AirMozillaIE(InfoExtractor):
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
-        video_id = self._html_search_regex(r'//vid.ly/(.*?)/embed', webpage, 'id')
+        video_id = self._html_search_regex(r'//vid\.ly/(.*?)/embed', webpage, 'id')
 
         embed_script = self._download_webpage('https://vid.ly/{0}/embed'.format(video_id), video_id)
-        jwconfig = self._search_regex(r'\svar jwconfig = (\{.*?\});\s', embed_script, 'metadata')
-        metadata = self._parse_json(jwconfig, video_id)
-
-        formats = [{
-            'url': source['file'],
-            'ext': source['type'],
-            'format_id': self._search_regex(r'&format=(.*)$', source['file'], 'video format'),
-            'format': source['label'],
-            'height': int(source['label'].rstrip('p')),
-        } for source in metadata['playlist'][0]['sources']]
-        self._sort_formats(formats)
+        jwconfig = self._parse_json(self._search_regex(
+            r'initCallback\((.*)\);', embed_script, 'metadata'), video_id)['config']
 
+        info_dict = self._parse_jwplayer_data(jwconfig, video_id)
         view_count = int_or_none(self._html_search_regex(
             r'Views since archived: ([0-9]+)',
             webpage, 'view count', fatal=False))
@@ -58,17 +50,17 @@ class AirMozillaIE(InfoExtractor):
             r'Duration:\s*(\d+\s*hours?\s*\d+\s*minutes?)',
             webpage, 'duration', fatal=False))
 
-        return {
+        info_dict.update({
             'id': video_id,
             'title': self._og_search_title(webpage),
-            'formats': formats,
             'url': self._og_search_url(webpage),
             'display_id': display_id,
-            'thumbnail': metadata['playlist'][0].get('image'),
             'description': self._og_search_description(webpage),
             'timestamp': timestamp,
             'location': self._html_search_regex(r'Location: (.*)', webpage, 'location', default=None),
             'duration': duration,
             'view_count': view_count,
             'categories': re.findall(r'<a href=".*?" class="channel">(.*?)</a>', webpage),
-        }
+        })
+
+        return info_dict

From f8f2da25ab83779f5a9521f327987d5e7bace85f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 16:01:56 +0700
Subject: [PATCH 0598/1696] [wshh] Extract html5 entries and delegate to
 generic extractor (closes #12676)

---
 youtube_dl/extractor/worldstarhiphop.py | 46 ++++++-------------------
 1 file changed, 11 insertions(+), 35 deletions(-)

diff --git a/youtube_dl/extractor/worldstarhiphop.py b/youtube_dl/extractor/worldstarhiphop.py
index 09415b589..82587b4ce 100644
--- a/youtube_dl/extractor/worldstarhiphop.py
+++ b/youtube_dl/extractor/worldstarhiphop.py
@@ -1,12 +1,10 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
 class WorldStarHipHopIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www|m)\.worldstar(?:candy|hiphop)\.com/(?:videos|android)/video\.php\?v=(?P<id>.*)'
+    _VALID_URL = r'https?://(?:www|m)\.worldstar(?:candy|hiphop)\.com/(?:videos|android)/video\.php\?.*?\bv=(?P<id>[^&]+)'
     _TESTS = [{
         'url': 'http://www.worldstarhiphop.com/videos/video.php?v=wshh6a7q1ny0G34ZwuIO',
         'md5': '9d04de741161603bf7071bbf4e883186',
@@ -17,48 +15,26 @@ class WorldStarHipHopIE(InfoExtractor):
         }
     }, {
         'url': 'http://m.worldstarhiphop.com/android/video.php?v=wshh6a7q1ny0G34ZwuIO',
-        'md5': 'dc1c76c83ecc4190bb1eb143899b87d3',
-        'info_dict': {
-            'id': 'wshh6a7q1ny0G34ZwuIO',
-            'ext': 'mp4',
-            'title': 'KO Of The Week: MMA Fighter Gets Knocked Out By Swift Head Kick!'
-        }
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        m_vevo_id = re.search(r'videoId=(.*?)&amp?', webpage)
-        if m_vevo_id is not None:
-            return self.url_result('vevo:%s' % m_vevo_id.group(1), ie='Vevo')
+        entries = self._parse_html5_media_entries(url, webpage, video_id)
 
-        video_url = self._search_regex(
-            [r'so\.addVariable\("file","(.*?)"\)',
-             r'<div class="artlist">\s*<a[^>]+href="([^"]+)">'],
-            webpage, 'video URL')
+        if not entries:
+            return self.url_result(url, 'Generic')
 
-        if 'youtube' in video_url:
-            return self.url_result(video_url, ie='Youtube')
-
-        video_title = self._html_search_regex(
+        title = self._html_search_regex(
             [r'(?s)<div class="content-heading">\s*<h1>(.*?)</h1>',
              r'<span[^>]+class="tc-sp-pinned-title">(.*)</span>'],
             webpage, 'title')
 
-        # Getting thumbnail and if not thumbnail sets correct title for WSHH candy video.
-        thumbnail = self._html_search_regex(
-            r'rel="image_src" href="(.*)" />', webpage, 'thumbnail',
-            default=None)
-        if not thumbnail:
-            _title = r'candytitles.*>(.*)</span>'
-            mobj = re.search(_title, webpage)
-            if mobj is not None:
-                video_title = mobj.group(1)
-
-        return {
+        info = entries[0]
+        info.update({
             'id': video_id,
-            'url': video_url,
-            'title': video_title,
-            'thumbnail': thumbnail,
-        }
+            'title': title,
+        })
+        return info

From a01825a5416174de56625a6ef202311e2d6c8cb1 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Sat, 8 Apr 2017 05:40:57 -0400
Subject: [PATCH 0599/1696] [kaltura] Add support for iframe embeds

---
 youtube_dl/extractor/generic.py | 15 +++++++++++++++
 youtube_dl/extractor/kaltura.py | 13 ++++++++++++-
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 73911940c..658533cf6 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1080,6 +1080,21 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['Kaltura'],
         },
+        {
+            # Kaltura iframe embed
+            'url': 'http://www.gsd.harvard.edu/event/i-m-pei-a-centennial-celebration/',
+            'md5': 'ae5ace8eb09dc1a35d03b579a9c2cc44',
+            'info_dict': {
+                'id': '0_f2cfbpwy',
+                'ext': 'mp4',
+                'title': 'I. M. Pei: A Centennial Celebration',
+                'description': 'md5:1db8f40c69edc46ca180ba30c567f37c',
+                'upload_date': '20170403',
+                'uploader_id': 'batchUser',
+                'timestamp': 1491232186,
+            },
+            'add_ie': ['Kaltura'],
+        },
         # Eagle.Platform embed (generic URL)
         {
             'url': 'http://lenta.ru/news/2015/03/06/navalny/',
diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 54374ea76..c6efdea93 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -91,6 +91,7 @@ class KalturaIE(InfoExtractor):
                     }],
                 },
             },
+            'skip': 'Gone. Maybe https://www.safaribooksonline.com/library/tutorials/introduction-to-python-anon/3469/',
             'params': {
                 'skip_download': True,
             },
@@ -108,6 +109,7 @@ class KalturaIE(InfoExtractor):
     @staticmethod
     def _extract_url(webpage):
         mobj = (
+            # Embed codes: https://knowledge.kaltura.com/embedding-kaltura-media-players-your-site
             re.search(
                 r"""(?xs)
                     kWidget\.(?:thumb)?[Ee]mbed\(
@@ -127,7 +129,16 @@ class KalturaIE(InfoExtractor):
                         (?P<q2>["\'])entry_?[Ii]d(?P=q2)
                     )\s*:\s*
                     (?P<q3>["\'])(?P<id>(?:(?!(?P=q3)).)+)(?P=q3)
-                ''', webpage))
+                ''', webpage) or
+            re.search(
+                r'''(?xs)
+                    <iframe[^>]+src=(?P<q1>["\'])
+                      (?:https?:)?//(?:www\.)?kaltura\.com/p/(?P<partner_id>\d+)/
+                      (?:(?!(?P=q1)).)*
+                      [\?&]entry_id=(?P<id>(?:(?!(?P=q1))[^&])+)
+                    (?P=q1)
+                ''', webpage)
+        )
         if mobj:
             embed_info = mobj.groupdict()
             url = 'kaltura:%(partner_id)s:%(id)s' % embed_info

From 78280352ca8e4af4e82fc92228200aef3f06ce53 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 16:48:27 +0700
Subject: [PATCH 0600/1696] [kaltura] Cleanup regexes from redundant escaping

---
 youtube_dl/extractor/kaltura.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index c6efdea93..41c1f3d96 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -108,34 +108,34 @@ class KalturaIE(InfoExtractor):
 
     @staticmethod
     def _extract_url(webpage):
+        # Embed codes: https://knowledge.kaltura.com/embedding-kaltura-media-players-your-site
         mobj = (
-            # Embed codes: https://knowledge.kaltura.com/embedding-kaltura-media-players-your-site
             re.search(
                 r"""(?xs)
                     kWidget\.(?:thumb)?[Ee]mbed\(
                     \{.*?
-                        (?P<q1>['\"])wid(?P=q1)\s*:\s*
-                        (?P<q2>['\"])_?(?P<partner_id>(?:(?!(?P=q2)).)+)(?P=q2),.*?
-                        (?P<q3>['\"])entry_?[Ii]d(?P=q3)\s*:\s*
-                        (?P<q4>['\"])(?P<id>(?:(?!(?P=q4)).)+)(?P=q4)(?:,|\s*\})
+                        (?P<q1>['"])wid(?P=q1)\s*:\s*
+                        (?P<q2>['"])_?(?P<partner_id>(?:(?!(?P=q2)).)+)(?P=q2),.*?
+                        (?P<q3>['"])entry_?[Ii]d(?P=q3)\s*:\s*
+                        (?P<q4>['"])(?P<id>(?:(?!(?P=q4)).)+)(?P=q4)(?:,|\s*\})
                 """, webpage) or
             re.search(
                 r'''(?xs)
-                    (?P<q1>["\'])
+                    (?P<q1>["'])
                         (?:https?:)?//cdnapi(?:sec)?\.kaltura\.com(?::\d+)?/(?:(?!(?P=q1)).)*\b(?:p|partner_id)/(?P<partner_id>\d+)(?:(?!(?P=q1)).)*
                     (?P=q1).*?
                     (?:
                         entry_?[Ii]d|
-                        (?P<q2>["\'])entry_?[Ii]d(?P=q2)
+                        (?P<q2>["'])entry_?[Ii]d(?P=q2)
                     )\s*:\s*
-                    (?P<q3>["\'])(?P<id>(?:(?!(?P=q3)).)+)(?P=q3)
+                    (?P<q3>["'])(?P<id>(?:(?!(?P=q3)).)+)(?P=q3)
                 ''', webpage) or
             re.search(
                 r'''(?xs)
-                    <iframe[^>]+src=(?P<q1>["\'])
-                      (?:https?:)?//(?:www\.)?kaltura\.com/p/(?P<partner_id>\d+)/
+                    <iframe[^>]+src=(?P<q1>["'])
+                      (?:https?:)?//(?:www\.)?kaltura\.com/(?:(?!(?P=q1)).)*\b(?:p|partner_id)/(?P<partner_id>\d+)
                       (?:(?!(?P=q1)).)*
-                      [\?&]entry_id=(?P<id>(?:(?!(?P=q1))[^&])+)
+                      [?&]entry_id=(?P<id>(?:(?!(?P=q1))[^&])+)
                     (?P=q1)
                 ''', webpage)
         )

From e18f1da97a8840522b0cefac3af0995751733d67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 19:41:14 +0700
Subject: [PATCH 0601/1696] [ceskateleveize] Improve extraction and remove URL
 replacement hacks

---
 youtube_dl/extractor/ceskatelevize.py | 61 +++++++++++----------------
 1 file changed, 24 insertions(+), 37 deletions(-)

diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index dd2529a6d..0daee313f 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -18,7 +18,7 @@ from ..utils import (
 
 
 class CeskaTelevizeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ceskatelevize\.cz/(porady|ivysilani)/(?:[^/]+/)*(?P<id>[^/#?]+)/*(?:[#?].*)?$'
+    _VALID_URL = r'https?://(?:www\.)?ceskatelevize\.cz/ivysilani/(?:[^/?#&]+/)*(?P<id>[^/#?]+)'
     _TESTS = [{
         'url': 'http://www.ceskatelevize.cz/ivysilani/ivysilani/10441294653-hyde-park-civilizace/214411058091220',
         'info_dict': {
@@ -62,40 +62,12 @@ class CeskaTelevizeIE(InfoExtractor):
         },
         'skip': 'Georestricted to Czech Republic',
     }, {
-        # video with 18+ caution trailer
-        'url': 'http://www.ceskatelevize.cz/porady/10520528904-queer/215562210900007-bogotart/',
-        'info_dict': {
-            'id': '215562210900007-bogotart',
-            'title': 'Queer: Bogotart',
-            'description': 'Alternativní průvodce současným queer světem',
-        },
-        'playlist': [{
-            'info_dict': {
-                'id': '61924494876844842',
-                'ext': 'mp4',
-                'title': 'Queer: Bogotart (Varování 18+)',
-                'duration': 10.2,
-            },
-        }, {
-            'info_dict': {
-                'id': '61924494877068022',
-                'ext': 'mp4',
-                'title': 'Queer: Bogotart (Queer)',
-                'thumbnail': r're:^https?://.*\.jpg',
-                'duration': 1558.3,
-            },
-        }],
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
+        'url': 'http://www.ceskatelevize.cz/ivysilani/embed/iFramePlayer.php?hash=d6a3e1370d2e4fa76296b90bad4dfc19673b641e&IDEC=217 562 22150/0004&channelID=1&width=100%25',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        url = url.replace('/porady/', '/ivysilani/').replace('/video/', '')
-
-        mobj = re.match(self._VALID_URL, url)
-        playlist_id = mobj.group('id')
+        playlist_id = self._match_id(url)
 
         webpage = self._download_webpage(url, playlist_id)
 
@@ -103,13 +75,28 @@ class CeskaTelevizeIE(InfoExtractor):
         if '%s</p>' % NOT_AVAILABLE_STRING in webpage:
             raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)
 
-        typ = self._html_search_regex(
-            r'getPlaylistUrl\(\[\{"type":"(.+?)","id":".+?"\}\],', webpage, 'type')
-        episode_id = self._html_search_regex(
-            r'getPlaylistUrl\(\[\{"type":".+?","id":"(.+?)"\}\],', webpage, 'episode_id')
+        type_ = None
+        episode_id = None
+
+        playlist = self._parse_json(
+            self._search_regex(
+                r'getPlaylistUrl\(\[({.+?})\]', webpage, 'playlist',
+                default='{}'), playlist_id)
+        if playlist:
+            type_ = playlist.get('type')
+            episode_id = playlist.get('id')
+
+        if not type_:
+            type_ = self._html_search_regex(
+                r'getPlaylistUrl\(\[\{"type":"(.+?)","id":".+?"\}\],',
+                webpage, 'type')
+        if not episode_id:
+            episode_id = self._html_search_regex(
+                r'getPlaylistUrl\(\[\{"type":".+?","id":"(.+?)"\}\],',
+                webpage, 'episode_id')
 
         data = {
-            'playlist[0][type]': typ,
+            'playlist[0][type]': type_,
             'playlist[0][id]': episode_id,
             'requestUrl': compat_urllib_parse_urlparse(url).path,
             'requestSource': 'iVysilani',

From 28b674ca238fbae29f2ee7bc85202fa2f115a876 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 19:42:09 +0700
Subject: [PATCH 0602/1696] [ceskateleveize:porady] Add extractor (closes
 #7411, closes #12645)

---
 youtube_dl/extractor/ceskatelevize.py | 45 +++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py    |  5 ++-
 2 files changed, 49 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index 0daee313f..e250de18c 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -12,6 +12,7 @@ from ..utils import (
     ExtractorError,
     float_or_none,
     sanitized_Request,
+    unescapeHTML,
     urlencode_postdata,
     USER_AGENTS,
 )
@@ -232,3 +233,47 @@ class CeskaTelevizeIE(InfoExtractor):
                     yield line
 
         return '\r\n'.join(_fix_subtitle(subtitles))
+
+
+class CeskaTelevizePoradyIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?ceskatelevize\.cz/porady/(?:[^/?#&]+/)*(?P<id>[^/#?]+)'
+    _TESTS = [{
+        # video with 18+ caution trailer
+        'url': 'http://www.ceskatelevize.cz/porady/10520528904-queer/215562210900007-bogotart/',
+        'info_dict': {
+            'id': '215562210900007-bogotart',
+            'title': 'Queer: Bogotart',
+            'description': 'Alternativní průvodce současným queer světem',
+        },
+        'playlist': [{
+            'info_dict': {
+                'id': '61924494876844842',
+                'ext': 'mp4',
+                'title': 'Queer: Bogotart (Varování 18+)',
+                'duration': 10.2,
+            },
+        }, {
+            'info_dict': {
+                'id': '61924494877068022',
+                'ext': 'mp4',
+                'title': 'Queer: Bogotart (Queer)',
+                'thumbnail': r're:^https?://.*\.jpg',
+                'duration': 1558.3,
+            },
+        }],
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        data_url = unescapeHTML(self._search_regex(
+            r'<span[^>]*\bdata-url=(["\'])(?P<url>(?:(?!\1).)+)\1',
+            webpage, 'iframe player url', group='url'))
+
+        return self.url_result(data_url, ie=CeskaTelevizeIE.ie_key())
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 2904dd4d1..72728d919 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -165,7 +165,10 @@ from .ccc import CCCIE
 from .ccma import CCMAIE
 from .cctv import CCTVIE
 from .cda import CDAIE
-from .ceskatelevize import CeskaTelevizeIE
+from .ceskatelevize import (
+    CeskaTelevizeIE,
+    CeskaTelevizePoradyIE,
+)
 from .channel9 import Channel9IE
 from .charlierose import CharlieRoseIE
 from .chaturbate import ChaturbateIE

From 5f3e0b69efa8ae80f536fb54ace4bb4d9c667d2e Mon Sep 17 00:00:00 2001
From: Entropy <0fiscalentropy@users.noreply.github.com>
Date: Fri, 7 Apr 2017 21:52:48 +0200
Subject: [PATCH 0603/1696] [TheSun] Add new extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/thesun.py     | 27 +++++++++++++++++++++++++++
 2 files changed, 28 insertions(+)
 create mode 100644 youtube_dl/extractor/thesun.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 72728d919..01865479c 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -983,6 +983,7 @@ from .theplatform import (
 from .thescene import TheSceneIE
 from .thesixtyone import TheSixtyOneIE
 from .thestar import TheStarIE
+from .thesun import TheSunIE
 from .theweatherchannel import TheWeatherChannelIE
 from .thisamericanlife import ThisAmericanLifeIE
 from .thisav import ThisAVIE
diff --git a/youtube_dl/extractor/thesun.py b/youtube_dl/extractor/thesun.py
new file mode 100644
index 000000000..7f96bf8c9
--- /dev/null
+++ b/youtube_dl/extractor/thesun.py
@@ -0,0 +1,27 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from .ooyala import OoyalaIE
+
+
+class TheSunIE(InfoExtractor):
+    _VALID_URL = r'https://(?:www\.)?thesun\.co\.uk/\w+/(?P<id>\d+)/[\w-]'
+    _TEST = {
+        'url': 'https://www.thesun.co.uk/tvandshowbiz/2261604/orlando-bloom-and-katy-perry-post-adorable-instagram-video-together-celebrating-thanksgiving-after-split-rumours/',
+        'md5': '5667123b24f25f43f4c4f381ef34c5c2',
+        'info_dict': {
+            'id': 'h4OXN0NzE6rv6ObkEifKcNA-gYUw4xFf',
+            'ext': 'mp4',
+            'title': 'Katy Perry and Orlando Bloom shut down split rumours with cute Thanksgiving video',
+            'description': 'Still going strong',
+            'duration': 31.28,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+        ooyala_id = self._search_regex(r'id\s*=\s*"thesun-ooyala-player-([^"]+)"', webpage, 'ooyala id')
+
+        return OoyalaIE._build_url_result(ooyala_id)

From 90e3f18fc1cff2c12361cbbd85303774bf7d59f7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 20:08:31 +0700
Subject: [PATCH 0604/1696] [thesun] Extract playlists (closes #11298, closes
 #12674)

---
 youtube_dl/extractor/thesun.py | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/thesun.py b/youtube_dl/extractor/thesun.py
index 7f96bf8c9..22d003776 100644
--- a/youtube_dl/extractor/thesun.py
+++ b/youtube_dl/extractor/thesun.py
@@ -1,27 +1,32 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from .ooyala import OoyalaIE
 
 
 class TheSunIE(InfoExtractor):
-    _VALID_URL = r'https://(?:www\.)?thesun\.co\.uk/\w+/(?P<id>\d+)/[\w-]'
+    _VALID_URL = r'https://(?:www\.)?thesun\.co\.uk/[^/]+/(?P<id>\d+)'
     _TEST = {
         'url': 'https://www.thesun.co.uk/tvandshowbiz/2261604/orlando-bloom-and-katy-perry-post-adorable-instagram-video-together-celebrating-thanksgiving-after-split-rumours/',
-        'md5': '5667123b24f25f43f4c4f381ef34c5c2',
         'info_dict': {
-            'id': 'h4OXN0NzE6rv6ObkEifKcNA-gYUw4xFf',
-            'ext': 'mp4',
-            'title': 'Katy Perry and Orlando Bloom shut down split rumours with cute Thanksgiving video',
-            'description': 'Still going strong',
-            'duration': 31.28,
-        }
+            'id': '2261604',
+            'title': 'md5:cba22f48bad9218b64d5bbe0e16afddf',
+        },
+        'playlist_count': 2,
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        article_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
-        ooyala_id = self._search_regex(r'id\s*=\s*"thesun-ooyala-player-([^"]+)"', webpage, 'ooyala id')
+        webpage = self._download_webpage(url, article_id)
 
-        return OoyalaIE._build_url_result(ooyala_id)
+        entries = []
+        for ooyala_id in re.findall(
+                r'<[^>]+\b(?:id\s*=\s*"thesun-ooyala-player-|data-content-id\s*=\s*")([^"]+)',
+                webpage):
+            entries.append(OoyalaIE._build_url_result(ooyala_id))
+
+        return self.playlist_result(
+            entries, article_id, self._og_search_title(webpage, fatal=False))

From c93c0fc2fd4162c06fac7b263484f50f14ac60b8 Mon Sep 17 00:00:00 2001
From: Kfir Breger <kfir@cubicalwave.com>
Date: Mon, 27 Mar 2017 20:56:37 +0200
Subject: [PATCH 0605/1696] [mixcloud:playlist] Fix title extraction

---
 youtube_dl/extractor/mixcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index a24b3165a..ba0bb190f 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -250,7 +250,7 @@ class MixcloudPlaylistIE(MixcloudPlaylistBaseIE):
 
         description = self._get_user_description(profile)
         playlist_title = self._html_search_regex(
-            r'<span[^>]+class="[^"]*list-playlist-title[^"]*"[^>]*>(.*?)</span>',
+            r'<a class="parent active" href="[^"]*"><b>\d+</b><span title="[^"]*">([^</]*?)</span></a>',
             profile, 'playlist title')
 
         entries = OnDemandPagedList(

From a66e25859a0f163446c009f9a71abab4083745f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 20:51:38 +0700
Subject: [PATCH 0606/1696] [mixcloud:playlist] Relax title extraction and fix
 description extraction (closes #12582)

---
 youtube_dl/extractor/mixcloud.py | 30 +++++++++++++-----------------
 1 file changed, 13 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index ba0bb190f..bc38a8663 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -138,7 +138,7 @@ class MixcloudPlaylistBaseIE(InfoExtractor):
 
     def _get_user_description(self, page_content):
         return self._html_search_regex(
-            r'<div[^>]+class="description-text"[^>]*>(.+?)</div>',
+            r'<div[^>]+class="profile-bio"[^>]*>(.+?)</div>',
             page_content, 'user description', fatal=False)
 
 
@@ -151,7 +151,7 @@ class MixcloudUserIE(MixcloudPlaylistBaseIE):
         'info_dict': {
             'id': 'dholbach_uploads',
             'title': 'Daniel Holbach (uploads)',
-            'description': 'md5:327af72d1efeb404a8216c27240d1370',
+            'description': 'md5:def36060ac8747b3aabca54924897e47',
         },
         'playlist_mincount': 11,
     }, {
@@ -159,7 +159,7 @@ class MixcloudUserIE(MixcloudPlaylistBaseIE):
         'info_dict': {
             'id': 'dholbach_uploads',
             'title': 'Daniel Holbach (uploads)',
-            'description': 'md5:327af72d1efeb404a8216c27240d1370',
+            'description': 'md5:def36060ac8747b3aabca54924897e47',
         },
         'playlist_mincount': 11,
     }, {
@@ -167,7 +167,7 @@ class MixcloudUserIE(MixcloudPlaylistBaseIE):
         'info_dict': {
             'id': 'dholbach_favorites',
             'title': 'Daniel Holbach (favorites)',
-            'description': 'md5:327af72d1efeb404a8216c27240d1370',
+            'description': 'md5:def36060ac8747b3aabca54924897e47',
         },
         'params': {
             'playlist_items': '1-100',
@@ -178,7 +178,7 @@ class MixcloudUserIE(MixcloudPlaylistBaseIE):
         'info_dict': {
             'id': 'dholbach_listens',
             'title': 'Daniel Holbach (listens)',
-            'description': 'md5:327af72d1efeb404a8216c27240d1370',
+            'description': 'md5:def36060ac8747b3aabca54924897e47',
         },
         'params': {
             'playlist_items': '1-100',
@@ -229,12 +229,7 @@ class MixcloudPlaylistIE(MixcloudPlaylistBaseIE):
         'playlist_mincount': 16,
     }, {
         'url': 'https://www.mixcloud.com/maxvibes/playlists/jazzcat-on-ness-radio/',
-        'info_dict': {
-            'id': 'maxvibes_jazzcat-on-ness-radio',
-            'title': 'Jazzcat on Ness Radio',
-            'description': 'md5:7bbbf0d6359a0b8cda85224be0f8f263',
-        },
-        'playlist_mincount': 23
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -243,15 +238,16 @@ class MixcloudPlaylistIE(MixcloudPlaylistBaseIE):
         playlist_id = mobj.group('playlist')
         video_id = '%s_%s' % (user_id, playlist_id)
 
-        profile = self._download_webpage(
+        webpage = self._download_webpage(
             url, user_id,
             note='Downloading playlist page',
             errnote='Unable to download playlist page')
 
-        description = self._get_user_description(profile)
-        playlist_title = self._html_search_regex(
-            r'<a class="parent active" href="[^"]*"><b>\d+</b><span title="[^"]*">([^</]*?)</span></a>',
-            profile, 'playlist title')
+        title = self._html_search_regex(
+            r'<a[^>]+class="parent active"[^>]*><b>\d+</b><span[^>]*>([^<]+)',
+            webpage, 'playlist title',
+            default=None) or self._og_search_title(webpage, fatal=False)
+        description = self._get_user_description(webpage)
 
         entries = OnDemandPagedList(
             functools.partial(
@@ -259,7 +255,7 @@ class MixcloudPlaylistIE(MixcloudPlaylistBaseIE):
                 '%s/playlists/%s' % (user_id, playlist_id), video_id, 'tracklist'),
             self._PAGE_SIZE)
 
-        return self.playlist_result(entries, video_id, playlist_title, description)
+        return self.playlist_result(entries, video_id, title, description)
 
 
 class MixcloudStreamIE(MixcloudPlaylistBaseIE):

From 29c6726646326fdbf17fca2f0e5600b6ee3b9f63 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 21:11:08 +0700
Subject: [PATCH 0607/1696] [mixcloud] Fix view count extraction and modernize

---
 youtube_dl/extractor/mixcloud.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index bc38a8663..0efbe660a 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -97,7 +97,7 @@ class MixcloudIE(InfoExtractor):
         view_count = str_to_int(self._search_regex(
             [r'<meta itemprop="interactionCount" content="UserPlays:([0-9]+)"',
              r'/listeners/?">([0-9,.]+)</a>',
-             r'm-tooltip=["\']([\d,.]+) plays'],
+             r'(?:m|data)-tooltip=["\']([\d,.]+) plays'],
             webpage, 'play count', default=None))
 
         return {
@@ -143,7 +143,7 @@ class MixcloudPlaylistBaseIE(InfoExtractor):
 
 
 class MixcloudUserIE(MixcloudPlaylistBaseIE):
-    _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/(?P<user>[^/]+)/(?P<type>uploads|favorites|listens)?/?$'
+    _VALID_URL = r'https?://(?:www\.)?mixcloud\.com/(?P<user>[^/]+)/(?P<type>uploads|favorites|listens)?/?$'
     IE_NAME = 'mixcloud:user'
 
     _TESTS = [{
@@ -216,7 +216,7 @@ class MixcloudUserIE(MixcloudPlaylistBaseIE):
 
 
 class MixcloudPlaylistIE(MixcloudPlaylistBaseIE):
-    _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/(?P<user>[^/]+)/playlists/(?P<playlist>[^/]+)/?$'
+    _VALID_URL = r'https?://(?:www\.)?mixcloud\.com/(?P<user>[^/]+)/playlists/(?P<playlist>[^/]+)/?$'
     IE_NAME = 'mixcloud:playlist'
 
     _TESTS = [{
@@ -259,7 +259,7 @@ class MixcloudPlaylistIE(MixcloudPlaylistBaseIE):
 
 
 class MixcloudStreamIE(MixcloudPlaylistBaseIE):
-    _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/(?P<id>[^/]+)/stream/?$'
+    _VALID_URL = r'https?://(?:www\.)?mixcloud\.com/(?P<id>[^/]+)/stream/?$'
     IE_NAME = 'mixcloud:stream'
 
     _TEST = {

From 1591ba258a49b682124e6e545c13a5da3bc660e0 Mon Sep 17 00:00:00 2001
From: Aldo Gunsing <dev@aldogunsing.nl>
Date: Sat, 25 Mar 2017 16:51:36 +0100
Subject: [PATCH 0608/1696] [npo:live] Add support for default url

---
 youtube_dl/extractor/npo.py | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 38fefe492..d1c9bc20e 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -313,9 +313,9 @@ class NPOIE(NPOBaseIE):
 
 class NPOLiveIE(NPOBaseIE):
     IE_NAME = 'npo.nl:live'
-    _VALID_URL = r'https?://(?:www\.)?npo\.nl/live/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?npo\.nl/live(/(?P<id>[^/?#&]+))?'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.npo.nl/live/npo-1',
         'info_dict': {
             'id': 'LI_NL1_4188102',
@@ -327,10 +327,22 @@ class NPOLiveIE(NPOBaseIE):
         'params': {
             'skip_download': True,
         }
-    }
+    }, {
+        'url': 'http://www.npo.nl/live',
+        'info_dict': {
+            'id': 'LI_NL1_4188102',
+            'display_id': 'npo-1',
+            'ext': 'mp4',
+            'title': 're:^NPO 1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }]
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
+        display_id = self._match_id(url) or 'npo-1'
 
         webpage = self._download_webpage(url, display_id)
 

From 04e431cf97e8b73190e8db77de0e3d2b5655c195 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 21:31:22 +0700
Subject: [PATCH 0609/1696] [npo:live] Improve (closes #12555)

---
 youtube_dl/extractor/npo.py | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index d1c9bc20e..79296f0ef 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -313,7 +313,7 @@ class NPOIE(NPOBaseIE):
 
 class NPOLiveIE(NPOBaseIE):
     IE_NAME = 'npo.nl:live'
-    _VALID_URL = r'https?://(?:www\.)?npo\.nl/live(/(?P<id>[^/?#&]+))?'
+    _VALID_URL = r'https?://(?:www\.)?npo\.nl/live(?:/(?P<id>[^/?#&]+))?'
 
     _TESTS = [{
         'url': 'http://www.npo.nl/live/npo-1',
@@ -329,16 +329,7 @@ class NPOLiveIE(NPOBaseIE):
         }
     }, {
         'url': 'http://www.npo.nl/live',
-        'info_dict': {
-            'id': 'LI_NL1_4188102',
-            'display_id': 'npo-1',
-            'ext': 'mp4',
-            'title': 're:^NPO 1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
-            'is_live': True,
-        },
-        'params': {
-            'skip_download': True,
-        }
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 95152630db22a648afefc7dc447da17f8989513c Mon Sep 17 00:00:00 2001
From: Steven Maude <StevenMaude@users.noreply.github.com>
Date: Sat, 8 Apr 2017 15:39:07 +0100
Subject: [PATCH 0610/1696] [rbmaradio] Add support for redbullradio.com URLs

---
 youtube_dl/extractor/rbmaradio.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rbmaradio.py b/youtube_dl/extractor/rbmaradio.py
index 53b82fba3..afa7b9161 100644
--- a/youtube_dl/extractor/rbmaradio.py
+++ b/youtube_dl/extractor/rbmaradio.py
@@ -13,15 +13,15 @@ from ..utils import (
 
 
 class RBMARadioIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?rbmaradio\.com/shows/(?P<show_id>[^/]+)/episodes/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:rbmaradio|redbullradio)\.com/shows/(?P<show_id>[^/]+)/episodes/(?P<id>[^/?#&]+)'
     _TEST = {
         'url': 'https://www.rbmaradio.com/shows/main-stage/episodes/ford-lopatin-live-at-primavera-sound-2011',
         'md5': '6bc6f9bcb18994b4c983bc3bf4384d95',
         'info_dict': {
             'id': 'ford-lopatin-live-at-primavera-sound-2011',
             'ext': 'mp3',
-            'title': 'Main Stage - Ford & Lopatin',
-            'description': 'md5:4f340fb48426423530af5a9d87bd7b91',
+            'title': 'Main Stage - Ford & Lopatin at Primavera Sound',
+            'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
             'thumbnail': r're:^https?://.*\.jpg',
             'duration': 2452,
             'timestamp': 1307103164,

From 2bd875edfe674e72ae935bacdbf0a055572d1d1b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 22:38:37 +0700
Subject: [PATCH 0611/1696] [medici] Add extractor (closes #3406)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/medici.py     | 70 ++++++++++++++++++++++++++++++
 2 files changed, 71 insertions(+)
 create mode 100644 youtube_dl/extractor/medici.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 01865479c..68e1a5cfc 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -541,6 +541,7 @@ from .mangomolo import (
 )
 from .matchtv import MatchTVIE
 from .mdr import MDRIE
+from .medici import MediciIE
 from .meipai import MeipaiIE
 from .melonvod import MelonVODIE
 from .meta import METAIE
diff --git a/youtube_dl/extractor/medici.py b/youtube_dl/extractor/medici.py
new file mode 100644
index 000000000..cd910238e
--- /dev/null
+++ b/youtube_dl/extractor/medici.py
@@ -0,0 +1,70 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    unified_strdate,
+    update_url_query,
+    urlencode_postdata,
+)
+
+
+class MediciIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?medici\.tv/#!/(?P<id>[^?#&]+)'
+    _TEST = {
+        'url': 'http://www.medici.tv/#!/daniel-harding-frans-helmerson-verbier-festival-music-camp',
+        'md5': '004c21bb0a57248085b6ff3fec72719d',
+        'info_dict': {
+            'id': '3059',
+            'ext': 'flv',
+            'title': 'Daniel Harding conducts the Verbier Festival Music Camp \u2013 With Frans Helmerson',
+            'description': 'md5:322a1e952bafb725174fd8c1a8212f58',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'upload_date': '20170408',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        # Sets csrftoken cookie
+        self._download_webpage(url, video_id)
+
+        MEDICI_URL = 'http://www.medici.tv/'
+
+        data = self._download_json(
+            MEDICI_URL, video_id,
+            data=urlencode_postdata({
+                'json': 'true',
+                'page': '/%s' % video_id,
+                'timezone_offset': -420,
+            }), headers={
+                'X-CSRFToken': self._get_cookies(url)['csrftoken'].value,
+                'X-Requested-With': 'XMLHttpRequest',
+                'Referer': MEDICI_URL,
+                'Content-Type': 'application/x-www-form-urlencoded',
+            })
+
+        video = data['video']['videos']['video1']
+
+        title = video.get('nom') or data['title']
+
+        video_id = video.get('id') or video_id
+        formats = self._extract_f4m_formats(
+            update_url_query(video['url_akamai'], {
+                'hdcore': '3.1.0',
+                'plugin=aasp': '3.1.0.43.124',
+            }), video_id, f4m_id='hds')
+
+        description = data.get('meta_description')
+        thumbnail = video.get('url_thumbnail') or data.get('main_image')
+        upload_date = unified_strdate(data['video'].get('date'))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'upload_date': upload_date,
+            'formats': formats,
+        }

From e129fa0846734da1ea5814934c0c4b8fa4dc21b8 Mon Sep 17 00:00:00 2001
From: midas02 <midas02@users.noreply.github.com>
Date: Sat, 8 Apr 2017 17:43:29 +0200
Subject: [PATCH 0612/1696] [vier] Add IE_DESC

---
 youtube_dl/extractor/vier.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py
index 5086f591e..5ef7635b6 100644
--- a/youtube_dl/extractor/vier.py
+++ b/youtube_dl/extractor/vier.py
@@ -9,6 +9,7 @@ from .common import InfoExtractor
 
 class VierIE(InfoExtractor):
     IE_NAME = 'vier'
+    IE_DESC = 'vier.be and vijf.be'
     _VALID_URL = r'https?://(?:www\.)?(?P<site>vier|vijf)\.be/(?:[^/]+/videos/(?P<display_id>[^/]+)(?:/(?P<id>\d+))?|video/v3/embed/(?P<embed_id>\d+))'
     _TESTS = [{
         'url': 'http://www.vier.be/planb/videos/het-wordt-warm-de-moestuin/16129',

From ca8fca9d9d8b3773762a950a029e0a5354779e2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 22:44:31 +0700
Subject: [PATCH 0613/1696] [vrt] Add IE_DESC (closes #12477)

---
 youtube_dl/extractor/vrt.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/vrt.py b/youtube_dl/extractor/vrt.py
index 00c72e346..444295d68 100644
--- a/youtube_dl/extractor/vrt.py
+++ b/youtube_dl/extractor/vrt.py
@@ -10,6 +10,7 @@ from ..utils import (
 
 
 class VRTIE(InfoExtractor):
+    IE_DESC = 'deredactie.be, sporza.be, cobra.be and cobra.canvas.be'
     _VALID_URL = r'https?://(?:deredactie|sporza|cobra(?:\.canvas)?)\.be/cm/(?:[^/]+/)+(?P<id>[^/]+)/*'
     _TESTS = [
         # deredactie.be

From 41b263ac8a2721a324ad21d489de7d1ea9b41164 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 22:45:45 +0700
Subject: [PATCH 0614/1696] [canvas] Add IE_DESC (closes #12478)

---
 youtube_dl/extractor/canvas.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py
index 544c6657c..aada02917 100644
--- a/youtube_dl/extractor/canvas.py
+++ b/youtube_dl/extractor/canvas.py
@@ -7,6 +7,7 @@ from ..utils import float_or_none
 
 
 class CanvasIE(InfoExtractor):
+    IE_DESC = 'canvas.be and een.be'
     _VALID_URL = r'https?://(?:www\.)?(?P<site_id>canvas|een)\.be/(?:[^/]+/)*(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://www.canvas.be/video/de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week',

From eb8cc8ea3bde565b48b7eecfb81c7c7a95ea2480 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Apr 2017 23:59:17 +0700
Subject: [PATCH 0615/1696] [ChangeLog] Actualize

---
 ChangeLog | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 47804428f..93e260606 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,23 @@
 version <unreleased>
 
 Extractors
++ [medici] Add support for medici.tv (#3406)
++ [rbmaradio] Add support for redbullradio.com URLs (#12687)
++ [npo:live] Add support for default URL (#12555)
+* [mixcloud:playlist] Fix title, description and view count extraction (#12582)
++ [thesun] Add suport for thesun.co.uk (#11298, #12674)
++ [ceskateleveize:porady] Add support for porady (#7411, #12645)
+* [ceskateleveize] Improve extraction and remove URL replacement hacks
++ [kaltura] Add support for iframe embeds (#12679)
 * [airmozilla] Fix extraction (#12670)
+* [wshh] Extract html5 entries and delegate to generic extractor (12676)
++ [raiplay] Extract subtitles
++ [xfileshare] Add support for vidlo.us (#12660)
++ [xfileshare] Add support for vidbom.com (#12661)
++ [aenetworks] Add more video URL regular expressions (#12657)
++ [odnoklassniki] Fix format sorting for 1080p quality
++ [rtl2] Add support for you.rtl2.de (#10257)
++ [vshare] Add support for vshare.io (#12278)
 
 
 version 2017.04.03

From 43724365048755adbea8bbff05708903ea52e1a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Apr 2017 00:01:28 +0700
Subject: [PATCH 0616/1696] release 2017.04.09

---
 .github/ISSUE_TEMPLATE.md |  6 +++---
 ChangeLog                 |  2 +-
 docs/supportedsites.md    | 16 +++++++++++-----
 youtube_dl/version.py     |  2 +-
 4 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 8461a6173..898d96b78 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.03*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.03**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.09*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.09**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.04.03
+[debug] youtube-dl version 2017.04.09
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 93e260606..143c688f5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.04.09
 
 Extractors
 + [medici] Add support for medici.tv (#3406)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index a075c7012..0157ce3ef 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -127,7 +127,7 @@
  - **CamWithHer**
  - **canalc2.tv**
  - **Canalplus**: canalplus.fr, piwiplus.fr and d8.tv
- - **Canvas**
+ - **Canvas**: canvas.be and een.be
  - **CarambaTV**
  - **CarambaTVPage**
  - **CartoonNetwork**
@@ -145,6 +145,7 @@
  - **CCTV**: 央视网
  - **CDA**
  - **CeskaTelevize**
+ - **CeskaTelevizePorady**
  - **channel9**: Channel 9
  - **CharlieRose**
  - **Chaturbate**
@@ -431,6 +432,7 @@
  - **MDR**: MDR.DE and KiKA
  - **media.ccc.de**
  - **Medialaan**
+ - **Medici**
  - **Meipai**: 美拍
  - **MelonVOD**
  - **META**
@@ -658,7 +660,9 @@
  - **rte**: Raidió Teilifís Éireann TV
  - **rte:radio**: Raidió Teilifís Éireann radio
  - **rtl.nl**: rtl.nl and rtlxl.nl
- - **RTL2**
+ - **rtl2**
+ - **rtl2:you**
+ - **rtl2:you:series**
  - **RTP**
  - **RTS**: RTS.ch
  - **rtve.es:alacarta**: RTVE a la carta
@@ -780,6 +784,7 @@
  - **TheScene**
  - **TheSixtyOne**
  - **TheStar**
+ - **TheSun**
  - **TheWeatherChannel**
  - **ThisAmericanLife**
  - **ThisAV**
@@ -893,7 +898,7 @@
  - **vidme:user**
  - **vidme:user:likes**
  - **Vidzi**
- - **vier**
+ - **vier**: vier.be and vijf.be
  - **vier:videos**
  - **ViewLift**
  - **ViewLiftEmbed**
@@ -930,9 +935,10 @@
  - **Vporn**
  - **vpro**: npo.nl and ntr.nl
  - **Vrak**
- - **VRT**
+ - **VRT**: deredactie.be, sporza.be, cobra.be and cobra.canvas.be
  - **vrv**
  - **vrv:series**
+ - **VShare**
  - **vube**: Vube.com
  - **VuClip**
  - **VVVVID**
@@ -960,7 +966,7 @@
  - **WSJ**: Wall Street Journal
  - **XBef**
  - **XboxClips**
- - **XFileShare**: XFileShare based sites: DaClips, FileHoot, GorillaVid, MovPod, PowerWatch, Rapidvideo.ws, TheVideoBee, Vidto, Streamin.To, XVIDSTAGE, Vid ABC
+ - **XFileShare**: XFileShare based sites: DaClips, FileHoot, GorillaVid, MovPod, PowerWatch, Rapidvideo.ws, TheVideoBee, Vidto, Streamin.To, XVIDSTAGE, Vid ABC, VidBom, vidlo
  - **XHamster**
  - **XHamsterEmbed**
  - **xiami:album**: 虾米音乐 - 专辑
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1b260064b..39268f734 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.04.03'
+__version__ = '2017.04.09'

From a6f7263cf421d9156048596cca0552e027e70788 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Apr 2017 01:39:40 +0700
Subject: [PATCH 0617/1696] [chaturbate] Fix extraction (closes #12665)

---
 youtube_dl/extractor/chaturbate.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/chaturbate.py b/youtube_dl/extractor/chaturbate.py
index 8fbc91c1f..e3eba4be9 100644
--- a/youtube_dl/extractor/chaturbate.py
+++ b/youtube_dl/extractor/chaturbate.py
@@ -33,10 +33,17 @@ class ChaturbateIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        m3u8_formats = [(m.group('id').lower(), m.group('url')) for m in re.finditer(
-            r'hlsSource(?P<id>.+?)\s*=\s*(?P<q>["\'])(?P<url>http.+?)(?P=q)', webpage)]
+        m3u8_urls = []
 
-        if not m3u8_formats:
+        for m in re.finditer(
+                r'(["\'])(?P<url>http.+?\.m3u8.*?)\1', webpage):
+            m3u8_fast_url, m3u8_no_fast_url = m.group('url'), m.group(
+                'url').replace('_fast', '')
+            for m3u8_url in (m3u8_fast_url, m3u8_no_fast_url):
+                if m3u8_url not in m3u8_urls:
+                    m3u8_urls.append(m3u8_url)
+
+        if not m3u8_urls:
             error = self._search_regex(
                 [r'<span[^>]+class=(["\'])desc_span\1[^>]*>(?P<error>[^<]+)</span>',
                  r'<div[^>]+id=(["\'])defchat\1[^>]*>\s*<p><strong>(?P<error>[^<]+)<'],
@@ -50,7 +57,8 @@ class ChaturbateIE(InfoExtractor):
             raise ExtractorError('Unable to find stream URL')
 
         formats = []
-        for m3u8_id, m3u8_url in m3u8_formats:
+        for m3u8_url in m3u8_urls:
+            m3u8_id = 'fast' if '_fast' in m3u8_url else 'slow'
             formats.extend(self._extract_m3u8_formats(
                 m3u8_url, video_id, ext='mp4',
                 # ffmpeg skips segments for fast m3u8

From 3f2ce6896acfd487ce915788255ddee1e04794b3 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 9 Apr 2017 12:32:26 +0100
Subject: [PATCH 0618/1696] [adn] update subtitle decryption key

---
 youtube_dl/extractor/adn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/adn.py b/youtube_dl/extractor/adn.py
index e44caa00b..66caf6a81 100644
--- a/youtube_dl/extractor/adn.py
+++ b/youtube_dl/extractor/adn.py
@@ -45,7 +45,7 @@ class ADNIE(InfoExtractor):
         # http://animedigitalnetwork.fr/components/com_vodvideo/videojs/adn-vjs.min.js
         dec_subtitles = intlist_to_bytes(aes_cbc_decrypt(
             bytes_to_intlist(base64.b64decode(enc_subtitles[24:])),
-            bytes_to_intlist(b'\xb5@\xcfq\xa3\x98"N\xe4\xf3\x12\x98}}\x16\xd8'),
+            bytes_to_intlist(b'\nd\xaf\xd2J\xd0\xfc\xe1\xfc\xdf\xb61\xe8\xe1\xf0\xcc'),
             bytes_to_intlist(base64.b64decode(enc_subtitles[:24]))
         ))
         subtitles_json = self._parse_json(

From a206ef62df7e45992ebb26bf41f86c01d904fe75 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 10 Apr 2017 13:50:08 +0100
Subject: [PATCH 0619/1696] [curiositystream] fix extraction(closes #12638)

---
 youtube_dl/extractor/curiositystream.py | 54 +++++++++++++++++++++----
 1 file changed, 47 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/curiositystream.py b/youtube_dl/extractor/curiositystream.py
index e3c99468c..145d9397c 100644
--- a/youtube_dl/extractor/curiositystream.py
+++ b/youtube_dl/extractor/curiositystream.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
@@ -49,6 +51,48 @@ class CuriosityStreamBaseIE(InfoExtractor):
         limelight_media_id = media['limelight_media_id']
         title = media['title']
 
+        formats = []
+        for encoding in media.get('encodings', []):
+            m3u8_url = encoding.get('master_playlist_url')
+            if m3u8_url:
+                formats.extend(self._extract_m3u8_formats(
+                    m3u8_url, video_id, 'mp4', 'm3u8_native',
+                    m3u8_id='hls', fatal=False))
+            encoding_url = encoding.get('url')
+            file_url = encoding.get('file_url')
+            if not encoding_url and not file_url:
+                continue
+            f = {
+                'width': int_or_none(encoding.get('width')),
+                'height': int_or_none(encoding.get('height')),
+                'vbr': int_or_none(encoding.get('video_bitrate')),
+                'abr': int_or_none(encoding.get('audio_bitrate')),
+                'filesize': int_or_none(encoding.get('size_in_bytes')),
+                'vcodec': encoding.get('video_codec'),
+                'acodec': encoding.get('audio_codec'),
+                'container': encoding.get('container_type'),
+            }
+            for f_url in (encoding_url, file_url):
+                if not f_url:
+                    continue
+                fmt = f.copy()
+                rtmp = re.search(r'^(?P<url>rtmpe?://(?P<host>[^/]+)/(?P<app>.+))/(?P<playpath>mp[34]:.+)$', f_url)
+                if rtmp:
+                    fmt.update({
+                        'url': rtmp.group('url'),
+                        'play_path': rtmp.group('playpath'),
+                        'app': rtmp.group('app'),
+                        'ext': 'flv',
+                        'format_id': 'rtmp',
+                    })
+                else:
+                    fmt.update({
+                        'url': f_url,
+                        'format_id': 'http',
+                    })
+                formats.append(fmt)
+        self._sort_formats(formats)
+
         subtitles = {}
         for closed_caption in media.get('closed_captions', []):
             sub_url = closed_caption.get('file')
@@ -60,16 +104,14 @@ class CuriosityStreamBaseIE(InfoExtractor):
             })
 
         return {
-            '_type': 'url_transparent',
             'id': video_id,
-            'url': 'limelight:media:' + limelight_media_id,
+            'formats': formats,
             'title': title,
             'description': media.get('description'),
             'thumbnail': media.get('image_large') or media.get('image_medium') or media.get('image_small'),
             'duration': int_or_none(media.get('duration')),
             'tags': media.get('tags'),
             'subtitles': subtitles,
-            'ie_key': 'LimelightMedia',
         }
 
 
@@ -78,14 +120,12 @@ class CuriosityStreamIE(CuriosityStreamBaseIE):
     _VALID_URL = r'https?://app\.curiositystream\.com/video/(?P<id>\d+)'
     _TEST = {
         'url': 'https://app.curiositystream.com/video/2',
-        'md5': 'a0074c190e6cddaf86900b28d3e9ee7a',
+        'md5': '262bb2f257ff301115f1973540de8983',
         'info_dict': {
             'id': '2',
             'ext': 'mp4',
             'title': 'How Did You Develop The Internet?',
             'description': 'Vint Cerf, Google\'s Chief Internet Evangelist, describes how he and Bob Kahn created the internet.',
-            'timestamp': 1448388615,
-            'upload_date': '20151124',
         }
     }
 
@@ -105,7 +145,7 @@ class CuriosityStreamCollectionIE(CuriosityStreamBaseIE):
             'title': 'Curious Minds: The Internet',
             'description': 'How is the internet shaping our lives in the 21st Century?',
         },
-        'playlist_mincount': 17,
+        'playlist_mincount': 12,
     }
 
     def _real_extract(self, url):

From 1af959ef9f8936a8e211f692a2da1a69e0076962 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 10 Apr 2017 22:53:06 +0700
Subject: [PATCH 0620/1696] [bbccouk] Add support for https protocol in media
 selector (closes #12701)

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 8a2ed0ab6..f0de07619 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -370,7 +370,7 @@ class BBCCoUkIE(InfoExtractor):
                                 'acodec': encoding,
                                 'vcodec': 'none',
                             })
-                        if protocol == 'http':
+                        if protocol in ('http', 'https'):
                             # Direct link
                             fmt.update({
                                 'url': href,

From 964744af952fc372e961a4b6b4b0f3ed6172a83f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 10 Apr 2017 22:53:51 +0700
Subject: [PATCH 0621/1696] [bbccouk] Skip unrecognized formats in media
 selector (#12701)

---
 youtube_dl/extractor/bbc.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index f0de07619..a0c77b0b0 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -389,6 +389,8 @@ class BBCCoUkIE(InfoExtractor):
                                 'rtmp_live': False,
                                 'ext': 'flv',
                             })
+                        else:
+                            continue
                         formats.append(fmt)
             elif kind == 'captions':
                 subtitles = self.extract_subtitles(media, programme_id)

From 6240925b4021a755a9deb9c4707ee5cabcb63c2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 10 Apr 2017 22:56:22 +0700
Subject: [PATCH 0622/1696] [bbccouk] Treat bitrate as audio+video bitrate in
 media selector

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index a0c77b0b0..0e05b782b 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -361,7 +361,7 @@ class BBCCoUkIE(InfoExtractor):
                             fmt.update({
                                 'width': width,
                                 'height': height,
-                                'vbr': bitrate,
+                                'tbr': bitrate,
                                 'vcodec': encoding,
                             })
                         else:

From 364a69e8c66f2259f21d2d67a0e766f0180cd217 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 10 Apr 2017 23:59:38 +0700
Subject: [PATCH 0623/1696] [test_download] Fix testing playlists with single
 video and add comments

---
 test/test_download.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/test/test_download.py b/test/test_download.py
index fd7d2e8be..0e9f293b5 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -199,9 +199,16 @@ def generator(test_case, tname):
                 self.assertEqual(
                     test_case['playlist_duration_sum'], got_duration)
 
+            # Generalize both playlists and single videos to unified format for
+            # simplicity
+            if 'entries' not in res_dict:
+                res_dict['entries'] = [res_dict]
+
             for tc_num, tc in enumerate(test_cases):
-                tc_res_dict = res_dict['entries'][tc_num] if is_playlist else res_dict
+                tc_res_dict = res_dict['entries'][tc_num]
+                # First, check test cases' data against extracted data alone
                 expect_info_dict(self, tc_res_dict, tc.get('info_dict', {}))
+                # Now, check downloaded file consistency
                 tc_filename = get_tc_filename(tc)
                 if not test_case.get('params', {}).get('skip_download', False):
                     self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename)
@@ -219,13 +226,14 @@ def generator(test_case, tname):
                     if 'md5' in tc:
                         md5_for_file = _file_md5(tc_filename)
                         self.assertEqual(md5_for_file, tc['md5'])
+                # Finally, check test cases' data again but this time against
+                # extracted data from info JSON file written during processing
                 info_json_fn = os.path.splitext(tc_filename)[0] + '.info.json'
                 self.assertTrue(
                     os.path.exists(info_json_fn),
                     'Missing info file %s' % info_json_fn)
                 with io.open(info_json_fn, encoding='utf-8') as infof:
                     info_dict = json.load(infof)
-
                 expect_info_dict(self, info_dict, tc.get('info_dict', {}))
         finally:
             try_rm_tcs_files()

From 61568e50cf999e334336ad9c2c6bbd62864aa3fc Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Mon, 10 Apr 2017 13:04:32 -0400
Subject: [PATCH 0624/1696] [generic] Add support for <object> youtube embeds
 (closes #12637)

---
 youtube_dl/extractor/generic.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 658533cf6..4690816cb 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -730,6 +730,21 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             }
         },
+        # YouTube <object> embed
+        {
+            'url': 'http://www.improbable.com/2017/04/03/untrained-modern-youths-and-ancient-masters-in-selfie-portraits/',
+            'md5': '516718101ec834f74318df76259fb3cc',
+            'info_dict': {
+                'id': 'msN87y-iEx0',
+                'ext': 'webm',
+                'title': 'Feynman: Mirrors FUN TO IMAGINE 6',
+                'upload_date': '20080526',
+                'description': 'md5:0ffc78ea3f01b2e2c247d5f8d1d3c18d',
+                'uploader': 'Christopher Sykes',
+                'uploader_id': 'ChristopherJSykes',
+            },
+            'add_ie': ['Youtube'],
+        },
         # Camtasia studio
         {
             'url': 'http://www.ll.mit.edu/workshops/education/videocourses/antennas/lecture1/video/',
@@ -1938,6 +1953,7 @@ class GenericIE(InfoExtractor):
                 data-video-url=|
                 <embed[^>]+?src=|
                 embedSWF\(?:\s*|
+                <object[^>]+data=|
                 new\s+SWFObject\(
             )
             (["\'])

From 6b9466de2f3f8bcaaf360a60625221c4bd7a50cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Apr 2017 02:05:53 +0700
Subject: [PATCH 0625/1696] [afreecatv] Fix extraction (closes #12706)

---
 youtube_dl/extractor/afreecatv.py | 126 ++++++++++++++++++++++++++----
 1 file changed, 112 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py
index b774d6db8..8ac88c2dc 100644
--- a/youtube_dl/extractor/afreecatv.py
+++ b/youtube_dl/extractor/afreecatv.py
@@ -6,6 +6,7 @@ import re
 from .common import InfoExtractor
 from ..compat import compat_xpath
 from ..utils import (
+    determine_ext,
     ExtractorError,
     int_or_none,
     xpath_text,
@@ -72,13 +73,54 @@ class AfreecaTVIE(InfoExtractor):
         'url': 'http://vod.afreecatv.com/PLAYER/STATION/18650793',
         'info_dict': {
             'id': '18650793',
-            'ext': 'flv',
+            'ext': 'mp4',
+            'title': '오늘은 다르다! 쏘님의 우월한 위아래~ 댄스리액션!',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': '윈아디',
             'uploader_id': 'badkids',
-            'title': '오늘은 다르다! 쏘님의 우월한 위아래~ 댄스리액션!',
+            'duration': 107,
         },
         'params': {
-            'skip_download': True,  # requires rtmpdump
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://vod.afreecatv.com/PLAYER/STATION/10481652',
+        'info_dict': {
+            'id': '10481652',
+            'title': "BJ유트루와 함께하는 '팅커벨 메이크업!'",
+            'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
+            'uploader': 'dailyapril',
+            'uploader_id': 'dailyapril',
+            'duration': 6492,
+        },
+        'playlist_count': 2,
+        'playlist': [{
+            'md5': 'd8b7c174568da61d774ef0203159bf97',
+            'info_dict': {
+                'id': '10481652_1',
+                'ext': 'mp4',
+                'title': "BJ유트루와 함께하는 '팅커벨 메이크업!' (part 1)",
+                'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
+                'uploader': 'dailyapril',
+                'uploader_id': 'dailyapril',
+                'upload_date': '20160502',
+                'duration': 3601,
+            },
+        }, {
+            'md5': '58f2ce7f6044e34439ab2d50612ab02b',
+            'info_dict': {
+                'id': '10481652_2',
+                'ext': 'mp4',
+                'title': "BJ유트루와 함께하는 '팅커벨 메이크업!' (part 2)",
+                'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
+                'uploader': 'dailyapril',
+                'uploader_id': 'dailyapril',
+                'upload_date': '20160502',
+                'duration': 2891,
+            },
+        }],
+        'params': {
+            'skip_download': True,
         },
     }, {
         'url': 'http://www.afreecatv.com/player/Player.swf?szType=szBjId=djleegoon&nStationNo=11273158&nBbsNo=13161095&nTitleNo=36327652',
@@ -94,7 +136,7 @@ class AfreecaTVIE(InfoExtractor):
         m = re.match(r'^(?P<upload_date>\d{8})_\w+_(?P<part>\d+)$', key)
         if m:
             video_key['upload_date'] = m.group('upload_date')
-            video_key['part'] = m.group('part')
+            video_key['part'] = int(m.group('part'))
         return video_key
 
     def _real_extract(self, url):
@@ -109,23 +151,64 @@ class AfreecaTVIE(InfoExtractor):
             raise ExtractorError('Specified AfreecaTV video does not exist',
                                  expected=True)
 
-        video_url_raw = video_element.text
-
-        app, playpath = video_url_raw.split('mp4:')
+        video_url = video_element.text.strip()
 
         title = xpath_text(video_xml, './track/title', 'title', fatal=True)
+
         uploader = xpath_text(video_xml, './track/nickname', 'uploader')
         uploader_id = xpath_text(video_xml, './track/bj_id', 'uploader id')
-        duration = int_or_none(xpath_text(video_xml, './track/duration',
-                                          'duration'))
+        duration = int_or_none(xpath_text(
+            video_xml, './track/duration', 'duration'))
         thumbnail = xpath_text(video_xml, './track/titleImage', 'thumbnail')
 
-        return {
+        common_entry = {
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'thumbnail': thumbnail,
+        }
+
+        info = common_entry.copy()
+        info.update({
+            'id': video_id,
+            'title': title,
+            'duration': duration,
+        })
+
+        if not video_url:
+            entries = []
+            for file_num, file_element in enumerate(
+                    video_element.findall(compat_xpath('./file')), start=1):
+                file_url = file_element.text
+                if not file_url:
+                    continue
+                video_key = self.parse_video_key(file_element.get('key', ''))
+                if not video_key:
+                    continue
+                file_duration = int_or_none(file_element.get('duration'))
+                part = video_key.get('part', file_num)
+                format_id = '%s_%s' % (video_id, part)
+                formats = self._extract_m3u8_formats(
+                    file_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls',
+                    note='Downloading part %d m3u8 information' % file_num)
+                file_info = common_entry.copy()
+                file_info.update({
+                    'id': format_id,
+                    'title': '%s (part %d)' % (title, part),
+                    'upload_date': video_key.get('upload_date'),
+                    'duration': file_duration,
+                    'formats': formats,
+                })
+                entries.append(file_info)
+            entries_info = info.copy()
+            entries_info.update({
+                '_type': 'multi_video',
+                'entries': entries,
+            })
+            return entries_info
+
+        info = {
             'id': video_id,
-            'url': app,
-            'ext': 'flv',
-            'play_path': 'mp4:' + playpath,
-            'rtmp_live': True,  # downloading won't end without this
             'title': title,
             'uploader': uploader,
             'uploader_id': uploader_id,
@@ -133,6 +216,21 @@ class AfreecaTVIE(InfoExtractor):
             'thumbnail': thumbnail,
         }
 
+        if determine_ext(video_url) == 'm3u8':
+            info['formats'] = self._extract_m3u8_formats(
+                video_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                m3u8_id='hls')
+        else:
+            app, playpath = video_url.split('mp4:')
+            info.update({
+                'url': app,
+                'ext': 'flv',
+                'play_path': 'mp4:' + playpath,
+                'rtmp_live': True,  # downloading won't end without this
+            })
+
+        return info
+
 
 class AfreecaTVGlobalIE(AfreecaTVIE):
     IE_NAME = 'afreecatv:global'

From 689cd458a67f87c9c9537191f017a0082fa17bc9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Apr 2017 02:16:20 +0700
Subject: [PATCH 0626/1696] [ChangeLog] Actualize

---
 ChangeLog | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 143c688f5..bfb812742 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,16 @@
+version <unreleased>
+
+Extractors
+* [afreecatv] Fix extraction (#12706)
++ [generic] Add support for <object> YouTube embeds (#12637)
+* [bbccouk] Treat bitrate as audio+video bitrate in media selector
++ [bbccouk] Skip unrecognized formats in media selector (#12701)
++ [bbccouk] Add support for https protocol in media selector (#12701)
+* [curiositystream] Fix extraction (#12638)
+* [adn] Update subtitle decryption key
+* [chaturbate] Fix extraction (#12665, #12688, #12690)
+
+
 version 2017.04.09
 
 Extractors

From 173087816710dc59a60a4cc81c316bb04dfa7d85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Apr 2017 02:17:53 +0700
Subject: [PATCH 0627/1696] release 2017.04.11

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 898d96b78..61af50e5c 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.09*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.09**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.11*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.11**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.04.09
+[debug] youtube-dl version 2017.04.11
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index bfb812742..f26372bc1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.04.11
 
 Extractors
 * [afreecatv] Fix extraction (#12706)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 39268f734..5918c10bc 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.04.09'
+__version__ = '2017.04.11'

From 6214611a4a01674d1f3cd5434d5ee4494110bd91 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Apr 2017 22:30:15 +0700
Subject: [PATCH 0628/1696] [canalplus] Bypass geo restriction

---
 youtube_dl/extractor/canalplus.py | 39 +++++++++++++++++--------------
 1 file changed, 22 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 4b9fa2ddf..90b454115 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -45,6 +45,9 @@ class CanalplusIE(InfoExtractor):
         'itele': 'itele',
     }
 
+    # Only works for direct mp4 URLs
+    _GEO_COUNTRIES = ['FR']
+
     _TESTS = [{
         'url': 'http://www.canalplus.fr/c-emissions/pid1830-c-zapping.html?vid=1192814',
         'info_dict': {
@@ -56,6 +59,7 @@ class CanalplusIE(InfoExtractor):
             'upload_date': '20160702',
         },
     }, {
+        # geo restricted, bypassed
         'url': 'http://www.piwiplus.fr/videos-piwi/pid1405-le-labyrinthe-boing-super-ranger.html?vid=1108190',
         'info_dict': {
             'id': '1108190',
@@ -65,19 +69,20 @@ class CanalplusIE(InfoExtractor):
             'description': 'md5:4cea7a37153be42c1ba2c1d3064376ff',
             'upload_date': '20140724',
         },
-        'skip': 'Only works from France',
+        'expected_warnings': ['HTTP Error 403: Forbidden'],
     }, {
-        'url': 'http://www.c8.fr/c8-divertissement/ms-touche-pas-a-mon-poste/pid6318-videos-integrales.html',
-        'md5': '4b47b12b4ee43002626b97fad8fb1de5',
+        # geo restricted, bypassed
+        'url': 'http://www.c8.fr/c8-divertissement/ms-touche-pas-a-mon-poste/pid6318-videos-integrales.html?vid=1443684',
+        'md5': 'bb6f9f343296ab7ebd88c97b660ecf8d',
         'info_dict': {
-            'id': '1420213',
+            'id': '1443684',
             'display_id': 'pid6318-videos-integrales',
             'ext': 'mp4',
-            'title': 'TPMP ! Même le matin - Les 35H de Baba - 14/10/2016',
-            'description': 'md5:f96736c1b0ffaa96fd5b9e60ad871799',
-            'upload_date': '20161014',
+            'title': 'Guess my iep ! - TPMP - 07/04/2017',
+            'description': 'md5:6f005933f6e06760a9236d9b3b5f17fa',
+            'upload_date': '20170407',
         },
-        'skip': 'Only works from France',
+        'expected_warnings': ['HTTP Error 403: Forbidden'],
     }, {
         'url': 'http://www.itele.fr/chroniques/invite-michael-darmon/rachida-dati-nicolas-sarkozy-est-le-plus-en-phase-avec-les-inquietudes-des-francais-171510',
         'info_dict': {
@@ -134,15 +139,15 @@ class CanalplusIE(InfoExtractor):
 
         preference = qualities(['MOBILE', 'BAS_DEBIT', 'HAUT_DEBIT', 'HD'])
 
-        fmt_url = next(iter(media.get('VIDEOS')))
-        if '/geo' in fmt_url.lower():
-            response = self._request_webpage(
-                HEADRequest(fmt_url), video_id,
-                'Checking if the video is georestricted')
-            if '/blocage' in response.geturl():
-                raise ExtractorError(
-                    'The video is not available in your country',
-                    expected=True)
+        # _, fmt_url = next(iter(media['VIDEOS'].items()))
+        # if '/geo' in fmt_url.lower():
+        #     response = self._request_webpage(
+        #         HEADRequest(fmt_url), video_id,
+        #         'Checking if the video is georestricted')
+        #     if '/blocage' in response.geturl():
+        #         raise ExtractorError(
+        #             'The video is not available in your country',
+        #             expected=True)
 
         formats = []
         for format_id, format_url in media['VIDEOS'].items():

From d23028a8fb3784735c11853f0d4144d8bad37ba0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Apr 2017 22:41:48 +0700
Subject: [PATCH 0629/1696] [youtube] Remove explicit preference for audio-only
 and video-only formats In order not to break sorting when new formats appear

---
 youtube_dl/extractor/youtube.py | 90 ++++++++++++++++-----------------
 1 file changed, 45 insertions(+), 45 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index ca40de522..2506cd27c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -317,60 +317,60 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 
         # DASH mp4 video
-        '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
-        '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
-        '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
-        '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
-        '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
-        '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},  # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
-        '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
-        '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
-        '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
-        '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
-        '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
-        '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
+        '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
+        '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
+        '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
+        '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
+        '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
+        '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
+        '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
+        '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
+        '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
+        '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
+        '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
+        '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
 
         # Dash mp4 audio
-        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'preference': -50, 'container': 'm4a_dash'},
-        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'preference': -50, 'container': 'm4a_dash'},
-        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'preference': -50, 'container': 'm4a_dash'},
-        '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'preference': -50, 'container': 'm4a_dash'},
-        '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'preference': -50, 'container': 'm4a_dash'},
-        '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'preference': -50, 'container': 'm4a_dash'},
-        '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'preference': -50, 'container': 'm4a_dash'},
+        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
+        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
+        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
+        '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
+        '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
+        '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
+        '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
 
         # Dash webm
-        '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
-        '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
-        '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
-        '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
-        '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
-        '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
-        '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9', 'preference': -40},
-        '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
-        '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
-        '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
-        '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
-        '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
-        '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
-        '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
-        '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
+        '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
+        '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
+        '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
+        '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
+        '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
+        '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
+        '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
+        '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
+        '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
+        '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
+        '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
+        '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
+        '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
+        '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
+        '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
-        '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
-        '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
-        '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
-        '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
-        '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
-        '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
+        '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
+        '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
+        '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
+        '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
+        '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
+        '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
 
         # Dash webm audio
-        '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},
-        '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256, 'preference': -50},
+        '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
+        '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
 
         # Dash webm audio with opus inside
-        '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50, 'preference': -50},
-        '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70, 'preference': -50},
-        '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160, 'preference': -50},
+        '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
+        '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
+        '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
 
         # RTMP (unnamed)
         '_rtmp': {'protocol': 'rtmp'},

From e109f1ff438f82305d91bb49944a616c53abc9bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 12 Apr 2017 02:15:37 +0700
Subject: [PATCH 0630/1696] [afreecatv] Fix extraction for videos with
 different key layout (closes #12718)

---
 youtube_dl/extractor/afreecatv.py | 39 ++++++++++++++++++++++---------
 1 file changed, 28 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py
index 8ac88c2dc..78d29c861 100644
--- a/youtube_dl/extractor/afreecatv.py
+++ b/youtube_dl/extractor/afreecatv.py
@@ -97,7 +97,7 @@ class AfreecaTVIE(InfoExtractor):
         'playlist': [{
             'md5': 'd8b7c174568da61d774ef0203159bf97',
             'info_dict': {
-                'id': '10481652_1',
+                'id': '20160502_c4c62b9d_174361386_1',
                 'ext': 'mp4',
                 'title': "BJ유트루와 함께하는 '팅커벨 메이크업!' (part 1)",
                 'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
@@ -109,7 +109,7 @@ class AfreecaTVIE(InfoExtractor):
         }, {
             'md5': '58f2ce7f6044e34439ab2d50612ab02b',
             'info_dict': {
-                'id': '10481652_2',
+                'id': '20160502_39e739bb_174361386_2',
                 'ext': 'mp4',
                 'title': "BJ유트루와 함께하는 '팅커벨 메이크업!' (part 2)",
                 'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
@@ -122,6 +122,22 @@ class AfreecaTVIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        # non standard key
+        'url': 'http://vod.afreecatv.com/PLAYER/STATION/20515605',
+        'info_dict': {
+            'id': '20170411_BE689A0E_190960999_1_2_h',
+            'ext': 'mp4',
+            'title': '혼자사는여자집',
+            'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
+            'uploader': '♥이슬이',
+            'uploader_id': 'dasl8121',
+            'upload_date': '20170411',
+            'duration': 213,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'http://www.afreecatv.com/player/Player.swf?szType=szBjId=djleegoon&nStationNo=11273158&nBbsNo=13161095&nTitleNo=36327652',
         'only_matching': True,
@@ -176,26 +192,27 @@ class AfreecaTVIE(InfoExtractor):
 
         if not video_url:
             entries = []
-            for file_num, file_element in enumerate(
-                    video_element.findall(compat_xpath('./file')), start=1):
+            file_elements = video_element.findall(compat_xpath('./file'))
+            one = len(file_elements) == 1
+            for file_num, file_element in enumerate(file_elements, start=1):
                 file_url = file_element.text
                 if not file_url:
                     continue
-                video_key = self.parse_video_key(file_element.get('key', ''))
-                if not video_key:
-                    continue
+                key = file_element.get('key', '')
+                upload_date = self._search_regex(
+                    r'^(\d{8})_', key, 'upload date', default=None)
                 file_duration = int_or_none(file_element.get('duration'))
-                part = video_key.get('part', file_num)
-                format_id = '%s_%s' % (video_id, part)
+                format_id = key if key else '%s_%s' % (video_id, file_num)
                 formats = self._extract_m3u8_formats(
                     file_url, video_id, 'mp4', entry_protocol='m3u8_native',
                     m3u8_id='hls',
                     note='Downloading part %d m3u8 information' % file_num)
+                title = title if one else '%s (part %d)' % (title, file_num)
                 file_info = common_entry.copy()
                 file_info.update({
                     'id': format_id,
-                    'title': '%s (part %d)' % (title, part),
-                    'upload_date': video_key.get('upload_date'),
+                    'title': title,
+                    'upload_date': upload_date,
                     'duration': file_duration,
                     'formats': formats,
                 })

From 89beedd31f8a00a811787b651a506001833da0e1 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 11 Apr 2017 21:55:59 +0100
Subject: [PATCH 0631/1696] [youtube] improve m3u8 format extraction

---
 youtube_dl/extractor/youtube.py | 47 +++++++++++----------------------
 1 file changed, 15 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2506cd27c..9e2b9115c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1253,21 +1253,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         video_id = mobj.group(2)
         return video_id
 
-    def _extract_from_m3u8(self, manifest_url, video_id):
-        url_map = {}
-
-        def _get_urls(_manifest):
-            lines = _manifest.split('\n')
-            urls = filter(lambda l: l and not l.startswith('#'),
-                          lines)
-            return urls
-        manifest = self._download_webpage(manifest_url, video_id, 'Downloading formats manifest')
-        formats_urls = _get_urls(manifest)
-        for format_url in formats_urls:
-            itag = self._search_regex(r'itag/(\d+?)/', format_url, 'itag')
-            url_map[itag] = format_url
-        return url_map
-
     def _extract_annotations(self, video_id):
         url = 'https://www.youtube.com/annotations_invideo?features=1&legacy=1&video_id=%s' % video_id
         return self._download_webpage(url, video_id, note='Searching for annotations.', errnote='Unable to download video annotations.')
@@ -1573,19 +1558,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         if self._downloader.params.get('writeannotations', False):
             video_annotations = self._extract_annotations(video_id)
 
-        def _map_to_format_list(urlmap):
-            formats = []
-            for itag, video_real_url in urlmap.items():
-                dct = {
-                    'format_id': itag,
-                    'url': video_real_url,
-                    'player_url': player_url,
-                }
-                if itag in self._formats:
-                    dct.update(self._formats[itag])
-                formats.append(dct)
-            return formats
-
         if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
             self.report_rtmp_download()
             formats = [{
@@ -1718,11 +1690,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 formats.append(dct)
         elif video_info.get('hlsvp'):
             manifest_url = video_info['hlsvp'][0]
-            url_map = self._extract_from_m3u8(manifest_url, video_id)
-            formats = _map_to_format_list(url_map)
-            # Accept-Encoding header causes failures in live streams on Youtube and Youtube Gaming
-            for a_format in formats:
+            formats = []
+            m3u8_formats = self._extract_m3u8_formats(
+                manifest_url, video_id, 'mp4', fatal=False)
+            for a_format in m3u8_formats:
+                itag = self._search_regex(
+                    r'/itag/(\d+)/', a_format['url'], 'itag', default=None)
+                if itag:
+                    a_format['format_id'] = itag
+                    if itag in self._formats:
+                        dct = self._formats[itag].copy()
+                        dct.update(a_format)
+                        a_format = dct
+                a_format['player_url'] = player_url
+                # Accept-Encoding header causes failures in live streams on Youtube and Youtube Gaming
                 a_format.setdefault('http_headers', {})['Youtubedl-no-compression'] = 'True'
+                formats.append(a_format)
         else:
             unavailable_message = self._html_search_regex(
                 r'(?s)<h1[^>]+id="unavailable-message"[^>]*>(.+?)</h1>',

From a9a346535bd855f4dd01aa32e54345cc741769c4 Mon Sep 17 00:00:00 2001
From: Daniel Twardowski <daniel.j.twardowski@gmail.com>
Date: Wed, 12 Apr 2017 01:18:57 -0400
Subject: [PATCH 0632/1696] [adobepass] Add Verizon support (#10803)

---
 youtube_dl/extractor/adobepass.py | 71 +++++++++++++++++++++++++++++++
 1 file changed, 71 insertions(+)

diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
index 1b2d364ca..a15843315 100644
--- a/youtube_dl/extractor/adobepass.py
+++ b/youtube_dl/extractor/adobepass.py
@@ -41,6 +41,11 @@ MSO_INFO = {
         'username_field': 'IDToken1',
         'password_field': 'IDToken2',
     },
+    'Verizon': {
+        'name': 'Verizon FiOS',
+        'username_field': 'IDToken1',
+        'password_field': 'IDToken2',
+    },
     'thr030': {
         'name': '3 Rivers Communications'
     },
@@ -1416,6 +1421,72 @@ class AdobePassIE(InfoExtractor):
                         if '<button class="submit" value="Resume">Resume</button>' in mvpd_confirm_page:
                             post_form(mvpd_confirm_page_res, 'Confirming Login')
 
+                elif mso_id == 'Verizon':
+                    # In general, if you're connecting from a Verizon-assigned IP,
+                    # you will not actually pass your credentials.
+                    provider_redirect_page, urlh = provider_redirect_page_res
+                    # print('provider_redirect_page_url', urlh.geturl())
+                    # https://signin.verizon.com/sso/VOLPortalLogin?SAMLRequest=<snip>
+                    if 'Please wait ...' in provider_redirect_page:
+                        # print('[debug] SAML Auto-Login is TRUE') # GO,
+                        saml_redirect_url = self._html_search_regex(
+                            r'self\.parent\.location=(["\'])(?P<url>.+?)\1',
+                            provider_redirect_page, 'Extracting SAML Redirect URL', group='url'
+                        )
+                        # print('saml_redirect_url', saml_redirect_url)
+                        # https://signin.verizon.com/sso/choice/tvpHandler.jsp?loginType=vzRedirect&partner=<snip>
+                        saml_login_page = self._download_webpage(
+                            saml_redirect_url, video_id, 'Downloading SAML Login Page'
+                        )
+                    else:
+                        if 'Please try again.' in provider_redirect_page:
+                            # print('[debug] SAML Auto-Login is FALSE') # NBC,
+                            provider_login_page_res = provider_redirect_page_res
+                        else:
+                            print('[error] SAML Auto-Login is UNKNOWN')
+                            import sys
+                            sys.exit(1)
+                        provider_login_page, urlh = provider_login_page_res
+                        # print('provider_login_page_url', urlh.geturl())
+                        # https://signin.verizon.com/sso/VOLPortalLogin?SAMLRequest=<snip>
+                        saml_login_page_res = post_form(provider_login_page_res, 'Logging in', {
+                            mso_info.get('username_field', 'username'): username,
+                            mso_info.get('password_field', 'password'): password,
+                        })
+                        saml_login_page, urlh = saml_login_page_res
+                        if 'Please try again.' in saml_login_page:
+                            print("[error] We're sorry, but either the User ID or Password entered is not correct.")
+                            import sys
+                            sys.exit(1)
+                        # elif 'Please wait - we are verifying your account...' in saml_login_page:
+                        # print('saml_login_page_url', urlh.geturl())
+                        # https://signin.verizon.com/sso/choice/tvpHandler.jsp?loginType=vzRedirect&partner=<snip>
+                    saml_login_url = self._html_search_regex(
+                        r'xmlHttp\.open\(\"POST\",\s(["\'])(?P<url>.+?)\1',
+                        saml_login_page, 'Extracting SAML Login URL', group='url')
+                    # print('saml_login_url', saml_login_url)
+                    # https://signin.verizon.com/sso/TVPHandlerServlet?loginType=vzRedirect&partner=<snip>
+                    saml_response_json = self._download_json(
+                        saml_login_url, video_id, 'Downloading SAML Response',
+                        headers={'Content-Type': 'text/xml'}
+                    )
+                    saml_target_url = saml_response_json['targetValue']
+                    saml_response = saml_response_json['SAMLResponse']
+                    saml_relay_state = saml_response_json['RelayState']
+                    # print('saml_target_url', saml_target_url)
+                    # https://sp.auth.adobe.com/sp/saml/SAMLAssertionConsumer
+                    form_data = {
+                        'SAMLResponse': saml_response,
+                        'RelayState': saml_relay_state
+                    }
+                    saml_autologin_res = self._download_webpage_handle(
+                        saml_target_url, video_id,
+                        'Auto-Login', data=urlencode_postdata(form_data),
+                        headers={'Content-Type': 'application/x-www-form-urlencoded'}
+                    )
+                    original_page, urlh = saml_autologin_res
+                    # print('original_page_url', urlh.geturl())
+
                 else:
                     # Normal, non-Comcast flow
                     provider_login_page_res = post_form(

From 79a51069e5c08afe7bf15bca7fcdf44fd0d93eb6 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 12 Apr 2017 15:17:18 +0100
Subject: [PATCH 0633/1696] [adobepass] improve comcast and verison login
 code(closes #10803)

---
 youtube_dl/extractor/adobepass.py | 98 ++++++++++---------------------
 1 file changed, 32 insertions(+), 66 deletions(-)

diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
index a15843315..100cf997f 100644
--- a/youtube_dl/extractor/adobepass.py
+++ b/youtube_dl/extractor/adobepass.py
@@ -1389,106 +1389,72 @@ class AdobePassIE(InfoExtractor):
                     # Comcast page flow varies by video site and whether you
                     # are on Comcast's network.
                     provider_redirect_page, urlh = provider_redirect_page_res
-                    # Check for Comcast auto login
                     if 'automatically signing you in' in provider_redirect_page:
                         oauth_redirect_url = self._html_search_regex(
                             r'window\.location\s*=\s*[\'"]([^\'"]+)',
                             provider_redirect_page, 'oauth redirect')
-                        # Just need to process the request. No useful data comes back
                         self._download_webpage(
                             oauth_redirect_url, video_id, 'Confirming auto login')
                     else:
                         if '<form name="signin"' in provider_redirect_page:
-                            # already have the form, just fill it
                             provider_login_page_res = provider_redirect_page_res
                         elif 'http-equiv="refresh"' in provider_redirect_page:
-                            # redirects to the login page
                             oauth_redirect_url = self._html_search_regex(
                                 r'content="0;\s*url=([^\'"]+)',
                                 provider_redirect_page, 'meta refresh redirect')
                             provider_login_page_res = self._download_webpage_handle(
-                                oauth_redirect_url,
-                                video_id, 'Downloading Provider Login Page')
+                                oauth_redirect_url, video_id,
+                                'Downloading Provider Login Page')
                         else:
                             provider_login_page_res = post_form(
-                                provider_redirect_page_res, 'Downloading Provider Login Page')
+                                provider_redirect_page_res,
+                                'Downloading Provider Login Page')
 
-                        mvpd_confirm_page_res = post_form(provider_login_page_res, 'Logging in', {
-                            mso_info.get('username_field', 'username'): username,
-                            mso_info.get('password_field', 'password'): password,
-                        })
+                        mvpd_confirm_page_res = post_form(
+                            provider_login_page_res, 'Logging in', {
+                                mso_info['username_field']: username,
+                                mso_info['password_field']: password,
+                            })
                         mvpd_confirm_page, urlh = mvpd_confirm_page_res
                         if '<button class="submit" value="Resume">Resume</button>' in mvpd_confirm_page:
                             post_form(mvpd_confirm_page_res, 'Confirming Login')
-
                 elif mso_id == 'Verizon':
                     # In general, if you're connecting from a Verizon-assigned IP,
                     # you will not actually pass your credentials.
                     provider_redirect_page, urlh = provider_redirect_page_res
-                    # print('provider_redirect_page_url', urlh.geturl())
-                    # https://signin.verizon.com/sso/VOLPortalLogin?SAMLRequest=<snip>
                     if 'Please wait ...' in provider_redirect_page:
-                        # print('[debug] SAML Auto-Login is TRUE') # GO,
                         saml_redirect_url = self._html_search_regex(
                             r'self\.parent\.location=(["\'])(?P<url>.+?)\1',
-                            provider_redirect_page, 'Extracting SAML Redirect URL', group='url'
-                        )
-                        # print('saml_redirect_url', saml_redirect_url)
-                        # https://signin.verizon.com/sso/choice/tvpHandler.jsp?loginType=vzRedirect&partner=<snip>
+                            provider_redirect_page,
+                            'SAML Redirect URL', group='url')
                         saml_login_page = self._download_webpage(
-                            saml_redirect_url, video_id, 'Downloading SAML Login Page'
-                        )
+                            saml_redirect_url, video_id,
+                            'Downloading SAML Login Page')
                     else:
-                        if 'Please try again.' in provider_redirect_page:
-                            # print('[debug] SAML Auto-Login is FALSE') # NBC,
-                            provider_login_page_res = provider_redirect_page_res
-                        else:
-                            print('[error] SAML Auto-Login is UNKNOWN')
-                            import sys
-                            sys.exit(1)
-                        provider_login_page, urlh = provider_login_page_res
-                        # print('provider_login_page_url', urlh.geturl())
-                        # https://signin.verizon.com/sso/VOLPortalLogin?SAMLRequest=<snip>
-                        saml_login_page_res = post_form(provider_login_page_res, 'Logging in', {
-                            mso_info.get('username_field', 'username'): username,
-                            mso_info.get('password_field', 'password'): password,
-                        })
+                        saml_login_page_res = post_form(
+                            provider_redirect_page_res, 'Logging in', {
+                                mso_info['username_field']: username,
+                                mso_info['password_field']: password,
+                            })
                         saml_login_page, urlh = saml_login_page_res
                         if 'Please try again.' in saml_login_page:
-                            print("[error] We're sorry, but either the User ID or Password entered is not correct.")
-                            import sys
-                            sys.exit(1)
-                        # elif 'Please wait - we are verifying your account...' in saml_login_page:
-                        # print('saml_login_page_url', urlh.geturl())
-                        # https://signin.verizon.com/sso/choice/tvpHandler.jsp?loginType=vzRedirect&partner=<snip>
-                    saml_login_url = self._html_search_regex(
-                        r'xmlHttp\.open\(\"POST\",\s(["\'])(?P<url>.+?)\1',
-                        saml_login_page, 'Extracting SAML Login URL', group='url')
-                    # print('saml_login_url', saml_login_url)
-                    # https://signin.verizon.com/sso/TVPHandlerServlet?loginType=vzRedirect&partner=<snip>
+                            raise ExtractorError(
+                                'We\'re sorry, but either the User ID or Password entered is not correct.')
+                    saml_login_url = self._search_regex(
+                        r'xmlHttp\.open\("POST"\s*,\s*(["\'])(?P<url>.+?)\1',
+                        saml_login_page, 'SAML Login URL', group='url')
                     saml_response_json = self._download_json(
                         saml_login_url, video_id, 'Downloading SAML Response',
-                        headers={'Content-Type': 'text/xml'}
-                    )
-                    saml_target_url = saml_response_json['targetValue']
-                    saml_response = saml_response_json['SAMLResponse']
-                    saml_relay_state = saml_response_json['RelayState']
-                    # print('saml_target_url', saml_target_url)
-                    # https://sp.auth.adobe.com/sp/saml/SAMLAssertionConsumer
-                    form_data = {
-                        'SAMLResponse': saml_response,
-                        'RelayState': saml_relay_state
-                    }
-                    saml_autologin_res = self._download_webpage_handle(
-                        saml_target_url, video_id,
-                        'Auto-Login', data=urlencode_postdata(form_data),
-                        headers={'Content-Type': 'application/x-www-form-urlencoded'}
-                    )
-                    original_page, urlh = saml_autologin_res
-                    # print('original_page_url', urlh.geturl())
-
+                        headers={'Content-Type': 'text/xml'})
+                    self._download_webpage(
+                        saml_response_json['targetValue'], video_id,
+                        'Confirming Login', data=urlencode_postdata({
+                            'SAMLResponse': saml_response_json['SAMLResponse'],
+                            'RelayState': saml_response_json['RelayState']
+                        }), headers={
+                            'Content-Type': 'application/x-www-form-urlencoded'
+                        })
                 else:
-                    # Normal, non-Comcast flow
                     provider_login_page_res = post_form(
                         provider_redirect_page_res, 'Downloading Provider Login Page')
                     mvpd_confirm_page_res = post_form(provider_login_page_res, 'Logging in', {

From 0ee79a378ad9536cfb580187c9a30a0463ad5d8b Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 12 Apr 2017 16:22:08 +0100
Subject: [PATCH 0634/1696] [generic] exclude urls with xml ext from valid
 video urls(closes #10768)(closes #11654)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 4690816cb..36d23d2f3 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2611,7 +2611,7 @@ class GenericIE(InfoExtractor):
                 return True
             vpath = compat_urlparse.urlparse(vurl).path
             vext = determine_ext(vpath)
-            return '.' in vpath and vext not in ('swf', 'png', 'jpg', 'srt', 'sbv', 'sub', 'vtt', 'ttml', 'js')
+            return '.' in vpath and vext not in ('swf', 'png', 'jpg', 'srt', 'sbv', 'sub', 'vtt', 'ttml', 'js', 'xml')
 
         def filter_video(urls):
             return list(filter(check_video, urls))

From 3962260b7d398d75ce456dbb08a85ae8fcecfd28 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 12 Apr 2017 16:54:53 +0100
Subject: [PATCH 0635/1696] Credit @SkiTheSlicer for Verizon adobe pass support
 (#12721)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 2d676b210..9abd5fdf4 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -210,3 +210,4 @@ Lars Vierbergen
 Juanjo Benages
 Xiao Di Guan
 Thomas Winant
+Daniel Twardowski

From 3ef1d0c73373000bf65851c7afe9c68e707e4a97 Mon Sep 17 00:00:00 2001
From: Vukkk <Vukkk@github.com>
Date: Wed, 31 Aug 2016 10:56:11 +0200
Subject: [PATCH 0636/1696] [tv2hu] Add new extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/tv2hu.py      | 78 ++++++++++++++++++++++++++++++
 2 files changed, 79 insertions(+)
 create mode 100644 youtube_dl/extractor/tv2hu.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 68e1a5cfc..7484099f6 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1031,6 +1031,7 @@ from .tv2 import (
     TV2IE,
     TV2ArticleIE,
 )
+from .tv2hu import TV2HUIE
 from .tv3 import TV3IE
 from .tv4 import TV4IE
 from .tv5mondeplus import TV5MondePlusIE
diff --git a/youtube_dl/extractor/tv2hu.py b/youtube_dl/extractor/tv2hu.py
new file mode 100644
index 000000000..cb1ce822a
--- /dev/null
+++ b/youtube_dl/extractor/tv2hu.py
@@ -0,0 +1,78 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+class TV2HUIE(InfoExtractor):
+    IE_NAME = 'tv2.hu'
+    _VALID_URL = r'https?://(?:www\.)?tv2\.hu/(?:musoraink/)?(?P<uploader>[^/]+)/(?:teljes_adasok/)?(?P<id>[0-9]+)_(.+?)\.html'
+    _JSON_URL = r'(?P<json_url>https?://.+?\.tv2\.hu/vod/(?P<upload_date>\d+)/id_(?P<upload_id>\d+).+?&type=json)'
+
+    _TESTS = [{
+        'url': 'http://tv2.hu/ezek_megorultek/217679_ezek-megorultek---1.-adas-1.-resz.html',
+        'info_dict': {
+            'id': '217679',
+            'ext': 'mp4',
+            'title': 'Ezek megőrültek! - 1. adás 1. rész',
+            'upload_id': '220289',
+            'upload_date': '20160826',
+            'uploader': 'ezek_megorultek',
+            'thumbnail': 're:^https?://.*\.jpg$'
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
+    }, {
+        'url': 'http://tv2.hu/ezek_megorultek/teljes_adasok/217677_ezek-megorultek---1.-adas-2.-resz.html',
+        'only_matching': True
+    }, {
+        'url': 'http://tv2.hu/musoraink/aktiv/aktiv_teljes_adas/217963_aktiv-teljes-adas---2016.08.30..html',
+        'only_matching': True
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(
+            url, video_id, 'Downloading info page')
+
+        json_url = re.search(self._JSON_URL, webpage)
+
+        json_data = self._download_json(
+            json_url.group('json_url'), video_id, 'Downloading video info')
+
+        manifest_url = json_data['bitrates']['hls']
+
+        formats = self._extract_m3u8_formats(
+            manifest_url, video_id, 'mp4', entry_protocol='m3u8_native')
+
+        for i in range(len(json_data['bitrates']['mp4'])):
+            quality = json_data.get('mp4Labels')[i]
+
+            if quality.lower() == 'auto':
+                continue
+
+            formats.append({
+                'protocol': 'http',
+                'url': json_data['bitrates']['mp4'][i],
+                'height': int(quality[:-1]),
+                'width': int(quality[:-1])/9*16,
+                'ext': 'mp4',
+                'format_id': quality,
+                'format_note': 'HTTP',
+                'preference': int(quality[:-1])
+            })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage).strip(),
+            'thumbnail': self._og_search_property('image', webpage),
+            'uploader': self._search_regex(self._VALID_URL, url, 'uploader'),
+            'upload_id': json_url.group('upload_id'),
+            'upload_date': json_url.group('upload_date'),
+            'formats': formats
+        }

From e4d74e2778ca283330460d893a0923820a74df66 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 12 Apr 2017 19:31:18 +0100
Subject: [PATCH 0637/1696] [tv2hu] improve extraction

---
 youtube_dl/extractor/extractors.py |  2 +-
 youtube_dl/extractor/tv2hu.py      | 74 ++++++++++++------------------
 2 files changed, 30 insertions(+), 46 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 7484099f6..45591f2a7 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1031,7 +1031,7 @@ from .tv2 import (
     TV2IE,
     TV2ArticleIE,
 )
-from .tv2hu import TV2HUIE
+from .tv2hu import TV2HuIE
 from .tv3 import TV3IE
 from .tv4 import TV4IE
 from .tv5mondeplus import TV5MondePlusIE
diff --git a/youtube_dl/extractor/tv2hu.py b/youtube_dl/extractor/tv2hu.py
index cb1ce822a..ce01cc80d 100644
--- a/youtube_dl/extractor/tv2hu.py
+++ b/youtube_dl/extractor/tv2hu.py
@@ -1,29 +1,22 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..utils import int_or_none
 
-class TV2HUIE(InfoExtractor):
+
+class TV2HuIE(InfoExtractor):
     IE_NAME = 'tv2.hu'
-    _VALID_URL = r'https?://(?:www\.)?tv2\.hu/(?:musoraink/)?(?P<uploader>[^/]+)/(?:teljes_adasok/)?(?P<id>[0-9]+)_(.+?)\.html'
-    _JSON_URL = r'(?P<json_url>https?://.+?\.tv2\.hu/vod/(?P<upload_date>\d+)/id_(?P<upload_id>\d+).+?&type=json)'
-
+    _VALID_URL = r'https?://(?:www\.)?tv2\.hu/(?:[^/]+/)+(?P<id>\d+)_[^/?#]+?\.html'
     _TESTS = [{
         'url': 'http://tv2.hu/ezek_megorultek/217679_ezek-megorultek---1.-adas-1.-resz.html',
+        'md5': '585e58e2e090f34603804bb2c48e98d8',
         'info_dict': {
             'id': '217679',
             'ext': 'mp4',
             'title': 'Ezek megőrültek! - 1. adás 1. rész',
-            'upload_id': '220289',
             'upload_date': '20160826',
-            'uploader': 'ezek_megorultek',
             'thumbnail': 're:^https?://.*\.jpg$'
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
         }
     }, {
         'url': 'http://tv2.hu/ezek_megorultek/teljes_adasok/217677_ezek-megorultek---1.-adas-2.-resz.html',
@@ -35,44 +28,35 @@ class TV2HUIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(
-            url, video_id, 'Downloading info page')
+        webpage = self._download_webpage(url, video_id)
+        json_url = self._search_regex(
+            r'jsonUrl\s*=\s*"([^"]+)"', webpage, 'json url')
+        json_data = self._download_json(json_url, video_id)
 
-        json_url = re.search(self._JSON_URL, webpage)
-
-        json_data = self._download_json(
-            json_url.group('json_url'), video_id, 'Downloading video info')
-
-        manifest_url = json_data['bitrates']['hls']
-
-        formats = self._extract_m3u8_formats(
-            manifest_url, video_id, 'mp4', entry_protocol='m3u8_native')
-
-        for i in range(len(json_data['bitrates']['mp4'])):
-            quality = json_data.get('mp4Labels')[i]
-
-            if quality.lower() == 'auto':
-                continue
-
-            formats.append({
-                'protocol': 'http',
-                'url': json_data['bitrates']['mp4'][i],
-                'height': int(quality[:-1]),
-                'width': int(quality[:-1])/9*16,
-                'ext': 'mp4',
-                'format_id': quality,
-                'format_note': 'HTTP',
-                'preference': int(quality[:-1])
-            })
+        formats = []
+        for b in ('bitrates', 'backupBitrates'):
+            bitrates = json_data.get(b, {})
+            m3u8_url = bitrates.get('hls')
+            if m3u8_url:
+                formats.extend(self._extract_wowza_formats(
+                    m3u8_url, video_id, skip_protocols=['rtmp', 'rtsp']))
 
+            for mp4_url in bitrates.get('mp4', []):
+                height = int_or_none(self._search_regex(
+                    r'\.(\d+)p\.mp4', mp4_url, 'height', default=None))
+                formats.append({
+                    'format_id': 'http' + ('-%d' % height if height else ''),
+                    'url': mp4_url,
+                    'height': height,
+                    'width': int_or_none(height / 9.0 * 16.0 if height else None),
+                })
         self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': self._og_search_title(webpage).strip(),
-            'thumbnail': self._og_search_property('image', webpage),
-            'uploader': self._search_regex(self._VALID_URL, url, 'uploader'),
-            'upload_id': json_url.group('upload_id'),
-            'upload_date': json_url.group('upload_date'),
-            'formats': formats
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'upload_date': self._search_regex(
+                r'/vod/(\d{8})/', json_url, 'upload_date', default=None),
+            'formats': formats,
         }

From 40fcba5edb0f54f09e33a193a0ffefb5668ca694 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 12 Apr 2017 20:38:43 +0100
Subject: [PATCH 0638/1696] improve coding style

---
 youtube_dl/YoutubeDL.py                       | 2 +-
 youtube_dl/compat.py                          | 2 +-
 youtube_dl/downloader/rtmp.py                 | 2 +-
 youtube_dl/extractor/bbc.py                   | 4 ++--
 youtube_dl/extractor/common.py                | 2 +-
 youtube_dl/extractor/rudo.py                  | 2 +-
 youtube_dl/extractor/viewlift.py              | 2 +-
 youtube_dl/extractor/vlive.py                 | 4 ++--
 youtube_dl/postprocessor/ffmpeg.py            | 2 +-
 youtube_dl/postprocessor/metadatafromtitle.py | 2 +-
 10 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 54bc8b06d..7953670a7 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -855,7 +855,7 @@ class YoutubeDL(object):
 
             return self.process_ie_result(
                 new_result, download=download, extra_info=extra_info)
-        elif result_type == 'playlist' or result_type == 'multi_video':
+        elif result_type in ('playlist', 'multi_video'):
             # We process each entry in the playlist
             playlist = ie_result.get('title') or ie_result.get('id')
             self.to_screen('[download] Downloading playlist: %s' % playlist)
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 0c119e417..39527117f 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2692,7 +2692,7 @@ else:
                 userhome = pwent.pw_dir
             userhome = userhome.rstrip('/')
             return (userhome + path[i:]) or '/'
-    elif compat_os_name == 'nt' or compat_os_name == 'ce':
+    elif compat_os_name in ('nt', 'ce'):
         def compat_expanduser(path):
             """Expand ~ and ~user constructs.
 
diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 9de6e70bb..b823b5171 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -169,7 +169,7 @@ class RtmpFD(FileDownloader):
             self.report_error('[rtmpdump] Could not connect to RTMP server.')
             return False
 
-        while (retval == RD_INCOMPLETE or retval == RD_FAILED) and not test and not live:
+        while retval in (RD_INCOMPLETE, RD_FAILED) and not test and not live:
             prevsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen('[rtmpdump] %s bytes' % prevsize)
             time.sleep(5.0)  # This seems to be needed
diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 0e05b782b..dd65b8d86 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -409,7 +409,7 @@ class BBCCoUkIE(InfoExtractor):
                 description = smp_config['summary']
                 for item in smp_config['items']:
                     kind = item['kind']
-                    if kind != 'programme' and kind != 'radioProgramme':
+                    if kind not in ('programme', 'radioProgramme'):
                         continue
                     programme_id = item.get('vpid')
                     duration = int_or_none(item.get('duration'))
@@ -450,7 +450,7 @@ class BBCCoUkIE(InfoExtractor):
 
         for item in self._extract_items(playlist):
             kind = item.get('kind')
-            if kind != 'programme' and kind != 'radioProgramme':
+            if kind not in ('programme', 'radioProgramme'):
                 continue
             title = playlist.find('./{%s}title' % self._EMP_PLAYLIST_NS).text
             description_el = playlist.find('./{%s}summary' % self._EMP_PLAYLIST_NS)
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index ae8af61de..dcc9d628a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1779,7 +1779,7 @@ class InfoExtractor(object):
                     if content_type == 'text':
                         # TODO implement WebVTT downloading
                         pass
-                    elif content_type == 'video' or content_type == 'audio':
+                    elif content_type in ('video', 'audio'):
                         base_url = ''
                         for element in (representation, adaptation_set, period, mpd_doc):
                             base_url_e = element.find(_add_ns('BaseURL'))
diff --git a/youtube_dl/extractor/rudo.py b/youtube_dl/extractor/rudo.py
index 51644011e..f036f6757 100644
--- a/youtube_dl/extractor/rudo.py
+++ b/youtube_dl/extractor/rudo.py
@@ -26,7 +26,7 @@ class RudoIE(InfoExtractor):
     }
 
     @classmethod
-    def _extract_url(self, webpage):
+    def _extract_url(cls, webpage):
         mobj = re.search(
             r'<iframe[^>]+src=(?P<q1>[\'"])(?P<url>(?:https?:)?//rudo\.video/vod/[0-9a-zA-Z]+)(?P=q1)',
             webpage)
diff --git a/youtube_dl/extractor/viewlift.py b/youtube_dl/extractor/viewlift.py
index 18735cfb2..1f29c273f 100644
--- a/youtube_dl/extractor/viewlift.py
+++ b/youtube_dl/extractor/viewlift.py
@@ -68,7 +68,7 @@ class ViewLiftEmbedIE(ViewLiftBaseIE):
             type_ = source.get('type')
             ext = determine_ext(file_)
             format_id = source.get('label') or ext
-            if all(v == 'm3u8' or v == 'hls' for v in (type_, ext)):
+            if all(v in ('m3u8', 'hls') for v in (type_, ext)):
                 formats.extend(self._extract_m3u8_formats(
                     file_, video_id, 'mp4', m3u8_id='hls'))
             else:
diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index b9718901b..e58940607 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -70,9 +70,9 @@ class VLiveIE(InfoExtractor):
         status, long_video_id, key = params[2], params[5], params[6]
         status = remove_start(status, 'PRODUCT_')
 
-        if status == 'LIVE_ON_AIR' or status == 'BIG_EVENT_ON_AIR':
+        if status in ('LIVE_ON_AIR', 'BIG_EVENT_ON_AIR'):
             return self._live(video_id, webpage)
-        elif status == 'VOD_ON_AIR' or status == 'BIG_EVENT_INTRO':
+        elif status in ('VOD_ON_AIR', 'BIG_EVENT_INTRO'):
             if long_video_id and key:
                 return self._replay(video_id, webpage, long_video_id, key)
             else:
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 7c162d92a..665109558 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -552,7 +552,7 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
             sub_filenames.append(old_file)
             new_file = subtitles_filename(filename, lang, new_ext)
 
-            if ext == 'dfxp' or ext == 'ttml' or ext == 'tt':
+            if ext in ('dfxp', 'ttml', 'tt'):
                 self._downloader.report_warning(
                     'You have requested to convert dfxp (TTML) subtitles into another format, '
                     'which results in style information loss')
diff --git a/youtube_dl/postprocessor/metadatafromtitle.py b/youtube_dl/postprocessor/metadatafromtitle.py
index 164edd3a8..a7d637a3c 100644
--- a/youtube_dl/postprocessor/metadatafromtitle.py
+++ b/youtube_dl/postprocessor/metadatafromtitle.py
@@ -26,7 +26,7 @@ class MetadataFromTitlePP(PostProcessor):
             regex += r'(?P<' + match.group(1) + '>.+)'
             lastpos = match.end()
         if lastpos < len(fmt):
-            regex += re.escape(fmt[lastpos:len(fmt)])
+            regex += re.escape(fmt[lastpos:])
         return regex
 
     def run(self, info):

From 34d98cc411d379cea02a67ba7fc9f52e74daff23 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 13 Apr 2017 16:52:52 +0800
Subject: [PATCH 0639/1696] [tv2hu] Fix invalid escape sequence on Py3.6

---
 youtube_dl/extractor/tv2hu.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tv2hu.py b/youtube_dl/extractor/tv2hu.py
index ce01cc80d..86017b757 100644
--- a/youtube_dl/extractor/tv2hu.py
+++ b/youtube_dl/extractor/tv2hu.py
@@ -16,7 +16,7 @@ class TV2HuIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Ezek megőrültek! - 1. adás 1. rész',
             'upload_date': '20160826',
-            'thumbnail': 're:^https?://.*\.jpg$'
+            'thumbnail': r're:^https?://.*\.jpg$'
         }
     }, {
         'url': 'http://tv2.hu/ezek_megorultek/teljes_adasok/217677_ezek-megorultek---1.-adas-2.-resz.html',

From f59746372a1aae3bc80462d34a7e96200be22ab5 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 13 Apr 2017 12:21:17 +0100
Subject: [PATCH 0640/1696] [downloader/hls] add basic support for
 EXT-X-BYTERANGE tag(#10955)

---
 youtube_dl/downloader/hls.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 4989abce1..d0a5f7ba4 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -34,7 +34,7 @@ class HlsFD(FragmentFD):
     def can_download(manifest, info_dict):
         UNSUPPORTED_FEATURES = (
             r'#EXT-X-KEY:METHOD=(?!NONE|AES-128)',  # encrypted streams [1]
-            r'#EXT-X-BYTERANGE',  # playlists composed of byte ranges of media files [2]
+            # r'#EXT-X-BYTERANGE',  # playlists composed of byte ranges of media files [2]
 
             # Live streams heuristic does not always work (e.g. geo restricted to Germany
             # http://hls-geo.daserste.de/i/videoportal/Film/c_620000/622873/format,716451,716457,716450,716458,716459,.mp4.csmil/index_4_av.m3u8?null=0)
@@ -52,7 +52,9 @@ class HlsFD(FragmentFD):
             # 4. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.5
         )
         check_results = [not re.search(feature, manifest) for feature in UNSUPPORTED_FEATURES]
-        check_results.append(can_decrypt_frag or '#EXT-X-KEY:METHOD=AES-128' not in manifest)
+        is_aes128_enc = '#EXT-X-KEY:METHOD=AES-128' in manifest
+        check_results.append(can_decrypt_frag or not is_aes128_enc)
+        check_results.append(not (is_aes128_enc and r'#EXT-X-BYTERANGE' in manifest))
         check_results.append(not info_dict.get('is_live'))
         return all(check_results)
 
@@ -100,6 +102,7 @@ class HlsFD(FragmentFD):
         i = 0
         media_sequence = 0
         decrypt_info = {'METHOD': 'NONE'}
+        byte_range = {}
         frags_filenames = []
         for line in s.splitlines():
             line = line.strip()
@@ -114,11 +117,14 @@ class HlsFD(FragmentFD):
                     if extra_query:
                         frag_url = update_url_query(frag_url, extra_query)
                     count = 0
+                    headers = info_dict.get('http_headers', {})
+                    if byte_range:
+                        headers['Range'] = 'bytes=%d-%d' % (byte_range['start'], byte_range['end'])
                     while count <= fragment_retries:
                         try:
                             success = ctx['dl'].download(frag_filename, {
                                 'url': frag_url,
-                                'http_headers': info_dict.get('http_headers'),
+                                'http_headers': headers,
                             })
                             if not success:
                                 return False
@@ -167,6 +173,13 @@ class HlsFD(FragmentFD):
                         decrypt_info['KEY'] = self.ydl.urlopen(decrypt_info['URI']).read()
                 elif line.startswith('#EXT-X-MEDIA-SEQUENCE'):
                     media_sequence = int(line[22:])
+                elif line.startswith('#EXT-X-BYTERANGE'):
+                    splitted_byte_range = line[17:].split('@')
+                    sub_range_start = int(splitted_byte_range[1]) if len(splitted_byte_range) == 2 else byte_range['end']
+                    byte_range = {
+                        'start': sub_range_start,
+                        'end': sub_range_start + int(splitted_byte_range[0]),
+                    }
 
         self._finish_frag_download(ctx)
 

From b2fff30817848c1761a53d49fbe49aabd596089e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 13 Apr 2017 12:27:28 +0100
Subject: [PATCH 0641/1696] [go90] Add new extractor(closes #10127)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/go90.py       | 92 ++++++++++++++++++++++++++++++
 2 files changed, 93 insertions(+)
 create mode 100644 youtube_dl/extractor/go90.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 45591f2a7..1671090f4 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -384,6 +384,7 @@ from .globo import (
     GloboArticleIE,
 )
 from .go import GoIE
+from .go90 import Go90IE
 from .godtube import GodTubeIE
 from .godtv import GodTVIE
 from .golem import GolemIE
diff --git a/youtube_dl/extractor/go90.py b/youtube_dl/extractor/go90.py
new file mode 100644
index 000000000..3550eca7c
--- /dev/null
+++ b/youtube_dl/extractor/go90.py
@@ -0,0 +1,92 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+)
+
+
+class Go90IE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?go90\.com/videos/(?P<id>[0-9a-zA-Z]+)'
+    _TEST = {
+        'url': 'https://www.go90.com/videos/84BUqjLpf9D',
+        'md5': 'efa7670dbbbf21a7b07b360652b24a32',
+        'info_dict': {
+            'id': '84BUqjLpf9D',
+            'ext': 'mp4',
+            'title': 'Inside The Utah Coalition Against Pornography Convention',
+            'description': 'VICE\'s Karley Sciortino meets with activists who discuss the state\'s strong anti-porn stance. Then, VICE Sports explains NFL contracts.',
+            'timestamp': 1491868800,
+            'upload_date': '20170411',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_data = self._download_json(
+            'https://www.go90.com/api/view/items/' + video_id,
+            video_id, headers={
+                'Content-Type': 'application/json; charset=utf-8',
+            }, data=b'{"client":"web","device_type":"pc"}')
+        title = video_data['title']
+        main_video_asset = video_data['main_video_asset']
+
+        thumbnails = []
+        formats = []
+        for asset in video_data.get('assets'):
+            if asset.get('id') == main_video_asset:
+                for source in asset.get('sources', []):
+                    source_location = source.get('location')
+                    if not source_location:
+                        continue
+                    source_type = source.get('type')
+                    if source_type == 'hls':
+                        m3u8_formats = self._extract_m3u8_formats(
+                            source_location, video_id, 'mp4',
+                            'm3u8_native', m3u8_id='hls', fatal=False)
+                        for f in m3u8_formats:
+                            mobj = re.search(r'/hls-(\d+)-(\d+)K', f['url'])
+                            if mobj:
+                                height, tbr = mobj.groups()
+                                height = int_or_none(height)
+                                f.update({
+                                    'height': f.get('height') or height,
+                                    'width': f.get('width') or int_or_none(height / 9.0 * 16.0 if height else None),
+                                    'tbr': f.get('tbr') or int_or_none(tbr),
+                                })
+                        formats.extend(m3u8_formats)
+                    elif source_type == 'dash':
+                        formats.extend(self._extract_mpd_formats(
+                            source_location, video_id, mpd_id='dash', fatal=False))
+                    else:
+                        formats.append({
+                            'format_id': source.get('name'),
+                            'url': source_location,
+                            'width': int_or_none(source.get('width')),
+                            'height': int_or_none(source.get('height')),
+                            'tbr': int_or_none(source.get('bitrate')),
+                        })
+            elif asset.get('type') == 'image':
+                asset_location = asset.get('location')
+                if not asset_location:
+                    continue
+                thumbnails.append({
+                    'url': asset_location,
+                    'width': int_or_none(asset.get('width')),
+                    'height': int_or_none(asset.get('height')),
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnails': thumbnails,
+            'description': video_data.get('short_description'),
+            'like_count': int_or_none(video_data.get('like_count')),
+            'timestamp': parse_iso8601(video_data.get('released_at')),
+        }

From 3f7409f745df1dbde394643fe40d661bc109de74 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 13 Apr 2017 12:28:00 +0100
Subject: [PATCH 0642/1696] [hbo] extract m3u8 formats

---
 youtube_dl/extractor/hbo.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/hbo.py b/youtube_dl/extractor/hbo.py
index 931f71a5a..859ad5429 100644
--- a/youtube_dl/extractor/hbo.py
+++ b/youtube_dl/extractor/hbo.py
@@ -92,12 +92,14 @@ class HBOBaseIE(InfoExtractor):
                         video_url.replace('.tar', '/base_index_w8.m3u8'),
                         video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
                 elif source.tag == 'hls':
-                    # #EXT-X-BYTERANGE is not supported by native hls downloader
-                    # and ffmpeg (#10955)
-                    # formats.extend(self._extract_m3u8_formats(
-                    #     video_url.replace('.tar', '/base_index.m3u8'),
-                    #     video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
-                    continue
+                    m3u8_formats = self._extract_m3u8_formats(
+                        video_url.replace('.tar', '/base_index.m3u8'),
+                        video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+                    for f in m3u8_formats:
+                        if f.get('vcodec') == 'none' and not f.get('tbr'):
+                            f['tbr'] = int_or_none(self._search_regex(
+                                r'-(\d+)k/', f['url'], 'tbr', default=None))
+                    formats.extend(m3u8_formats)
                 elif source.tag == 'dash':
                     formats.extend(self._extract_mpd_formats(
                         video_url.replace('.tar', '/manifest.mpd'),
@@ -110,7 +112,7 @@ class HBOBaseIE(InfoExtractor):
                         'width': format_info.get('width'),
                         'height': format_info.get('height'),
                     })
-        self._sort_formats(formats, ('width', 'height', 'tbr', 'format_id'))
+        self._sort_formats(formats)
 
         thumbnails = []
         card_sizes = xpath_element(video_data, 'titleCardSizes')

From 95728fda7059265374e92d92004bd75e0beb630d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Apr 2017 23:39:55 +0700
Subject: [PATCH 0643/1696] [aenetworks] Add support for specials (closes
 #12723)

---
 youtube_dl/extractor/aenetworks.py | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index 9a5074037..c01c67303 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -23,7 +23,19 @@ class AENetworksBaseIE(ThePlatformIE):
 class AENetworksIE(AENetworksBaseIE):
     IE_NAME = 'aenetworks'
     IE_DESC = 'A+E Networks: A&E, Lifetime, History.com, FYI Network'
-    _VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:history|aetv|mylifetime|lifetimemovieclub)\.com|fyi\.tv)/(?:shows/(?P<show_path>[^/]+(?:/[^/]+){0,2})|movies/(?P<movie_display_id>[^/]+)(?:/full-movie)?)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:www\.)?
+                        (?P<domain>
+                            (?:history|aetv|mylifetime|lifetimemovieclub)\.com|
+                            fyi\.tv
+                        )/
+                        (?:
+                            shows/(?P<show_path>[^/]+(?:/[^/]+){0,2})|
+                            movies/(?P<movie_display_id>[^/]+)(?:/full-movie)?|
+                            specials/(?P<special_display_id>[^/]+)/full-special
+                        )
+                    '''
     _TESTS = [{
         'url': 'http://www.history.com/shows/mountain-men/season-1/episode-1',
         'md5': 'a97a65f7e823ae10e9244bc5433d5fe6',
@@ -65,6 +77,9 @@ class AENetworksIE(AENetworksBaseIE):
     }, {
         'url': 'https://www.lifetimemovieclub.com/movies/a-killer-among-us',
         'only_matching': True
+    }, {
+        'url': 'http://www.history.com/specials/sniper-into-the-kill-zone/full-special',
+        'only_matching': True
     }]
     _DOMAIN_TO_REQUESTOR_ID = {
         'history.com': 'HISTORY',
@@ -75,8 +90,8 @@ class AENetworksIE(AENetworksBaseIE):
     }
 
     def _real_extract(self, url):
-        domain, show_path, movie_display_id = re.match(self._VALID_URL, url).groups()
-        display_id = show_path or movie_display_id
+        domain, show_path, movie_display_id, special_display_id = re.match(self._VALID_URL, url).groups()
+        display_id = show_path or movie_display_id or special_display_id
         webpage = self._download_webpage(url, display_id)
         if show_path:
             url_parts = show_path.split('/')

From 6f4dd6667b2d7ee4d3927090c4bc3e1ee6c85ce7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 14 Apr 2017 00:29:36 +0700
Subject: [PATCH 0644/1696] [ChangeLog] Actualize

---
 ChangeLog | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index f26372bc1..c61b0f5e5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,23 @@
+version <unreleased>
+
+Core
++ [downloader/hls] Add basic support for EXT-X-BYTERANGE tag (#10955)
++ [adobepass] Improve Comcast and Verison login code (#10803)
++ [adobepass] Add support for Verizon (#10803)
+
+Extractors
++ [aenetworks] Add support for specials (#12723)
++ [hbo] Extract HLS formats
++ [go90] Add support for go90.com (#10127)
++ [tv2hu] Add support for tv2.hu (#10509)
++ [generic] Exclude URLs with xml ext from valid video URLs (#10768, #11654)
+* [youtube] Improve HLS formats extraction
+* [afreecatv] Fix extraction for videos with different key layout (#12718)
+- [youtube] Remove explicit preference for audio-only and video-only formats in
+  order not to break sorting when new formats appear
+* [canalplus] Bypass geo restriction
+
+
 version 2017.04.11
 
 Extractors

From 38d2f8325f8d30cdb16c4ddea277772cbca6c8d0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 14 Apr 2017 00:31:49 +0700
Subject: [PATCH 0645/1696] release 2017.04.14

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 ++
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 61af50e5c..02e9481c1 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.11*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.11**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.14*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.14**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.04.11
+[debug] youtube-dl version 2017.04.14
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index c61b0f5e5..8d5553ead 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.04.14
 
 Core
 + [downloader/hls] Add basic support for EXT-X-BYTERANGE tag (#10955)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 0157ce3ef..b29b50c8d 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -308,6 +308,7 @@
  - **Globo**
  - **GloboArticle**
  - **Go**
+ - **Go90**
  - **GodTube**
  - **GodTV**
  - **Golem**
@@ -818,6 +819,7 @@
  - **Tutv**
  - **tv.dfb.de**
  - **TV2**
+ - **tv2.hu**
  - **TV2Article**
  - **TV3**
  - **TV4**: tv4.se and tv4play.se
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 5918c10bc..f64ad97bb 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.04.11'
+__version__ = '2017.04.14'

From e5106ba402031f619503b3cecb3405433258eece Mon Sep 17 00:00:00 2001
From: xuhaomin <xhmgreat@sina.com>
Date: Sat, 15 Apr 2017 02:23:02 +0800
Subject: [PATCH 0646/1696] [youku] Fix fileid extraction

---
 youtube_dl/extractor/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index e37f237c7..73f6f0e90 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -105,7 +105,7 @@ class YoukuIE(InfoExtractor):
             if stream.get('channel_type') == 'tail':
                 continue
             format = stream.get('stream_type')
-            fileid = stream['stream_fileid']
+            fileid = stream['segs'][0]['fileid']
             fileid_dict[format] = fileid
 
         def get_fileid(format, n):

From 34c3bbe1bd1def4e4407d3170cb583ffcb2674dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Apr 2017 01:25:32 +0700
Subject: [PATCH 0647/1696] [youku] Keep old fileid extraction code as fallback
 (#12741)

---
 youtube_dl/extractor/youku.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 73f6f0e90..73ebe5759 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -10,12 +10,14 @@ import time
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse_urlencode,
     compat_ord,
+    compat_str,
+    compat_urllib_parse_urlencode,
 )
 from ..utils import (
     ExtractorError,
     get_element_by_attribute,
+    try_get,
 )
 
 
@@ -105,7 +107,9 @@ class YoukuIE(InfoExtractor):
             if stream.get('channel_type') == 'tail':
                 continue
             format = stream.get('stream_type')
-            fileid = stream['segs'][0]['fileid']
+            fileid = try_get(
+                stream, lambda x: x['segs'][0]['fileid'],
+                compat_str) or stream['stream_fileid']
             fileid_dict[format] = fileid
 
         def get_fileid(format, n):

From b9859e2735e754f90136117e5ac9d0645b9bc359 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Apr 2017 01:37:26 +0700
Subject: [PATCH 0648/1696] [ChangeLog] Actualize

---
 ChangeLog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 8d5553ead..9f958dd91 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [youku] Fix fileid extraction (#12741, #12743)
+
+
 version 2017.04.14
 
 Core

From d24bd4ffdb64b6b2c0568bd5eb79920b053e95fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Apr 2017 01:39:33 +0700
Subject: [PATCH 0649/1696] release 2017.04.15

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 02e9481c1..70f6b51ed 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.14*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.14**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.15*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.15**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.04.14
+[debug] youtube-dl version 2017.04.15
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 9f958dd91..cf5ee84a4 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.04.15
 
 Extractors
 * [youku] Fix fileid extraction (#12741, #12743)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index f64ad97bb..612b50f7b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.04.14'
+__version__ = '2017.04.15'

From 6f1b2374af37b6a290080036edae9361d2e5a66d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Apr 2017 02:33:12 +0700
Subject: [PATCH 0650/1696] [canalplus] PEP 8

---
 youtube_dl/extractor/canalplus.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 90b454115..d8bf073f4 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -7,8 +7,8 @@ from .common import InfoExtractor
 from ..compat import compat_urllib_parse_urlparse
 from ..utils import (
     dict_get,
-    ExtractorError,
-    HEADRequest,
+    # ExtractorError,
+    # HEADRequest,
     int_or_none,
     qualities,
     remove_end,

From 57e2cb6c3ad05720cd8e4cd707c62f6fadc18be8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Apr 2017 02:33:26 +0700
Subject: [PATCH 0651/1696] [curiositystream] PEP 8

---
 youtube_dl/extractor/curiositystream.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/curiositystream.py b/youtube_dl/extractor/curiositystream.py
index 145d9397c..8e45923e3 100644
--- a/youtube_dl/extractor/curiositystream.py
+++ b/youtube_dl/extractor/curiositystream.py
@@ -48,7 +48,6 @@ class CuriosityStreamBaseIE(InfoExtractor):
 
     def _extract_media_info(self, media):
         video_id = compat_str(media['id'])
-        limelight_media_id = media['limelight_media_id']
         title = media['title']
 
         formats = []

From 3dfceb286c5ffc39a0025d3728448c6d572e022b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Apr 2017 03:26:19 +0700
Subject: [PATCH 0652/1696] [udemy] Add support for react rendition (closes
 #12744)

---
 youtube_dl/extractor/udemy.py | 79 +++++++++++++++++++++++++----------
 1 file changed, 58 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index cce29c6e0..dae1aa3c6 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -212,12 +212,15 @@ class UdemyIE(InfoExtractor):
         thumbnail = asset.get('thumbnail_url') or asset.get('thumbnailUrl')
         duration = float_or_none(asset.get('data', {}).get('duration'))
 
+        subtitles = {}
+        automatic_captions = {}
+
         formats = []
 
-        def extract_output_format(src):
+        def extract_output_format(src, f_id):
             return {
                 'url': src['url'],
-                'format_id': '%sp' % (src.get('height') or format_id),
+                'format_id': '%sp' % (src.get('height') or f_id),
                 'width': int_or_none(src.get('width')),
                 'height': int_or_none(src.get('height')),
                 'vbr': int_or_none(src.get('video_bitrate_in_kbps')),
@@ -237,30 +240,33 @@ class UdemyIE(InfoExtractor):
         def add_output_format_meta(f, key):
             output = outputs.get(key)
             if isinstance(output, dict):
-                output_format = extract_output_format(output)
+                output_format = extract_output_format(output, key)
                 output_format.update(f)
                 return output_format
             return f
 
+        def extract_formats(source_list):
+            if not isinstance(source_list, list):
+                return
+            for source in source_list:
+                video_url = source.get('file') or source.get('src')
+                if not video_url or not isinstance(video_url, compat_str):
+                    continue
+                format_id = source.get('label')
+                f = {
+                    'url': video_url,
+                    'format_id': '%sp' % format_id,
+                    'height': int_or_none(format_id),
+                }
+                if format_id:
+                    # Some videos contain additional metadata (e.g.
+                    # https://www.udemy.com/ios9-swift/learn/#/lecture/3383208)
+                    f = add_output_format_meta(f, format_id)
+                formats.append(f)
+
         download_urls = asset.get('download_urls')
         if isinstance(download_urls, dict):
-            video = download_urls.get('Video')
-            if isinstance(video, list):
-                for format_ in video:
-                    video_url = format_.get('file')
-                    if not video_url:
-                        continue
-                    format_id = format_.get('label')
-                    f = {
-                        'url': format_['file'],
-                        'format_id': '%sp' % format_id,
-                        'height': int_or_none(format_id),
-                    }
-                    if format_id:
-                        # Some videos contain additional metadata (e.g.
-                        # https://www.udemy.com/ios9-swift/learn/#/lecture/3383208)
-                        f = add_output_format_meta(f, format_id)
-                    formats.append(f)
+            extract_formats(download_urls.get('Video'))
 
         view_html = lecture.get('view_html')
         if view_html:
@@ -294,6 +300,35 @@ class UdemyIE(InfoExtractor):
                         'height': height,
                     }, res))
 
+            # react rendition since 2017.04.15 (see
+            # https://github.com/rg3/youtube-dl/issues/12744)
+            data = self._parse_json(
+                self._search_regex(
+                    r'videojs-setup-data=(["\'])(?P<data>{.+?})\1', view_html,
+                    'setup data', default='{}', group='data'), video_id,
+                transform_source=unescapeHTML, fatal=False)
+            if data and isinstance(data, dict):
+                extract_formats(data.get('sources'))
+                if not duration:
+                    duration = int_or_none(data.get('duration'))
+                tracks = data.get('tracks')
+                if isinstance(tracks, list):
+                    for track in tracks:
+                        if not isinstance(track, dict):
+                            continue
+                        if track.get('kind') != 'captions':
+                            continue
+                        src = track.get('src')
+                        if not src or not isinstance(src, compat_str):
+                            continue
+                        lang = track.get('language') or track.get(
+                            'srclang') or track.get('label')
+                        sub_dict = automatic_captions if track.get(
+                            'autogenerated') is True else subtitles
+                        sub_dict.setdefault(lang, []).append({
+                            'url': src,
+                        })
+
         self._sort_formats(formats, field_preference=('height', 'width', 'tbr', 'format_id'))
 
         return {
@@ -302,7 +337,9 @@ class UdemyIE(InfoExtractor):
             'description': description,
             'thumbnail': thumbnail,
             'duration': duration,
-            'formats': formats
+            'formats': formats,
+            'subtitles': subtitles,
+            'automatic_captions': automatic_captions,
         }
 
 

From 40158f55c906720ccfb6177c0949e0e7beaa085e Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Sun, 12 Feb 2017 08:26:36 -0500
Subject: [PATCH 0653/1696] [brightcove] Relax video tag embeds extraction

BrightcoveNewIE.extract_urls(): Handle player data when it is all in
attributes of the <video> tag, and also the id is
data-brightcove-video-id not data-video-id.

Add test to generic extractor, note HEAD gives 404, no checksum computed(?).
---
 youtube_dl/extractor/brightcove.py | 58 ++++++++++++++++++++----------
 youtube_dl/extractor/generic.py    | 54 +++++++++++++++++++++++++++-
 2 files changed, 92 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 46ef8e605..c7059a12a 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -17,6 +17,7 @@ from ..compat import (
 from ..utils import (
     determine_ext,
     ExtractorError,
+    extract_attributes,
     find_xpath_attr,
     fix_xml_ampersands,
     float_or_none,
@@ -109,6 +110,7 @@ class BrightcoveLegacyIE(InfoExtractor):
                 'upload_date': '20140827',
                 'uploader_id': '710858724001',
             },
+            'skip': 'Video gone',
         },
         {
             # playlist with 'videoList'
@@ -490,9 +492,10 @@ class BrightcoveNewIE(InfoExtractor):
     def _extract_urls(webpage):
         # Reference:
         # 1. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideoiniframe
-        # 2. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideousingjavascript
-        # 3. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/embed-in-page.html
-        # 4. https://support.brightcove.com/en/video-cloud/docs/dynamically-assigning-videos-player
+        # 2. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#tag
+        # 3. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideousingjavascript
+        # 4. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/in-page-embed-player-implementation.html
+        # 5. https://support.brightcove.com/en/video-cloud/docs/dynamically-assigning-videos-player
 
         entries = []
 
@@ -501,22 +504,39 @@ class BrightcoveNewIE(InfoExtractor):
                 r'<iframe[^>]+src=(["\'])((?:https?:)?//players\.brightcove\.net/\d+/[^/]+/index\.html.+?)\1', webpage):
             entries.append(url if url.startswith('http') else 'http:' + url)
 
-        # Look for embed_in_page embeds [2]
-        for video_id, account_id, player_id, embed in re.findall(
-                # According to examples from [3] it's unclear whether video id
-                # may be optional and what to do when it is
-                # According to [4] data-video-id may be prefixed with ref:
-                r'''(?sx)
-                    <video[^>]+
-                        data-video-id=["\'](\d+|ref:[^"\']+)["\'][^>]*>.*?
-                    </video>.*?
-                    <script[^>]+
-                        src=["\'](?:https?:)?//players\.brightcove\.net/
-                        (\d+)/([^/]+)_([^/]+)/index(?:\.min)?\.js
-                ''', webpage):
-            entries.append(
-                'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
-                % (account_id, player_id, embed, video_id))
+        # Look for <video> tags [2] and embed_in_page embeds [3]
+        # [2] looks like:
+        # <video data-video-id="5320421710001" data-account="245991542" data-player="SJWAiyYWg" data-embed="default" class="video-js" controls itemscope itemtype="http://schema.org/VideoObject">
+
+        for video, script_tag, account_id, player_id, embed in re.findall(
+            r'''(?isx)
+                (<video[^>]+>)
+                (?:.*?
+                  (<script[^>]+
+                    src=["\'](?:https?:)?//players\.brightcove\.net/
+                    (\d+)/([^/]+)_([^/]+)/index(?:\.min)?\.js
+                  )
+                )?
+            ''', webpage
+        ):
+            attrs = extract_attributes(video)
+
+            # According to examples from [4] it's unclear whether video id
+            # may be optional and what to do when it is
+            video_id = attrs.get('data-video-id')
+            # See PR#12099/bostonglobe.py for 'data-brightcove-video-id' variant
+
+            if not account_id:
+                account_id = attrs.get('data-account')
+            if not player_id:
+                player_id = attrs.get('data-player')
+            if not embed:
+                embed = attrs.get('data-embed')
+
+            if video_id and account_id and player_id and embed:
+                entries.append(
+                    'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
+                    % (account_id, player_id, embed, video_id))
 
         return entries
 
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 36d23d2f3..dc60d77f5 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -449,6 +449,59 @@ class GenericIE(InfoExtractor):
                 },
             }],
         },
+        {
+            # Brightcove with UUID in videoPlayer
+            'url': 'http://www8.hp.com/cn/zh/home.html',
+            'info_dict': {
+                'id': '5255815316001',
+                'ext': 'mp4',
+                'title': 'Sprocket Video - China',
+                'description': 'Sprocket Video - China',
+                'uploader': 'HP-Video Gallery',
+                'timestamp': 1482263210,
+                'upload_date': '20161220',
+                'uploader_id': '1107601872001',
+            },
+            'params': {
+                'skip_download': True,  # m3u8 download
+            },
+            'skip': 'video rotates...weekly?',
+        },
+        {
+            # Brightcove:new type [2].
+            'url': 'http://www.delawaresportszone.com/video-st-thomas-more-earns-first-trip-to-basketball-semis',
+            'md5': '2b35148fcf48da41c9fb4591650784f3',
+            'info_dict': {
+                'id': '5348741021001',
+                'ext': 'mp4',
+                'upload_date': '20170306',
+                'uploader_id': '4191638492001',
+                'timestamp': 1488769918,
+                'title': 'VIDEO:  St. Thomas More earns first trip to basketball semis',
+
+            },
+        },
+        {
+            # Alternative brightcove <video> attributes
+            'url': 'http://www.programme-tv.net/videos/extraits/81095-guillaume-canet-evoque-les-rumeurs-d-infidelite-de-marion-cotillard-avec-brad-pitt-dans-vivement-dimanche/',
+            'info_dict': {
+                'id': '81095-guillaume-canet-evoque-les-rumeurs-d-infidelite-de-marion-cotillard-avec-brad-pitt-dans-vivement-dimanche',
+                'title': "Guillaume Canet évoque les rumeurs d'infidélité de Marion Cotillard avec Brad Pitt dans Vivement Dimanche, Extraits : toutes les vidéos avec Télé-Loisirs",
+            },
+            'playlist': [{
+                'md5': '732d22ba3d33f2f3fc253c39f8f36523',
+                'info_dict': {
+                    'id': '5311302538001',
+                    'ext': 'mp4',
+                    'title': "Guillaume Canet évoque les rumeurs d'infidélité de Marion Cotillard avec Brad Pitt dans Vivement Dimanche",
+                    'description': "Guillaume Canet évoque les rumeurs d'infidélité de Marion Cotillard avec Brad Pitt dans Vivement Dimanche (France 2, 5 février 2017)",
+                    'timestamp': 1486321708,
+                    'upload_date': '20170205',
+                    'uploader_id': '800000640001',
+                },
+                'only_matching': True,
+            }],
+        },
         {
             # Brightcove with UUID in videoPlayer
             'url': 'http://www8.hp.com/cn/zh/home.html',
@@ -1900,7 +1953,6 @@ class GenericIE(InfoExtractor):
         # Look for Brightcove Legacy Studio embeds
         bc_urls = BrightcoveLegacyIE._extract_brightcove_urls(webpage)
         if bc_urls:
-            self.to_screen('Brightcove video detected.')
             entries = [{
                 '_type': 'url',
                 'url': smuggle_url(bc_url, {'Referer': url}),

From 0254f93b08ef5f8af8ea0f1b9f7558892626900b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Apr 2017 20:30:51 +0700
Subject: [PATCH 0654/1696] [brightcove] Validate ambiguous embeds' URLs
 (closes #9163, closes #12005, closes #12178, closes #12480)

---
 youtube_dl/extractor/brightcove.py | 55 +++++++++++++++++-------------
 youtube_dl/extractor/generic.py    |  2 +-
 2 files changed, 33 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index c7059a12a..124497e95 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -489,7 +489,7 @@ class BrightcoveNewIE(InfoExtractor):
         return urls[0] if urls else None
 
     @staticmethod
-    def _extract_urls(webpage):
+    def _extract_urls(ie, webpage):
         # Reference:
         # 1. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideoiniframe
         # 2. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#tag
@@ -506,37 +506,46 @@ class BrightcoveNewIE(InfoExtractor):
 
         # Look for <video> tags [2] and embed_in_page embeds [3]
         # [2] looks like:
-        # <video data-video-id="5320421710001" data-account="245991542" data-player="SJWAiyYWg" data-embed="default" class="video-js" controls itemscope itemtype="http://schema.org/VideoObject">
-
         for video, script_tag, account_id, player_id, embed in re.findall(
-            r'''(?isx)
-                (<video[^>]+>)
-                (?:.*?
-                  (<script[^>]+
-                    src=["\'](?:https?:)?//players\.brightcove\.net/
-                    (\d+)/([^/]+)_([^/]+)/index(?:\.min)?\.js
-                  )
-                )?
-            ''', webpage
-        ):
+                r'''(?isx)
+                    (<video\s+[^>]+>)
+                    (?:.*?
+                        (<script[^>]+
+                            src=["\'](?:https?:)?//players\.brightcove\.net/
+                            (\d+)/([^/]+)_([^/]+)/index(?:\.min)?\.js
+                        )
+                    )?
+                ''', webpage):
             attrs = extract_attributes(video)
 
             # According to examples from [4] it's unclear whether video id
             # may be optional and what to do when it is
             video_id = attrs.get('data-video-id')
-            # See PR#12099/bostonglobe.py for 'data-brightcove-video-id' variant
+            if not video_id:
+                continue
 
+            account_id = account_id or attrs.get('data-account')
             if not account_id:
-                account_id = attrs.get('data-account')
-            if not player_id:
-                player_id = attrs.get('data-player')
-            if not embed:
-                embed = attrs.get('data-embed')
+                continue
 
-            if video_id and account_id and player_id and embed:
-                entries.append(
-                    'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
-                    % (account_id, player_id, embed, video_id))
+            player_id = player_id or attrs.get('data-player') or 'default'
+            embed = embed or attrs.get('data-embed') or 'default'
+
+            bc_url = 'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s' % (
+                account_id, player_id, embed, video_id)
+
+            # Some brightcove videos may be embedded with video tag only and
+            # without script tag or any mentioning of brightcove at all. Such
+            # embeds are considered ambiguous since they are matched based only
+            # on data-video-id and data-account attributes and in the wild may
+            # not be brightcove embeds at all. Let's check reconstructed
+            # brightcove URLs in case of such embeds and only process valid
+            # ones. By this we ensure there is indeed a brightcove embed.
+            if not script_tag and not ie._is_valid_url(
+                    bc_url, video_id, 'possible brightcove video'):
+                continue
+
+            entries.append(bc_url)
 
         return entries
 
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index dc60d77f5..bc7c21f7a 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1967,7 +1967,7 @@ class GenericIE(InfoExtractor):
             }
 
         # Look for Brightcove New Studio embeds
-        bc_urls = BrightcoveNewIE._extract_urls(webpage)
+        bc_urls = BrightcoveNewIE._extract_urls(self, webpage)
         if bc_urls:
             return self.playlist_from_matches(bc_urls, video_id, video_title, ie='BrightcoveNew')
 

From 3266d08af29bbd6078aca172741458ddee180ab9 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Sat, 25 Mar 2017 19:47:48 -0400
Subject: [PATCH 0655/1696] [wsj:article] Add extractor

---
 youtube_dl/extractor/extractors.py |  5 ++++-
 youtube_dl/extractor/wsj.py        | 28 +++++++++++++++++++++++++---
 2 files changed, 29 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 1671090f4..72d28a7e6 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1233,7 +1233,10 @@ from .wrzuta import (
     WrzutaIE,
     WrzutaPlaylistIE,
 )
-from .wsj import WSJIE
+from .wsj import (
+    WSJIE,
+    WSJArticleIE,
+)
 from .xbef import XBefIE
 from .xboxclips import XboxClipsIE
 from .xfileshare import XFileShareIE
diff --git a/youtube_dl/extractor/wsj.py b/youtube_dl/extractor/wsj.py
index deb7483ae..ec38a2ad8 100644
--- a/youtube_dl/extractor/wsj.py
+++ b/youtube_dl/extractor/wsj.py
@@ -10,10 +10,11 @@ from ..utils import (
 
 
 class WSJIE(InfoExtractor):
-    _VALID_URL = r'''(?x)https?://
+    _VALID_URL = r'''(?x)
         (?:
-            video-api\.wsj\.com/api-video/player/iframe\.html\?guid=|
-            (?:www\.)?wsj\.com/video/[^/]+/
+            https?://video-api\.wsj\.com/api-video/player/iframe\.html\?guid=|
+            https?://(?:www\.)?wsj\.com/video/[^/]+/|
+            wsj:
         )
         (?P<id>[a-zA-Z0-9-]+)'''
     IE_DESC = 'Wall Street Journal'
@@ -87,3 +88,24 @@ class WSJIE(InfoExtractor):
             'title': title,
             'categories': info.get('keywords'),
         }
+
+
+class WSJArticleIE(InfoExtractor):
+    _VALID_URL = r'(?i)https?://(?:www\.)?wsj\.com/articles/(?P<id>\w[^/]+)'
+    _TESTS = [{
+        'url': 'https://www.wsj.com/articles/dont-like-china-no-pandas-for-you-1490366939?',
+        'info_dict': {
+            'id': '4B13FA62-1D8C-45DB-8EA1-4105CB20B362',
+            'ext': 'mp4',
+            'upload_date': '20170221',
+            'uploader_id': 'ralcaraz',
+            'title': 'Bao Bao the Panda Leaves for China',
+        }
+    }]
+
+    def _real_extract(self, url):
+        article_id = self._match_id(url)
+        webpage = self._download_webpage(url, article_id)
+        video_id = self._search_regex(r'data-src=["\']([A-Z0-9\-]+)',
+                                      webpage, 'video id')
+        return self.url_result('wsj:%s' % video_id, WSJIE.ie_key(), video_id)

From b2a19e38293206a4ff687315baf0369c205bcd6b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Apr 2017 20:51:47 +0700
Subject: [PATCH 0656/1696] [wsj] Improve and modernize (closes #12558)

---
 youtube_dl/extractor/wsj.py | 40 +++++++++++++++++++++----------------
 1 file changed, 23 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/wsj.py b/youtube_dl/extractor/wsj.py
index ec38a2ad8..45cfca7c5 100644
--- a/youtube_dl/extractor/wsj.py
+++ b/youtube_dl/extractor/wsj.py
@@ -11,12 +11,13 @@ from ..utils import (
 
 class WSJIE(InfoExtractor):
     _VALID_URL = r'''(?x)
-        (?:
-            https?://video-api\.wsj\.com/api-video/player/iframe\.html\?guid=|
-            https?://(?:www\.)?wsj\.com/video/[^/]+/|
-            wsj:
-        )
-        (?P<id>[a-zA-Z0-9-]+)'''
+                        (?:
+                            https?://video-api\.wsj\.com/api-video/player/iframe\.html\?.*?\bguid=|
+                            https?://(?:www\.)?wsj\.com/video/[^/]+/|
+                            wsj:
+                        )
+                        (?P<id>[a-fA-F0-9-]{36})
+                    '''
     IE_DESC = 'Wall Street Journal'
     _TESTS = [{
         'url': 'http://video-api.wsj.com/api-video/player/iframe.html?guid=1BD01A4C-BFE8-40A5-A42F-8A8AF9898B1A',
@@ -39,12 +40,17 @@ class WSJIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        api_url = (
-            'http://video-api.wsj.com/api-video/find_all_videos.asp?'
-            'type=guid&count=1&query=%s&fields=type,hls,videoMP4List,'
-            'thumbnailList,author,description,name,duration,videoURL,'
-            'titletag,formattedCreationDate,keywords,editor' % video_id)
-        info = self._download_json(api_url, video_id)['items'][0]
+        info = self._download_json(
+            'http://video-api.wsj.com/api-video/find_all_videos.asp', video_id,
+            query={
+                'type': 'guid',
+                'count': 1,
+                'query': video_id,
+                'fields': ','.join((
+                    'type', 'hls', 'videoMP4List', 'thumbnailList', 'author',
+                    'description', 'name', 'duration', 'videoURL', 'titletag',
+                    'formattedCreationDate', 'keywords', 'editor')),
+            })['items'][0]
         title = info.get('name', info.get('titletag'))
 
         formats = []
@@ -91,8 +97,8 @@ class WSJIE(InfoExtractor):
 
 
 class WSJArticleIE(InfoExtractor):
-    _VALID_URL = r'(?i)https?://(?:www\.)?wsj\.com/articles/(?P<id>\w[^/]+)'
-    _TESTS = [{
+    _VALID_URL = r'(?i)https?://(?:www\.)?wsj\.com/articles/(?P<id>[^/?#&]+)'
+    _TEST = {
         'url': 'https://www.wsj.com/articles/dont-like-china-no-pandas-for-you-1490366939?',
         'info_dict': {
             'id': '4B13FA62-1D8C-45DB-8EA1-4105CB20B362',
@@ -101,11 +107,11 @@ class WSJArticleIE(InfoExtractor):
             'uploader_id': 'ralcaraz',
             'title': 'Bao Bao the Panda Leaves for China',
         }
-    }]
+    }
 
     def _real_extract(self, url):
         article_id = self._match_id(url)
         webpage = self._download_webpage(url, article_id)
-        video_id = self._search_regex(r'data-src=["\']([A-Z0-9\-]+)',
-                                      webpage, 'video id')
+        video_id = self._search_regex(
+            r'data-src=["\']([a-fA-F0-9-]{36})', webpage, 'video id')
         return self.url_result('wsj:%s' % video_id, WSJIE.ie_key(), video_id)

From 4db79fa1bc482ef5d97c8b73f0bf3683d5cc383c Mon Sep 17 00:00:00 2001
From: Marvin Ewald <marvin.e@protonmail.ch>
Date: Tue, 4 Apr 2017 19:18:23 +0200
Subject: [PATCH 0657/1696] [streamango] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/streamango.py | 54 ++++++++++++++++++++++++++++++
 2 files changed, 55 insertions(+)
 create mode 100644 youtube_dl/extractor/streamango.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 72d28a7e6..a92cbefed 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -939,6 +939,7 @@ from .srmediathek import SRMediathekIE
 from .stanfordoc import StanfordOpenClassroomIE
 from .steam import SteamIE
 from .streamable import StreamableIE
+from .streamango import StreamangoIE
 from .streamcloud import StreamcloudIE
 from .streamcz import StreamCZIE
 from .streetvoice import StreetVoiceIE
diff --git a/youtube_dl/extractor/streamango.py b/youtube_dl/extractor/streamango.py
new file mode 100644
index 000000000..a4ef06b66
--- /dev/null
+++ b/youtube_dl/extractor/streamango.py
@@ -0,0 +1,54 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class StreamangoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?streamango\.com/(?:f|embed)/(?P<id>.+?)/(?:.+)'
+    _TESTS = [{
+        'url': 'https://streamango.com/f/clapasobsptpkdfe/20170315_150006_mp4',
+        'md5': 'e992787515a182f55e38fc97588d802a',
+        'info_dict': {
+            'id': 'clapasobsptpkdfe',
+            'ext': 'mp4',
+            'title': '20170315_150006.mp4',
+            'url': r're:https://streamango\.com/v/d/clapasobsptpkdfe~[0-9]{10}~(?:[0-9]+\.){3}[0-9]+~.{8}/720',
+        }
+    }, {
+        'url': 'https://streamango.com/embed/clapasobsptpkdfe/20170315_150006_mp4',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        def extract_url(urltype):
+            return self._search_regex(
+                r'type\s*:\s*["\']{}["\']\s*,\s*src\s*:\s*["\'](?P<url>.+?)["\'].*'.format(urltype),
+                webpage, 'video URL', group='url')
+
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._og_search_title(webpage)
+        url = 'https:' + extract_url('video/mp4')
+        dashurl = extract_url(r'application/dash\+xml')
+
+        formats = [{
+            'url': url,
+            'ext': 'mp4',
+            'width': 1280,
+            'height': 720,
+            'format_id': 'mp4',
+        }]
+
+        formats.extend(self._extract_mpd_formats(
+            dashurl, video_id, mpd_id='dash', fatal=False))
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'url': url,
+            'title': title,
+            'formats': formats,
+        }

From 8068296276657c9d888338c2211c112d69de6fc4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Apr 2017 21:50:15 +0700
Subject: [PATCH 0658/1696] [streamango] Improve extraction (closes #12643)

---
 youtube_dl/extractor/streamango.py | 50 ++++++++++++++++++------------
 1 file changed, 30 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/streamango.py b/youtube_dl/extractor/streamango.py
index a4ef06b66..aa4fad162 100644
--- a/youtube_dl/extractor/streamango.py
+++ b/youtube_dl/extractor/streamango.py
@@ -1,11 +1,18 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    int_or_none,
+    js_to_json,
+)
 
 
 class StreamangoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?streamango\.com/(?:f|embed)/(?P<id>.+?)/(?:.+)'
+    _VALID_URL = r'https?://(?:www\.)?streamango\.com/(?:f|embed)/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'https://streamango.com/f/clapasobsptpkdfe/20170315_150006_mp4',
         'md5': 'e992787515a182f55e38fc97588d802a',
@@ -13,7 +20,6 @@ class StreamangoIE(InfoExtractor):
             'id': 'clapasobsptpkdfe',
             'ext': 'mp4',
             'title': '20170315_150006.mp4',
-            'url': r're:https://streamango\.com/v/d/clapasobsptpkdfe~[0-9]{10}~(?:[0-9]+\.){3}[0-9]+~.{8}/720',
         }
     }, {
         'url': 'https://streamango.com/embed/clapasobsptpkdfe/20170315_150006_mp4',
@@ -21,29 +27,33 @@ class StreamangoIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        def extract_url(urltype):
-            return self._search_regex(
-                r'type\s*:\s*["\']{}["\']\s*,\s*src\s*:\s*["\'](?P<url>.+?)["\'].*'.format(urltype),
-                webpage, 'video URL', group='url')
-
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
 
         title = self._og_search_title(webpage)
-        url = 'https:' + extract_url('video/mp4')
-        dashurl = extract_url(r'application/dash\+xml')
-
-        formats = [{
-            'url': url,
-            'ext': 'mp4',
-            'width': 1280,
-            'height': 720,
-            'format_id': 'mp4',
-        }]
-
-        formats.extend(self._extract_mpd_formats(
-            dashurl, video_id, mpd_id='dash', fatal=False))
 
+        formats = []
+        for format_ in re.findall(r'({[^}]*\bsrc\s*:\s*[^}]*})', webpage):
+            video = self._parse_json(
+                format_, video_id, transform_source=js_to_json, fatal=False)
+            if not video:
+                continue
+            src = video.get('src')
+            if not src:
+                continue
+            ext = determine_ext(src, default_ext=None)
+            if video.get('type') == 'application/dash+xml' or ext == 'mpd':
+                formats.extend(self._extract_mpd_formats(
+                    src, video_id, mpd_id='dash', fatal=False))
+            else:
+                formats.append({
+                    'url': src,
+                    'ext': ext or 'mp4',
+                    'width': int_or_none(video.get('width')),
+                    'height': int_or_none(video.get('height')),
+                    'tbr': int_or_none(video.get('bitrate')),
+                })
         self._sort_formats(formats)
 
         return {

From 413c1f8e2fc9ef8a9fae2767ef79905510e8c37f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Apr 2017 00:52:15 +0700
Subject: [PATCH 0659/1696] [extractor/generic] Extract RSS entries as
 url_transparent (#11163)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index bc7c21f7a..6a34c2491 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1693,7 +1693,7 @@ class GenericIE(InfoExtractor):
                 continue
 
             entries.append({
-                '_type': 'url',
+                '_type': 'url_transparent',
                 'url': next_url,
                 'title': it.find('title').text,
             })

From 0563f7ac6ed49929e1b488d92439928271c403df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Apr 2017 00:56:53 +0700
Subject: [PATCH 0660/1696] [YoutubeDL] Propagate overridden metadata to IE
 results of type url (closes #11163)

---
 youtube_dl/YoutubeDL.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 7953670a7..3da5200d7 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -851,7 +851,14 @@ class YoutubeDL(object):
             new_result = info.copy()
             new_result.update(force_properties)
 
-            assert new_result.get('_type') != 'url_transparent'
+            # Extracted info may not be a video result (i.e.
+            # info.get('_type', 'video') != video) but rather an url or
+            # url_transparent. In such cases outer metadata (from ie_result)
+            # should be propagated to inner one (info). For this to happen
+            # _type of info should be overridden with url_transparent. This
+            # fixes issue from https://github.com/rg3/youtube-dl/pull/11163.
+            if new_result.get('_type') == 'url':
+                new_result['_type'] = 'url_transparent'
 
             return self.process_ie_result(
                 new_result, download=download, extra_info=extra_info)

From 51350db5a395c45cb181b1813a71b5e1aff86993 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Apr 2017 01:14:05 +0700
Subject: [PATCH 0661/1696] [test_YoutubeDL] Add test for #11163

---
 test/test_YoutubeDL.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 8491a88bd..75945e38f 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -755,6 +755,7 @@ class TestYoutubeDL(unittest.TestCase):
                     '_type': 'url_transparent',
                     'url': 'foo2:',
                     'ie_key': 'Foo2',
+                    'title': 'foo1 title'
                 }
 
         class Foo2IE(InfoExtractor):
@@ -771,7 +772,7 @@ class TestYoutubeDL(unittest.TestCase):
             _VALID_URL = r'foo3:'
 
             def _real_extract(self, url):
-                return _make_result([{'url': TEST_URL}])
+                return _make_result([{'url': TEST_URL}], title='foo3 title')
 
         ydl.add_info_extractor(Foo1IE(ydl))
         ydl.add_info_extractor(Foo2IE(ydl))
@@ -779,6 +780,7 @@ class TestYoutubeDL(unittest.TestCase):
         ydl.extract_info('foo1:')
         downloaded = ydl.downloaded_info_dicts[0]
         self.assertEqual(downloaded['url'], TEST_URL)
+        self.assertEqual(downloaded['title'], 'foo1 title')
 
 
 if __name__ == '__main__':

From d35dc344af8ca0b4cc9ea738a290c04842adb9db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Apr 2017 01:39:07 +0700
Subject: [PATCH 0662/1696] [YoutubeDL] Apply expand_path after output template
 substitution os.path.expandvars translates '%%' into '%' making output
 template invalid. Before: '%%(ext)s' -(expand path)-> '%(ext)s' -(outtmpl
 subst.)-> 'mp4' After: '%%(ext)s' -(outtmpl subst.)-> '%(ext)s' -(expand
 path)-> '%(ext)s'

---
 youtube_dl/YoutubeDL.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 3da5200d7..819b374ef 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -672,8 +672,7 @@ class YoutubeDL(object):
                         FORMAT_RE.format(numeric_field),
                         r'%({0})s'.format(numeric_field), outtmpl)
 
-            tmpl = expand_path(outtmpl)
-            filename = tmpl % template_dict
+            filename = expand_path(outtmpl % template_dict)
             # Temporary fix for #4787
             # 'Treat' all problem characters by passing filename through preferredencoding
             # to workaround encoding issues with subprocess on python2 @ Windows

From 76b5f99617f5b1ecdf64bb6445962f4db9685282 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Apr 2017 01:47:58 +0700
Subject: [PATCH 0663/1696] [ChangeLog] Actualize

---
 ChangeLog | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index cf5ee84a4..34ef32c1d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,19 @@
+version <unreleased>
+
+Core
+* [YoutubeDL] Apply expand_path after output template substitution
++ [YoutubeDL] Propagate overridden meta fields to extraction results of type
+  url (#11163)
+
+Extractors
++ [generic] Extract RSS entries as url_transparent (#11163)
++ [streamango] Add support for streamango.com (#12643)
++ [wsj:article] Add support for articles (#12558)
+* [brightcove] Relax video tag embeds extraction and validate ambiguous embeds'
+  URLs (#9163, #12005, #12178, #12480)
++ [udemy] Add support for react rendition (#12744)
+
+
 version 2017.04.15
 
 Extractors

From 022250a594750cf6d2e000e9ea142146d028bfe9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Apr 2017 01:49:02 +0700
Subject: [PATCH 0664/1696] release 2017.04.16

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 ++
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 70f6b51ed..5d5adb199 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.15*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.15**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.16*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.16**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.04.15
+[debug] youtube-dl version 2017.04.16
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 34ef32c1d..6be86a090 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.04.16
 
 Core
 * [YoutubeDL] Apply expand_path after output template substitution
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index b29b50c8d..afae82214 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -745,6 +745,7 @@
  - **Steam**
  - **Stitcher**
  - **Streamable**
+ - **Streamango**
  - **streamcloud.eu**
  - **StreamCZ**
  - **StreetVoice**
@@ -966,6 +967,7 @@
  - **wrzuta.pl**
  - **wrzuta.pl:playlist**
  - **WSJ**: Wall Street Journal
+ - **WSJArticle**
  - **XBef**
  - **XboxClips**
  - **XFileShare**: XFileShare based sites: DaClips, FileHoot, GorillaVid, MovPod, PowerWatch, Rapidvideo.ws, TheVideoBee, Vidto, Streamin.To, XVIDSTAGE, Vid ABC, VidBom, vidlo
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 612b50f7b..8b01fbc0a 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.04.15'
+__version__ = '2017.04.16'

From 334f41e0d827a361b4b8d39f55b66221281d92ae Mon Sep 17 00:00:00 2001
From: "Jeremie J. Jarosh" <jeremie@jarosh.org>
Date: Sat, 15 Apr 2017 10:12:08 -0500
Subject: [PATCH 0665/1696] [go90] Improve extraction

- add metadata for 'series', 'episode', 'season', 'season_id',
  'season_number', and 'episode_number'
- integrate series title into the title
- extract subtitles  (fallback to `vtt` if the subtitle file type
  detection fails as that is the most likely extension)
---
 youtube_dl/extractor/go90.py | 38 ++++++++++++++++++++++++++++++++++--
 1 file changed, 36 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/go90.py b/youtube_dl/extractor/go90.py
index 3550eca7c..9b2e1c164 100644
--- a/youtube_dl/extractor/go90.py
+++ b/youtube_dl/extractor/go90.py
@@ -5,6 +5,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    determine_ext,
     int_or_none,
     parse_iso8601,
 )
@@ -18,7 +19,7 @@ class Go90IE(InfoExtractor):
         'info_dict': {
             'id': '84BUqjLpf9D',
             'ext': 'mp4',
-            'title': 'Inside The Utah Coalition Against Pornography Convention',
+            'title': 'Daily VICE - Inside The Utah Coalition Against Pornography Convention',
             'description': 'VICE\'s Karley Sciortino meets with activists who discuss the state\'s strong anti-porn stance. Then, VICE Sports explains NFL contracts.',
             'timestamp': 1491868800,
             'upload_date': '20170411',
@@ -32,11 +33,28 @@ class Go90IE(InfoExtractor):
             video_id, headers={
                 'Content-Type': 'application/json; charset=utf-8',
             }, data=b'{"client":"web","device_type":"pc"}')
-        title = video_data['title']
         main_video_asset = video_data['main_video_asset']
 
+        episode_number = int_or_none(video_data.get('episode_number'))
+        series = None
+        season = None
+        season_id = None
+        season_number = None
+        for metadata in video_data.get('__children', {}).get('Item', {}).values():
+            if metadata.get('type') == 'show':
+                series = metadata.get('title')
+            elif metadata.get('type') == 'season':
+                season = metadata.get('title')
+                season_id = metadata.get('id')
+                season_number = int_or_none(metadata.get('season_number'))
+
+        title = episode = video_data.get('title') or series
+        if series and series != title:
+            title = '%s - %s' % (series, title)
+
         thumbnails = []
         formats = []
+        subtitles = {}
         for asset in video_data.get('assets'):
             if asset.get('id') == main_video_asset:
                 for source in asset.get('sources', []):
@@ -70,6 +88,15 @@ class Go90IE(InfoExtractor):
                             'height': int_or_none(source.get('height')),
                             'tbr': int_or_none(source.get('bitrate')),
                         })
+
+                for caption in asset.get('caption_metadata', []):
+                    caption_url = caption.get('source_url')
+                    if not caption_url:
+                        continue
+                    subtitles.setdefault(caption.get('language', 'en'), []).append({
+                        'url': caption_url,
+                        'ext': determine_ext(caption_url, 'vtt'),
+                    })
             elif asset.get('type') == 'image':
                 asset_location = asset.get('location')
                 if not asset_location:
@@ -89,4 +116,11 @@ class Go90IE(InfoExtractor):
             'description': video_data.get('short_description'),
             'like_count': int_or_none(video_data.get('like_count')),
             'timestamp': parse_iso8601(video_data.get('released_at')),
+            'series': series,
+            'episode': episode,
+            'season': season,
+            'season_id': season_id,
+            'season_number': season_number,
+            'episode_number': episode_number,
+            'subtitles': subtitles,
         }

From 1183e22c7e107938a64a3f4d920999ea00871d86 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 15 Apr 2017 23:19:23 +0100
Subject: [PATCH 0666/1696] Credit @triple-j for extracting more metadata from
 go90.com(#12721)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 9abd5fdf4..1bdb74285 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -211,3 +211,4 @@ Juanjo Benages
 Xiao Di Guan
 Thomas Winant
 Daniel Twardowski
+Jeremie Jarosh

From 47e0cef46e9a76e589a2b1cde1b2dfa8bcf01d8e Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Sun, 16 Apr 2017 00:34:34 +0200
Subject: [PATCH 0667/1696] [openload] rewrite extractor

---
 youtube_dl/extractor/openload.py | 110 +++++++++++++++++++------------
 1 file changed, 67 insertions(+), 43 deletions(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index d8036b54a..789bf997e 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -1,12 +1,16 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import os
 import re
+import subprocess
+import tempfile
 
 from .common import InfoExtractor
-from ..compat import compat_chr
 from ..utils import (
+    check_executable,
     determine_ext,
+    encodeArgument,
     ExtractorError,
 )
 
@@ -58,6 +62,39 @@ class OpenloadIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    _PHANTOMJS_SCRIPT = r'''
+        phantom.onError = function(msg, trace) {
+          var msgStack = ['PHANTOM ERROR: ' + msg];
+          if(trace && trace.length) {
+            msgStack.push('TRACE:');
+            trace.forEach(function(t) {
+              msgStack.push(' -> ' + (t.file || t.sourceURL) + ': ' + t.line
+                + (t.function ? ' (in function ' + t.function +')' : ''));
+            });
+          }
+          console.error(msgStack.join('\n'));
+          phantom.exit(1);
+        };
+        var page = require('webpage').create();
+        page.settings.resourceTimeout = 10000;
+        page.onInitialized = function() {
+          page.evaluate(function() {
+            delete window._phantom;
+            delete window.callPhantom;
+          });
+        };
+        page.open('https://openload.co/embed/%s/', function(status) {
+          var info = page.evaluate(function() {
+            return {
+              decoded_id: document.getElementById('streamurl').innerHTML,
+              title: document.querySelector('meta[name="og:title"],'
+                + 'meta[name=description]').content
+            };
+          });
+          console.log(info.decoded_id + ' ' + info.title);
+          phantom.exit();
+        });'''
+
     @staticmethod
     def _extract_urls(webpage):
         return re.findall(
@@ -65,61 +102,48 @@ class OpenloadIE(InfoExtractor):
             webpage)
 
     def _real_extract(self, url):
+        exe = check_executable('phantomjs', ['-v'])
+        if not exe:
+            raise ExtractorError('PhantomJS executable not found in PATH, '
+                                 'download it from http://phantomjs.org',
+                                 expected=True)
+
         video_id = self._match_id(url)
-        webpage = self._download_webpage('https://openload.co/embed/%s/' % video_id, video_id)
+        url = 'https://openload.co/embed/%s/' % video_id
+        webpage = self._download_webpage(url, video_id)
 
         if 'File not found' in webpage or 'deleted by the owner' in webpage:
-            raise ExtractorError('File not found', expected=True)
+            raise ExtractorError('File not found', expected=True, video_id=video_id)
 
-        ol_id = self._search_regex(
-            '<span[^>]+id="[^"]+"[^>]*>([0-9A-Za-z]+)</span>',
-            webpage, 'openload ID')
+        script_file = tempfile.NamedTemporaryFile(mode='w', delete=False)
 
-        decoded = ''
-        a = ol_id[0:24]
-        b = []
-        for i in range(0, len(a), 8):
-            b.append(int(a[i:i + 8] or '0', 16))
-        ol_id = ol_id[24:]
-        j = 0
-        k = 0
-        while j < len(ol_id):
-            c = 128
-            d = 0
-            e = 0
-            f = 0
-            _more = True
-            while _more:
-                if j + 1 >= len(ol_id):
-                    c = 143
-                f = int(ol_id[j:j + 2] or '0', 16)
-                j += 2
-                d += (f & 127) << e
-                e += 7
-                _more = f >= c
-            g = d ^ b[k % 3]
-            for i in range(4):
-                char_dec = (g >> 8 * i) & (c + 127)
-                char = compat_chr(char_dec)
-                if char != '#':
-                    decoded += char
-            k += 1
+        # write JS script to file and close it
+        with script_file:
+            script_file.write(self._PHANTOMJS_SCRIPT % video_id)
 
-        video_url = 'https://openload.co/stream/%s?mime=true'
-        video_url = video_url % decoded
+        self.to_screen('%s: Decoding video ID with PhantomJS' % video_id)
 
-        title = self._og_search_title(webpage, default=None) or self._search_regex(
-            r'<span[^>]+class=["\']title["\'][^>]*>([^<]+)', webpage,
-            'title', default=None) or self._html_search_meta(
-            'description', webpage, 'title', fatal=True)
+        p = subprocess.Popen([exe, '--ssl-protocol=any', script_file.name],
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        output, err = p.communicate()
+        if p.returncode != 0:
+            raise ExtractorError('Decoding failed\n:'
+                                 + encodeArgument(err))
+        else:
+            decoded_id, title = encodeArgument(output).strip().split(' ', 1)
+
+        os.remove(script_file.name)
+
+        video_url = 'https://openload.co/stream/%s?mime=true' % decoded_id
 
         entries = self._parse_html5_media_entries(url, webpage, video_id)
-        subtitles = entries[0]['subtitles'] if entries else None
+        entry = entries[0] if entries else {}
+        subtitles = entry.get('subtitles')
 
         info_dict = {
             'id': video_id,
             'title': title,
-            'thumbnail': self._og_search_thumbnail(webpage, default=None),
+            'thumbnail': entry.get('thumbnail') or self._og_search_thumbnail(webpage, default=None),
             'url': video_url,
             # Seems all videos have extensions in their titles
             'ext': determine_ext(title, 'mp4'),

From 5935ef3c5d9c70c412a0c49f03feb3d0da8dedc6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Apr 2017 21:52:07 +0700
Subject: [PATCH 0668/1696] [itv] Lower preference for rtmp formats (closes
 #12759)

---
 youtube_dl/extractor/itv.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py
index 021c6b278..7f24f14ce 100644
--- a/youtube_dl/extractor/itv.py
+++ b/youtube_dl/extractor/itv.py
@@ -122,6 +122,8 @@ class ITVIE(InfoExtractor):
                 'play_path': play_path,
                 'tbr': tbr,
                 'ext': 'flv',
+                # rtmp formats are now stop downloading at ~72MiB
+                'preference': -10,
             })
 
         ios_playlist_url = params.get('data-video-playlist')

From f67177cae8a2e616d265bbbafe1ceabc7df8dbd0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Apr 2017 21:52:45 +0700
Subject: [PATCH 0669/1696] [itv] Use native hls

---
 youtube_dl/extractor/itv.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py
index 7f24f14ce..5ff7e1aaa 100644
--- a/youtube_dl/extractor/itv.py
+++ b/youtube_dl/extractor/itv.py
@@ -174,7 +174,9 @@ class ITVIE(InfoExtractor):
                         href = ios_base_url + href
                     ext = determine_ext(href)
                     if ext == 'm3u8':
-                        formats.extend(self._extract_m3u8_formats(href, video_id, 'mp4', m3u8_id='hls', fatal=False))
+                        formats.extend(self._extract_m3u8_formats(
+                            href, video_id, 'mp4', entry_protocol='m3u8_native',
+                            m3u8_id='hls', fatal=False))
                     else:
                         formats.append({
                             'url': href,

From c2d7d76efd37c16d01100635f308f03deb9c04aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Apr 2017 23:15:24 +0700
Subject: [PATCH 0670/1696] [itv] Fix rtmp formats (#12759)

---
 youtube_dl/extractor/itv.py | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py
index 5ff7e1aaa..7442c24d9 100644
--- a/youtube_dl/extractor/itv.py
+++ b/youtube_dl/extractor/itv.py
@@ -116,15 +116,25 @@ class ITVIE(InfoExtractor):
             if not play_path:
                 continue
             tbr = int_or_none(media_file.get('bitrate'), 1000)
-            formats.append({
+            f = {
                 'format_id': 'rtmp' + ('-%d' % tbr if tbr else ''),
-                'url': rtmp_url,
                 'play_path': play_path,
+                # Providing this swfVfy allows to avoid truncated downloads
+                'player_url': 'http://www.itv.com/mercury/Mercury_VideoPlayer.swf',
+                'page_url': url,
                 'tbr': tbr,
                 'ext': 'flv',
-                # rtmp formats are now stop downloading at ~72MiB
-                'preference': -10,
-            })
+            }
+            app = self._search_regex(
+                'rtmpe?://[^/]+/(.+)$', rtmp_url, 'app', default=None)
+            if app:
+                f.update({
+                    'url': rtmp_url.split('?', 1)[0],
+                    'app': app,
+                })
+            else:
+                f['url'] = rtmp_url
+            formats.append(f)
 
         ios_playlist_url = params.get('data-video-playlist')
         hmac = params.get('data-video-hmac')

From 751c89a27d68c54375e96789cc90d4c8a3ce3dbc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Apr 2017 23:19:20 +0700
Subject: [PATCH 0671/1696] [itv] Extract series metadata

---
 youtube_dl/extractor/itv.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py
index 7442c24d9..f3156804d 100644
--- a/youtube_dl/extractor/itv.py
+++ b/youtube_dl/extractor/itv.py
@@ -203,7 +203,8 @@ class ITVIE(InfoExtractor):
                 'ext': 'ttml' if ext == 'xml' else ext,
             })
 
-        return {
+        info = self._search_json_ld(webpage, video_id, default={})
+        info.update({
             'id': video_id,
             'title': title,
             'formats': formats,
@@ -212,4 +213,5 @@ class ITVIE(InfoExtractor):
             'episode_number': int_or_none(xpath_text(playlist, 'EpisodeNumber')),
             'series': xpath_text(playlist, 'ProgrammeTitle'),
             'duartion': parse_duration(xpath_text(playlist, 'Duration')),
-        }
+        })
+        return info

From e5d39886ec8e4e40b2b7257d16cc5d8505cc1f69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Apr 2017 00:23:16 +0700
Subject: [PATCH 0672/1696] [limelight] Improve embeds extraction (closes
 #12761) * Move extraction code to extractor * Add extraction for
 LimelightEmbeddedPlayerFlash embeds * Extract multiple video

---
 youtube_dl/extractor/generic.py   |  6 +++++
 youtube_dl/extractor/limelight.py | 37 +++++++++++++++++++++++++++++++
 2 files changed, 43 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6a34c2491..c523abb25 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -85,6 +85,7 @@ from .ustream import UstreamIE
 from .openload import OpenloadIE
 from .videopress import VideoPressIE
 from .rutube import RutubeIE
+from .limelight import LimelightBaseIE
 
 
 class GenericIE(InfoExtractor):
@@ -2483,6 +2484,11 @@ class GenericIE(InfoExtractor):
             return self.url_result(piksel_url, PikselIE.ie_key())
 
         # Look for Limelight embeds
+        limelight_urls = LimelightBaseIE._extract_urls(webpage, url)
+        if limelight_urls:
+            return self.playlist_result(
+                limelight_urls, video_id, video_title, video_description)
+
         mobj = re.search(r'LimelightPlayer\.doLoad(Media|Channel|ChannelList)\(["\'](?P<id>[a-z0-9]{32})', webpage)
         if mobj:
             lm = {
diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index f52c2e169..0041453af 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -9,6 +9,7 @@ from ..utils import (
     determine_ext,
     float_or_none,
     int_or_none,
+    smuggle_url,
     unsmuggle_url,
     ExtractorError,
 )
@@ -18,6 +19,42 @@ class LimelightBaseIE(InfoExtractor):
     _PLAYLIST_SERVICE_URL = 'http://production-ps.lvp.llnw.net/r/PlaylistService/%s/%s/%s'
     _API_URL = 'http://api.video.limelight.com/rest/organizations/%s/%s/%s/%s.json'
 
+    @classmethod
+    def _extract_urls(cls, webpage, source_url):
+        lm = {
+            'Media': 'media',
+            'Channel': 'channel',
+            'ChannelList': 'channel_list',
+        }
+        entries = []
+        for kind, video_id in re.findall(
+                r'LimelightPlayer\.doLoad(Media|Channel|ChannelList)\(["\'](?P<id>[a-z0-9]{32})',
+                webpage):
+            print('video_id', video_id)
+            entries.append(cls.url_result(
+                smuggle_url(
+                    'limelight:%s:%s' % (lm[kind], video_id),
+                    {'source_url': source_url}),
+                'Limelight%s' % kind, video_id))
+        for mobj in re.finditer(
+                # As per [1] class attribute should be exactly equal to
+                # LimelightEmbeddedPlayerFlash but numerous examples seen
+                # that don't exactly match it (e.g. [2]).
+                # 1. http://support.3playmedia.com/hc/en-us/articles/227732408-Limelight-Embedding-the-Captions-Plugin-with-the-Limelight-Player-on-Your-Webpage
+                # 2. http://www.sedona.com/FacilitatorTraining2017
+                r'''(?sx)
+                    <object[^>]+class=(["\'])(?:(?!\1).)*\bLimelightEmbeddedPlayerFlash\b(?:(?!\1).)*\1[^>]*>.*?
+                        <param[^>]+
+                            name=(["\'])flashVars\2[^>]+
+                            value=(["\'])(?:(?!\3).)*mediaId=(?P<id>[a-z0-9]{32})
+                ''', webpage):
+            entries.append(cls.url_result(
+                smuggle_url(
+                    'limelight:media:%s' % mobj.group('id'),
+                    {'source_url': source_url}),
+                'LimelightMedia', mobj.group('id')))
+        return entries
+
     def _call_playlist_service(self, item_id, method, fatal=True, referer=None):
         headers = {}
         if referer:

From 91bc57e4c527f918c7735d67f4bceb3cf1e2332c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Apr 2017 00:33:47 +0700
Subject: [PATCH 0673/1696] [limelight] Add support for channels and channels
 lists embedded using LimelightEmbeddedPlayerFlash (#12761)

---
 youtube_dl/extractor/limelight.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index 0041453af..0a5a3956c 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -30,7 +30,6 @@ class LimelightBaseIE(InfoExtractor):
         for kind, video_id in re.findall(
                 r'LimelightPlayer\.doLoad(Media|Channel|ChannelList)\(["\'](?P<id>[a-z0-9]{32})',
                 webpage):
-            print('video_id', video_id)
             entries.append(cls.url_result(
                 smuggle_url(
                     'limelight:%s:%s' % (lm[kind], video_id),
@@ -46,13 +45,14 @@ class LimelightBaseIE(InfoExtractor):
                     <object[^>]+class=(["\'])(?:(?!\1).)*\bLimelightEmbeddedPlayerFlash\b(?:(?!\1).)*\1[^>]*>.*?
                         <param[^>]+
                             name=(["\'])flashVars\2[^>]+
-                            value=(["\'])(?:(?!\3).)*mediaId=(?P<id>[a-z0-9]{32})
+                            value=(["\'])(?:(?!\3).)*(?P<kind>media|channel(?:List)?)Id=(?P<id>[a-z0-9]{32})
                 ''', webpage):
+            kind, video_id = mobj.group('kind'), mobj.group('id')
             entries.append(cls.url_result(
                 smuggle_url(
-                    'limelight:media:%s' % mobj.group('id'),
+                    'limelight:%s:%s' % (kind, video_id),
                     {'source_url': source_url}),
-                'LimelightMedia', mobj.group('id')))
+                'Limelight%s' % kind.capitalize(), video_id))
         return entries
 
     def _call_playlist_service(self, item_id, method, fatal=True, referer=None):

From ab87c2600922bf093fb1fbb81f263c2acada8824 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Apr 2017 00:36:59 +0700
Subject: [PATCH 0674/1696] [extractor/generic] Add test for #12761

---
 youtube_dl/extractor/generic.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c523abb25..b22c3122a 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1652,6 +1652,15 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': [SenateISVPIE.ie_key()],
         },
+        {
+            # Limelight embeds (1 channel embed + 4 media embeds)
+            'url': 'http://www.sedona.com/FacilitatorTraining2017',
+            'info_dict': {
+                'id': 'FacilitatorTraining2017',
+                'title': 'Facilitator Training 2017',
+            },
+            'playlist_mincount': 5,
+        },
         # {
         #     # TODO: find another test
         #     # http://schema.org/VideoObject

From fa7a6e6de66a8d8ce9af09d85a678ca5d91988f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Apr 2017 00:43:56 +0700
Subject: [PATCH 0675/1696] [ChangeLog] Actualize

---
 ChangeLog | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 6be86a090..e923372c8 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,16 @@
+version <unreleased>
+
+Extractors
+* [limelight] Improve extraction LimelightEmbeddedPlayerFlash media embeds and
+  add support for channel and channelList embeds
+* [generic] Extract multiple Limelight embeds (#12761)
++ [itv] Extract series metadata
+* [itv] Fix RTMP formats downloading (#12759)
+* [itv] Use native HLS downloader by default
++ [go90] Extract subtitles (#12752)
++ [go90] Extract series metadata (#12752)
+
+
 version 2017.04.16
 
 Core

From 16a09aefe33ba2d0122926d761f89a93c0c7e7d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Apr 2017 00:46:32 +0700
Subject: [PATCH 0676/1696] release 2017.04.17

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 5d5adb199..45f8db721 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.16*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.16**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.17*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.17**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.04.16
+[debug] youtube-dl version 2017.04.17
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index e923372c8..03ef0363b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.04.17
 
 Extractors
 * [limelight] Improve extraction LimelightEmbeddedPlayerFlash media embeds and
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 8b01fbc0a..480e04e06 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.04.16'
+__version__ = '2017.04.17'

From 1c35b3da44a087890b7144e5e626a573f2a80bf7 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 16 Apr 2017 21:24:34 +0100
Subject: [PATCH 0677/1696] [odnoklassniki] extract m3u8 formats

---
 youtube_dl/extractor/odnoklassniki.py | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 0ee56a45b..b9369cea5 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_etree_fromstring,
     compat_parse_qs,
     compat_urllib_parse_unquote,
     compat_urllib_parse_urlparse,
@@ -176,14 +177,32 @@ class OdnoklassnikiIE(InfoExtractor):
             })
             return info
 
-        quality = qualities(('mobile', 'lowest', 'low', 'sd', 'hd', 'full'))
+        quality = qualities(('4', '0', '1', '2', '3', '5'))
 
         formats = [{
             'url': f['url'],
             'ext': 'mp4',
             'format_id': f['name'],
-            'quality': quality(f['name']),
         } for f in metadata['videos']]
+
+        m3u8_url = metadata.get('hlsManifestUrl')
+        if m3u8_url:
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', 'm3u8_native',
+                m3u8_id='hls', fatal=False))
+
+        dash_manifest = metadata.get('metadataEmbedded')
+        if dash_manifest:
+            formats.extend(self._parse_mpd_formats(
+                compat_etree_fromstring(dash_manifest), 'mpd'))
+
+        for fmt in formats:
+            fmt_type = self._search_regex(
+                r'\btype[/=](\d)', fmt['url'],
+                'format type', default=None)
+            if fmt_type:
+                fmt['quality'] = quality(fmt_type)
+
         self._sort_formats(formats)
 
         info['formats'] = formats

From bf1b87cd919f07b7fef204838be73981e122ee11 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 17 Apr 2017 08:48:24 +0100
Subject: [PATCH 0678/1696] [common] Relax JWPlayer regex and remove duplicate
 urls(#12768)

---
 youtube_dl/extractor/common.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index dcc9d628a..12e010a0d 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2182,7 +2182,7 @@ class InfoExtractor(object):
 
     def _find_jwplayer_data(self, webpage, video_id=None, transform_source=js_to_json):
         mobj = re.search(
-            r'jwplayer\((?P<quote>[\'"])[^\'" ]+(?P=quote)\)\.setup\s*\((?P<options>[^)]+)\)',
+            r'(?s)jwplayer\((?P<quote>[\'"])[^\'" ]+(?P=quote)\).*?\.setup\s*\((?P<options>[^)]+)\)',
             webpage)
         if mobj:
             try:
@@ -2258,11 +2258,17 @@ class InfoExtractor(object):
 
     def _parse_jwplayer_formats(self, jwplayer_sources_data, video_id=None,
                                 m3u8_id=None, mpd_id=None, rtmp_params=None, base_url=None):
+        urls = []
         formats = []
         for source in jwplayer_sources_data:
-            source_url = self._proto_relative_url(source['file'])
+            source_url = self._proto_relative_url(source.get('file'))
+            if not source_url:
+                continue
             if base_url:
                 source_url = compat_urlparse.urljoin(base_url, source_url)
+            if source_url in urls:
+                continue
+            urls.append(source_url)
             source_type = source.get('type') or ''
             ext = mimetype2ext(source_type) or determine_ext(source_url)
             if source_type == 'hls' or ext == 'm3u8':

From f631b557915eae6c72f2f503255903c92481e85e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 18 Apr 2017 21:46:25 +0700
Subject: [PATCH 0679/1696] [brightcove] Fix _extract_url (closes #12782)

---
 youtube_dl/extractor/brightcove.py | 4 ++--
 youtube_dl/extractor/nowness.py    | 2 +-
 youtube_dl/extractor/yahoo.py      | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 124497e95..c4fd96b3d 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -484,8 +484,8 @@ class BrightcoveNewIE(InfoExtractor):
     }]
 
     @staticmethod
-    def _extract_url(webpage):
-        urls = BrightcoveNewIE._extract_urls(webpage)
+    def _extract_url(ie, webpage):
+        urls = BrightcoveNewIE._extract_urls(ie, webpage)
         return urls[0] if urls else None
 
     @staticmethod
diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index b6c5ee6e4..f26dafb8f 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -28,7 +28,7 @@ class NownessBaseIE(InfoExtractor):
                         bc_url = BrightcoveLegacyIE._extract_brightcove_url(player_code)
                         if bc_url:
                             return self.url_result(bc_url, BrightcoveLegacyIE.ie_key())
-                        bc_url = BrightcoveNewIE._extract_url(player_code)
+                        bc_url = BrightcoveNewIE._extract_url(self, player_code)
                         if bc_url:
                             return self.url_result(bc_url, BrightcoveNewIE.ie_key())
                         raise ExtractorError('Could not find player definition')
diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 4951414e9..38f82bf44 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -258,7 +258,7 @@ class YahooIE(InfoExtractor):
             return self.url_result(bc_url, BrightcoveLegacyIE.ie_key())
 
         # Look for Brightcove New Studio embeds
-        bc_url = BrightcoveNewIE._extract_url(webpage)
+        bc_url = BrightcoveNewIE._extract_url(self, webpage)
         if bc_url:
             return self.url_result(bc_url, BrightcoveNewIE.ie_key())
 

From 06d0ad9a4e2266b1cc74b45a59a53fad3f23fe15 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 18 Apr 2017 23:03:03 +0800
Subject: [PATCH 0680/1696] [brightcove] Support URLs with bcpid instead of
 playerID

Fixes #12482
---
 ChangeLog                          |  7 +++++++
 youtube_dl/extractor/brightcove.py | 20 +++++++++++++++++---
 youtube_dl/extractor/generic.py    | 16 ++++++++++++++++
 3 files changed, 40 insertions(+), 3 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 03ef0363b..f8084fda5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,10 @@
+version <unreleased>
+
+Extractors
++ [generic] Support Brightcove videos in <iframe> (#12482)
++ [brightcove] Support URLs with bcpid instead of playerID (#12482)
+
+
 version 2017.04.17
 
 Extractors
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index c4fd96b3d..a2db705b0 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -131,6 +131,12 @@ class BrightcoveLegacyIE(InfoExtractor):
             },
             'playlist_mincount': 10,
         },
+        {
+            # playerID inferred from bcpid
+            # from http://www.un.org/chinese/News/story.asp?NewsID=27724
+            'url': 'https://link.brightcove.com/services/player/bcpid1722935254001/?bctid=5360463607001&autoStart=false&secureConnections=true&width=650&height=350',
+            'only_matching': True,  # Tested in GenericIE
+        }
     ]
     FLV_VCODECS = {
         1: 'SORENSON',
@@ -266,9 +272,13 @@ class BrightcoveLegacyIE(InfoExtractor):
         if matches:
             return list(filter(None, [cls._build_brighcove_url(m) for m in matches]))
 
-        return list(filter(None, [
-            cls._build_brighcove_url_from_js(custom_bc)
-            for custom_bc in re.findall(r'(customBC\.createVideo\(.+?\);)', webpage)]))
+        matches = re.findall(r'(customBC\.createVideo\(.+?\);)', webpage)
+        if matches:
+            return list(filter(None, [
+                cls._build_brighcove_url_from_js(custom_bc)
+                for custom_bc in matches]))
+        return [url for _, url in re.findall(
+            r'<iframe[^>]+src=([\'"])((?:https?:)?//link\.brightcove\.com/services/player/(?!\1).+)\1', webpage)]
 
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})
@@ -285,6 +295,10 @@ class BrightcoveLegacyIE(InfoExtractor):
         if videoPlayer:
             # We set the original url as the default 'Referer' header
             referer = smuggled_data.get('Referer', url)
+            if 'playerID' not in query:
+                mobj = re.search(r'/bcpid(\d+)', url)
+                if mobj is not None:
+                    query['playerID'] = [mobj.group(1)]
             return self._get_video_info(
                 videoPlayer[0], query, referer=referer)
         elif 'playerKey' in query:
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index b22c3122a..67184bc5d 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -430,6 +430,22 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,  # m3u8 download
             },
         },
+        {
+            # Brightcove video in <iframe>
+            'url': 'http://www.un.org/chinese/News/story.asp?NewsID=27724',
+            'md5': '36d74ef5e37c8b4a2ce92880d208b968',
+            'info_dict': {
+                'id': '5360463607001',
+                'ext': 'mp4',
+                'title': '叙利亚失明儿童在废墟上演唱《心跳》  呼吁获得正常童年生活',
+                'description': '联合国儿童基金会中东和北非区域大使、作曲家扎德·迪拉尼（Zade Dirani）在3月15日叙利亚冲突爆发7周年纪念日之际发布了为叙利亚谱写的歌曲《心跳》（HEARTBEAT），为受到六年冲突影响的叙利亚儿童发出强烈呐喊，呼吁世界做出共同努力，使叙利亚儿童重新获得享有正常童年生活的权利。',
+                'uploader': 'United Nations',
+                'uploader_id': '1362235914001',
+                'timestamp': 1489593889,
+                'upload_date': '20170315',
+            },
+            'add_ie': ['BrightcoveLegacy'],
+        },
         {
             # Brightcove with alternative playerID key
             'url': 'http://www.nature.com/nmeth/journal/v9/n7/fig_tab/nmeth.2062_SV1.html',

From bae1404893341ed89f4c9b556aa4068c13ed9f7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 18 Apr 2017 22:21:38 +0700
Subject: [PATCH 0681/1696] [extractor/common] Add support for video of WebPage
 context in _json_ld (closes #12778)

---
 youtube_dl/extractor/common.py | 33 +++++++++++++++++++++------------
 1 file changed, 21 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 12e010a0d..61d97ab72 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -976,6 +976,22 @@ class InfoExtractor(object):
             return info
         if isinstance(json_ld, dict):
             json_ld = [json_ld]
+
+        def extract_video_object(e):
+            assert e['@type'] == 'VideoObject'
+            info.update({
+                'url': e.get('contentUrl'),
+                'title': unescapeHTML(e.get('name')),
+                'description': unescapeHTML(e.get('description')),
+                'thumbnail': e.get('thumbnailUrl') or e.get('thumbnailURL'),
+                'duration': parse_duration(e.get('duration')),
+                'timestamp': unified_timestamp(e.get('uploadDate')),
+                'filesize': float_or_none(e.get('contentSize')),
+                'tbr': int_or_none(e.get('bitrate')),
+                'width': int_or_none(e.get('width')),
+                'height': int_or_none(e.get('height')),
+            })
+
         for e in json_ld:
             if e.get('@context') == 'http://schema.org':
                 item_type = e.get('@type')
@@ -1000,18 +1016,11 @@ class InfoExtractor(object):
                         'description': unescapeHTML(e.get('articleBody')),
                     })
                 elif item_type == 'VideoObject':
-                    info.update({
-                        'url': e.get('contentUrl'),
-                        'title': unescapeHTML(e.get('name')),
-                        'description': unescapeHTML(e.get('description')),
-                        'thumbnail': e.get('thumbnailUrl') or e.get('thumbnailURL'),
-                        'duration': parse_duration(e.get('duration')),
-                        'timestamp': unified_timestamp(e.get('uploadDate')),
-                        'filesize': float_or_none(e.get('contentSize')),
-                        'tbr': int_or_none(e.get('bitrate')),
-                        'width': int_or_none(e.get('width')),
-                        'height': int_or_none(e.get('height')),
-                    })
+                    extract_video_object(e)
+                elif item_type == 'WebPage':
+                    video = e.get('video')
+                    if isinstance(video, dict) and video.get('@type') == 'VideoObject':
+                        extract_video_object(video)
                 break
         return dict((k, v) for k, v in info.items() if v is not None)
 

From a32a9a7ef5f64bcec29ef71c8e5b6dc509232c13 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 18 Apr 2017 22:39:58 +0700
Subject: [PATCH 0682/1696] [extractor/common] Add support multiple getters in
 try_get

---
 youtube_dl/utils.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 2340bc306..79a337df4 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2103,13 +2103,16 @@ def dict_get(d, key_or_keys, default=None, skip_false_values=True):
 
 
 def try_get(src, getter, expected_type=None):
-    try:
-        v = getter(src)
-    except (AttributeError, KeyError, TypeError, IndexError):
-        pass
-    else:
-        if expected_type is None or isinstance(v, expected_type):
-            return v
+    if not isinstance(getter, (list, tuple)):
+        getter = [getter]
+    for get in getter:
+        try:
+            v = get(src)
+        except (AttributeError, KeyError, TypeError, IndexError):
+            pass
+        else:
+            if expected_type is None or isinstance(v, expected_type):
+                return v
 
 
 def encode_compat_str(string, encoding=preferredencoding(), errors='strict'):

From 18848d226a254a20df2b772ac3c881caccf2f260 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 18 Apr 2017 22:40:26 +0700
Subject: [PATCH 0683/1696] [instagram] Fix extraction (closes #12777)

---
 youtube_dl/extractor/instagram.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index c1921cbcf..4667335e0 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -112,7 +112,8 @@ class InstagramIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         (video_url, description, thumbnail, timestamp, uploader,
-         uploader_id, like_count, comment_count, height, width) = [None] * 10
+         uploader_id, like_count, comment_count, comments, height,
+         width) = [None] * 11
 
         shared_data = self._parse_json(
             self._search_regex(
@@ -121,7 +122,10 @@ class InstagramIE(InfoExtractor):
             video_id, fatal=False)
         if shared_data:
             media = try_get(
-                shared_data, lambda x: x['entry_data']['PostPage'][0]['media'], dict)
+                shared_data,
+                (lambda x: x['entry_data']['PostPage'][0]['graphql']['shortcode_media'],
+                 lambda x: x['entry_data']['PostPage'][0]['media']),
+                dict)
             if media:
                 video_url = media.get('video_url')
                 height = int_or_none(media.get('dimensions', {}).get('height'))

From 2dc48df5bc52a5c64d0fc699189fbcb162d62d9b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 18 Apr 2017 23:54:36 +0700
Subject: [PATCH 0684/1696] [xfileshare] Add support for gorillavid.com and
 daclips.com (closes #12776)

---
 youtube_dl/extractor/xfileshare.py | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index 1b5cd122a..13f8be6cb 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -17,24 +17,24 @@ from ..utils import (
 
 class XFileShareIE(InfoExtractor):
     _SITES = (
-        ('daclips.in', 'DaClips'),
-        ('filehoot.com', 'FileHoot'),
-        ('gorillavid.in', 'GorillaVid'),
-        ('movpod.in', 'MovPod'),
-        ('powerwatch.pw', 'PowerWatch'),
-        ('rapidvideo.ws', 'Rapidvideo.ws'),
-        ('thevideobee.to', 'TheVideoBee'),
-        ('vidto.me', 'Vidto'),
-        ('streamin.to', 'Streamin.To'),
-        ('xvidstage.com', 'XVIDSTAGE'),
-        ('vidabc.com', 'Vid ABC'),
-        ('vidbom.com', 'VidBom'),
-        ('vidlo.us', 'vidlo'),
+        (r'daclips\.(?:in|com)', 'DaClips'),
+        (r'filehoot\.com', 'FileHoot'),
+        (r'gorillavid\.(?:in|com)', 'GorillaVid'),
+        (r'movpod\.in', 'MovPod'),
+        (r'powerwatch\.pw', 'PowerWatch'),
+        (r'rapidvideo\.ws', 'Rapidvideo.ws'),
+        (r'thevideobee\.to', 'TheVideoBee'),
+        (r'vidto\.me', 'Vidto'),
+        (r'streamin\.to', 'Streamin.To'),
+        (r'xvidstage\.com', 'XVIDSTAGE'),
+        (r'vidabc\.com', 'Vid ABC'),
+        (r'vidbom\.com', 'VidBom'),
+        (r'vidlo\.us', 'vidlo'),
     )
 
     IE_DESC = 'XFileShare based sites: %s' % ', '.join(list(zip(*_SITES))[1])
     _VALID_URL = (r'https?://(?P<host>(?:www\.)?(?:%s))/(?:embed-)?(?P<id>[0-9a-zA-Z]+)'
-                  % '|'.join(re.escape(site) for site in list(zip(*_SITES))[0]))
+                  % '|'.join(site for site in list(zip(*_SITES))[0]))
 
     _FILE_NOT_FOUND_REGEXES = (
         r'>(?:404 - )?File Not Found<',

From 58f6ab72ed74c2ab1b6d3885ef2c407465d8c6c2 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 19 Apr 2017 00:16:31 +0100
Subject: [PATCH 0685/1696] [odnoklassniki] update tests

---
 youtube_dl/extractor/odnoklassniki.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index b9369cea5..854b6800c 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -38,7 +38,7 @@ class OdnoklassnikiIE(InfoExtractor):
     }, {
         # metadataUrl
         'url': 'http://ok.ru/video/63567059965189-0?fromTime=5',
-        'md5': '9676cf86eff5391d35dea675d224e131',
+        'md5': '6ff470ea2dd51d5d18c295a355b0b6bc',
         'info_dict': {
             'id': '63567059965189-0',
             'ext': 'mp4',
@@ -54,7 +54,7 @@ class OdnoklassnikiIE(InfoExtractor):
     }, {
         # YouTube embed (metadataUrl, provider == USER_YOUTUBE)
         'url': 'http://ok.ru/video/64211978996595-1',
-        'md5': '5d7475d428845cd2e13bae6f1a992278',
+        'md5': '2f206894ffb5dbfcce2c5a14b909eea5',
         'info_dict': {
             'id': '64211978996595-1',
             'ext': 'mp4',
@@ -62,8 +62,8 @@ class OdnoklassnikiIE(InfoExtractor):
             'description': 'md5:848eb8b85e5e3471a3a803dae1343ed0',
             'duration': 440,
             'upload_date': '20150826',
-            'uploader_id': '750099571',
-            'uploader': 'Алина П',
+            'uploader_id': 'tvroscosmos',
+            'uploader': 'Телестудия Роскосмоса',
             'age_limit': 0,
         },
     }, {
@@ -82,6 +82,7 @@ class OdnoklassnikiIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+        'skip': 'Video has not been found',
     }, {
         'url': 'http://ok.ru/web-api/video/moviePlayer/20079905452',
         'only_matching': True,

From 75a24854073e590f4efc9f037b57dee348f52b61 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 28 Jun 2016 18:07:50 +0100
Subject: [PATCH 0686/1696] [fragment,hls,f4m,dash,ism] improve fragment
 downloading

- resume immediately
- no need to concatenate segments and decrypt them on every resume
- no need to save temp files for segments

and for hls downloader:
- no need to download keys for segments that already downloaded
---
 youtube_dl/downloader/common.py   | 29 ++++++++--------
 youtube_dl/downloader/dash.py     | 43 ++++++-----------------
 youtube_dl/downloader/f4m.py      | 33 ++++++------------
 youtube_dl/downloader/fragment.py | 57 +++++++++++++++++++++++++------
 youtube_dl/downloader/hls.py      | 34 +++++++-----------
 youtube_dl/downloader/http.py     |  8 +++--
 youtube_dl/downloader/ism.py      | 30 +++++-----------
 7 files changed, 111 insertions(+), 123 deletions(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 2c4470a95..fdb77b620 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -327,21 +327,22 @@ class FileDownloader(object):
             os.path.exists(encodeFilename(filename))
         )
 
-        continuedl_and_exists = (
-            self.params.get('continuedl', True) and
-            os.path.isfile(encodeFilename(filename)) and
-            not self.params.get('nopart', False)
-        )
+        if not hasattr(filename, 'write'):
+            continuedl_and_exists = (
+                self.params.get('continuedl', True) and
+                os.path.isfile(encodeFilename(filename)) and
+                not self.params.get('nopart', False)
+            )
 
-        # Check file already present
-        if filename != '-' and (nooverwrites_and_exists or continuedl_and_exists):
-            self.report_file_already_downloaded(filename)
-            self._hook_progress({
-                'filename': filename,
-                'status': 'finished',
-                'total_bytes': os.path.getsize(encodeFilename(filename)),
-            })
-            return True
+            # Check file already present
+            if filename != '-' and (nooverwrites_and_exists or continuedl_and_exists):
+                self.report_file_already_downloaded(filename)
+                self._hook_progress({
+                    'filename': filename,
+                    'status': 'finished',
+                    'total_bytes': os.path.getsize(encodeFilename(filename)),
+                })
+                return True
 
         min_sleep_interval = self.params.get('sleep_interval')
         if min_sleep_interval:
diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index e2ddc369e..94a13a543 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -1,13 +1,7 @@
 from __future__ import unicode_literals
 
-import os
-
 from .fragment import FragmentFD
 from ..compat import compat_urllib_error
-from ..utils import (
-    sanitize_open,
-    encodeFilename,
-)
 
 
 class DashSegmentsFD(FragmentFD):
@@ -28,31 +22,24 @@ class DashSegmentsFD(FragmentFD):
 
         self._prepare_and_start_frag_download(ctx)
 
-        segments_filenames = []
-
         fragment_retries = self.params.get('fragment_retries', 0)
         skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
 
-        def process_segment(segment, tmp_filename, num):
-            segment_url = segment['url']
-            segment_name = 'Frag%d' % num
-            target_filename = '%s-%s' % (tmp_filename, segment_name)
+        frag_index = 0
+        for i, segment in enumerate(segments):
+            frag_index += 1
+            if frag_index <= ctx['frag_index']:
+                continue
             # In DASH, the first segment contains necessary headers to
             # generate a valid MP4 file, so always abort for the first segment
-            fatal = num == 0 or not skip_unavailable_fragments
+            fatal = i == 0 or not skip_unavailable_fragments
             count = 0
             while count <= fragment_retries:
                 try:
-                    success = ctx['dl'].download(target_filename, {
-                        'url': segment_url,
-                        'http_headers': info_dict.get('http_headers'),
-                    })
+                    success, frag_content = self._download_fragment(ctx, segment['url'], info_dict)
                     if not success:
                         return False
-                    down, target_sanitized = sanitize_open(target_filename, 'rb')
-                    ctx['dest_stream'].write(down.read())
-                    down.close()
-                    segments_filenames.append(target_sanitized)
+                    self._append_fragment(ctx, frag_content)
                     break
                 except compat_urllib_error.HTTPError as err:
                     # YouTube may often return 404 HTTP error for a fragment causing the
@@ -63,22 +50,14 @@ class DashSegmentsFD(FragmentFD):
                     # HTTP error.
                     count += 1
                     if count <= fragment_retries:
-                        self.report_retry_fragment(err, segment_name, count, fragment_retries)
+                        self.report_retry_fragment(err, frag_index, count, fragment_retries)
             if count > fragment_retries:
                 if not fatal:
-                    self.report_skip_fragment(segment_name)
-                    return True
+                    self.report_skip_fragment(frag_index)
+                    continue
                 self.report_error('giving up after %s fragment retries' % fragment_retries)
                 return False
-            return True
-
-        for i, segment in enumerate(segments):
-            if not process_segment(segment, ctx['tmpfilename'], i):
-                return False
 
         self._finish_frag_download(ctx)
 
-        for segment_file in segments_filenames:
-            os.remove(encodeFilename(segment_file))
-
         return True
diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 688e086eb..e456ed58f 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -3,7 +3,6 @@ from __future__ import division, unicode_literals
 import base64
 import io
 import itertools
-import os
 import time
 
 from .fragment import FragmentFD
@@ -16,9 +15,7 @@ from ..compat import (
     compat_struct_unpack,
 )
 from ..utils import (
-    encodeFilename,
     fix_xml_ampersands,
-    sanitize_open,
     xpath_text,
 )
 
@@ -366,17 +363,21 @@ class F4mFD(FragmentFD):
 
         dest_stream = ctx['dest_stream']
 
-        write_flv_header(dest_stream)
-        if not live:
-            write_metadata_tag(dest_stream, metadata)
+        if ctx['complete_frags_downloaded_bytes'] == 0:
+            write_flv_header(dest_stream)
+            if not live:
+                write_metadata_tag(dest_stream, metadata)
 
         base_url_parsed = compat_urllib_parse_urlparse(base_url)
 
         self._start_frag_download(ctx)
 
-        frags_filenames = []
+        frag_index = 0
         while fragments_list:
             seg_i, frag_i = fragments_list.pop(0)
+            frag_index += 1
+            if frag_index <= ctx['frag_index']:
+                continue
             name = 'Seg%d-Frag%d' % (seg_i, frag_i)
             query = []
             if base_url_parsed.query:
@@ -386,17 +387,10 @@ class F4mFD(FragmentFD):
             if info_dict.get('extra_param_to_segment_url'):
                 query.append(info_dict['extra_param_to_segment_url'])
             url_parsed = base_url_parsed._replace(path=base_url_parsed.path + name, query='&'.join(query))
-            frag_filename = '%s-%s' % (ctx['tmpfilename'], name)
             try:
-                success = ctx['dl'].download(frag_filename, {
-                    'url': url_parsed.geturl(),
-                    'http_headers': info_dict.get('http_headers'),
-                })
+                success, down_data = self._download_fragment(ctx, url_parsed.geturl(), info_dict)
                 if not success:
                     return False
-                (down, frag_sanitized) = sanitize_open(frag_filename, 'rb')
-                down_data = down.read()
-                down.close()
                 reader = FlvReader(down_data)
                 while True:
                     try:
@@ -411,12 +405,8 @@ class F4mFD(FragmentFD):
                             break
                         raise
                     if box_type == b'mdat':
-                        dest_stream.write(box_data)
+                        self._append_fragment(ctx, box_data)
                         break
-                if live:
-                    os.remove(encodeFilename(frag_sanitized))
-                else:
-                    frags_filenames.append(frag_sanitized)
             except (compat_urllib_error.HTTPError, ) as err:
                 if live and (err.code == 404 or err.code == 410):
                     # We didn't keep up with the live window. Continue
@@ -436,7 +426,4 @@ class F4mFD(FragmentFD):
 
         self._finish_frag_download(ctx)
 
-        for frag_file in frags_filenames:
-            os.remove(encodeFilename(frag_file))
-
         return True
diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 56f975266..44a3c1040 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -2,6 +2,7 @@ from __future__ import division, unicode_literals
 
 import os
 import time
+import io
 
 from .common import FileDownloader
 from .http import HttpFD
@@ -10,6 +11,7 @@ from ..utils import (
     encodeFilename,
     sanitize_open,
     sanitized_Request,
+    compat_str,
 )
 
 
@@ -30,13 +32,13 @@ class FragmentFD(FileDownloader):
                         Skip unavailable fragments (DASH and hlsnative only)
     """
 
-    def report_retry_fragment(self, err, fragment_name, count, retries):
+    def report_retry_fragment(self, err, frag_index, count, retries):
         self.to_screen(
-            '[download] Got server HTTP error: %s. Retrying fragment %s (attempt %d of %s)...'
-            % (error_to_compat_str(err), fragment_name, count, self.format_retries(retries)))
+            '[download] Got server HTTP error: %s. Retrying fragment %d (attempt %d of %s)...'
+            % (error_to_compat_str(err), frag_index, count, self.format_retries(retries)))
 
-    def report_skip_fragment(self, fragment_name):
-        self.to_screen('[download] Skipping fragment %s...' % fragment_name)
+    def report_skip_fragment(self, frag_index):
+        self.to_screen('[download] Skipping fragment %d...' % frag_index)
 
     def _prepare_url(self, info_dict, url):
         headers = info_dict.get('http_headers')
@@ -46,6 +48,25 @@ class FragmentFD(FileDownloader):
         self._prepare_frag_download(ctx)
         self._start_frag_download(ctx)
 
+    def _download_fragment(self, ctx, frag_url, info_dict, headers=None):
+        down = io.BytesIO()
+        success = ctx['dl'].download(down, {
+            'url': frag_url,
+            'http_headers': headers or info_dict.get('http_headers'),
+        })
+        if not success:
+            return False, None
+        frag_content = down.getvalue()
+        down.close()
+        return True, frag_content
+
+    def _append_fragment(self, ctx, frag_content):
+        ctx['dest_stream'].write(frag_content)
+        if not (ctx.get('live') or ctx['tmpfilename'] == '-'):
+            frag_index_stream, _ = sanitize_open(ctx['tmpfilename'] + '.fragindex', 'w')
+            frag_index_stream.write(compat_str(ctx['frag_index']))
+            frag_index_stream.close()
+
     def _prepare_frag_download(self, ctx):
         if 'live' not in ctx:
             ctx['live'] = False
@@ -66,11 +87,26 @@ class FragmentFD(FileDownloader):
             }
         )
         tmpfilename = self.temp_name(ctx['filename'])
-        dest_stream, tmpfilename = sanitize_open(tmpfilename, 'wb')
+        open_mode = 'wb'
+        resume_len = 0
+        frag_index = 0
+        # Establish possible resume length
+        if os.path.isfile(encodeFilename(tmpfilename)):
+            open_mode = 'ab'
+            resume_len = os.path.getsize(encodeFilename(tmpfilename))
+            if os.path.isfile(encodeFilename(tmpfilename + '.fragindex')):
+                frag_index_stream, _ = sanitize_open(tmpfilename + '.fragindex', 'r')
+                frag_index = int(frag_index_stream.read())
+                frag_index_stream.close()
+        dest_stream, tmpfilename = sanitize_open(tmpfilename, open_mode)
+
         ctx.update({
             'dl': dl,
             'dest_stream': dest_stream,
             'tmpfilename': tmpfilename,
+            'frag_index': frag_index,
+            # Total complete fragments downloaded so far in bytes
+            'complete_frags_downloaded_bytes': resume_len,
         })
 
     def _start_frag_download(self, ctx):
@@ -79,8 +115,8 @@ class FragmentFD(FileDownloader):
         # hook
         state = {
             'status': 'downloading',
-            'downloaded_bytes': 0,
-            'frag_index': 0,
+            'downloaded_bytes': ctx['complete_frags_downloaded_bytes'],
+            'frag_index': ctx['frag_index'],
             'frag_count': total_frags,
             'filename': ctx['filename'],
             'tmpfilename': ctx['tmpfilename'],
@@ -89,8 +125,6 @@ class FragmentFD(FileDownloader):
         start = time.time()
         ctx.update({
             'started': start,
-            # Total complete fragments downloaded so far in bytes
-            'complete_frags_downloaded_bytes': 0,
             # Amount of fragment's bytes downloaded by the time of the previous
             # frag progress hook invocation
             'prev_frag_downloaded_bytes': 0,
@@ -111,6 +145,7 @@ class FragmentFD(FileDownloader):
 
             if s['status'] == 'finished':
                 state['frag_index'] += 1
+                ctx['frag_index'] = state['frag_index']
                 state['downloaded_bytes'] += frag_total_bytes - ctx['prev_frag_downloaded_bytes']
                 ctx['complete_frags_downloaded_bytes'] = state['downloaded_bytes']
                 ctx['prev_frag_downloaded_bytes'] = 0
@@ -132,6 +167,8 @@ class FragmentFD(FileDownloader):
 
     def _finish_frag_download(self, ctx):
         ctx['dest_stream'].close()
+        if os.path.isfile(encodeFilename(ctx['tmpfilename'] + '.fragindex')):
+            os.remove(encodeFilename(ctx['tmpfilename'] + '.fragindex'))
         elapsed = time.time() - ctx['started']
         self.try_rename(ctx['tmpfilename'], ctx['filename'])
         fsize = os.path.getsize(encodeFilename(ctx['filename']))
diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index d0a5f7ba4..9a87d7ca8 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import os.path
 import re
 import binascii
 try:
@@ -18,8 +17,6 @@ from ..compat import (
     compat_struct_pack,
 )
 from ..utils import (
-    encodeFilename,
-    sanitize_open,
     parse_m3u8_attributes,
     update_url_query,
 )
@@ -103,17 +100,18 @@ class HlsFD(FragmentFD):
         media_sequence = 0
         decrypt_info = {'METHOD': 'NONE'}
         byte_range = {}
-        frags_filenames = []
+        frag_index = 0
         for line in s.splitlines():
             line = line.strip()
             if line:
                 if not line.startswith('#'):
+                    frag_index += 1
+                    if frag_index <= ctx['frag_index']:
+                        continue
                     frag_url = (
                         line
                         if re.match(r'^https?://', line)
                         else compat_urlparse.urljoin(man_url, line))
-                    frag_name = 'Frag%d' % i
-                    frag_filename = '%s-%s' % (ctx['tmpfilename'], frag_name)
                     if extra_query:
                         frag_url = update_url_query(frag_url, extra_query)
                     count = 0
@@ -122,15 +120,10 @@ class HlsFD(FragmentFD):
                         headers['Range'] = 'bytes=%d-%d' % (byte_range['start'], byte_range['end'])
                     while count <= fragment_retries:
                         try:
-                            success = ctx['dl'].download(frag_filename, {
-                                'url': frag_url,
-                                'http_headers': headers,
-                            })
+                            success, frag_content = self._download_fragment(
+                                ctx, frag_url, info_dict, headers)
                             if not success:
                                 return False
-                            down, frag_sanitized = sanitize_open(frag_filename, 'rb')
-                            frag_content = down.read()
-                            down.close()
                             break
                         except compat_urllib_error.HTTPError as err:
                             # Unavailable (possibly temporary) fragments may be served.
@@ -139,28 +132,29 @@ class HlsFD(FragmentFD):
                             # https://github.com/rg3/youtube-dl/issues/10448).
                             count += 1
                             if count <= fragment_retries:
-                                self.report_retry_fragment(err, frag_name, count, fragment_retries)
+                                self.report_retry_fragment(err, frag_index, count, fragment_retries)
                     if count > fragment_retries:
                         if skip_unavailable_fragments:
                             i += 1
                             media_sequence += 1
-                            self.report_skip_fragment(frag_name)
+                            self.report_skip_fragment(frag_index)
                             continue
                         self.report_error(
                             'giving up after %s fragment retries' % fragment_retries)
                         return False
                     if decrypt_info['METHOD'] == 'AES-128':
                         iv = decrypt_info.get('IV') or compat_struct_pack('>8xq', media_sequence)
+                        decrypt_info['KEY'] = decrypt_info.get('KEY') or self.ydl.urlopen(decrypt_info['URI']).read()
                         frag_content = AES.new(
                             decrypt_info['KEY'], AES.MODE_CBC, iv).decrypt(frag_content)
-                    ctx['dest_stream'].write(frag_content)
-                    frags_filenames.append(frag_sanitized)
+                    self._append_fragment(ctx, frag_content)
                     # We only download the first fragment during the test
                     if test:
                         break
                     i += 1
                     media_sequence += 1
                 elif line.startswith('#EXT-X-KEY'):
+                    decrypt_url = decrypt_info.get('URI')
                     decrypt_info = parse_m3u8_attributes(line[11:])
                     if decrypt_info['METHOD'] == 'AES-128':
                         if 'IV' in decrypt_info:
@@ -170,7 +164,8 @@ class HlsFD(FragmentFD):
                                 man_url, decrypt_info['URI'])
                         if extra_query:
                             decrypt_info['URI'] = update_url_query(decrypt_info['URI'], extra_query)
-                        decrypt_info['KEY'] = self.ydl.urlopen(decrypt_info['URI']).read()
+                        if decrypt_url != decrypt_info['URI']:
+                            decrypt_info['KEY'] = None
                 elif line.startswith('#EXT-X-MEDIA-SEQUENCE'):
                     media_sequence = int(line[22:])
                 elif line.startswith('#EXT-X-BYTERANGE'):
@@ -183,7 +178,4 @@ class HlsFD(FragmentFD):
 
         self._finish_frag_download(ctx)
 
-        for frag_file in frags_filenames:
-            os.remove(encodeFilename(frag_file))
-
         return True
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index af405b950..2896a17af 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -20,10 +20,14 @@ from ..utils import (
 
 
 class HttpFD(FileDownloader):
-    def real_download(self, filename, info_dict):
+    def real_download(self, filename_or_stream, info_dict):
         url = info_dict['url']
-        tmpfilename = self.temp_name(filename)
+        filename = filename_or_stream
         stream = None
+        if hasattr(filename_or_stream, 'write'):
+            stream = filename_or_stream
+            filename = '-'
+        tmpfilename = self.temp_name(filename)
 
         # Do not include the Accept-Encoding header
         headers = {'Youtubedl-no-compression': 'True'}
diff --git a/youtube_dl/downloader/ism.py b/youtube_dl/downloader/ism.py
index 63a636cb7..9f0fc36b3 100644
--- a/youtube_dl/downloader/ism.py
+++ b/youtube_dl/downloader/ism.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import os
 import time
 import struct
 import binascii
@@ -8,10 +7,6 @@ import io
 
 from .fragment import FragmentFD
 from ..compat import compat_urllib_error
-from ..utils import (
-    sanitize_open,
-    encodeFilename,
-)
 
 
 u8 = struct.Struct(b'>B')
@@ -225,16 +220,15 @@ class IsmFD(FragmentFD):
 
         self._prepare_and_start_frag_download(ctx)
 
-        segments_filenames = []
-
         fragment_retries = self.params.get('fragment_retries', 0)
         skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
 
         track_written = False
+        frag_index = 0
         for i, segment in enumerate(segments):
-            segment_url = segment['url']
-            segment_name = 'Frag%d' % i
-            target_filename = '%s-%s' % (ctx['tmpfilename'], segment_name)
+            frag_index += 1
+            if frag_index <= ctx['frag_index']:
+                continue
             count = 0
             while count <= fragment_retries:
                 try:
@@ -242,33 +236,27 @@ class IsmFD(FragmentFD):
                         'url': segment_url,
                         'http_headers': info_dict.get('http_headers'),
                     })
+                    success, frag_content = self._download_fragment(ctx, segment['url'], info_dict)
                     if not success:
                         return False
-                    down, target_sanitized = sanitize_open(target_filename, 'rb')
-                    down_data = down.read()
                     if not track_written:
-                        tfhd_data = extract_box_data(down_data, [b'moof', b'traf', b'tfhd'])
+                        tfhd_data = extract_box_data(frag_content, [b'moof', b'traf', b'tfhd'])
                         info_dict['_download_params']['track_id'] = u32.unpack(tfhd_data[4:8])[0]
                         write_piff_header(ctx['dest_stream'], info_dict['_download_params'])
                         track_written = True
-                    ctx['dest_stream'].write(down_data)
-                    down.close()
-                    segments_filenames.append(target_sanitized)
+                    self._append_fragment(ctx, frag_content)
                     break
                 except compat_urllib_error.HTTPError as err:
                     count += 1
                     if count <= fragment_retries:
-                        self.report_retry_fragment(err, segment_name, count, fragment_retries)
+                        self.report_retry_fragment(err, frag_index, count, fragment_retries)
             if count > fragment_retries:
                 if skip_unavailable_fragments:
-                    self.report_skip_fragment(segment_name)
+                    self.report_skip_fragment(frag_index)
                     continue
                 self.report_error('giving up after %s fragment retries' % fragment_retries)
                 return False
 
         self._finish_frag_download(ctx)
 
-        for segment_file in segments_filenames:
-            os.remove(encodeFilename(segment_file))
-
         return True

From 5b995f713bf45e9e373029e1411811dfde855088 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 23 Feb 2017 18:46:20 +0100
Subject: [PATCH 0687/1696] [utils] add support for ttml styles

---
 test/test_utils.py  |  41 +++++++++++++++
 youtube_dl/utils.py | 120 ++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 152 insertions(+), 9 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index aa4569b81..4cd818850 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -1069,6 +1069,47 @@ The first line
 '''
         self.assertEqual(dfxp2srt(dfxp_data_no_default_namespace), srt_data)
 
+        dfxp_data_with_style = '''<?xml version="1.0" encoding="utf-8"?>
+<tt xmlns="http://www.w3.org/2006/10/ttaf1" xmlns:ttp="http://www.w3.org/2006/10/ttaf1#parameter" ttp:timeBase="media" xmlns:tts="http://www.w3.org/2006/10/ttaf1#style" xml:lang="en" xmlns:ttm="http://www.w3.org/2006/10/ttaf1#metadata">
+  <head>
+    <styling>
+      <style id="s2" style="s0" tts:color="cyan" tts:fontWeight="bold" />
+      <style id="s1" style="s0" tts:color="yellow" tts:fontStyle="italic" />
+      <style id="s3" style="s0" tts:color="lime" tts:textDecoration="underline" />
+      <style id="s0" tts:backgroundColor="black" tts:fontStyle="normal" tts:fontSize="16" tts:fontFamily="sansSerif" tts:color="white" />
+    </styling>
+  </head>
+  <body tts:textAlign="center" style="s0">
+    <div>
+      <p begin="00:00:02.08" id="p0" end="00:00:05.84">default style<span tts:color="red">custom style</span></p>
+      <p style="s2" begin="00:00:02.08" id="p0" end="00:00:05.84"><span tts:color="lime">part 1<br /></span><span tts:color="cyan">part 2</span></p>
+      <p style="s3" begin="00:00:05.84" id="p1" end="00:00:09.56">line 3<br />part 3</p>
+      <p style="s1" tts:textDecoration="underline" begin="00:00:09.56" id="p2" end="00:00:12.36"><span style="s2" tts:color="lime">inner<br /> </span>style</p>
+    </div>
+  </body>
+</tt>'''
+        srt_data = '''1
+00:00:02,080 --> 00:00:05,839
+<font color="white" face="sansSerif" size="16">default style<font color="red">custom style</font></font>
+
+2
+00:00:02,080 --> 00:00:05,839
+<b><font color="cyan" face="sansSerif" size="16"><font color="lime">part 1
+</font>part 2</font></b>
+
+3
+00:00:05,839 --> 00:00:09,560
+<u><font color="lime">line 3
+part 3</font></u>
+
+4
+00:00:09,560 --> 00:00:12,359
+<i><u><font color="yellow"><font color="lime">inner
+ </font>style</font></u></i>
+
+'''
+        self.assertEqual(dfxp2srt(dfxp_data_with_style), srt_data)
+
     def test_cli_option(self):
         self.assertEqual(cli_option({'proxy': '127.0.0.1:3128'}, '--proxy', 'proxy'), ['--proxy', '127.0.0.1:3128'])
         self.assertEqual(cli_option({'proxy': None}, '--proxy', 'proxy'), [])
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 79a337df4..91e235ff2 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2511,27 +2511,97 @@ def srt_subtitles_timecode(seconds):
 
 
 def dfxp2srt(dfxp_data):
+    LEGACY_NAMESPACES = (
+        ('http://www.w3.org/ns/ttml', [
+            'http://www.w3.org/2004/11/ttaf1',
+            'http://www.w3.org/2006/04/ttaf1',
+            'http://www.w3.org/2006/10/ttaf1',
+        ]),
+        ('http://www.w3.org/ns/ttml#styling', [
+            'http://www.w3.org/ns/ttml#style',
+        ]),
+    )
+
+    SUPPORTED_STYLING = [
+        'color',
+        'fontFamily',
+        'fontSize',
+        'fontStyle',
+        'fontWeight',
+        'textDecoration'
+    ]
+
     _x = functools.partial(xpath_with_ns, ns_map={
         'ttml': 'http://www.w3.org/ns/ttml',
-        'ttaf1': 'http://www.w3.org/2006/10/ttaf1',
-        'ttaf1_0604': 'http://www.w3.org/2006/04/ttaf1',
+        'tts': 'http://www.w3.org/ns/ttml#styling',
     })
 
+    styles = {}
+    default_style = {}
+
     class TTMLPElementParser(object):
-        out = ''
+        _out = ''
+        _unclosed_elements = []
+        _applied_styles = []
 
         def start(self, tag, attrib):
-            if tag in (_x('ttml:br'), _x('ttaf1:br'), 'br'):
-                self.out += '\n'
+            if tag in (_x('ttml:br'), 'br'):
+                self._out += '\n'
+            else:
+                unclosed_elements = []
+                style = {}
+                element_style_id = attrib.get('style')
+                if default_style:
+                    style.update(default_style)
+                if element_style_id:
+                    style.update(styles.get(element_style_id, {}))
+                for prop in SUPPORTED_STYLING:
+                    prop_val = attrib.get(_x('tts:' + prop))
+                    if prop_val:
+                        style[prop] = prop_val
+                if style:
+                    font = ''
+                    for k, v in sorted(style.items()):
+                        if self._applied_styles and self._applied_styles[-1].get(k) == v:
+                            continue
+                        if k == 'color':
+                            font += ' color="%s"' % v
+                        elif k == 'fontSize':
+                            font += ' size="%s"' % v
+                        elif k == 'fontFamily':
+                            font += ' face="%s"' % v
+                        elif k == 'fontWeight' and v == 'bold':
+                            self._out += '<b>'
+                            unclosed_elements.append('b')
+                        elif k == 'fontStyle' and v == 'italic':
+                            self._out += '<i>'
+                            unclosed_elements.append('i')
+                        elif k == 'textDecoration' and v == 'underline':
+                            self._out += '<u>'
+                            unclosed_elements.append('u')
+                    if font:
+                        self._out += '<font' + font + '>'
+                        unclosed_elements.append('font')
+                    applied_style = {}
+                    if self._applied_styles:
+                        applied_style.update(self._applied_styles[-1])
+                    applied_style.update(style)
+                    self._applied_styles.append(applied_style)
+                self._unclosed_elements.append(unclosed_elements)
 
         def end(self, tag):
-            pass
+            if tag not in (_x('ttml:br'), 'br'):
+                unclosed_elements = self._unclosed_elements.pop()
+                for element in reversed(unclosed_elements):
+                    self._out += '</%s>' % element
+                if unclosed_elements and self._applied_styles:
+                    self._applied_styles.pop()
 
         def data(self, data):
-            self.out += data
+            self._out += data
 
         def close(self):
-            return self.out.strip()
+            return self._out.strip()
 
     def parse_node(node):
         target = TTMLPElementParser()
@@ -2539,13 +2609,45 @@ def dfxp2srt(dfxp_data):
         parser.feed(xml.etree.ElementTree.tostring(node))
         return parser.close()
 
+    for k, v in LEGACY_NAMESPACES:
+        for ns in v:
+            dfxp_data = dfxp_data.replace(ns, k)
+
     dfxp = compat_etree_fromstring(dfxp_data.encode('utf-8'))
     out = []
-    paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall(_x('.//ttaf1:p')) or dfxp.findall(_x('.//ttaf1_0604:p')) or dfxp.findall('.//p')
+    paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall('.//p')
 
     if not paras:
         raise ValueError('Invalid dfxp/TTML subtitle')
 
+    repeat = False
+    while True:
+        for style in dfxp.findall(_x('.//ttml:style')):
+            style_id = style.get('id')
+            parent_style_id = style.get('style')
+            if parent_style_id:
+                if parent_style_id not in styles:
+                    repeat = True
+                    continue
+                styles[style_id] = styles[parent_style_id].copy()
+            for prop in SUPPORTED_STYLING:
+                prop_val = style.get(_x('tts:' + prop))
+                if prop_val:
+                    styles.setdefault(style_id, {})[prop] = prop_val
+        if repeat:
+            repeat = False
+        else:
+            break
+
+    for p in ('body', 'div'):
+        ele = xpath_element(dfxp, [_x('.//ttml:' + p), './/' + p])
+        if ele is None:
+            continue
+        style = styles.get(ele.get('style'))
+        if not style:
+            continue
+        default_style.update(style)
+
     for para, index in zip(paras, itertools.count(1)):
         begin_time = parse_dfxp_time_expr(para.attrib.get('begin'))
         end_time = parse_dfxp_time_expr(para.attrib.get('end'))

From 481ef51e2345b1b34b16148eb6e5e58e9fb45cfb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 19 Apr 2017 21:47:03 +0700
Subject: [PATCH 0688/1696] [brightcove] PEP 8

---
 youtube_dl/extractor/brightcove.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index a2db705b0..183ac7dd0 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -277,7 +277,7 @@ class BrightcoveLegacyIE(InfoExtractor):
             return list(filter(None, [
                 cls._build_brighcove_url_from_js(custom_bc)
                 for custom_bc in matches]))
-        return [url for _, url in re.findall(
+        return [src for _, src in re.findall(
             r'<iframe[^>]+src=([\'"])((?:https?:)?//link\.brightcove\.com/services/player/(?!\1).+)\1', webpage)]
 
     def _real_extract(self, url):

From ea0c2f219c219de8f59f1ae82e106cec5911c56c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 19 Apr 2017 18:34:25 +0100
Subject: [PATCH 0689/1696] [downloader/fragment] use a general file to store
 fragment download context

---
 youtube_dl/downloader/common.py   |  3 +++
 youtube_dl/downloader/fragment.py | 19 ++++++++++++-------
 2 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index fdb77b620..5d6621147 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -187,6 +187,9 @@ class FileDownloader(object):
             return filename[:-len('.part')]
         return filename
 
+    def ytdl_filename(self, filename):
+        return filename + '.ytdl'
+
     def try_rename(self, old_filename, new_filename):
         try:
             if old_filename == new_filename:
diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 44a3c1040..80bb14d61 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -3,6 +3,7 @@ from __future__ import division, unicode_literals
 import os
 import time
 import io
+import json
 
 from .common import FileDownloader
 from .http import HttpFD
@@ -63,8 +64,10 @@ class FragmentFD(FileDownloader):
     def _append_fragment(self, ctx, frag_content):
         ctx['dest_stream'].write(frag_content)
         if not (ctx.get('live') or ctx['tmpfilename'] == '-'):
-            frag_index_stream, _ = sanitize_open(ctx['tmpfilename'] + '.fragindex', 'w')
-            frag_index_stream.write(compat_str(ctx['frag_index']))
+            frag_index_stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'w')
+            frag_index_stream.write(json.dumps({
+                'frag_index': ctx['frag_index']
+            }))
             frag_index_stream.close()
 
     def _prepare_frag_download(self, ctx):
@@ -94,9 +97,10 @@ class FragmentFD(FileDownloader):
         if os.path.isfile(encodeFilename(tmpfilename)):
             open_mode = 'ab'
             resume_len = os.path.getsize(encodeFilename(tmpfilename))
-            if os.path.isfile(encodeFilename(tmpfilename + '.fragindex')):
-                frag_index_stream, _ = sanitize_open(tmpfilename + '.fragindex', 'r')
-                frag_index = int(frag_index_stream.read())
+            ytdl_filename = encodeFilename(self.ytdl_filename(ctx['filename']))
+            if os.path.isfile(ytdl_filename):
+                frag_index_stream, _ = sanitize_open(ytdl_filename, 'r')
+                frag_index = json.loads(frag_index_stream.read())['frag_index']
                 frag_index_stream.close()
         dest_stream, tmpfilename = sanitize_open(tmpfilename, open_mode)
 
@@ -167,8 +171,9 @@ class FragmentFD(FileDownloader):
 
     def _finish_frag_download(self, ctx):
         ctx['dest_stream'].close()
-        if os.path.isfile(encodeFilename(ctx['tmpfilename'] + '.fragindex')):
-            os.remove(encodeFilename(ctx['tmpfilename'] + '.fragindex'))
+        ytdl_filename = encodeFilename(self.ytdl_filename(ctx['filename']))
+        if os.path.isfile(ytdl_filename):
+            os.remove(ytdl_filename)
         elapsed = time.time() - ctx['started']
         self.try_rename(ctx['tmpfilename'], ctx['filename'])
         fsize = os.path.getsize(encodeFilename(ctx['filename']))

From 8abc7dca39236462c0652932df23862acda92d51 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 20 Apr 2017 05:16:41 +0100
Subject: [PATCH 0690/1696] [amp] extract error message(closes #12795)

---
 youtube_dl/extractor/amp.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/amp.py b/youtube_dl/extractor/amp.py
index e8e40126b..98f8e69cd 100644
--- a/youtube_dl/extractor/amp.py
+++ b/youtube_dl/extractor/amp.py
@@ -7,15 +7,19 @@ from ..utils import (
     parse_iso8601,
     mimetype2ext,
     determine_ext,
+    ExtractorError,
 )
 
 
 class AMPIE(InfoExtractor):
     # parse Akamai Adaptive Media Player feed
     def _extract_feed_info(self, url):
-        item = self._download_json(
+        feed = self._download_json(
             url, None, 'Downloading Akamai AMP feed',
-            'Unable to download Akamai AMP feed')['channel']['item']
+            'Unable to download Akamai AMP feed')
+        item = feed.get('channel', {}).get('item')
+        if not item:
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, feed['error']))
 
         video_id = item['guid']
 

From f7799582500ca2b00c371c287028298f187c0477 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 21 Apr 2017 23:37:06 +0700
Subject: [PATCH 0691/1696] [vidzi] Fix extraction (closes #12793)

---
 youtube_dl/extractor/vidzi.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vidzi.py b/youtube_dl/extractor/vidzi.py
index d0556297e..e64873bce 100644
--- a/youtube_dl/extractor/vidzi.py
+++ b/youtube_dl/extractor/vidzi.py
@@ -42,14 +42,15 @@ class VidziIE(InfoExtractor):
         title = self._html_search_regex(
             r'(?s)<h2 class="video-title">(.*?)</h2>', webpage, 'title')
 
-        packed_codes = [mobj.group(0) for mobj in re.finditer(
-            PACKED_CODES_RE, webpage)]
-        for num, pc in enumerate(packed_codes, 1):
-            code = decode_packed_codes(pc).replace('\\\'', '\'')
+        codes = [webpage]
+        codes.extend([
+            decode_packed_codes(mobj.group(0)).replace('\\\'', '\'')
+            for mobj in re.finditer(PACKED_CODES_RE, webpage)])
+        for num, code in enumerate(codes, 1):
             jwplayer_data = self._parse_json(
                 self._search_regex(
                     r'setup\(([^)]+)\)', code, 'jwplayer data',
-                    default=NO_DEFAULT if num == len(packed_codes) else '{}'),
+                    default=NO_DEFAULT if num == len(codes) else '{}'),
                 video_id, transform_source=js_to_json)
             if jwplayer_data:
                 break

From cb2520802d7b8efdc71d6d97aeca984b5f878716 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 22 Apr 2017 07:01:00 +0700
Subject: [PATCH 0692/1696] [extractor/common] Improve m3u8 extraction (closes
 #12211) * Extract m3u8 parsing to separate method * Improve rendition groups
 extraction * Build stream name according stream GROUP-ID * Ignore reference
 to AUDIO group without URI when stream has no CODECS + Add test coverage for
 parsing m3u8 from #11507, #11995, #12211 and twitch vod

---
 .gitignore                                   |   1 -
 test/test_InfoExtractor.py                   | 308 ++++++++++++++++++-
 test/testdata/m3u8/pluzz_francetv_11507.m3u8 |  14 +
 test/testdata/m3u8/teamcoco_11995.m3u8       |  16 +
 test/testdata/m3u8/toggle_mobile_12211.m3u8  |  13 +
 test/testdata/m3u8/twitch_vod.m3u8           |  20 ++
 youtube_dl/extractor/common.py               | 154 ++++++----
 7 files changed, 466 insertions(+), 60 deletions(-)
 create mode 100644 test/testdata/m3u8/pluzz_francetv_11507.m3u8
 create mode 100644 test/testdata/m3u8/teamcoco_11995.m3u8
 create mode 100644 test/testdata/m3u8/toggle_mobile_12211.m3u8
 create mode 100644 test/testdata/m3u8/twitch_vod.m3u8

diff --git a/.gitignore b/.gitignore
index 9ce4b5e2d..bef084baa 100644
--- a/.gitignore
+++ b/.gitignore
@@ -36,7 +36,6 @@ updates_key.pem
 *.swf
 *.part
 *.swp
-test/testdata
 test/local_parameters.json
 .tox
 youtube-dl.zsh
diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 881197afb..57205f6ae 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -3,12 +3,13 @@
 from __future__ import unicode_literals
 
 # Allow direct execution
+import io
 import os
 import sys
 import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from test.helper import FakeYDL, expect_dict
+from test.helper import FakeYDL, expect_dict, expect_value
 from youtube_dl.extractor.common import InfoExtractor
 from youtube_dl.extractor import YoutubeIE, get_info_extractor
 from youtube_dl.utils import encode_data_uri, strip_jsonp, ExtractorError, RegexNotFoundError
@@ -175,6 +176,311 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 }]
             })
 
+    def test_parse_m3u8_formats(self):
+        _TEST_CASES = [
+            (
+                # https://github.com/rg3/youtube-dl/issues/11507
+                # http://pluzz.francetv.fr/videos/le_ministere.html
+                'pluzz_francetv_11507',
+                'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/master.m3u8?caption=2017%2F16%2F156589847-1492488987.m3u8%3Afra%3AFrancais&audiotrack=0%3Afra%3AFrancais',
+                [{
+                    'url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/master.m3u8?caption=2017%2F16%2F156589847-1492488987.m3u8%3Afra%3AFrancais&audiotrack=0%3Afra%3AFrancais',
+                    'ext': 'mp4',
+                    'format_id': 'meta',
+                    'format_note': 'Quality selection URL',
+                    'protocol': 'm3u8',
+                    'preference': -100,
+                    'resolution': 'multiple'
+                }, {
+                    'url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_0_av.m3u8?null=0',
+                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_0_av.m3u8?null=0',
+                    'ext': 'mp4',
+                    'format_id': '180',
+                    'protocol': 'm3u8',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc1.66.30',
+                    'tbr': 180,
+                    'width': 256,
+                    'height': 144,
+                }, {
+                    'url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_1_av.m3u8?null=0',
+                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_1_av.m3u8?null=0',
+                    'ext': 'mp4',
+                    'format_id': '303',
+                    'protocol': 'm3u8',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc1.66.30',
+                    'tbr': 303,
+                    'width': 320,
+                    'height': 180,
+                }, {
+                    'url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_2_av.m3u8?null=0',
+                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_2_av.m3u8?null=0',
+                    'ext': 'mp4',
+                    'format_id': '575',
+                    'protocol': 'm3u8',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc1.66.30',
+                    'tbr': 575,
+                    'width': 512,
+                    'height': 288,
+                }, {
+                    'url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_3_av.m3u8?null=0',
+                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_3_av.m3u8?null=0',
+                    'ext': 'mp4',
+                    'format_id': '831',
+                    'protocol': 'm3u8',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc1.77.30',
+                    'tbr': 831,
+                    'width': 704,
+                    'height': 396,
+                }, {
+                    'url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_4_av.m3u8?null=0',
+                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_4_av.m3u8?null=0',
+                    'ext': 'mp4',
+                    'protocol': 'm3u8',
+                    'format_id': '1467',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc1.77.30',
+                    'tbr': 1467,
+                    'width': 1024,
+                    'height': 576,
+                }]
+            ),
+            (
+                # https://github.com/rg3/youtube-dl/issues/11995
+                # http://teamcoco.com/video/clueless-gamer-super-bowl-for-honor
+                'teamcoco_11995',
+                'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/main.m3u8',
+                [{
+                    'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/main.m3u8',
+                    'ext': 'mp4',
+                    'format_id': 'meta',
+                    'format_note': 'Quality selection URL',
+                    'protocol': 'm3u8',
+                    'preference': -100,
+                    'resolution': 'multiple',
+                }, {
+                    'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-audio-160k_v4.m3u8',
+                    'ext': 'mp4',
+                    'format_id': 'audio-0-Default',
+                    'protocol': 'm3u8',
+                    'vcodec': 'none',
+                }, {
+                    'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-audio-64k_v4.m3u8',
+                    'ext': 'mp4',
+                    'format_id': 'audio-1-Default',
+                    'protocol': 'm3u8',
+                    'vcodec': 'none',
+                }, {
+                    'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-audio-64k_v4.m3u8',
+                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-audio-64k_v4.m3u8',
+                    'ext': 'mp4',
+                    'format_id': '71',
+                    'protocol': 'm3u8',
+                    'acodec': 'mp4a.40.5',
+                    'vcodec': 'none',
+                    'tbr': 71,
+                }, {
+                    'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-400k_v4.m3u8',
+                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-400k_v4.m3u8',
+                    'ext': 'mp4',
+                    'format_id': '413',
+                    'protocol': 'm3u8',
+                    'acodec': 'none',
+                    'vcodec': 'avc1.42001e',
+                    'tbr': 413,
+                    'width': 400,
+                    'height': 224,
+                }, {
+                    'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-400k_v4.m3u8',
+                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-400k_v4.m3u8',
+                    'ext': 'mp4',
+                    'format_id': '522',
+                    'protocol': 'm3u8',
+                    'acodec': 'none',
+                    'vcodec': 'avc1.42001e',
+                    'tbr': 522,
+                    'width': 400,
+                    'height': 224,
+                }, {
+                    'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-1m_v4.m3u8',
+                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-1m_v4.m3u8',
+                    'ext': 'mp4',
+                    'format_id': '1205',
+                    'protocol': 'm3u8',
+                    'acodec': 'none',
+                    'vcodec': 'avc1.4d001e',
+                    'tbr': 1205,
+                    'width': 640,
+                    'height': 360,
+                }, {
+                    'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-2m_v4.m3u8',
+                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-2m_v4.m3u8',
+                    'ext': 'mp4',
+                    'format_id': '2374',
+                    'protocol': 'm3u8',
+                    'acodec': 'none',
+                    'vcodec': 'avc1.4d001f',
+                    'tbr': 2374,
+                    'width': 1024,
+                    'height': 576,
+                }]
+            ),
+            (
+                # https://github.com/rg3/youtube-dl/issues/12211
+                # http://video.toggle.sg/en/series/whoopie-s-world/ep3/478601
+                'toggle_mobile_12211',
+                'http://cdnapi.kaltura.com/p/2082311/sp/208231100/playManifest/protocol/http/entryId/0_89q6e8ku/format/applehttp/tags/mobile_sd/f/a.m3u8',
+                [{
+                    'url': 'http://cdnapi.kaltura.com/p/2082311/sp/208231100/playManifest/protocol/http/entryId/0_89q6e8ku/format/applehttp/tags/mobile_sd/f/a.m3u8',
+                    'ext': 'mp4',
+                    'format_id': 'meta',
+                    'format_note': 'Quality selection URL',
+                    'protocol': 'm3u8',
+                    'preference': -100,
+                    'resolution': 'multiple'
+                }, {
+                    'url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_sa2ntrdg/name/a.mp4/index.m3u8',
+                    'ext': 'mp4',
+                    'format_id': 'audio-English',
+                    'protocol': 'm3u8',
+                    'language': 'eng',
+                    'vcodec': 'none',
+                }, {
+                    'url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_r7y0nitg/name/a.mp4/index.m3u8',
+                    'ext': 'mp4',
+                    'format_id': 'audio-Undefined',
+                    'protocol': 'm3u8',
+                    'language': 'und',
+                    'vcodec': 'none',
+                }, {
+                    'url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_qlk9hlzr/name/a.mp4/index.m3u8',
+                    'manifest_url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_qlk9hlzr/name/a.mp4/index.m3u8',
+                    'ext': 'mp4',
+                    'format_id': '155',
+                    'protocol': 'm3u8',
+                    'tbr': 155,
+                    'width': 320,
+                    'height': 180,
+                }, {
+                    'url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_oefackmi/name/a.mp4/index.m3u8',
+                    'manifest_url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_oefackmi/name/a.mp4/index.m3u8',
+                    'ext': 'mp4',
+                    'format_id': '502',
+                    'protocol': 'm3u8',
+                    'tbr': 502,
+                    'width': 480,
+                    'height': 270,
+                }, {
+                    'url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/12/pv/1/flavorId/0_vyg9pj7k/name/a.mp4/index.m3u8',
+                    'manifest_url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/12/pv/1/flavorId/0_vyg9pj7k/name/a.mp4/index.m3u8',
+                    'ext': 'mp4',
+                    'format_id': '827',
+                    'protocol': 'm3u8',
+                    'tbr': 827,
+                    'width': 640,
+                    'height': 360,
+                }, {
+                    'url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/12/pv/1/flavorId/0_50n4psvx/name/a.mp4/index.m3u8',
+                    'manifest_url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/12/pv/1/flavorId/0_50n4psvx/name/a.mp4/index.m3u8',
+                    'ext': 'mp4',
+                    'format_id': '1396',
+                    'protocol': 'm3u8',
+                    'tbr': 1396,
+                    'width': 854,
+                    'height': 480,
+                }]
+            ),
+            (
+                # http://www.twitch.tv/riotgames/v/6528877
+                'twitch_vod',
+                'https://usher.ttvnw.net/vod/6528877?allow_source=true&allow_audio_only=true&allow_spectre=true&player=twitchweb&nauth=%7B%22user_id%22%3Anull%2C%22vod_id%22%3A6528877%2C%22expires%22%3A1492887874%2C%22chansub%22%3A%7B%22restricted_bitrates%22%3A%5B%5D%7D%2C%22privileged%22%3Afalse%2C%22https_required%22%3Afalse%7D&nauthsig=3e29296a6824a0f48f9e731383f77a614fc79bee',
+                [{
+                    'url': 'https://usher.ttvnw.net/vod/6528877?allow_source=true&allow_audio_only=true&allow_spectre=true&player=twitchweb&nauth=%7B%22user_id%22%3Anull%2C%22vod_id%22%3A6528877%2C%22expires%22%3A1492887874%2C%22chansub%22%3A%7B%22restricted_bitrates%22%3A%5B%5D%7D%2C%22privileged%22%3Afalse%2C%22https_required%22%3Afalse%7D&nauthsig=3e29296a6824a0f48f9e731383f77a614fc79bee',
+                    'ext': 'mp4',
+                    'format_id': 'meta',
+                    'format_note': 'Quality selection URL',
+                    'protocol': 'm3u8',
+                    'preference': -100,
+                    'resolution': 'multiple'
+                }, {
+                    'url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/audio_only/index-muted-HM49I092CC.m3u8',
+                    'manifest_url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/audio_only/index-muted-HM49I092CC.m3u8',
+                    'ext': 'mp4',
+                    'format_id': 'Audio Only',
+                    'protocol': 'm3u8',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'none',
+                    'tbr': 182,
+                }, {
+                    'url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/mobile/index-muted-HM49I092CC.m3u8',
+                    'manifest_url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/mobile/index-muted-HM49I092CC.m3u8',
+                    'ext': 'mp4',
+                    'format_id': 'Mobile',
+                    'protocol': 'm3u8',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc1.42C00D',
+                    'tbr': 280,
+                    'width': 400,
+                    'height': 226,
+                }, {
+                    'url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/low/index-muted-HM49I092CC.m3u8',
+                    'manifest_url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/low/index-muted-HM49I092CC.m3u8',
+                    'ext': 'mp4',
+                    'format_id': 'Low',
+                    'protocol': 'm3u8',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc1.42C01E',
+                    'tbr': 628,
+                    'width': 640,
+                    'height': 360,
+                }, {
+                    'url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/medium/index-muted-HM49I092CC.m3u8',
+                    'manifest_url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/medium/index-muted-HM49I092CC.m3u8',
+                    'ext': 'mp4',
+                    'format_id': 'Medium',
+                    'protocol': 'm3u8',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc1.42C01E',
+                    'tbr': 893,
+                    'width': 852,
+                    'height': 480,
+                }, {
+                    'url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/high/index-muted-HM49I092CC.m3u8',
+                    'manifest_url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/high/index-muted-HM49I092CC.m3u8',
+                    'ext': 'mp4',
+                    'format_id': 'High',
+                    'protocol': 'm3u8',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc1.42C01F',
+                    'tbr': 1603,
+                    'width': 1280,
+                    'height': 720,
+                }, {
+                    'url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/chunked/index-muted-HM49I092CC.m3u8',
+                    'manifest_url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/chunked/index-muted-HM49I092CC.m3u8',
+                    'ext': 'mp4',
+                    'format_id': 'Source',
+                    'protocol': 'm3u8',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc1.100.31',
+                    'tbr': 3214,
+                    'width': 1280,
+                    'height': 720,
+                }]
+            )
+        ]
+
+        for m3u8_file, m3u8_url, expected_formats in _TEST_CASES:
+            with io.open('./test/testdata/m3u8/%s.m3u8' % m3u8_file,
+                         mode='r', encoding='utf-8') as f:
+                formats = self.ie._parse_m3u8_formats(
+                    f.read(), m3u8_url, ext='mp4')
+                self.ie._sort_formats(formats)
+                expect_value(self, formats, expected_formats, None)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/testdata/m3u8/pluzz_francetv_11507.m3u8 b/test/testdata/m3u8/pluzz_francetv_11507.m3u8
new file mode 100644
index 000000000..0809f5aa0
--- /dev/null
+++ b/test/testdata/m3u8/pluzz_francetv_11507.m3u8
@@ -0,0 +1,14 @@
+#EXTM3U
+    
#EXT-X-VERSION:5
+    
#EXT-X-MEDIA:TYPE=SUBTITLES,GROUP-ID="subs",NAME="Francais",DEFAULT=NO,FORCED=NO,URI="http://replayftv-pmd.francetv.fr/subtitles/2017/16/156589847-1492488987.m3u8",LANGUAGE="fra"
+    
#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="aac",LANGUAGE="fra",NAME="Francais",DEFAULT=YES, AUTOSELECT=YES
+#EXT-X-STREAM-INF:SUBTITLES="subs",AUDIO="aac",PROGRAM-ID=1,BANDWIDTH=180000,RESOLUTION=256x144,CODECS="avc1.66.30, mp4a.40.2"
+http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_0_av.m3u8?null=0
+#EXT-X-STREAM-INF:SUBTITLES="subs",AUDIO="aac",PROGRAM-ID=1,BANDWIDTH=303000,RESOLUTION=320x180,CODECS="avc1.66.30, mp4a.40.2"
+http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_1_av.m3u8?null=0
+#EXT-X-STREAM-INF:SUBTITLES="subs",AUDIO="aac",PROGRAM-ID=1,BANDWIDTH=575000,RESOLUTION=512x288,CODECS="avc1.66.30, mp4a.40.2"
+http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_2_av.m3u8?null=0
+#EXT-X-STREAM-INF:SUBTITLES="subs",AUDIO="aac",PROGRAM-ID=1,BANDWIDTH=831000,RESOLUTION=704x396,CODECS="avc1.77.30, mp4a.40.2"
+http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_3_av.m3u8?null=0
+#EXT-X-STREAM-INF:SUBTITLES="subs",AUDIO="aac",PROGRAM-ID=1,BANDWIDTH=1467000,RESOLUTION=1024x576,CODECS="avc1.77.30, mp4a.40.2"
+http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_4_av.m3u8?null=0
diff --git a/test/testdata/m3u8/teamcoco_11995.m3u8 b/test/testdata/m3u8/teamcoco_11995.m3u8
new file mode 100644
index 000000000..a6e421697
--- /dev/null
+++ b/test/testdata/m3u8/teamcoco_11995.m3u8
@@ -0,0 +1,16 @@
+#EXTM3U
+#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio-0",NAME="Default",AUTOSELECT=YES,DEFAULT=YES,URI="hls/CONAN_020217_Highlight_show-audio-160k_v4.m3u8"
+#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio-1",NAME="Default",AUTOSELECT=YES,DEFAULT=YES,URI="hls/CONAN_020217_Highlight_show-audio-64k_v4.m3u8"
+#EXT-X-I-FRAME-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=37862000,CODECS="avc1.4d001f",URI="hls/CONAN_020217_Highlight_show-2m_iframe.m3u8"
+#EXT-X-I-FRAME-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=18750000,CODECS="avc1.4d001e",URI="hls/CONAN_020217_Highlight_show-1m_iframe.m3u8"
+#EXT-X-I-FRAME-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=6535000,CODECS="avc1.42001e",URI="hls/CONAN_020217_Highlight_show-400k_iframe.m3u8"
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=2374000,RESOLUTION=1024x576,CODECS="avc1.4d001f,mp4a.40.2",AUDIO="audio-0"
+hls/CONAN_020217_Highlight_show-2m_v4.m3u8
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=1205000,RESOLUTION=640x360,CODECS="avc1.4d001e,mp4a.40.2",AUDIO="audio-0"
+hls/CONAN_020217_Highlight_show-1m_v4.m3u8
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=522000,RESOLUTION=400x224,CODECS="avc1.42001e,mp4a.40.2",AUDIO="audio-0"
+hls/CONAN_020217_Highlight_show-400k_v4.m3u8
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=413000,RESOLUTION=400x224,CODECS="avc1.42001e,mp4a.40.5",AUDIO="audio-1"
+hls/CONAN_020217_Highlight_show-400k_v4.m3u8
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=71000,CODECS="mp4a.40.5",AUDIO="audio-1"
+hls/CONAN_020217_Highlight_show-audio-64k_v4.m3u8
diff --git a/test/testdata/m3u8/toggle_mobile_12211.m3u8 b/test/testdata/m3u8/toggle_mobile_12211.m3u8
new file mode 100644
index 000000000..69604e683
--- /dev/null
+++ b/test/testdata/m3u8/toggle_mobile_12211.m3u8
@@ -0,0 +1,13 @@
+#EXTM3U
+#EXT-X-VERSION:4
+#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",LANGUAGE="eng",NAME="English",URI="http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_sa2ntrdg/name/a.mp4/index.m3u8"
+#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",LANGUAGE="und",NAME="Undefined",URI="http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_r7y0nitg/name/a.mp4/index.m3u8"
+
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=155648,RESOLUTION=320x180,AUDIO="audio"
+http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_qlk9hlzr/name/a.mp4/index.m3u8
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=502784,RESOLUTION=480x270,AUDIO="audio"
+http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_oefackmi/name/a.mp4/index.m3u8
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=827392,RESOLUTION=640x360,AUDIO="audio"
+http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/12/pv/1/flavorId/0_vyg9pj7k/name/a.mp4/index.m3u8
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=1396736,RESOLUTION=854x480,AUDIO="audio"
+http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/12/pv/1/flavorId/0_50n4psvx/name/a.mp4/index.m3u8
diff --git a/test/testdata/m3u8/twitch_vod.m3u8 b/test/testdata/m3u8/twitch_vod.m3u8
new file mode 100644
index 000000000..7617277ca
--- /dev/null
+++ b/test/testdata/m3u8/twitch_vod.m3u8
@@ -0,0 +1,20 @@
+#EXTM3U
+#EXT-X-TWITCH-INFO:ORIGIN="s3",CLUSTER="edgecast_vod",REGION="EU",MANIFEST-CLUSTER="edgecast_vod",USER-IP="109.171.17.81"
+#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID="chunked",NAME="Source",AUTOSELECT=YES,DEFAULT=YES
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=3214134,CODECS="avc1.100.31,mp4a.40.2",RESOLUTION="1280x720",VIDEO="chunked"
+https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/chunked/index-muted-HM49I092CC.m3u8
+#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID="high",NAME="High",AUTOSELECT=YES,DEFAULT=YES
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=1603789,CODECS="avc1.42C01F,mp4a.40.2",RESOLUTION="1280x720",VIDEO="high"
+https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/high/index-muted-HM49I092CC.m3u8
+#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID="medium",NAME="Medium",AUTOSELECT=YES,DEFAULT=YES
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=893387,CODECS="avc1.42C01E,mp4a.40.2",RESOLUTION="852x480",VIDEO="medium"
+https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/medium/index-muted-HM49I092CC.m3u8
+#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID="low",NAME="Low",AUTOSELECT=YES,DEFAULT=YES
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=628347,CODECS="avc1.42C01E,mp4a.40.2",RESOLUTION="640x360",VIDEO="low"
+https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/low/index-muted-HM49I092CC.m3u8
+#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID="mobile",NAME="Mobile",AUTOSELECT=YES,DEFAULT=YES
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=280474,CODECS="avc1.42C00D,mp4a.40.2",RESOLUTION="400x226",VIDEO="mobile"
+https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/mobile/index-muted-HM49I092CC.m3u8
+#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID="audio_only",NAME="Audio Only",AUTOSELECT=NO,DEFAULT=NO
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=182725,CODECS="mp4a.40.2",VIDEO="audio_only"
+https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/audio_only/index-muted-HM49I092CC.m3u8
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 61d97ab72..359c549c5 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1312,17 +1312,25 @@ class InfoExtractor(object):
                               entry_protocol='m3u8', preference=None,
                               m3u8_id=None, note=None, errnote=None,
                               fatal=True, live=False):
-
         res = self._download_webpage_handle(
             m3u8_url, video_id,
             note=note or 'Downloading m3u8 information',
             errnote=errnote or 'Failed to download m3u8 information',
             fatal=fatal)
+
         if res is False:
             return []
+
         m3u8_doc, urlh = res
         m3u8_url = urlh.geturl()
 
+        return self._parse_m3u8_formats(
+            m3u8_doc, m3u8_url, ext=ext, entry_protocol=entry_protocol,
+            preference=preference, m3u8_id=m3u8_id, live=live)
+
+    def _parse_m3u8_formats(self, m3u8_doc, m3u8_url, ext=None,
+                            entry_protocol='m3u8', preference=None,
+                            m3u8_id=None, live=False):
         if '#EXT-X-FAXS-CM:' in m3u8_doc:  # Adobe Flash Access
             return []
 
@@ -1333,19 +1341,21 @@ class InfoExtractor(object):
             if re.match(r'^https?://', u)
             else compat_urlparse.urljoin(m3u8_url, u))
 
-        # We should try extracting formats only from master playlists [1], i.e.
-        # playlists that describe available qualities. On the other hand media
-        # playlists [2] should be returned as is since they contain just the media
-        # without qualities renditions.
+        # References:
+        # 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-21
+        # 2. https://github.com/rg3/youtube-dl/issues/12211
+
+        # We should try extracting formats only from master playlists [1, 4.3.4],
+        # i.e. playlists that describe available qualities. On the other hand
+        # media playlists [1, 4.3.3] should be returned as is since they contain
+        # just the media without qualities renditions.
         # Fortunately, master playlist can be easily distinguished from media
-        # playlist based on particular tags availability. As of [1, 2] master
-        # playlist tags MUST NOT appear in a media playist and vice versa.
-        # As of [3] #EXT-X-TARGETDURATION tag is REQUIRED for every media playlist
-        # and MUST NOT appear in master playlist thus we can clearly detect media
-        # playlist with this criterion.
-        # 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.4
-        # 2. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3
-        # 3. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.1
+        # playlist based on particular tags availability. As of [1, 4.3.3, 4.3.4]
+        # master playlist tags MUST NOT appear in a media playist and vice versa.
+        # As of [1, 4.3.3.1] #EXT-X-TARGETDURATION tag is REQUIRED for every
+        # media playlist and MUST NOT appear in master playlist thus we can
+        # clearly detect media playlist with this criterion.
+
         if '#EXT-X-TARGETDURATION' in m3u8_doc:  # media playlist, return as is
             return [{
                 'url': m3u8_url,
@@ -1354,52 +1364,67 @@ class InfoExtractor(object):
                 'protocol': entry_protocol,
                 'preference': preference,
             }]
-        audio_in_video_stream = {}
-        last_info = {}
-        last_media = {}
+
+        groups = {}
+        last_stream_inf = {}
+
+        def extract_media(x_media_line):
+            media = parse_m3u8_attributes(x_media_line)
+            # As per [1, 4.3.4.1] TYPE, GROUP-ID and NAME are REQUIRED
+            media_type, group_id, name = media.get('TYPE'), media.get('GROUP-ID'), media.get('NAME')
+            if not (media_type and group_id and name):
+                return
+            groups.setdefault(group_id, []).append(media)
+            if media_type not in ('VIDEO', 'AUDIO'):
+                return
+            media_url = media.get('URI')
+            if media_url:
+                format_id = []
+                for v in (group_id, name):
+                    if v:
+                        format_id.append(v)
+                f = {
+                    'format_id': '-'.join(format_id),
+                    'url': format_url(media_url),
+                    'language': media.get('LANGUAGE'),
+                    'ext': ext,
+                    'protocol': entry_protocol,
+                    'preference': preference,
+                }
+                if media_type == 'AUDIO':
+                    f['vcodec'] = 'none'
+                formats.append(f)
+
+        def build_stream_name():
+            # Despite specification does not mention NAME attribute for
+            # EXT-X-STREAM-INF it still sometimes may be present
+            stream_name = last_stream_inf.get('NAME')
+            if stream_name:
+                return stream_name
+            # If there is no NAME in EXT-X-STREAM-INF it will be obtained
+            # from corresponding rendition group
+            stream_group_id = last_stream_inf.get('VIDEO')
+            if not stream_group_id:
+                return
+            stream_group = groups.get(stream_group_id)
+            if not stream_group:
+                return stream_group_id
+            rendition = stream_group[0]
+            return rendition.get('NAME') or stream_group_id
+
         for line in m3u8_doc.splitlines():
             if line.startswith('#EXT-X-STREAM-INF:'):
-                last_info = parse_m3u8_attributes(line)
+                last_stream_inf = parse_m3u8_attributes(line)
             elif line.startswith('#EXT-X-MEDIA:'):
-                media = parse_m3u8_attributes(line)
-                media_type = media.get('TYPE')
-                if media_type in ('VIDEO', 'AUDIO'):
-                    group_id = media.get('GROUP-ID')
-                    media_url = media.get('URI')
-                    if media_url:
-                        format_id = []
-                        for v in (group_id, media.get('NAME')):
-                            if v:
-                                format_id.append(v)
-                        f = {
-                            'format_id': '-'.join(format_id),
-                            'url': format_url(media_url),
-                            'language': media.get('LANGUAGE'),
-                            'ext': ext,
-                            'protocol': entry_protocol,
-                            'preference': preference,
-                        }
-                        if media_type == 'AUDIO':
-                            f['vcodec'] = 'none'
-                            if group_id and not audio_in_video_stream.get(group_id):
-                                audio_in_video_stream[group_id] = False
-                        formats.append(f)
-                    else:
-                        # When there is no URI in EXT-X-MEDIA let this tag's
-                        # data be used by regular URI lines below
-                        last_media = media
-                        if media_type == 'AUDIO' and group_id:
-                            audio_in_video_stream[group_id] = True
+                extract_media(line)
             elif line.startswith('#') or not line.strip():
                 continue
             else:
-                tbr = int_or_none(last_info.get('AVERAGE-BANDWIDTH') or last_info.get('BANDWIDTH'), scale=1000)
+                tbr = int_or_none(last_stream_inf.get('AVERAGE-BANDWIDTH') or last_stream_inf.get('BANDWIDTH'), scale=1000)
                 format_id = []
                 if m3u8_id:
                     format_id.append(m3u8_id)
-                # Despite specification does not mention NAME attribute for
-                # EXT-X-STREAM-INF it still sometimes may be present
-                stream_name = last_info.get('NAME') or last_media.get('NAME')
+                stream_name = build_stream_name()
                 # Bandwidth of live streams may differ over time thus making
                 # format_id unpredictable. So it's better to keep provided
                 # format_id intact.
@@ -1412,11 +1437,11 @@ class InfoExtractor(object):
                     'manifest_url': manifest_url,
                     'tbr': tbr,
                     'ext': ext,
-                    'fps': float_or_none(last_info.get('FRAME-RATE')),
+                    'fps': float_or_none(last_stream_inf.get('FRAME-RATE')),
                     'protocol': entry_protocol,
                     'preference': preference,
                 }
-                resolution = last_info.get('RESOLUTION')
+                resolution = last_stream_inf.get('RESOLUTION')
                 if resolution:
                     mobj = re.search(r'(?P<width>\d+)[xX](?P<height>\d+)', resolution)
                     if mobj:
@@ -1432,13 +1457,26 @@ class InfoExtractor(object):
                         'vbr': vbr,
                         'abr': abr,
                     })
-                f.update(parse_codecs(last_info.get('CODECS')))
-                if audio_in_video_stream.get(last_info.get('AUDIO')) is False and f['vcodec'] != 'none':
-                    # TODO: update acodec for audio only formats with the same GROUP-ID
-                    f['acodec'] = 'none'
+                codecs = parse_codecs(last_stream_inf.get('CODECS'))
+                f.update(codecs)
+                audio_group_id = last_stream_inf.get('AUDIO')
+                # As per [1, 4.3.4.1.1] any EXT-X-STREAM-INF tag which
+                # references a rendition group MUST have a CODECS attribute.
+                # However, this is not always respected, for example, [2]
+                # contains EXT-X-STREAM-INF tag which references AUDIO
+                # rendition group but does not have CODECS and despite
+                # referencing audio group an audio group, it represents
+                # a complete (with audio and video) format. So, for such cases
+                # we will ignore references to rendition groups and treat them
+                # as complete formats.
+                if audio_group_id and codecs and f.get('vcodec') != 'none':
+                    audio_group = groups.get(audio_group_id)
+                    if audio_group and audio_group[0].get('URI'):
+                        # TODO: update acodec for audio only formats with
+                        # the same GROUP-ID
+                        f['acodec'] = 'none'
                 formats.append(f)
-                last_info = {}
-                last_media = {}
+                last_stream_inf = {}
         return formats
 
     @staticmethod

From 3110bb937d7ab43bd998448f51c29a157ef0dfbd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 22 Apr 2017 21:16:36 +0800
Subject: [PATCH 0693/1696] [porn91] Fix extraction (closes #12814)

---
 ChangeLog                      |  1 +
 youtube_dl/extractor/porn91.py | 32 ++++++--------------------------
 2 files changed, 7 insertions(+), 26 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index f8084fda5..1d49f8c8f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [porn91] Fix extraction (#12814)
 + [generic] Support Brightcove videos in <iframe> (#12482)
 + [brightcove] Support URLs with bcpid instead of playerID (#12482)
 
diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index 073fc3e21..24c3600fe 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -1,10 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from ..compat import (
-    compat_urllib_parse_unquote,
-    compat_urllib_parse_urlencode,
-)
 from .common import InfoExtractor
 from ..utils import (
     parse_duration,
@@ -19,7 +15,7 @@ class Porn91IE(InfoExtractor):
 
     _TEST = {
         'url': 'http://91porn.com/view_video.php?viewkey=7e42283b4f5ab36da134',
-        'md5': '6df8f6d028bc8b14f5dbd73af742fb20',
+        'md5': '7fcdb5349354f40d41689bd0fa8db05a',
         'info_dict': {
             'id': '7e42283b4f5ab36da134',
             'title': '18岁大一漂亮学妹，水嫩性感，再爽一次！',
@@ -43,24 +39,7 @@ class Porn91IE(InfoExtractor):
             r'<div id="viewvideo-title">([^<]+)</div>', webpage, 'title')
         title = title.replace('\n', '')
 
-        # get real url
-        file_id = self._search_regex(
-            r'so.addVariable\(\'file\',\'(\d+)\'', webpage, 'file id')
-        sec_code = self._search_regex(
-            r'so.addVariable\(\'seccode\',\'([^\']+)\'', webpage, 'sec code')
-        max_vid = self._search_regex(
-            r'so.addVariable\(\'max_vid\',\'(\d+)\'', webpage, 'max vid')
-        url_params = compat_urllib_parse_urlencode({
-            'VID': file_id,
-            'mp4': '1',
-            'seccode': sec_code,
-            'max_vid': max_vid,
-        })
-        info_cn = self._download_webpage(
-            'http://91porn.com/getfile.php?' + url_params, video_id,
-            'Downloading real video url')
-        video_url = compat_urllib_parse_unquote(self._search_regex(
-            r'file=([^&]+)&', info_cn, 'url'))
+        info_dict = self._parse_html5_media_entries(url, webpage, video_id)[0]
 
         duration = parse_duration(self._search_regex(
             r'时长:\s*</span>\s*(\d+:\d+)', webpage, 'duration', fatal=False))
@@ -68,11 +47,12 @@ class Porn91IE(InfoExtractor):
         comment_count = int_or_none(self._search_regex(
             r'留言:\s*</span>\s*(\d+)', webpage, 'comment count', fatal=False))
 
-        return {
+        info_dict.update({
             'id': video_id,
             'title': title,
-            'url': video_url,
             'duration': duration,
             'comment_count': comment_count,
             'age_limit': self._rta_search(webpage),
-        }
+        })
+
+        return info_dict

From facfd79f9ad574db7bfcc90cbef01b3a7e7b1262 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 22 Apr 2017 21:20:25 +0800
Subject: [PATCH 0694/1696] [azubu] Remove extractor as the site is gone
 (closes #12813)

---
 ChangeLog                          |   1 +
 youtube_dl/extractor/azubu.py      | 140 -----------------------------
 youtube_dl/extractor/extractors.py |   1 -
 3 files changed, 1 insertion(+), 141 deletions(-)
 delete mode 100644 youtube_dl/extractor/azubu.py

diff --git a/ChangeLog b/ChangeLog
index 1d49f8c8f..d699acb31 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+- [azubu] Remove extractor (#12813)
 * [porn91] Fix extraction (#12814)
 + [generic] Support Brightcove videos in <iframe> (#12482)
 + [brightcove] Support URLs with bcpid instead of playerID (#12482)
diff --git a/youtube_dl/extractor/azubu.py b/youtube_dl/extractor/azubu.py
deleted file mode 100644
index 3ba2f00d3..000000000
--- a/youtube_dl/extractor/azubu.py
+++ /dev/null
@@ -1,140 +0,0 @@
-from __future__ import unicode_literals
-
-import json
-
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    float_or_none,
-    sanitized_Request,
-)
-
-
-class AzubuIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?azubu\.(?:tv|uol.com.br)/[^/]+#!/play/(?P<id>\d+)'
-    _TESTS = [
-        {
-            'url': 'http://www.azubu.tv/GSL#!/play/15575/2014-hot6-cup-last-big-match-ro8-day-1',
-            'md5': 'a88b42fcf844f29ad6035054bd9ecaf4',
-            'info_dict': {
-                'id': '15575',
-                'ext': 'mp4',
-                'title': '2014 HOT6 CUP LAST BIG MATCH Ro8 Day 1',
-                'description': 'md5:d06bdea27b8cc4388a90ad35b5c66c01',
-                'thumbnail': r're:^https?://.*\.jpe?g',
-                'timestamp': 1417523507.334,
-                'upload_date': '20141202',
-                'duration': 9988.7,
-                'uploader': 'GSL',
-                'uploader_id': 414310,
-                'view_count': int,
-            },
-        },
-        {
-            'url': 'http://www.azubu.tv/FnaticTV#!/play/9344/-fnatic-at-worlds-2014:-toyz---%22i-love-rekkles,-he-has-amazing-mechanics%22-',
-            'md5': 'b72a871fe1d9f70bd7673769cdb3b925',
-            'info_dict': {
-                'id': '9344',
-                'ext': 'mp4',
-                'title': 'Fnatic at Worlds 2014: Toyz - "I love Rekkles, he has amazing mechanics"',
-                'description': 'md5:4a649737b5f6c8b5c5be543e88dc62af',
-                'thumbnail': r're:^https?://.*\.jpe?g',
-                'timestamp': 1410530893.320,
-                'upload_date': '20140912',
-                'duration': 172.385,
-                'uploader': 'FnaticTV',
-                'uploader_id': 272749,
-                'view_count': int,
-            },
-            'skip': 'Channel offline',
-        },
-    ]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        data = self._download_json(
-            'http://www.azubu.tv/api/video/%s' % video_id, video_id)['data']
-
-        title = data['title'].strip()
-        description = data.get('description')
-        thumbnail = data.get('thumbnail')
-        view_count = data.get('view_count')
-        user = data.get('user', {})
-        uploader = user.get('username')
-        uploader_id = user.get('id')
-
-        stream_params = json.loads(data['stream_params'])
-
-        timestamp = float_or_none(stream_params.get('creationDate'), 1000)
-        duration = float_or_none(stream_params.get('length'), 1000)
-
-        renditions = stream_params.get('renditions') or []
-        video = stream_params.get('FLVFullLength') or stream_params.get('videoFullLength')
-        if video:
-            renditions.append(video)
-
-        if not renditions and not user.get('channel', {}).get('is_live', True):
-            raise ExtractorError('%s said: channel is offline.' % self.IE_NAME, expected=True)
-
-        formats = [{
-            'url': fmt['url'],
-            'width': fmt['frameWidth'],
-            'height': fmt['frameHeight'],
-            'vbr': float_or_none(fmt['encodingRate'], 1000),
-            'filesize': fmt['size'],
-            'vcodec': fmt['videoCodec'],
-            'container': fmt['videoContainer'],
-        } for fmt in renditions if fmt['url']]
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'timestamp': timestamp,
-            'duration': duration,
-            'uploader': uploader,
-            'uploader_id': uploader_id,
-            'view_count': view_count,
-            'formats': formats,
-        }
-
-
-class AzubuLiveIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?azubu\.(?:tv|uol.com.br)/(?P<id>[^/]+)$'
-
-    _TESTS = [{
-        'url': 'http://www.azubu.tv/MarsTVMDLen',
-        'only_matching': True,
-    }, {
-        'url': 'http://azubu.uol.com.br/adolfz',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        user = self._match_id(url)
-
-        info = self._download_json(
-            'http://api.azubu.tv/public/modules/last-video/{0}/info'.format(user),
-            user)['data']
-        if info['type'] != 'STREAM':
-            raise ExtractorError('{0} is not streaming live'.format(user), expected=True)
-
-        req = sanitized_Request(
-            'https://edge-elb.api.brightcove.com/playback/v1/accounts/3361910549001/videos/ref:' + info['reference_id'])
-        req.add_header('Accept', 'application/json;pk=BCpkADawqM1gvI0oGWg8dxQHlgT8HkdE2LnAlWAZkOlznO39bSZX726u4JqnDsK3MDXcO01JxXK2tZtJbgQChxgaFzEVdHRjaDoxaOu8hHOO8NYhwdxw9BzvgkvLUlpbDNUuDoc4E4wxDToV')
-        bc_info = self._download_json(req, user)
-        m3u8_url = next(source['src'] for source in bc_info['sources'] if source['container'] == 'M2TS')
-        formats = self._extract_m3u8_formats(m3u8_url, user, ext='mp4')
-        self._sort_formats(formats)
-
-        return {
-            'id': info['id'],
-            'title': self._live_title(info['title']),
-            'uploader_id': user,
-            'formats': formats,
-            'is_live': True,
-            'thumbnail': bc_info['poster'],
-        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index a92cbefed..355a4e56f 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -87,7 +87,6 @@ from .azmedien import (
     AZMedienPlaylistIE,
     AZMedienShowPlaylistIE,
 )
-from .azubu import AzubuIE, AzubuLiveIE
 from .baidu import BaiduVideoIE
 from .bambuser import BambuserIE, BambuserChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE

From 54f54fcca7af7e5bbc779cfc73d58d1ed2b4b6ae Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 22 Apr 2017 21:44:01 +0800
Subject: [PATCH 0695/1696] [socks] Report errors elegantly when credentails
 are required but missing

In some non-standard implementations, the server may respond AUTH_USER_PASS
even if's not listed in available authentication methods. (it should
respond AUTH_NO_ACCEPTABLE per standards)
---
 ChangeLog           | 4 ++++
 youtube_dl/socks.py | 5 +++--
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index d699acb31..8dc6e6857 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,9 @@
 version <unreleased>
 
+Core
+
+* [socks] Handle cases where credentials are required but missing
+
 Extractors
 - [azubu] Remove extractor (#12813)
 * [porn91] Fix extraction (#12814)
diff --git a/youtube_dl/socks.py b/youtube_dl/socks.py
index 0f5d7bdb2..5d4adbe72 100644
--- a/youtube_dl/socks.py
+++ b/youtube_dl/socks.py
@@ -193,9 +193,10 @@ class sockssocket(socket.socket):
 
         self._check_response_version(SOCKS5_VERSION, version)
 
-        if method == Socks5Auth.AUTH_NO_ACCEPTABLE:
+        if method == Socks5Auth.AUTH_NO_ACCEPTABLE or (
+                method == Socks5Auth.AUTH_USER_PASS and (not self._proxy.username or not self._proxy.password)):
             self.close()
-            raise Socks5Error(method)
+            raise Socks5Error(Socks5Auth.AUTH_NO_ACCEPTABLE)
 
         if method == Socks5Auth.AUTH_USER_PASS:
             username = self._proxy.username.encode('utf-8')

From fbf56be213021669363b69c5d0866b2cf22ecf2a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 22 Apr 2017 22:11:37 +0800
Subject: [PATCH 0696/1696] [iqiyi] Fix playlist detection (#12504)

---
 ChangeLog                     |  1 +
 youtube_dl/extractor/iqiyi.py | 15 +++++++++------
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 8dc6e6857..50daa1a69 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -5,6 +5,7 @@ Core
 * [socks] Handle cases where credentials are required but missing
 
 Extractors
+* [iqiyi] Fix playlist detection (#12504)
 - [azubu] Remove extractor (#12813)
 * [porn91] Fix extraction (#12814)
 + [generic] Support Brightcove videos in <iframe> (#12482)
diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 2af6a6db4..8a711afb9 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -337,13 +337,16 @@ class IqiyiIE(InfoExtractor):
             url, 'temp_id', note='download video page')
 
         # There's no simple way to determine whether an URL is a playlist or not
-        # So detect it
-        playlist_result = self._extract_playlist(webpage)
-        if playlist_result:
-            return playlist_result
-
+        # Sometimes there are playlist links in individual videos, so treat it
+        # as a single video first
         tvid = self._search_regex(
-            r'data-player-tvid\s*=\s*[\'"](\d+)', webpage, 'tvid')
+            r'data-player-tvid\s*=\s*[\'"](\d+)', webpage, 'tvid', default=None)
+        if tvid is None:
+            playlist_result = self._extract_playlist(webpage)
+            if playlist_result:
+                return playlist_result
+            raise ExtractorError('Can\'t find any video')
+
         video_id = self._search_regex(
             r'data-player-videoid\s*=\s*[\'"]([a-f\d]+)', webpage, 'video_id')
 

From 3e0304fe6e3a194cfb04f21aa261effb0850da40 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 22 Apr 2017 16:42:24 +0100
Subject: [PATCH 0697/1696] [downloader/fragment] use the documented names for
 fragment progress_hooks fields

---
 youtube_dl/downloader/dash.py     |  2 +-
 youtube_dl/downloader/f4m.py      |  2 +-
 youtube_dl/downloader/fragment.py | 18 ++++++++++--------
 youtube_dl/downloader/hls.py      |  2 +-
 youtube_dl/downloader/ism.py      |  2 +-
 5 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index 94a13a543..7491fdad8 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -28,7 +28,7 @@ class DashSegmentsFD(FragmentFD):
         frag_index = 0
         for i, segment in enumerate(segments):
             frag_index += 1
-            if frag_index <= ctx['frag_index']:
+            if frag_index <= ctx['fragment_index']:
                 continue
             # In DASH, the first segment contains necessary headers to
             # generate a valid MP4 file, so always abort for the first segment
diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index e456ed58f..c8fde9a89 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -376,7 +376,7 @@ class F4mFD(FragmentFD):
         while fragments_list:
             seg_i, frag_i = fragments_list.pop(0)
             frag_index += 1
-            if frag_index <= ctx['frag_index']:
+            if frag_index <= ctx['fragment_index']:
                 continue
             name = 'Seg%d-Frag%d' % (seg_i, frag_i)
             query = []
diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 80bb14d61..6c02cfc98 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -66,7 +66,9 @@ class FragmentFD(FileDownloader):
         if not (ctx.get('live') or ctx['tmpfilename'] == '-'):
             frag_index_stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'w')
             frag_index_stream.write(json.dumps({
-                'frag_index': ctx['frag_index']
+                'download': {
+                    'last_fragment_index': ctx['fragment_index']
+                },
             }))
             frag_index_stream.close()
 
@@ -100,7 +102,7 @@ class FragmentFD(FileDownloader):
             ytdl_filename = encodeFilename(self.ytdl_filename(ctx['filename']))
             if os.path.isfile(ytdl_filename):
                 frag_index_stream, _ = sanitize_open(ytdl_filename, 'r')
-                frag_index = json.loads(frag_index_stream.read())['frag_index']
+                frag_index = json.loads(frag_index_stream.read())['download']['last_fragment_index']
                 frag_index_stream.close()
         dest_stream, tmpfilename = sanitize_open(tmpfilename, open_mode)
 
@@ -108,7 +110,7 @@ class FragmentFD(FileDownloader):
             'dl': dl,
             'dest_stream': dest_stream,
             'tmpfilename': tmpfilename,
-            'frag_index': frag_index,
+            'fragment_index': frag_index,
             # Total complete fragments downloaded so far in bytes
             'complete_frags_downloaded_bytes': resume_len,
         })
@@ -120,8 +122,8 @@ class FragmentFD(FileDownloader):
         state = {
             'status': 'downloading',
             'downloaded_bytes': ctx['complete_frags_downloaded_bytes'],
-            'frag_index': ctx['frag_index'],
-            'frag_count': total_frags,
+            'fragment_index': ctx['fragment_index'],
+            'fragment_count': total_frags,
             'filename': ctx['filename'],
             'tmpfilename': ctx['tmpfilename'],
         }
@@ -144,12 +146,12 @@ class FragmentFD(FileDownloader):
             if not ctx['live']:
                 estimated_size = (
                     (ctx['complete_frags_downloaded_bytes'] + frag_total_bytes) /
-                    (state['frag_index'] + 1) * total_frags)
+                    (state['fragment_index'] + 1) * total_frags)
                 state['total_bytes_estimate'] = estimated_size
 
             if s['status'] == 'finished':
-                state['frag_index'] += 1
-                ctx['frag_index'] = state['frag_index']
+                state['fragment_index'] += 1
+                ctx['fragment_index'] = state['fragment_index']
                 state['downloaded_bytes'] += frag_total_bytes - ctx['prev_frag_downloaded_bytes']
                 ctx['complete_frags_downloaded_bytes'] = state['downloaded_bytes']
                 ctx['prev_frag_downloaded_bytes'] = 0
diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 9a87d7ca8..0e29c8a2a 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -106,7 +106,7 @@ class HlsFD(FragmentFD):
             if line:
                 if not line.startswith('#'):
                     frag_index += 1
-                    if frag_index <= ctx['frag_index']:
+                    if frag_index <= ctx['fragment_index']:
                         continue
                     frag_url = (
                         line
diff --git a/youtube_dl/downloader/ism.py b/youtube_dl/downloader/ism.py
index 9f0fc36b3..338820e71 100644
--- a/youtube_dl/downloader/ism.py
+++ b/youtube_dl/downloader/ism.py
@@ -227,7 +227,7 @@ class IsmFD(FragmentFD):
         frag_index = 0
         for i, segment in enumerate(segments):
             frag_index += 1
-            if frag_index <= ctx['frag_index']:
+            if frag_index <= ctx['fragment_index']:
                 continue
             count = 0
             while count <= fragment_retries:

From 4abdba643cc2d610fbbfe7ece9ee97a3df98ad33 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 22 Apr 2017 18:19:47 +0100
Subject: [PATCH 0698/1696] [downloader/fragment] remove unused code

---
 youtube_dl/downloader/fragment.py | 1 -
 youtube_dl/downloader/ism.py      | 4 ----
 2 files changed, 5 deletions(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 6c02cfc98..0a3b1ece0 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -12,7 +12,6 @@ from ..utils import (
     encodeFilename,
     sanitize_open,
     sanitized_Request,
-    compat_str,
 )
 
 
diff --git a/youtube_dl/downloader/ism.py b/youtube_dl/downloader/ism.py
index 338820e71..5f6f9faef 100644
--- a/youtube_dl/downloader/ism.py
+++ b/youtube_dl/downloader/ism.py
@@ -232,10 +232,6 @@ class IsmFD(FragmentFD):
             count = 0
             while count <= fragment_retries:
                 try:
-                    success = ctx['dl'].download(target_filename, {
-                        'url': segment_url,
-                        'http_headers': info_dict.get('http_headers'),
-                    })
                     success, frag_content = self._download_fragment(ctx, segment['url'], info_dict)
                     if not success:
                         return False

From ffbc8386b9298ef343a71bf54d7426a19755fef8 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 22 Apr 2017 22:26:20 +0100
Subject: [PATCH 0699/1696] [brightcove] match only video elements with
 data-video-id attribute

---
 youtube_dl/extractor/brightcove.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 183ac7dd0..97602ca30 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -522,7 +522,7 @@ class BrightcoveNewIE(InfoExtractor):
         # [2] looks like:
         for video, script_tag, account_id, player_id, embed in re.findall(
                 r'''(?isx)
-                    (<video\s+[^>]+>)
+                    (<video\s+[^>]*data-video-id=['"]?[^>]+>)
                     (?:.*?
                         (<script[^>]+
                             src=["\'](?:https?:)?//players\.brightcove\.net/

From 9c99bef704d185783b61a20ea1f5b58c4c55aba6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Apr 2017 11:33:19 +0700
Subject: [PATCH 0700/1696] [extractor/common] Use float for scaled tbr

---
 youtube_dl/extractor/common.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 359c549c5..2a099480f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1420,7 +1420,9 @@ class InfoExtractor(object):
             elif line.startswith('#') or not line.strip():
                 continue
             else:
-                tbr = int_or_none(last_stream_inf.get('AVERAGE-BANDWIDTH') or last_stream_inf.get('BANDWIDTH'), scale=1000)
+                tbr = float_or_none(
+                    last_stream_inf.get('AVERAGE-BANDWIDTH') or
+                    last_stream_inf.get('BANDWIDTH'), scale=1000)
                 format_id = []
                 if m3u8_id:
                     format_id.append(m3u8_id)
@@ -1850,7 +1852,7 @@ class InfoExtractor(object):
                             'ext': mimetype2ext(mime_type),
                             'width': int_or_none(representation_attrib.get('width')),
                             'height': int_or_none(representation_attrib.get('height')),
-                            'tbr': int_or_none(bandwidth, 1000),
+                            'tbr': float_or_none(bandwidth, 1000),
                             'asr': int_or_none(representation_attrib.get('audioSamplingRate')),
                             'fps': int_or_none(representation_attrib.get('frameRate')),
                             'language': lang if lang not in ('mul', 'und', 'zxx', 'mis') else None,

From 07ad0cf34fdeba447213c993210db1f012a812b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Apr 2017 11:48:51 +0700
Subject: [PATCH 0701/1696] [vidio] Improve and sort formats

---
 youtube_dl/extractor/vidio.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vidio.py b/youtube_dl/extractor/vidio.py
index 4e4b4e38c..701bb1d01 100644
--- a/youtube_dl/extractor/vidio.py
+++ b/youtube_dl/extractor/vidio.py
@@ -49,8 +49,11 @@ class VidioIE(InfoExtractor):
             thumbnail = clip.get('image')
 
         m3u8_url = m3u8_url or self._search_regex(
-            r'data(?:-vjs)?-clip-hls-url=(["\'])(?P<url>.+?)\1', webpage, 'hls url')
-        formats = self._extract_m3u8_formats(m3u8_url, display_id, 'mp4', entry_protocol='m3u8_native')
+            r'data(?:-vjs)?-clip-hls-url=(["\'])(?P<url>(?!\1).+)\1',
+            webpage, 'hls url')
+        formats = self._extract_m3u8_formats(
+            m3u8_url, display_id, 'mp4', entry_protocol='m3u8_native')
+        self._sort_formats(formats)
 
         duration = int_or_none(duration or self._search_regex(
             r'data-video-duration=(["\'])(?P<duartion>\d+)\1', webpage, 'duration'))

From ddd258f92270f48649b57ba4288027a2433af079 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Apr 2017 11:49:57 +0700
Subject: [PATCH 0702/1696] [test_InfoExtractor] Add m3u8 parsing test for NAME
 attribute in EXT-X-STREAM-INF tag

---
 test/test_InfoExtractor.py     | 63 ++++++++++++++++++++++++++++------
 youtube_dl/extractor/common.py |  4 ++-
 2 files changed, 56 insertions(+), 11 deletions(-)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 57205f6ae..2a3c3f7cb 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -361,7 +361,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'ext': 'mp4',
                     'format_id': '155',
                     'protocol': 'm3u8',
-                    'tbr': 155,
+                    'tbr': 155.648,
                     'width': 320,
                     'height': 180,
                 }, {
@@ -370,7 +370,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'ext': 'mp4',
                     'format_id': '502',
                     'protocol': 'm3u8',
-                    'tbr': 502,
+                    'tbr': 502.784,
                     'width': 480,
                     'height': 270,
                 }, {
@@ -379,7 +379,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'ext': 'mp4',
                     'format_id': '827',
                     'protocol': 'm3u8',
-                    'tbr': 827,
+                    'tbr': 827.392,
                     'width': 640,
                     'height': 360,
                 }, {
@@ -388,7 +388,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'ext': 'mp4',
                     'format_id': '1396',
                     'protocol': 'm3u8',
-                    'tbr': 1396,
+                    'tbr': 1396.736,
                     'width': 854,
                     'height': 480,
                 }]
@@ -413,7 +413,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'protocol': 'm3u8',
                     'acodec': 'mp4a.40.2',
                     'vcodec': 'none',
-                    'tbr': 182,
+                    'tbr': 182.725,
                 }, {
                     'url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/mobile/index-muted-HM49I092CC.m3u8',
                     'manifest_url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/mobile/index-muted-HM49I092CC.m3u8',
@@ -422,7 +422,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'protocol': 'm3u8',
                     'acodec': 'mp4a.40.2',
                     'vcodec': 'avc1.42C00D',
-                    'tbr': 280,
+                    'tbr': 280.474,
                     'width': 400,
                     'height': 226,
                 }, {
@@ -433,7 +433,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'protocol': 'm3u8',
                     'acodec': 'mp4a.40.2',
                     'vcodec': 'avc1.42C01E',
-                    'tbr': 628,
+                    'tbr': 628.347,
                     'width': 640,
                     'height': 360,
                 }, {
@@ -444,7 +444,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'protocol': 'm3u8',
                     'acodec': 'mp4a.40.2',
                     'vcodec': 'avc1.42C01E',
-                    'tbr': 893,
+                    'tbr': 893.387,
                     'width': 852,
                     'height': 480,
                 }, {
@@ -455,7 +455,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'protocol': 'm3u8',
                     'acodec': 'mp4a.40.2',
                     'vcodec': 'avc1.42C01F',
-                    'tbr': 1603,
+                    'tbr': 1603.789,
                     'width': 1280,
                     'height': 720,
                 }, {
@@ -466,7 +466,50 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'protocol': 'm3u8',
                     'acodec': 'mp4a.40.2',
                     'vcodec': 'avc1.100.31',
-                    'tbr': 3214,
+                    'tbr': 3214.134,
+                    'width': 1280,
+                    'height': 720,
+                }]
+            ),
+            (
+                # http://www.vidio.com/watch/165683-dj_ambred-booyah-live-2015
+                # EXT-X-STREAM-INF tag with NAME attribute that is not defined
+                # in HLS specification
+                'vidio',
+                'https://www.vidio.com/videos/165683/playlist.m3u8',
+                [{
+                    'url': 'https://www.vidio.com/videos/165683/playlist.m3u8',
+                    'ext': 'mp4',
+                    'format_id': 'meta',
+                    'format_note': 'Quality selection URL',
+                    'protocol': 'm3u8',
+                    'preference': -100,
+                    'resolution': 'multiple'
+                }, {
+                    'url': 'https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b300.mp4.m3u8',
+                    'manifest_url': 'https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b300.mp4.m3u8',
+                    'ext': 'mp4',
+                    'format_id': '270p 3G',
+                    'protocol': 'm3u8',
+                    'tbr': 300,
+                    'width': 480,
+                    'height': 270,
+                }, {
+                    'url': 'https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b600.mp4.m3u8',
+                    'manifest_url': 'https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b600.mp4.m3u8',
+                    'ext': 'mp4',
+                    'format_id': '360p SD',
+                    'protocol': 'm3u8',
+                    'tbr': 600,
+                    'width': 640,
+                    'height': 360,
+                }, {
+                    'url': 'https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b1200.mp4.m3u8',
+                    'manifest_url': 'https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b1200.mp4.m3u8',
+                    'ext': 'mp4',
+                    'format_id': '720p HD',
+                    'protocol': 'm3u8',
+                    'tbr': 1200,
                     'width': 1280,
                     'height': 720,
                 }]
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 2a099480f..6d023106e 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1397,7 +1397,9 @@ class InfoExtractor(object):
 
         def build_stream_name():
             # Despite specification does not mention NAME attribute for
-            # EXT-X-STREAM-INF it still sometimes may be present
+            # EXT-X-STREAM-INF tag (see [1] or vidio test in
+            # test_parse_m3u8_formats) it still sometimes may be present
+            # 1. http://www.vidio.com/watch/165683-dj_ambred-booyah-live-2015
             stream_name = last_stream_inf.get('NAME')
             if stream_name:
                 return stream_name

From 3019cb0c9976481cf2afaf0a2005d90040204fa0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Apr 2017 11:51:53 +0700
Subject: [PATCH 0703/1696] [extractor/common] Rephrase comment

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 6d023106e..9184e53e9 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1397,8 +1397,8 @@ class InfoExtractor(object):
 
         def build_stream_name():
             # Despite specification does not mention NAME attribute for
-            # EXT-X-STREAM-INF tag (see [1] or vidio test in
-            # test_parse_m3u8_formats) it still sometimes may be present
+            # EXT-X-STREAM-INF tag it still sometimes may be present (see [1]
+            # or vidio test in TestInfoExtractor.test_parse_m3u8_formats)
             # 1. http://www.vidio.com/watch/165683-dj_ambred-booyah-live-2015
             stream_name = last_stream_inf.get('NAME')
             if stream_name:

From a4d6cf970ca17f1112465353ec1c2aa857dd2e69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 24 Apr 2017 00:50:39 +0700
Subject: [PATCH 0704/1696] [YoutubeDL] Fix output template for missing
 timestamp (closes #12796)

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 819b374ef..eb465c425 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -640,7 +640,7 @@ class YoutubeDL(object):
 
             NUMERIC_FIELDS = set((
                 'width', 'height', 'tbr', 'abr', 'asr', 'vbr', 'fps', 'filesize', 'filesize_approx',
-                'upload_year', 'upload_month', 'upload_day',
+                'timestamp', 'upload_year', 'upload_month', 'upload_day',
                 'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
                 'average_rating', 'comment_count', 'age_limit',
                 'start_time', 'end_time',

From d3f0687cf7b049b976420056e02c26b5d96adeed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 24 Apr 2017 02:54:17 +0700
Subject: [PATCH 0705/1696] [downloader/fragment] Use temp file for current
 fragment

---
 youtube_dl/downloader/fragment.py | 59 ++++++++++++++++++++-----------
 youtube_dl/downloader/http.py     |  8 ++---
 2 files changed, 41 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 0a3b1ece0..62de0a75b 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -2,7 +2,6 @@ from __future__ import division, unicode_literals
 
 import os
 import time
-import io
 import json
 
 from .common import FileDownloader
@@ -48,28 +47,42 @@ class FragmentFD(FileDownloader):
         self._prepare_frag_download(ctx)
         self._start_frag_download(ctx)
 
+    def _read_ytdl_file(self, ctx):
+        stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'r')
+        ctx['fragment_index'] = json.loads(stream.read())['download']['current_fragment_index']
+        stream.close()
+
+    def _write_ytdl_file(self, ctx):
+        frag_index_stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'w')
+        frag_index_stream.write(json.dumps({
+            'download': {
+                'current_fragment_index': ctx['fragment_index']
+            },
+        }))
+        frag_index_stream.close()
+
     def _download_fragment(self, ctx, frag_url, info_dict, headers=None):
-        down = io.BytesIO()
-        success = ctx['dl'].download(down, {
+        fragment_filename = '%s-Frag%d' % (ctx['tmpfilename'], ctx['fragment_index'])
+        success = ctx['dl'].download(fragment_filename, {
             'url': frag_url,
             'http_headers': headers or info_dict.get('http_headers'),
         })
         if not success:
             return False, None
-        frag_content = down.getvalue()
+        down, frag_sanitized = sanitize_open(fragment_filename, 'rb')
+        ctx['fragment_filename_sanitized'] = frag_sanitized
+        frag_content = down.read()
         down.close()
         return True, frag_content
 
     def _append_fragment(self, ctx, frag_content):
-        ctx['dest_stream'].write(frag_content)
-        if not (ctx.get('live') or ctx['tmpfilename'] == '-'):
-            frag_index_stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'w')
-            frag_index_stream.write(json.dumps({
-                'download': {
-                    'last_fragment_index': ctx['fragment_index']
-                },
-            }))
-            frag_index_stream.close()
+        try:
+            ctx['dest_stream'].write(frag_content)
+        finally:
+            if not (ctx.get('live') or ctx['tmpfilename'] == '-'):
+                self._write_ytdl_file(ctx)
+            os.remove(ctx['fragment_filename_sanitized'])
+            del ctx['fragment_filename_sanitized']
 
     def _prepare_frag_download(self, ctx):
         if 'live' not in ctx:
@@ -93,23 +106,29 @@ class FragmentFD(FileDownloader):
         tmpfilename = self.temp_name(ctx['filename'])
         open_mode = 'wb'
         resume_len = 0
-        frag_index = 0
+
         # Establish possible resume length
         if os.path.isfile(encodeFilename(tmpfilename)):
             open_mode = 'ab'
             resume_len = os.path.getsize(encodeFilename(tmpfilename))
-            ytdl_filename = encodeFilename(self.ytdl_filename(ctx['filename']))
-            if os.path.isfile(ytdl_filename):
-                frag_index_stream, _ = sanitize_open(ytdl_filename, 'r')
-                frag_index = json.loads(frag_index_stream.read())['download']['last_fragment_index']
-                frag_index_stream.close()
+
+        ctx['fragment_index'] = 0
+        if os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename']))):
+            self._read_ytdl_file(ctx)
+        else:
+            self._write_ytdl_file(ctx)
+
+        if ctx['fragment_index'] > 0:
+            assert resume_len > 0
+        else:
+            assert resume_len == 0
+
         dest_stream, tmpfilename = sanitize_open(tmpfilename, open_mode)
 
         ctx.update({
             'dl': dl,
             'dest_stream': dest_stream,
             'tmpfilename': tmpfilename,
-            'fragment_index': frag_index,
             # Total complete fragments downloaded so far in bytes
             'complete_frags_downloaded_bytes': resume_len,
         })
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 2896a17af..af405b950 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -20,14 +20,10 @@ from ..utils import (
 
 
 class HttpFD(FileDownloader):
-    def real_download(self, filename_or_stream, info_dict):
+    def real_download(self, filename, info_dict):
         url = info_dict['url']
-        filename = filename_or_stream
-        stream = None
-        if hasattr(filename_or_stream, 'write'):
-            stream = filename_or_stream
-            filename = '-'
         tmpfilename = self.temp_name(filename)
+        stream = None
 
         # Do not include the Accept-Encoding header
         headers = {'Youtubedl-no-compression': 'True'}

From 0eee52f34bfa55ba9d4ebfc4b4ba508c989f05b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 24 Apr 2017 03:09:08 +0700
Subject: [PATCH 0706/1696] Introduce --keep-fragments

---
 youtube_dl/__init__.py            | 1 +
 youtube_dl/downloader/fragment.py | 5 ++++-
 youtube_dl/options.py             | 4 ++++
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index f15606568..c4589411e 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -343,6 +343,7 @@ def _real_main(argv=None):
         'retries': opts.retries,
         'fragment_retries': opts.fragment_retries,
         'skip_unavailable_fragments': opts.skip_unavailable_fragments,
+        'keep_fragments': opts.keep_fragments,
         'buffersize': opts.buffersize,
         'noresizebuffer': opts.noresizebuffer,
         'continuedl': opts.continue_dl,
diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 62de0a75b..bcff94cbe 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -29,6 +29,8 @@ class FragmentFD(FileDownloader):
                         and hlsnative only)
     skip_unavailable_fragments:
                         Skip unavailable fragments (DASH and hlsnative only)
+    keep_fragments:     Keep downloaded fragments on disk after downloading is
+                        finished
     """
 
     def report_retry_fragment(self, err, frag_index, count, retries):
@@ -81,7 +83,8 @@ class FragmentFD(FileDownloader):
         finally:
             if not (ctx.get('live') or ctx['tmpfilename'] == '-'):
                 self._write_ytdl_file(ctx)
-            os.remove(ctx['fragment_filename_sanitized'])
+            if not self.params.get('keep_fragments', False):
+                os.remove(ctx['fragment_filename_sanitized'])
             del ctx['fragment_filename_sanitized']
 
     def _prepare_frag_download(self, ctx):
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 2d2f5e47b..52309fb84 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -468,6 +468,10 @@ def parseOpts(overrideArguments=None):
         '--abort-on-unavailable-fragment',
         action='store_false', dest='skip_unavailable_fragments',
         help='Abort downloading when some fragment is not available')
+    downloader.add_option(
+        '--keep-fragments',
+        action='store_true', dest='keep_fragments', default=False,
+        help='Keep downloaded fragments on disk after downloading is finished; fragments are erased by default')
     downloader.add_option(
         '--buffer-size',
         dest='buffersize', metavar='SIZE', default='1024',

From adb4b03cd583857d00fa03c050d1472bf44a0d15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 24 Apr 2017 23:05:56 +0700
Subject: [PATCH 0707/1696] [downloader/fragment] Don't process ytdl file when
 it's not needed yet

---
 youtube_dl/downloader/fragment.py | 36 ++++++++++++++++++++-----------
 1 file changed, 23 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index bcff94cbe..25c8f18ec 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -49,6 +49,10 @@ class FragmentFD(FileDownloader):
         self._prepare_frag_download(ctx)
         self._start_frag_download(ctx)
 
+    @staticmethod
+    def __do_ytdl_file(ctx):
+        return not ctx['live'] and not ctx['tmpfilename'] == '-'
+
     def _read_ytdl_file(self, ctx):
         stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'r')
         ctx['fragment_index'] = json.loads(stream.read())['download']['current_fragment_index']
@@ -81,7 +85,7 @@ class FragmentFD(FileDownloader):
         try:
             ctx['dest_stream'].write(frag_content)
         finally:
-            if not (ctx.get('live') or ctx['tmpfilename'] == '-'):
+            if self.__do_ytdl_file(ctx):
                 self._write_ytdl_file(ctx)
             if not self.params.get('keep_fragments', False):
                 os.remove(ctx['fragment_filename_sanitized'])
@@ -115,16 +119,21 @@ class FragmentFD(FileDownloader):
             open_mode = 'ab'
             resume_len = os.path.getsize(encodeFilename(tmpfilename))
 
-        ctx['fragment_index'] = 0
-        if os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename']))):
-            self._read_ytdl_file(ctx)
-        else:
-            self._write_ytdl_file(ctx)
+        # Should be initialized before ytdl file check
+        ctx.update({
+            'tmpfilename': tmpfilename,
+            'fragment_index': 0,
+        })
 
-        if ctx['fragment_index'] > 0:
-            assert resume_len > 0
-        else:
-            assert resume_len == 0
+        if self.__do_ytdl_file(ctx):
+            if os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename']))):
+                self._read_ytdl_file(ctx)
+            else:
+                self._write_ytdl_file(ctx)
+            if ctx['fragment_index'] > 0:
+                assert resume_len > 0
+            else:
+                assert resume_len == 0
 
         dest_stream, tmpfilename = sanitize_open(tmpfilename, open_mode)
 
@@ -194,9 +203,10 @@ class FragmentFD(FileDownloader):
 
     def _finish_frag_download(self, ctx):
         ctx['dest_stream'].close()
-        ytdl_filename = encodeFilename(self.ytdl_filename(ctx['filename']))
-        if os.path.isfile(ytdl_filename):
-            os.remove(ytdl_filename)
+        if self.__do_ytdl_file(ctx):
+            ytdl_filename = encodeFilename(self.ytdl_filename(ctx['filename']))
+            if os.path.isfile(ytdl_filename):
+                os.remove(ytdl_filename)
         elapsed = time.time() - ctx['started']
         self.try_rename(ctx['tmpfilename'], ctx['filename'])
         fsize = os.path.getsize(encodeFilename(ctx['filename']))

From 290f64dbaa3a8c187d47b72b3c808bd52d15a6e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 24 Apr 2017 23:50:20 +0700
Subject: [PATCH 0708/1696] [downloader/fragment] Improve .ytdl format and
 start documenting

---
 youtube_dl/downloader/fragment.py | 31 ++++++++++++++++++++++++++-----
 1 file changed, 26 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 25c8f18ec..50c8254a4 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -31,6 +31,24 @@ class FragmentFD(FileDownloader):
                         Skip unavailable fragments (DASH and hlsnative only)
     keep_fragments:     Keep downloaded fragments on disk after downloading is
                         finished
+
+    For each incomplete fragment download youtube-dl keeps on disk a special
+    bookkeeping file with download state and metadata (in future such files will
+    be used for any incomplete download handled by youtube-dl). This file is
+    used to properly handle resuming, check download file consistency and detect
+    potential errors. The file has a .ytdl extension and represents a standard
+    JSON file of the following format:
+
+    extractor:
+        Dictionary of extractor related data. TBD.
+
+    downloader:
+        Dictionary of downloader related data. May contain following data:
+            current_fragment:
+                Dictionary with current (being downloaded) fragment data:
+                index:  Index of current fragment among all fragments
+            fragment_count:
+                Total count of fragments
     """
 
     def report_retry_fragment(self, err, frag_index, count, retries):
@@ -55,16 +73,19 @@ class FragmentFD(FileDownloader):
 
     def _read_ytdl_file(self, ctx):
         stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'r')
-        ctx['fragment_index'] = json.loads(stream.read())['download']['current_fragment_index']
+        ctx['fragment_index'] = json.loads(stream.read())['downloader']['current_fragment']['index']
         stream.close()
 
     def _write_ytdl_file(self, ctx):
         frag_index_stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'w')
-        frag_index_stream.write(json.dumps({
-            'download': {
-                'current_fragment_index': ctx['fragment_index']
+        downloader = {
+            'current_fragment': {
+                'index': ctx['fragment_index'],
             },
-        }))
+        }
+        if ctx.get('fragment_count') is not None:
+            downloader['fragment_count'] = ctx['fragment_count']
+        frag_index_stream.write(json.dumps({'downloader': downloader}))
         frag_index_stream.close()
 
     def _download_fragment(self, ctx, frag_url, info_dict, headers=None):

From da57ebaf84225240b356530cdf02d12596f0dce8 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Tue, 25 Apr 2017 01:06:14 +0200
Subject: [PATCH 0709/1696] [openload] separate PhantomJS code from extractor

---
 youtube_dl/extractor/openload.py |  78 ++++-------------
 youtube_dl/utils.py              | 141 +++++++++++++++++++++++++++++++
 2 files changed, 157 insertions(+), 62 deletions(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 789bf997e..ac5e0bb08 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -1,17 +1,14 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import os
 import re
-import subprocess
-import tempfile
 
 from .common import InfoExtractor
 from ..utils import (
-    check_executable,
     determine_ext,
-    encodeArgument,
     ExtractorError,
+    get_element_by_id,
+    PhantomJSwrapper,
 )
 
 
@@ -62,38 +59,7 @@ class OpenloadIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    _PHANTOMJS_SCRIPT = r'''
-        phantom.onError = function(msg, trace) {
-          var msgStack = ['PHANTOM ERROR: ' + msg];
-          if(trace && trace.length) {
-            msgStack.push('TRACE:');
-            trace.forEach(function(t) {
-              msgStack.push(' -> ' + (t.file || t.sourceURL) + ': ' + t.line
-                + (t.function ? ' (in function ' + t.function +')' : ''));
-            });
-          }
-          console.error(msgStack.join('\n'));
-          phantom.exit(1);
-        };
-        var page = require('webpage').create();
-        page.settings.resourceTimeout = 10000;
-        page.onInitialized = function() {
-          page.evaluate(function() {
-            delete window._phantom;
-            delete window.callPhantom;
-          });
-        };
-        page.open('https://openload.co/embed/%s/', function(status) {
-          var info = page.evaluate(function() {
-            return {
-              decoded_id: document.getElementById('streamurl').innerHTML,
-              title: document.querySelector('meta[name="og:title"],'
-                + 'meta[name=description]').content
-            };
-          });
-          console.log(info.decoded_id + ' ' + info.title);
-          phantom.exit();
-        });'''
+    _USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
 
     @staticmethod
     def _extract_urls(webpage):
@@ -102,40 +68,27 @@ class OpenloadIE(InfoExtractor):
             webpage)
 
     def _real_extract(self, url):
-        exe = check_executable('phantomjs', ['-v'])
-        if not exe:
-            raise ExtractorError('PhantomJS executable not found in PATH, '
-                                 'download it from http://phantomjs.org',
-                                 expected=True)
-
         video_id = self._match_id(url)
         url = 'https://openload.co/embed/%s/' % video_id
-        webpage = self._download_webpage(url, video_id)
+        headers = {
+            'User-Agent': self._USER_AGENT,
+        }
+
+        phantom = PhantomJSwrapper(self)
+        webpage, _ = phantom.get(url, video_id=video_id, headers=headers)
 
         if 'File not found' in webpage or 'deleted by the owner' in webpage:
             raise ExtractorError('File not found', expected=True, video_id=video_id)
 
-        script_file = tempfile.NamedTemporaryFile(mode='w', delete=False)
-
-        # write JS script to file and close it
-        with script_file:
-            script_file.write(self._PHANTOMJS_SCRIPT % video_id)
-
-        self.to_screen('%s: Decoding video ID with PhantomJS' % video_id)
-
-        p = subprocess.Popen([exe, '--ssl-protocol=any', script_file.name],
-            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        output, err = p.communicate()
-        if p.returncode != 0:
-            raise ExtractorError('Decoding failed\n:'
-                                 + encodeArgument(err))
-        else:
-            decoded_id, title = encodeArgument(output).strip().split(' ', 1)
-
-        os.remove(script_file.name)
+        decoded_id = get_element_by_id('streamurl', webpage)
 
         video_url = 'https://openload.co/stream/%s?mime=true' % decoded_id
 
+        title = self._og_search_title(webpage, default=None) or self._search_regex(
+            r'<span[^>]+class=["\']title["\'][^>]*>([^<]+)', webpage,
+            'title', default=None) or self._html_search_meta(
+            'description', webpage, 'title', fatal=True)
+
         entries = self._parse_html5_media_entries(url, webpage, video_id)
         entry = entries[0] if entries else {}
         subtitles = entry.get('subtitles')
@@ -148,5 +101,6 @@ class OpenloadIE(InfoExtractor):
             # Seems all videos have extensions in their titles
             'ext': determine_ext(title, 'mp4'),
             'subtitles': subtitles,
+            'http_headers': headers,
         }
         return info_dict
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 2340bc306..94e1b07a6 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3652,3 +3652,144 @@ def write_xattr(path, key, value):
                         "Couldn't find a tool to set the xattrs. "
                         "Install either the python 'xattr' module, "
                         "or the 'xattr' binary.")
+
+
+class PhantomJSwrapper(object):
+    """PhantomJS wrapper class"""
+
+    _TEMPLATE = r'''
+        phantom.onError = function(msg, trace) {{
+          var msgStack = ['PHANTOM ERROR: ' + msg];
+          if(trace && trace.length) {{
+            msgStack.push('TRACE:');
+            trace.forEach(function(t) {{
+              msgStack.push(' -> ' + (t.file || t.sourceURL) + ': ' + t.line
+                + (t.function ? ' (in function ' + t.function +')' : ''));
+            }});
+          }}
+          console.error(msgStack.join('\n'));
+          phantom.exit(1);
+        }};
+        var page = require('webpage').create();
+        var fs = require('fs');
+        var read = {{ mode: 'r', charset: 'utf-8' }};
+        var write = {{ mode: 'w', charset: 'utf-8' }};
+        page.settings.resourceTimeout = {timeout};
+        page.settings.userAgent = "{ua}";
+        page.onLoadStarted = function() {{
+          page.evaluate(function() {{
+            delete window._phantom;
+            delete window.callPhantom;
+          }});
+        }};
+        var saveAndExit = function() {{
+          fs.write("{html}", page.content, write);
+          phantom.exit();
+        }};
+        page.onLoadFinished = function(status) {{
+          if(page.url === "") {{
+            page.setContent(fs.read("{html}", read), "{url}");
+          }}
+          else {{
+            {jscode}
+          }}
+        }};
+        page.open("");
+    '''
+
+    _TMP_FILE_NAMES = ['script', 'html']
+
+    def __init__(self, extractor, timeout=10000):
+        self.exe = check_executable('phantomjs', ['-v'])
+        if not self.exe:
+            raise ExtractorError('PhantomJS executable not found in PATH, '
+                                 'download it from http://phantomjs.org',
+                                 expected=True)
+        self.extractor = extractor
+        self.options = {
+            'timeout': timeout,
+        }
+        self._TMP_FILES = {}
+        for name in self._TMP_FILE_NAMES:
+            tmp = tempfile.NamedTemporaryFile(delete=False)
+            tmp.close()
+            self._TMP_FILES[name] = tmp
+
+    def __del__(self):
+        for name in self._TMP_FILE_NAMES:
+            try:
+                os.remove(self._TMP_FILES[name].name)
+            except:
+                pass
+
+    def get(self, url, html=None, video_id=None, note=None, note2='Executing JS on webpage', headers={}, jscode='saveAndExit();'):
+        """
+        Downloads webpage (if needed) and executes JS
+        
+        Params:
+            url: website url
+            html: optional, html code of website
+            video_id: video id
+            note: optional, displayed when downloading webpage
+            note2: optional, displayed when executing JS
+            headers: custom http headers
+            jscode: code to be executed when page is loaded
+        
+        Returns tuple with:
+            * downloaded website (after JS execution)
+            * anything you print with `console.log` (but not inside `page.execute`!)
+        
+        In most cases you don't need to add any `jscode`.
+        It is executed in `page.onLoadFinished`.
+        `saveAndExit();` is mandatory, use it instead of `phantom.exit()`
+        It is possible to wait for some element on the webpage, for example:
+            var check = function() {
+              var elementFound = page.evaluate(function() {
+                return document.querySelector('#b.done') !== null;
+              });
+              if(elementFound)
+                saveAndExit();
+              else
+                window.setTimeout(check, 500);
+            }
+            
+            page.evaluate(function(){
+              document.querySelector('#a').click();
+            });
+            check();
+        """
+        if 'saveAndExit();' not in jscode:
+            raise ExtractorError('`saveAndExit();` not found in `jscode`')
+        if not html:
+            html = self.extractor._download_webpage(url, video_id, note=note, headers=headers)
+        with open(self._TMP_FILES['html'].name, 'wb') as f:
+            f.write(html.encode('utf-8'))
+
+        replaces = self.options
+        replaces['url'] = url
+        user_agent = headers.get('User-Agent') or std_headers['User-Agent']
+        replaces['ua'] = user_agent.replace('"', '\\"')
+        replaces['jscode'] = jscode
+
+        for x in self._TMP_FILE_NAMES:
+            replaces[x] = self._TMP_FILES[x].name.replace('\\', '\\\\').replace('"', '\\"')
+
+        with open(self._TMP_FILES['script'].name, 'wb') as f:
+            f.write(self._TEMPLATE.format(**replaces).encode('utf-8'))
+
+        if video_id is None:
+            self.extractor.to_screen('%s' % (note2,))
+        else:
+            self.extractor.to_screen('%s: %s' % (video_id, note2))
+
+        p = subprocess.Popen([self.exe, '--ssl-protocol=any',
+            self._TMP_FILES['script'].name], stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE)
+        out, err = p.communicate()
+        if p.returncode != 0:
+            raise ExtractorError('Executing JS failed\n:'
+                                 + encodeArgument(err))
+        with open(self._TMP_FILES['html'].name, 'rb') as f:
+            html = f.read().decode('utf-8')
+        return (html, encodeArgument(out))
+

From 40e41780f1d770a355f01e3c1e6fb09ff392f97e Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Tue, 25 Apr 2017 15:12:54 +0200
Subject: [PATCH 0710/1696] [phantomjs] add cookie support

---
 youtube_dl/extractor/common.py |  8 +++--
 youtube_dl/utils.py            | 62 +++++++++++++++++++++++++++++++++-
 2 files changed, 66 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index dcc9d628a..e54adc9f0 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2343,10 +2343,12 @@ class InfoExtractor(object):
                 self._downloader.report_warning(msg)
         return res
 
-    def _set_cookie(self, domain, name, value, expire_time=None):
+    def _set_cookie(self, domain, name, value, expire_time=None, port=None,
+                    path='/', secure=False, discard=False, rest={}, **kwargs):
         cookie = compat_cookiejar.Cookie(
-            0, name, value, None, None, domain, None,
-            None, '/', True, False, expire_time, '', None, None, None)
+            0, name, value, port, not port is None, domain, True,
+            domain.startswith('.'), path, True, secure, expire_time,
+            discard, None, None, rest)
         self._downloader.cookiejar.set_cookie(cookie)
 
     def _get_cookies(self, url):
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 94e1b07a6..9c94b7ec9 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3654,6 +3654,37 @@ def write_xattr(path, key, value):
                         "or the 'xattr' binary.")
 
 
+def cookie_to_dict(cookie):
+    cookie_dict = {
+        'name': cookie.name,
+        'value': cookie.value,
+    };
+    if cookie.port_specified:
+        cookie_dict['port'] = cookie.port
+    if cookie.domain_specified:
+        cookie_dict['domain'] = cookie.domain
+    if cookie.path_specified:
+        cookie_dict['path'] = cookie.path
+    if not cookie.expires is None:
+        cookie_dict['expires'] = cookie.expires
+    if not cookie.secure is None:
+        cookie_dict['secure'] = cookie.secure
+    if not cookie.discard is None:
+        cookie_dict['discard'] = cookie.discard
+    try:
+        if (cookie.has_nonstandard_attr('httpOnly') or
+            cookie.has_nonstandard_attr('httponly') or
+            cookie.has_nonstandard_attr('HttpOnly')):
+            cookie_dict['httponly'] = True
+    except TypeError:
+        pass
+    return cookie_dict
+
+
+def cookie_jar_to_list(cookie_jar):
+    return [cookie_to_dict(cookie) for cookie in cookie_jar]
+
+
 class PhantomJSwrapper(object):
     """PhantomJS wrapper class"""
 
@@ -3674,6 +3705,9 @@ class PhantomJSwrapper(object):
         var fs = require('fs');
         var read = {{ mode: 'r', charset: 'utf-8' }};
         var write = {{ mode: 'w', charset: 'utf-8' }};
+        JSON.parse(fs.read("{cookies}", read)).forEach(function(x) {{
+          phantom.addCookie(x);
+        }});
         page.settings.resourceTimeout = {timeout};
         page.settings.userAgent = "{ua}";
         page.onLoadStarted = function() {{
@@ -3684,6 +3718,7 @@ class PhantomJSwrapper(object):
         }};
         var saveAndExit = function() {{
           fs.write("{html}", page.content, write);
+          fs.write("{cookies}", JSON.stringify(phantom.cookies), write);
           phantom.exit();
         }};
         page.onLoadFinished = function(status) {{
@@ -3697,7 +3732,7 @@ class PhantomJSwrapper(object):
         page.open("");
     '''
 
-    _TMP_FILE_NAMES = ['script', 'html']
+    _TMP_FILE_NAMES = ['script', 'html', 'cookies']
 
     def __init__(self, extractor, timeout=10000):
         self.exe = check_executable('phantomjs', ['-v'])
@@ -3722,6 +3757,26 @@ class PhantomJSwrapper(object):
             except:
                 pass
 
+    def _save_cookies(self, url):
+        cookies = cookie_jar_to_list(self.extractor._downloader.cookiejar)
+        for cookie in cookies:
+            if 'path' not in cookie:
+                cookie['path'] = '/'
+            if 'domain' not in cookie:
+                cookie['domain'] = compat_urlparse.urlparse(url).netloc
+        with open(self._TMP_FILES['cookies'].name, 'wb') as f:
+            f.write(json.dumps(cookies).encode('utf-8'))
+
+    def _load_cookies(self):
+        with open(self._TMP_FILES['cookies'].name, 'rb') as f:
+            cookies = json.loads(f.read().decode('utf-8'))
+        for cookie in cookies:
+            if cookie['httponly'] is True:
+                cookie['rest'] = { 'httpOnly': None }
+            if 'expiry' in cookie:
+                cookie['expire_time'] = cookie['expiry']
+            self.extractor._set_cookie(**cookie)
+
     def get(self, url, html=None, video_id=None, note=None, note2='Executing JS on webpage', headers={}, jscode='saveAndExit();'):
         """
         Downloads webpage (if needed) and executes JS
@@ -3765,6 +3820,8 @@ class PhantomJSwrapper(object):
         with open(self._TMP_FILES['html'].name, 'wb') as f:
             f.write(html.encode('utf-8'))
 
+        self._save_cookies(url)
+
         replaces = self.options
         replaces['url'] = url
         user_agent = headers.get('User-Agent') or std_headers['User-Agent']
@@ -3791,5 +3848,8 @@ class PhantomJSwrapper(object):
                                  + encodeArgument(err))
         with open(self._TMP_FILES['html'].name, 'rb') as f:
             html = f.read().decode('utf-8')
+
+        self._load_cookies()
+
         return (html, encodeArgument(out))
 

From 01cb57016ff747ab07c506a708093dfd633d58f6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 25 Apr 2017 22:23:57 +0800
Subject: [PATCH 0711/1696] [iqiyi] Fix extraction of Yule videos

---
 ChangeLog                     |  1 +
 youtube_dl/extractor/iqiyi.py | 13 +++++++++----
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 50daa1a69..d38039663 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -5,6 +5,7 @@ Core
 * [socks] Handle cases where credentials are required but missing
 
 Extractors
+* [iqiyi] Fix extraction of Yule videos
 * [iqiyi] Fix playlist detection (#12504)
 - [azubu] Remove extractor (#12813)
 * [porn91] Fix extraction (#12814)
diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 8a711afb9..fdfa7de9e 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -189,7 +189,11 @@ class IqiyiIE(InfoExtractor):
         'only_matching': True,
     }, {
         'url': 'http://yule.iqiyi.com/pcb.html',
-        'only_matching': True,
+        'info_dict': {
+            'id': '4a0af228fddb55ec96398a364248ed7f',
+            'ext': 'mp4',
+            'title': '第2017-04-21期 女艺人频遭极端粉丝骚扰',
+        },
     }, {
         # VIP-only video. The first 2 parts (6 minutes) are available without login
         # MD5 sums omitted as values are different on Travis CI and my machine
@@ -340,7 +344,7 @@ class IqiyiIE(InfoExtractor):
         # Sometimes there are playlist links in individual videos, so treat it
         # as a single video first
         tvid = self._search_regex(
-            r'data-player-tvid\s*=\s*[\'"](\d+)', webpage, 'tvid', default=None)
+            r'data-(?:player|shareplattrigger)-tvid\s*=\s*[\'"](\d+)', webpage, 'tvid', default=None)
         if tvid is None:
             playlist_result = self._extract_playlist(webpage)
             if playlist_result:
@@ -348,7 +352,7 @@ class IqiyiIE(InfoExtractor):
             raise ExtractorError('Can\'t find any video')
 
         video_id = self._search_regex(
-            r'data-player-videoid\s*=\s*[\'"]([a-f\d]+)', webpage, 'video_id')
+            r'data-(?:player|shareplattrigger)-videoid\s*=\s*[\'"]([a-f\d]+)', webpage, 'video_id')
 
         formats = []
         for _ in range(5):
@@ -380,7 +384,8 @@ class IqiyiIE(InfoExtractor):
 
         self._sort_formats(formats)
         title = (get_element_by_id('widget-videotitle', webpage) or
-                 clean_html(get_element_by_attribute('class', 'mod-play-tit', webpage)))
+                 clean_html(get_element_by_attribute('class', 'mod-play-tit', webpage)) or
+                 self._html_search_regex(r'<span[^>]+data-videochanged-title="word"[^>]*>([^<]+)</span>', webpage, 'title'))
 
         return {
             'id': video_id,

From 9f54ae2873f8091c3f2512d307b96785f40a14ca Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 25 Apr 2017 22:42:55 +0800
Subject: [PATCH 0712/1696] Ignore and clean *.ytdl files

---
 .gitignore | 1 +
 Makefile   | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index bef084baa..a5b585f43 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,6 +35,7 @@ updates_key.pem
 *.mkv
 *.swf
 *.part
+*.ytdl
 *.swp
 test/local_parameters.json
 .tox
diff --git a/Makefile b/Makefile
index 9d1ddc9d1..023556391 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
 all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites
 
 clean:
-	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish youtube_dl/extractor/lazy_extractors.py *.dump *.part* *.info.json *.mp4 *.m4a *.flv *.mp3 *.avi *.mkv *.webm *.3gp *.wav *.ape *.swf *.jpg *.png CONTRIBUTING.md.tmp ISSUE_TEMPLATE.md.tmp youtube-dl youtube-dl.exe
+	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish youtube_dl/extractor/lazy_extractors.py *.dump *.part* *.ytdl *.info.json *.mp4 *.m4a *.flv *.mp3 *.avi *.mkv *.webm *.3gp *.wav *.ape *.swf *.jpg *.png CONTRIBUTING.md.tmp ISSUE_TEMPLATE.md.tmp youtube-dl youtube-dl.exe
 	find . -name "*.pyc" -delete
 	find . -name "*.class" -delete
 

From 538eee7b6a0399b71b8cd3f344c2e21e7f248a99 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 25 Apr 2017 22:25:40 +0700
Subject: [PATCH 0713/1696] Add missing test m3u8 file

---
 test/testdata/m3u8/vidio.m3u8 | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 test/testdata/m3u8/vidio.m3u8

diff --git a/test/testdata/m3u8/vidio.m3u8 b/test/testdata/m3u8/vidio.m3u8
new file mode 100644
index 000000000..89c244469
--- /dev/null
+++ b/test/testdata/m3u8/vidio.m3u8
@@ -0,0 +1,10 @@
+#EXTM3U
+
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=300000,RESOLUTION=480x270,NAME="270p 3G"
+https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b300.mp4.m3u8
+
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=600000,RESOLUTION=640x360,NAME="360p SD"
+https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b600.mp4.m3u8
+
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=1200000,RESOLUTION=1280x720,NAME="720p HD"
+https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b1200.mp4.m3u8

From 85f6de25e46562f57be4447e9109be964e6515ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 25 Apr 2017 23:33:35 +0700
Subject: [PATCH 0714/1696] [downloader/fragment] Clarify current_fragment's
 index and mark as experimental

---
 youtube_dl/downloader/fragment.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 50c8254a4..bb9e82578 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -46,9 +46,11 @@ class FragmentFD(FileDownloader):
         Dictionary of downloader related data. May contain following data:
             current_fragment:
                 Dictionary with current (being downloaded) fragment data:
-                index:  Index of current fragment among all fragments
+                index:  0-based index of current fragment among all fragments
             fragment_count:
                 Total count of fragments
+                
+    This feature is experimental and file format may change in future.
     """
 
     def report_retry_fragment(self, err, frag_index, count, retries):

From ac9c69ace7ee22e59a44d25c87b9b53d18762ff7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 25 Apr 2017 23:46:05 +0700
Subject: [PATCH 0715/1696] [extractor/common] Improve jwplayer regex

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9184e53e9..8b3f04c61 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2233,7 +2233,7 @@ class InfoExtractor(object):
 
     def _find_jwplayer_data(self, webpage, video_id=None, transform_source=js_to_json):
         mobj = re.search(
-            r'(?s)jwplayer\((?P<quote>[\'"])[^\'" ]+(?P=quote)\).*?\.setup\s*\((?P<options>[^)]+)\)',
+            r'(?s)jwplayer\((?P<quote>[\'"])[^\'" ]+(?P=quote)\)(?!</script>).*?\.setup\s*\((?P<options>[^)]+)\)',
             webpage)
         if mobj:
             try:

From 30a4ab191a553d4b946731864fd02dba9a6eb0c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Apr 2017 00:03:13 +0700
Subject: [PATCH 0716/1696] [ChangeLog] Actualize

---
 ChangeLog | 31 ++++++++++++++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index d38039663..55775ba1c 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,16 +1,45 @@
 version <unreleased>
 
 Core
-
+* Introduce --keep-fragments for keeping fragments of fragmented download
+  on disk after download is finished
+* [YoutubeDL] Fix output template for missing timestamp (#12796)
 * [socks] Handle cases where credentials are required but missing
+* [extractor/common] Improve HLS extraction (#12211)
+    - Extract m3u8 parsing to separate method
+    - Improve rendition groups extraction
+    - Build stream name according stream GROUP-ID
+    - Ignore reference to AUDIO group without URI when stream has no CODECS
+    - Use float for scaled tbr in _parse_m3u8_formats
+* [utils] Add support for TTML styles in dfxp2srt
+* [downloader/hls] No need to download keys for fragments that have been
+  already downloaded
+* [downloader/fragment] Improve fragment downloading
+    - Resume immediately
+    - Don't concatenate fragments and decrypt them on every resume
+    - Optimize disk storage usage, don't store intermediate fragments on disk
+    - Store bookkeeping download state file
++ [extractor/common] Add support for multiple getters in try_get
++ [extractor/common] Add support for video of WebPage context in _json_ld
+  (#12778)
++ [extractor/common] Relax JWPlayer regular expression and remove
+  duplicate URLs (#12768)
 
 Extractors
 * [iqiyi] Fix extraction of Yule videos
+* [vidio] Improve extraction and sort formats
++ [brightcove] Match only video elements with data-video-id attribute
 * [iqiyi] Fix playlist detection (#12504)
 - [azubu] Remove extractor (#12813)
 * [porn91] Fix extraction (#12814)
+* [vidzi] Fix extraction (#12793)
++ [amp] Extract error message (#12795)
++ [xfileshare] Add support for gorillavid.com and daclips.com (#12776)
+* [instagram] Fix extraction (#12777)
 + [generic] Support Brightcove videos in <iframe> (#12482)
 + [brightcove] Support URLs with bcpid instead of playerID (#12482)
+* [brightcove] Fix _extract_url (#12782)
++ [odnoklassniki] Extract HLS formats
 
 
 version 2017.04.17

From a82f41841d3268e5107cf0462fded1119229ebbf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Apr 2017 00:06:12 +0700
Subject: [PATCH 0717/1696] release 2017.04.26

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 README.md                 | 3 +++
 docs/supportedsites.md    | 2 --
 youtube_dl/version.py     | 2 +-
 5 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 45f8db721..dedc07604 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.17*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.17**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.26*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.26**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.04.17
+[debug] youtube-dl version 2017.04.26
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 55775ba1c..b23aa10d5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.04.26
 
 Core
 * Introduce --keep-fragments for keeping fragments of fragmented download
diff --git a/README.md b/README.md
index 41f647aaa..1ecd2005a 100644
--- a/README.md
+++ b/README.md
@@ -187,6 +187,9 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
                                      and ISM)
     --abort-on-unavailable-fragment  Abort downloading when some fragment is not
                                      available
+    --keep-fragments                 Keep downloaded fragments on disk after
+                                     downloading is finished; fragments are
+                                     erased by default
     --buffer-size SIZE               Size of download buffer (e.g. 1024 or 16K)
                                      (default is 1024)
     --no-resize-buffer               Do not automatically adjust the buffer
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index afae82214..0be7af8c4 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -81,8 +81,6 @@
  - **AZMedien**: AZ Medien videos
  - **AZMedienPlaylist**: AZ Medien playlists
  - **AZMedienShowPlaylist**: AZ Medien show playlists
- - **Azubu**
- - **AzubuLive**
  - **BaiduVideo**: 百度视频
  - **bambuser**
  - **bambuser:channel**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 480e04e06..e206501e1 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.04.17'
+__version__ = '2017.04.26'

From 3dc8b61b7f5b455e4f82626b10776d669082460f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Apr 2017 01:55:29 +0700
Subject: [PATCH 0718/1696] [arte:+7] Relax _VALID_URL (closes #12837)

---
 youtube_dl/extractor/arte.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 69a23e88c..56baef29d 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -180,7 +180,7 @@ class ArteTVBaseIE(InfoExtractor):
 
 class ArteTVPlus7IE(ArteTVBaseIE):
     IE_NAME = 'arte.tv:+7'
-    _VALID_URL = r'https?://(?:(?:www|sites)\.)?arte\.tv/[^/]+/(?P<lang>fr|de|en|es)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:(?:www|sites)\.)?arte\.tv/(?:[^/]+/)?(?P<lang>fr|de|en|es)/(?:videos/)?(?:[^/]+/)*(?P<id>[^/?#&]+)'
 
     _TESTS = [{
         'url': 'http://www.arte.tv/guide/de/sendungen/XEN/xenius/?vid=055918-015_PLUS7-D',
@@ -188,6 +188,9 @@ class ArteTVPlus7IE(ArteTVBaseIE):
     }, {
         'url': 'http://sites.arte.tv/karambolage/de/video/karambolage-22',
         'only_matching': True,
+    }, {
+        'url': 'http://www.arte.tv/de/videos/048696-000-A/der-kluge-bauch-unser-zweites-gehirn',
+        'only_matching': True,
     }]
 
     @classmethod

From e8bfe2a946eda74f16af8734c35fba8c223e41fa Mon Sep 17 00:00:00 2001
From: Lucas M <lawlmart@users.noreply.github.com>
Date: Wed, 26 Apr 2017 12:39:53 -0400
Subject: [PATCH 0719/1696] [streamable] Add support for new embedded URL
 schema

---
 youtube_dl/extractor/streamable.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/streamable.py b/youtube_dl/extractor/streamable.py
index 9f5c237ef..34725274e 100644
--- a/youtube_dl/extractor/streamable.py
+++ b/youtube_dl/extractor/streamable.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class StreamableIE(InfoExtractor):
-    _VALID_URL = r'https?://streamable\.com/(?:e/)?(?P<id>\w+)'
+    _VALID_URL = r'https?://streamable\.com/(?:[es]/)?(?P<id>\w+)'
     _TESTS = [
         {
             'url': 'https://streamable.com/dnd1',
@@ -47,6 +47,10 @@ class StreamableIE(InfoExtractor):
         {
             'url': 'https://streamable.com/e/dnd1',
             'only_matching': True,
+        },
+        {
+            'url': 'https://streamable.com/s/okkqk/drxjds',
+            'only_matching': True,
         }
     ]
 

From 76c19510368d61acd3e1315d93ac6f16dbeb2c65 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Thu, 27 Apr 2017 10:04:21 +0200
Subject: [PATCH 0720/1696] [appletrailers] update test cases

---
 youtube_dl/extractor/appletrailers.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index a6801f3d4..b45b431e1 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -70,7 +70,8 @@ class AppleTrailersIE(InfoExtractor):
     }, {
         'url': 'http://trailers.apple.com/trailers/magnolia/blackthorn/',
         'info_dict': {
-            'id': 'blackthorn',
+            'id': '4489',
+            'title': 'Blackthorn',
         },
         'playlist_mincount': 2,
         'expected_warnings': ['Unable to download JSON metadata'],
@@ -261,7 +262,7 @@ class AppleTrailersSectionIE(InfoExtractor):
             'title': 'Most Popular',
             'id': 'mostpopular',
         },
-        'playlist_mincount': 80,
+        'playlist_mincount': 30,
     }, {
         'url': 'http://trailers.apple.com/#section=moviestudios',
         'info_dict': {

From d68afc5bc9f0d6cbf49751f95db6f1b8e7c1403e Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Thu, 27 Apr 2017 15:27:01 +0200
Subject: [PATCH 0721/1696] [cbsinteractive] fix extractor

---
 youtube_dl/extractor/cbsinteractive.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cbsinteractive.py b/youtube_dl/extractor/cbsinteractive.py
index 57b18e81d..3d48a7cb1 100644
--- a/youtube_dl/extractor/cbsinteractive.py
+++ b/youtube_dl/extractor/cbsinteractive.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .theplatform import ThePlatformIE
+from ..compat import compat_urllib_parse
 from ..utils import int_or_none
 
 
@@ -80,7 +81,7 @@ class CBSInteractiveIE(ThePlatformIE):
 
         media_guid_path = 'media/guid/%d/%s' % (self.MPX_ACCOUNTS[site], vdata['mpxRefId'])
         formats, subtitles = [], {}
-        for (fkey, vid) in vdata['files'].items():
+        for (fkey, vid) in vdata.get('files', {}).items():
             if fkey == 'hls_phone' and 'hls_tablet' in vdata['files']:
                 continue
             release_url = self.TP_RELEASE_URL_TEMPLATE % vid
@@ -89,6 +90,23 @@ class CBSInteractiveIE(ThePlatformIE):
             tp_formats, tp_subtitles = self._extract_theplatform_smil(release_url, video_id, 'Downloading %s SMIL data' % fkey)
             formats.extend(tp_formats)
             subtitles = self._merge_subtitles(subtitles, tp_subtitles)
+
+        if 'm3u8' in vdata:
+            parsed_url = compat_urllib_parse.urlparse(url)
+            m3u8_url = ('%s://%s%s'
+                % (parsed_url.scheme, parsed_url.netloc, vdata['m3u8']))
+            m3u8_formats = self._extract_m3u8_formats(m3u8_url, video_id)
+            for format in m3u8_formats:
+                format['url'] = format['url'].replace('https://', 'http://')
+            formats.extend(m3u8_formats)
+
+        if 'mp4' in vdata:
+            formats.append({
+                'url': vdata['mp4'],
+                'format_id': 'mp4',
+                'ext': 'mp4',
+            })
+
         self._sort_formats(formats)
 
         info = self._extract_theplatform_metadata('kYEXFC/%s' % media_guid_path, video_id)

From e095109da16696d7ee66823db1c2ec2d4c11c1a8 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Thu, 27 Apr 2017 15:40:17 +0200
Subject: [PATCH 0722/1696] [cbsinteractive] update test cases

---
 youtube_dl/extractor/cbsinteractive.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/cbsinteractive.py b/youtube_dl/extractor/cbsinteractive.py
index 3d48a7cb1..f35f686ef 100644
--- a/youtube_dl/extractor/cbsinteractive.py
+++ b/youtube_dl/extractor/cbsinteractive.py
@@ -12,9 +12,10 @@ class CBSInteractiveIE(ThePlatformIE):
     _VALID_URL = r'https?://(?:www\.)?(?P<site>cnet|zdnet)\.com/(?:videos|video/share)/(?P<id>[^/?]+)'
     _TESTS = [{
         'url': 'http://www.cnet.com/videos/hands-on-with-microsofts-windows-8-1-update/',
+        'md5': '041233212a0d06b179c87cbcca1577b8',
         'info_dict': {
             'id': '56f4ea68-bd21-4852-b08c-4de5b8354c60',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Hands-on with Microsoft Windows 8.1 Update',
             'description': 'The new update to the Windows 8 OS brings improved performance for mouse and keyboard users.',
             'uploader_id': '6085384d-619e-11e3-b231-14feb5ca9861',
@@ -23,11 +24,15 @@ class CBSInteractiveIE(ThePlatformIE):
             'timestamp': 1396479627,
             'upload_date': '20140402',
         },
+        'params': {
+            'format': 'mp4',
+        },
     }, {
         'url': 'http://www.cnet.com/videos/whiny-pothole-tweets-at-local-government-when-hit-by-cars-tomorrow-daily-187/',
+        'md5': 'f2b16d73e08d69591dd9e25564695c0c',
         'info_dict': {
             'id': '56527b93-d25d-44e3-b738-f989ce2e49ba',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Whiny potholes tweet at local government when hit by cars (Tomorrow Daily 187)',
             'description': 'Khail and Ashley wonder what other civic woes can be solved by self-tweeting objects, investigate a new kind of VR camera and watch an origami robot self-assemble, walk, climb, dig and dissolve. #TDPothole',
             'uploader_id': 'b163284d-6b73-44fc-b3e6-3da66c392d40',
@@ -36,6 +41,9 @@ class CBSInteractiveIE(ThePlatformIE):
             'timestamp': 1433289889,
             'upload_date': '20150603',
         },
+        'params': {
+            'format': 'mp4',
+        },
     }, {
         'url': 'http://www.zdnet.com/video/share/video-keeping-android-smartphones-and-tablets-secure/',
         'info_dict': {
@@ -45,8 +53,8 @@ class CBSInteractiveIE(ThePlatformIE):
             'description': 'Here\'s the best way to keep Android devices secure, and what you do when they\'ve come to the end of their lives.',
             'uploader_id': 'f2d97ea2-8175-11e2-9d12-0018fe8a00b0',
             'uploader': 'Adrian Kingsley-Hughes',
-            'timestamp': 1448961720,
-            'upload_date': '20151201',
+            'timestamp': 1449129925,
+            'upload_date': '20151203',
         },
         'params': {
             # m3u8 download

From ff99fe529e52b2465f1d973e69df01a6391568d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 25 Apr 2017 22:07:10 +0700
Subject: [PATCH 0723/1696] Don't list master m3u8 playlists in format list
 (closes #12832)

---
 test/test_InfoExtractor.py         | 40 ------------------------------
 youtube_dl/extractor/anvato.py     |  7 +-----
 youtube_dl/extractor/common.py     |  4 +--
 youtube_dl/extractor/funnyordie.py |  3 +--
 youtube_dl/extractor/gamespot.py   |  3 +--
 youtube_dl/extractor/lego.py       |  2 +-
 youtube_dl/extractor/pbs.py        |  2 +-
 youtube_dl/extractor/r7.py         |  3 +--
 youtube_dl/extractor/ted.py        |  2 +-
 youtube_dl/extractor/tvp.py        |  3 +--
 youtube_dl/extractor/viewster.py   |  3 +--
 11 files changed, 11 insertions(+), 61 deletions(-)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 2a3c3f7cb..14abf03c4 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -184,14 +184,6 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 'pluzz_francetv_11507',
                 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/master.m3u8?caption=2017%2F16%2F156589847-1492488987.m3u8%3Afra%3AFrancais&audiotrack=0%3Afra%3AFrancais',
                 [{
-                    'url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/master.m3u8?caption=2017%2F16%2F156589847-1492488987.m3u8%3Afra%3AFrancais&audiotrack=0%3Afra%3AFrancais',
-                    'ext': 'mp4',
-                    'format_id': 'meta',
-                    'format_note': 'Quality selection URL',
-                    'protocol': 'm3u8',
-                    'preference': -100,
-                    'resolution': 'multiple'
-                }, {
                     'url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_0_av.m3u8?null=0',
                     'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_0_av.m3u8?null=0',
                     'ext': 'mp4',
@@ -254,14 +246,6 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 'teamcoco_11995',
                 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/main.m3u8',
                 [{
-                    'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/main.m3u8',
-                    'ext': 'mp4',
-                    'format_id': 'meta',
-                    'format_note': 'Quality selection URL',
-                    'protocol': 'm3u8',
-                    'preference': -100,
-                    'resolution': 'multiple',
-                }, {
                     'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-audio-160k_v4.m3u8',
                     'ext': 'mp4',
                     'format_id': 'audio-0-Default',
@@ -334,14 +318,6 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 'toggle_mobile_12211',
                 'http://cdnapi.kaltura.com/p/2082311/sp/208231100/playManifest/protocol/http/entryId/0_89q6e8ku/format/applehttp/tags/mobile_sd/f/a.m3u8',
                 [{
-                    'url': 'http://cdnapi.kaltura.com/p/2082311/sp/208231100/playManifest/protocol/http/entryId/0_89q6e8ku/format/applehttp/tags/mobile_sd/f/a.m3u8',
-                    'ext': 'mp4',
-                    'format_id': 'meta',
-                    'format_note': 'Quality selection URL',
-                    'protocol': 'm3u8',
-                    'preference': -100,
-                    'resolution': 'multiple'
-                }, {
                     'url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_sa2ntrdg/name/a.mp4/index.m3u8',
                     'ext': 'mp4',
                     'format_id': 'audio-English',
@@ -398,14 +374,6 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 'twitch_vod',
                 'https://usher.ttvnw.net/vod/6528877?allow_source=true&allow_audio_only=true&allow_spectre=true&player=twitchweb&nauth=%7B%22user_id%22%3Anull%2C%22vod_id%22%3A6528877%2C%22expires%22%3A1492887874%2C%22chansub%22%3A%7B%22restricted_bitrates%22%3A%5B%5D%7D%2C%22privileged%22%3Afalse%2C%22https_required%22%3Afalse%7D&nauthsig=3e29296a6824a0f48f9e731383f77a614fc79bee',
                 [{
-                    'url': 'https://usher.ttvnw.net/vod/6528877?allow_source=true&allow_audio_only=true&allow_spectre=true&player=twitchweb&nauth=%7B%22user_id%22%3Anull%2C%22vod_id%22%3A6528877%2C%22expires%22%3A1492887874%2C%22chansub%22%3A%7B%22restricted_bitrates%22%3A%5B%5D%7D%2C%22privileged%22%3Afalse%2C%22https_required%22%3Afalse%7D&nauthsig=3e29296a6824a0f48f9e731383f77a614fc79bee',
-                    'ext': 'mp4',
-                    'format_id': 'meta',
-                    'format_note': 'Quality selection URL',
-                    'protocol': 'm3u8',
-                    'preference': -100,
-                    'resolution': 'multiple'
-                }, {
                     'url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/audio_only/index-muted-HM49I092CC.m3u8',
                     'manifest_url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/audio_only/index-muted-HM49I092CC.m3u8',
                     'ext': 'mp4',
@@ -478,14 +446,6 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 'vidio',
                 'https://www.vidio.com/videos/165683/playlist.m3u8',
                 [{
-                    'url': 'https://www.vidio.com/videos/165683/playlist.m3u8',
-                    'ext': 'mp4',
-                    'format_id': 'meta',
-                    'format_note': 'Quality selection URL',
-                    'protocol': 'm3u8',
-                    'preference': -100,
-                    'resolution': 'multiple'
-                }, {
                     'url': 'https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b300.mp4.m3u8',
                     'manifest_url': 'https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b300.mp4.m3u8',
                     'ext': 'mp4',
diff --git a/youtube_dl/extractor/anvato.py b/youtube_dl/extractor/anvato.py
index 623f44dce..9fd91c2f6 100644
--- a/youtube_dl/extractor/anvato.py
+++ b/youtube_dl/extractor/anvato.py
@@ -178,12 +178,7 @@ class AnvatoIE(InfoExtractor):
             }
 
             if ext == 'm3u8' or media_format in ('m3u8', 'm3u8-variant'):
-                # Not using _extract_m3u8_formats here as individual media
-                # playlists are also included in published_urls.
-                if tbr is None:
-                    formats.append(self._m3u8_meta_format(video_url, ext='mp4', m3u8_id='hls'))
-                    continue
-                else:
+                if tbr is not None:
                     a_format.update({
                         'format_id': '-'.join(filter(None, ['hls', compat_str(tbr)])),
                         'ext': 'mp4',
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 8b3f04c61..6d01f0800 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1334,7 +1334,7 @@ class InfoExtractor(object):
         if '#EXT-X-FAXS-CM:' in m3u8_doc:  # Adobe Flash Access
             return []
 
-        formats = [self._m3u8_meta_format(m3u8_url, ext, preference, m3u8_id)]
+        formats = []
 
         format_url = lambda u: (
             u
@@ -1438,7 +1438,7 @@ class InfoExtractor(object):
                 f = {
                     'format_id': '-'.join(format_id),
                     'url': manifest_url,
-                    'manifest_url': manifest_url,
+                    'manifest_url': m3u8_url,
                     'tbr': tbr,
                     'ext': ext,
                     'fps': float_or_none(last_stream_inf.get('FRAME-RATE')),
diff --git a/youtube_dl/extractor/funnyordie.py b/youtube_dl/extractor/funnyordie.py
index 81c0ce9a3..49409369c 100644
--- a/youtube_dl/extractor/funnyordie.py
+++ b/youtube_dl/extractor/funnyordie.py
@@ -58,8 +58,7 @@ class FunnyOrDieIE(InfoExtractor):
             m3u8_url, video_id, 'mp4', 'm3u8_native',
             m3u8_id='hls', fatal=False)
         source_formats = list(filter(
-            lambda f: f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
-            m3u8_formats))
+            lambda f: f.get('vcodec') != 'none', m3u8_formats))
 
         bitrates = [int(bitrate) for bitrate in re.findall(r'[,/]v(\d+)(?=[,/])', m3u8_url)]
         bitrates.sort()
diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index 682c49e79..00d311158 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -78,8 +78,7 @@ class GameSpotIE(OnceIE):
                     if m3u8_formats:
                         self._sort_formats(m3u8_formats)
                         m3u8_formats = list(filter(
-                            lambda f: f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
-                            m3u8_formats))
+                            lambda f: f.get('vcodec') != 'none', m3u8_formats))
                     if len(qualities) == len(m3u8_formats):
                         for q, m3u8_format in zip(qualities, m3u8_formats):
                             f = m3u8_format.copy()
diff --git a/youtube_dl/extractor/lego.py b/youtube_dl/extractor/lego.py
index d3bca6435..b312e77f1 100644
--- a/youtube_dl/extractor/lego.py
+++ b/youtube_dl/extractor/lego.py
@@ -86,7 +86,7 @@ class LEGOIE(InfoExtractor):
         formats = self._extract_akamai_formats(
             '%si/s/public/%s_,%s,.mp4.csmil/master.m3u8' % (streaming_base, path, streaming_path), video_id)
         m3u8_formats = list(filter(
-            lambda f: f.get('protocol') == 'm3u8_native' and f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
+            lambda f: f.get('protocol') == 'm3u8_native' and f.get('vcodec') != 'none',
             formats))
         if len(m3u8_formats) == len(self._BITRATES):
             self._sort_formats(m3u8_formats)
diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 3e51b4dd7..6166dc2ad 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -515,7 +515,7 @@ class PBSIE(InfoExtractor):
                     http_url = format_url
         self._remove_duplicate_formats(formats)
         m3u8_formats = list(filter(
-            lambda f: f.get('protocol') == 'm3u8' and f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
+            lambda f: f.get('protocol') == 'm3u8' and f.get('vcodec') != 'none',
             formats))
         if http_url:
             for m3u8_format in m3u8_formats:
diff --git a/youtube_dl/extractor/r7.py b/youtube_dl/extractor/r7.py
index ed38c77eb..e2202d603 100644
--- a/youtube_dl/extractor/r7.py
+++ b/youtube_dl/extractor/r7.py
@@ -62,8 +62,7 @@ class R7IE(InfoExtractor):
             # m3u8 format always matches the http format, let's copy metadata from
             # one to another
             m3u8_formats = list(filter(
-                lambda f: f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
-                formats))
+                lambda f: f.get('vcodec') != 'none', formats))
             if len(m3u8_formats) == 1:
                 f_copy = m3u8_formats[0].copy()
                 f_copy.update(f)
diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 1b1afab32..3f3c681ae 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -210,7 +210,7 @@ class TEDIE(InfoExtractor):
                     resources.get('stream'), video_name, 'mp4', m3u8_id=format_id, fatal=False))
 
         m3u8_formats = list(filter(
-            lambda f: f.get('protocol') == 'm3u8' and f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
+            lambda f: f.get('protocol') == 'm3u8' and f.get('vcodec') != 'none',
             formats))
         if http_url:
             for m3u8_format in m3u8_formats:
diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index 06ea2b40a..c5b3288ad 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -150,8 +150,7 @@ class TVPEmbedIE(InfoExtractor):
                 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
             self._sort_formats(m3u8_formats)
             m3u8_formats = list(filter(
-                lambda f: f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
-                m3u8_formats))
+                lambda f: f.get('vcodec') != 'none', m3u8_formats))
             formats.extend(m3u8_formats)
             for i, m3u8_format in enumerate(m3u8_formats, 2):
                 http_url = '%s-%d.mp4' % (video_url_base, i)
diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index fcf0cb100..d5d5b4c69 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -176,8 +176,7 @@ class ViewsterIE(InfoExtractor):
                     if m3u8_formats:
                         self._sort_formats(m3u8_formats)
                         m3u8_formats = list(filter(
-                            lambda f: f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
-                            m3u8_formats))
+                            lambda f: f.get('vcodec') != 'none', m3u8_formats))
                     if len(qualities) == len(m3u8_formats):
                         for q, m3u8_format in zip(qualities, m3u8_formats):
                             f = m3u8_format.copy()

From 374560f0181423be530660fd4b2e1a685e6648f8 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Thu, 27 Apr 2017 11:04:21 +0200
Subject: [PATCH 0724/1696] [test_download] Fix order when testing file's md5

---
 test/test_download.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_download.py b/test/test_download.py
index 0e9f293b5..209f5f6d6 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -225,7 +225,7 @@ def generator(test_case, tname):
                                 format_bytes(got_fsize)))
                     if 'md5' in tc:
                         md5_for_file = _file_md5(tc_filename)
-                        self.assertEqual(md5_for_file, tc['md5'])
+                        self.assertEqual(tc['md5'], md5_for_file)
                 # Finally, check test cases' data again but this time against
                 # extracted data from info JSON file written during processing
                 info_json_fn = os.path.splitext(tc_filename)[0] + '.info.json'

From c95e2b59115bff314ae833f7d3fd08ef1768c6f8 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Thu, 27 Apr 2017 18:07:07 +0200
Subject: [PATCH 0725/1696] [cbc] update test cases

---
 youtube_dl/extractor/cbc.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/cbc.py b/youtube_dl/extractor/cbc.py
index cf678e7f8..87ad14e91 100644
--- a/youtube_dl/extractor/cbc.py
+++ b/youtube_dl/extractor/cbc.py
@@ -96,6 +96,7 @@ class CBCIE(InfoExtractor):
         'info_dict': {
             'title': 'Keep Rover active during the deep freeze with doggie pushups and other fun indoor tasks',
             'id': 'dog-indoor-exercise-winter-1.3928238',
+            'description': 'md5:c18552e41726ee95bd75210d1ca9194c',
         },
         'playlist_mincount': 6,
     }]
@@ -165,12 +166,11 @@ class CBCPlayerIE(InfoExtractor):
             'uploader': 'CBCC-NEW',
         },
     }, {
-        # available only when we add `formats=MPEG4,FLV,MP3` to theplatform url
         'url': 'http://www.cbc.ca/player/play/2164402062',
-        'md5': '17a61eb813539abea40618d6323a7f82',
+        'md5': '33fcd8f6719b9dd60a5e73adcb83b9f6',
         'info_dict': {
             'id': '2164402062',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Cancer survivor four times over',
             'description': 'Tim Mayer has beaten three different forms of cancer four times in five years.',
             'timestamp': 1320410746,

From 96820c1c6bd2a9613725fc17fd44fd00d3301475 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Thu, 27 Apr 2017 20:23:52 +0200
Subject: [PATCH 0726/1696] [cbsinteractive] extract formats with `CBSIE`

---
 youtube_dl/extractor/cbs.py            |  6 +--
 youtube_dl/extractor/cbsinteractive.py | 69 +++++++-------------------
 2 files changed, 22 insertions(+), 53 deletions(-)

diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index 58f258c54..1268e38ef 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -49,13 +49,13 @@ class CBSIE(CBSBaseIE):
         'only_matching': True,
     }]
 
-    def _extract_video_info(self, content_id):
+    def _extract_video_info(self, content_id, site='cbs', mpx_acc=2198311517):
         items_data = self._download_xml(
             'http://can.cbs.com/thunder/player/videoPlayerService.php',
-            content_id, query={'partner': 'cbs', 'contentId': content_id})
+            content_id, query={'partner': site, 'contentId': content_id})
         video_data = xpath_element(items_data, './/item')
         title = xpath_text(video_data, 'videoTitle', 'title', True)
-        tp_path = 'dJ5BDC/media/guid/2198311517/%s' % content_id
+        tp_path = 'dJ5BDC/media/guid/%d/%s' % (mpx_acc, content_id)
         tp_release_url = 'http://link.theplatform.com/s/' + tp_path
 
         asset_types = []
diff --git a/youtube_dl/extractor/cbsinteractive.py b/youtube_dl/extractor/cbsinteractive.py
index f35f686ef..86f7eee20 100644
--- a/youtube_dl/extractor/cbsinteractive.py
+++ b/youtube_dl/extractor/cbsinteractive.py
@@ -3,18 +3,17 @@ from __future__ import unicode_literals
 
 import re
 
-from .theplatform import ThePlatformIE
-from ..compat import compat_urllib_parse
+from .cbs import CBSIE
 from ..utils import int_or_none
 
 
-class CBSInteractiveIE(ThePlatformIE):
+class CBSInteractiveIE(CBSIE):
     _VALID_URL = r'https?://(?:www\.)?(?P<site>cnet|zdnet)\.com/(?:videos|video/share)/(?P<id>[^/?]+)'
     _TESTS = [{
         'url': 'http://www.cnet.com/videos/hands-on-with-microsofts-windows-8-1-update/',
-        'md5': '041233212a0d06b179c87cbcca1577b8',
         'info_dict': {
-            'id': '56f4ea68-bd21-4852-b08c-4de5b8354c60',
+            'id': 'R49SYt__yAfmlXR85z4f7gNmCBDcN_00',
+            'display_id': 'hands-on-with-microsofts-windows-8-1-update',
             'ext': 'mp4',
             'title': 'Hands-on with Microsoft Windows 8.1 Update',
             'description': 'The new update to the Windows 8 OS brings improved performance for mouse and keyboard users.',
@@ -25,45 +24,46 @@ class CBSInteractiveIE(ThePlatformIE):
             'upload_date': '20140402',
         },
         'params': {
-            'format': 'mp4',
+            # m3u8 download
+            'skip_download': True,
         },
     }, {
         'url': 'http://www.cnet.com/videos/whiny-pothole-tweets-at-local-government-when-hit-by-cars-tomorrow-daily-187/',
-        'md5': 'f2b16d73e08d69591dd9e25564695c0c',
+        'md5': 'f11d27b2fa18597fbf92444d2a9ed386',
         'info_dict': {
-            'id': '56527b93-d25d-44e3-b738-f989ce2e49ba',
+            'id': 'kjOJd_OoVJqbg_ZD8MZCOk8Wekb9QccK',
+            'display_id': 'whiny-pothole-tweets-at-local-government-when-hit-by-cars-tomorrow-daily-187',
             'ext': 'mp4',
             'title': 'Whiny potholes tweet at local government when hit by cars (Tomorrow Daily 187)',
-            'description': 'Khail and Ashley wonder what other civic woes can be solved by self-tweeting objects, investigate a new kind of VR camera and watch an origami robot self-assemble, walk, climb, dig and dissolve. #TDPothole',
+            'description': 'md5:d2b9a95a5ffe978ae6fbd4cf944d618f',
             'uploader_id': 'b163284d-6b73-44fc-b3e6-3da66c392d40',
             'uploader': 'Ashley Esqueda',
             'duration': 1482,
             'timestamp': 1433289889,
             'upload_date': '20150603',
         },
-        'params': {
-            'format': 'mp4',
-        },
     }, {
         'url': 'http://www.zdnet.com/video/share/video-keeping-android-smartphones-and-tablets-secure/',
         'info_dict': {
-            'id': 'bc1af9f0-a2b5-4e54-880d-0d95525781c0',
+            'id': 'k0r4T_ehht4xW_hAOqiVQPuBDPZ8SRjt',
+            'display_id': 'video-keeping-android-smartphones-and-tablets-secure',
             'ext': 'mp4',
             'title': 'Video: Keeping Android smartphones and tablets secure',
             'description': 'Here\'s the best way to keep Android devices secure, and what you do when they\'ve come to the end of their lives.',
             'uploader_id': 'f2d97ea2-8175-11e2-9d12-0018fe8a00b0',
             'uploader': 'Adrian Kingsley-Hughes',
+            'duration': 731,
             'timestamp': 1449129925,
             'upload_date': '20151203',
         },
         'params': {
             # m3u8 download
             'skip_download': True,
-        }
+        },
     }]
-    TP_RELEASE_URL_TEMPLATE = 'http://link.theplatform.com/s/kYEXFC/%s?mbr=true'
+
     MPX_ACCOUNTS = {
-        'cnet': 2288573011,
+        'cnet': 2198311517,
         'zdnet': 2387448114,
     }
 
@@ -77,7 +77,8 @@ class CBSInteractiveIE(ThePlatformIE):
         data = self._parse_json(data_json, display_id)
         vdata = data.get('video') or data['videos'][0]
 
-        video_id = vdata['id']
+        video_id = vdata['mpxRefId']
+
         title = vdata['title']
         author = vdata.get('author')
         if author:
@@ -87,37 +88,7 @@ class CBSInteractiveIE(ThePlatformIE):
             uploader = None
             uploader_id = None
 
-        media_guid_path = 'media/guid/%d/%s' % (self.MPX_ACCOUNTS[site], vdata['mpxRefId'])
-        formats, subtitles = [], {}
-        for (fkey, vid) in vdata.get('files', {}).items():
-            if fkey == 'hls_phone' and 'hls_tablet' in vdata['files']:
-                continue
-            release_url = self.TP_RELEASE_URL_TEMPLATE % vid
-            if fkey == 'hds':
-                release_url += '&manifest=f4m'
-            tp_formats, tp_subtitles = self._extract_theplatform_smil(release_url, video_id, 'Downloading %s SMIL data' % fkey)
-            formats.extend(tp_formats)
-            subtitles = self._merge_subtitles(subtitles, tp_subtitles)
-
-        if 'm3u8' in vdata:
-            parsed_url = compat_urllib_parse.urlparse(url)
-            m3u8_url = ('%s://%s%s'
-                % (parsed_url.scheme, parsed_url.netloc, vdata['m3u8']))
-            m3u8_formats = self._extract_m3u8_formats(m3u8_url, video_id)
-            for format in m3u8_formats:
-                format['url'] = format['url'].replace('https://', 'http://')
-            formats.extend(m3u8_formats)
-
-        if 'mp4' in vdata:
-            formats.append({
-                'url': vdata['mp4'],
-                'format_id': 'mp4',
-                'ext': 'mp4',
-            })
-
-        self._sort_formats(formats)
-
-        info = self._extract_theplatform_metadata('kYEXFC/%s' % media_guid_path, video_id)
+        info = self._extract_video_info(video_id, site, self.MPX_ACCOUNTS[site])
         info.update({
             'id': video_id,
             'display_id': display_id,
@@ -125,7 +96,5 @@ class CBSInteractiveIE(ThePlatformIE):
             'duration': int_or_none(vdata.get('duration')),
             'uploader': uploader,
             'uploader_id': uploader_id,
-            'subtitles': subtitles,
-            'formats': formats,
         })
         return info

From f5edd7ae5105c5e7c58f220f326f5d18ca2c035b Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 27 Apr 2017 21:51:30 +0200
Subject: [PATCH 0727/1696] [clipfish] Update test

---
 youtube_dl/extractor/clipfish.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/clipfish.py b/youtube_dl/extractor/clipfish.py
index bb52e0c6f..0920f6219 100644
--- a/youtube_dl/extractor/clipfish.py
+++ b/youtube_dl/extractor/clipfish.py
@@ -12,7 +12,7 @@ class ClipfishIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?clipfish\.de/(?:[^/]+/)+video/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.clipfish.de/special/ugly-americans/video/4343170/s01-e01-ugly-americans-date-in-der-hoelle/',
-        'md5': '720563e467b86374c194bdead08d207d',
+        'md5': 'b9a5dc46294154c1193e2d10e0c95693',
         'info_dict': {
             'id': '4343170',
             'ext': 'mp4',

From 6f4a8884164d7df2cae638d36b2e3dc400c81ba8 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 27 Apr 2017 21:53:11 +0200
Subject: [PATCH 0728/1696] [br] Update test

---
 youtube_dl/extractor/br.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/br.py b/youtube_dl/extractor/br.py
index ff0aa11b1..2c32b6ae2 100644
--- a/youtube_dl/extractor/br.py
+++ b/youtube_dl/extractor/br.py
@@ -77,7 +77,7 @@ class BRIE(InfoExtractor):
                 'description': 'md5:bb659990e9e59905c3d41e369db1fbe3',
                 'duration': 893,
                 'uploader': 'Eva Maria Steimle',
-                'upload_date': '20140117',
+                'upload_date': '20170208',
             }
         },
     ]

From c89b49f7432ebaed7c5032194df9240f5da4a84f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 28 Apr 2017 03:00:14 +0700
Subject: [PATCH 0729/1696] [extractor/common] Add manifest_url for explicit
 group rendition formats

---
 youtube_dl/extractor/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 6d01f0800..2cb55d6af 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1386,6 +1386,7 @@ class InfoExtractor(object):
                 f = {
                     'format_id': '-'.join(format_id),
                     'url': format_url(media_url),
+                    'manifest_url': m3u8_url,
                     'language': media.get('LANGUAGE'),
                     'ext': ext,
                     'protocol': entry_protocol,

From 30bb6ce1a49f37f11afdca96fa7fd62ec56395c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 28 Apr 2017 03:01:43 +0700
Subject: [PATCH 0730/1696] [test_InfoExtractor] Fix test_parse_m3u8_formats

---
 test/test_InfoExtractor.py | 50 ++++++++++++++++++++------------------
 1 file changed, 27 insertions(+), 23 deletions(-)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 14abf03c4..6f52e11f7 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -185,7 +185,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/master.m3u8?caption=2017%2F16%2F156589847-1492488987.m3u8%3Afra%3AFrancais&audiotrack=0%3Afra%3AFrancais',
                 [{
                     'url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_0_av.m3u8?null=0',
-                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_0_av.m3u8?null=0',
+                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/master.m3u8?caption=2017%2F16%2F156589847-1492488987.m3u8%3Afra%3AFrancais&audiotrack=0%3Afra%3AFrancais',
                     'ext': 'mp4',
                     'format_id': '180',
                     'protocol': 'm3u8',
@@ -196,7 +196,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 144,
                 }, {
                     'url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_1_av.m3u8?null=0',
-                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_1_av.m3u8?null=0',
+                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/master.m3u8?caption=2017%2F16%2F156589847-1492488987.m3u8%3Afra%3AFrancais&audiotrack=0%3Afra%3AFrancais',
                     'ext': 'mp4',
                     'format_id': '303',
                     'protocol': 'm3u8',
@@ -207,7 +207,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 180,
                 }, {
                     'url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_2_av.m3u8?null=0',
-                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_2_av.m3u8?null=0',
+                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/master.m3u8?caption=2017%2F16%2F156589847-1492488987.m3u8%3Afra%3AFrancais&audiotrack=0%3Afra%3AFrancais',
                     'ext': 'mp4',
                     'format_id': '575',
                     'protocol': 'm3u8',
@@ -218,7 +218,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 288,
                 }, {
                     'url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_3_av.m3u8?null=0',
-                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_3_av.m3u8?null=0',
+                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/master.m3u8?caption=2017%2F16%2F156589847-1492488987.m3u8%3Afra%3AFrancais&audiotrack=0%3Afra%3AFrancais',
                     'ext': 'mp4',
                     'format_id': '831',
                     'protocol': 'm3u8',
@@ -229,7 +229,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 396,
                 }, {
                     'url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_4_av.m3u8?null=0',
-                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_4_av.m3u8?null=0',
+                    'manifest_url': 'http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/master.m3u8?caption=2017%2F16%2F156589847-1492488987.m3u8%3Afra%3AFrancais&audiotrack=0%3Afra%3AFrancais',
                     'ext': 'mp4',
                     'protocol': 'm3u8',
                     'format_id': '1467',
@@ -247,19 +247,21 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/main.m3u8',
                 [{
                     'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-audio-160k_v4.m3u8',
+                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/main.m3u8',
                     'ext': 'mp4',
                     'format_id': 'audio-0-Default',
                     'protocol': 'm3u8',
                     'vcodec': 'none',
                 }, {
                     'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-audio-64k_v4.m3u8',
+                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/main.m3u8',
                     'ext': 'mp4',
                     'format_id': 'audio-1-Default',
                     'protocol': 'm3u8',
                     'vcodec': 'none',
                 }, {
                     'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-audio-64k_v4.m3u8',
-                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-audio-64k_v4.m3u8',
+                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/main.m3u8',
                     'ext': 'mp4',
                     'format_id': '71',
                     'protocol': 'm3u8',
@@ -268,7 +270,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'tbr': 71,
                 }, {
                     'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-400k_v4.m3u8',
-                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-400k_v4.m3u8',
+                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/main.m3u8',
                     'ext': 'mp4',
                     'format_id': '413',
                     'protocol': 'm3u8',
@@ -279,7 +281,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 224,
                 }, {
                     'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-400k_v4.m3u8',
-                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-400k_v4.m3u8',
+                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/main.m3u8',
                     'ext': 'mp4',
                     'format_id': '522',
                     'protocol': 'm3u8',
@@ -290,7 +292,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 224,
                 }, {
                     'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-1m_v4.m3u8',
-                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-1m_v4.m3u8',
+                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/main.m3u8',
                     'ext': 'mp4',
                     'format_id': '1205',
                     'protocol': 'm3u8',
@@ -301,7 +303,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 360,
                 }, {
                     'url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-2m_v4.m3u8',
-                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/hls/CONAN_020217_Highlight_show-2m_v4.m3u8',
+                    'manifest_url': 'http://ak.storage-w.teamcococdn.com/cdn/2017-02/98599/ed8f/main.m3u8',
                     'ext': 'mp4',
                     'format_id': '2374',
                     'protocol': 'm3u8',
@@ -319,6 +321,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 'http://cdnapi.kaltura.com/p/2082311/sp/208231100/playManifest/protocol/http/entryId/0_89q6e8ku/format/applehttp/tags/mobile_sd/f/a.m3u8',
                 [{
                     'url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_sa2ntrdg/name/a.mp4/index.m3u8',
+                    'manifest_url': 'http://cdnapi.kaltura.com/p/2082311/sp/208231100/playManifest/protocol/http/entryId/0_89q6e8ku/format/applehttp/tags/mobile_sd/f/a.m3u8',
                     'ext': 'mp4',
                     'format_id': 'audio-English',
                     'protocol': 'm3u8',
@@ -326,6 +329,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'vcodec': 'none',
                 }, {
                     'url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_r7y0nitg/name/a.mp4/index.m3u8',
+                    'manifest_url': 'http://cdnapi.kaltura.com/p/2082311/sp/208231100/playManifest/protocol/http/entryId/0_89q6e8ku/format/applehttp/tags/mobile_sd/f/a.m3u8',
                     'ext': 'mp4',
                     'format_id': 'audio-Undefined',
                     'protocol': 'm3u8',
@@ -333,7 +337,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'vcodec': 'none',
                 }, {
                     'url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_qlk9hlzr/name/a.mp4/index.m3u8',
-                    'manifest_url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_qlk9hlzr/name/a.mp4/index.m3u8',
+                    'manifest_url': 'http://cdnapi.kaltura.com/p/2082311/sp/208231100/playManifest/protocol/http/entryId/0_89q6e8ku/format/applehttp/tags/mobile_sd/f/a.m3u8',
                     'ext': 'mp4',
                     'format_id': '155',
                     'protocol': 'm3u8',
@@ -342,7 +346,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 180,
                 }, {
                     'url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_oefackmi/name/a.mp4/index.m3u8',
-                    'manifest_url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_oefackmi/name/a.mp4/index.m3u8',
+                    'manifest_url': 'http://cdnapi.kaltura.com/p/2082311/sp/208231100/playManifest/protocol/http/entryId/0_89q6e8ku/format/applehttp/tags/mobile_sd/f/a.m3u8',
                     'ext': 'mp4',
                     'format_id': '502',
                     'protocol': 'm3u8',
@@ -351,7 +355,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 270,
                 }, {
                     'url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/12/pv/1/flavorId/0_vyg9pj7k/name/a.mp4/index.m3u8',
-                    'manifest_url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/12/pv/1/flavorId/0_vyg9pj7k/name/a.mp4/index.m3u8',
+                    'manifest_url': 'http://cdnapi.kaltura.com/p/2082311/sp/208231100/playManifest/protocol/http/entryId/0_89q6e8ku/format/applehttp/tags/mobile_sd/f/a.m3u8',
                     'ext': 'mp4',
                     'format_id': '827',
                     'protocol': 'm3u8',
@@ -360,7 +364,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 360,
                 }, {
                     'url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/12/pv/1/flavorId/0_50n4psvx/name/a.mp4/index.m3u8',
-                    'manifest_url': 'http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/12/pv/1/flavorId/0_50n4psvx/name/a.mp4/index.m3u8',
+                    'manifest_url': 'http://cdnapi.kaltura.com/p/2082311/sp/208231100/playManifest/protocol/http/entryId/0_89q6e8ku/format/applehttp/tags/mobile_sd/f/a.m3u8',
                     'ext': 'mp4',
                     'format_id': '1396',
                     'protocol': 'm3u8',
@@ -375,7 +379,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 'https://usher.ttvnw.net/vod/6528877?allow_source=true&allow_audio_only=true&allow_spectre=true&player=twitchweb&nauth=%7B%22user_id%22%3Anull%2C%22vod_id%22%3A6528877%2C%22expires%22%3A1492887874%2C%22chansub%22%3A%7B%22restricted_bitrates%22%3A%5B%5D%7D%2C%22privileged%22%3Afalse%2C%22https_required%22%3Afalse%7D&nauthsig=3e29296a6824a0f48f9e731383f77a614fc79bee',
                 [{
                     'url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/audio_only/index-muted-HM49I092CC.m3u8',
-                    'manifest_url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/audio_only/index-muted-HM49I092CC.m3u8',
+                    'manifest_url': 'https://usher.ttvnw.net/vod/6528877?allow_source=true&allow_audio_only=true&allow_spectre=true&player=twitchweb&nauth=%7B%22user_id%22%3Anull%2C%22vod_id%22%3A6528877%2C%22expires%22%3A1492887874%2C%22chansub%22%3A%7B%22restricted_bitrates%22%3A%5B%5D%7D%2C%22privileged%22%3Afalse%2C%22https_required%22%3Afalse%7D&nauthsig=3e29296a6824a0f48f9e731383f77a614fc79bee',
                     'ext': 'mp4',
                     'format_id': 'Audio Only',
                     'protocol': 'm3u8',
@@ -384,7 +388,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'tbr': 182.725,
                 }, {
                     'url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/mobile/index-muted-HM49I092CC.m3u8',
-                    'manifest_url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/mobile/index-muted-HM49I092CC.m3u8',
+                    'manifest_url': 'https://usher.ttvnw.net/vod/6528877?allow_source=true&allow_audio_only=true&allow_spectre=true&player=twitchweb&nauth=%7B%22user_id%22%3Anull%2C%22vod_id%22%3A6528877%2C%22expires%22%3A1492887874%2C%22chansub%22%3A%7B%22restricted_bitrates%22%3A%5B%5D%7D%2C%22privileged%22%3Afalse%2C%22https_required%22%3Afalse%7D&nauthsig=3e29296a6824a0f48f9e731383f77a614fc79bee',
                     'ext': 'mp4',
                     'format_id': 'Mobile',
                     'protocol': 'm3u8',
@@ -395,7 +399,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 226,
                 }, {
                     'url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/low/index-muted-HM49I092CC.m3u8',
-                    'manifest_url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/low/index-muted-HM49I092CC.m3u8',
+                    'manifest_url': 'https://usher.ttvnw.net/vod/6528877?allow_source=true&allow_audio_only=true&allow_spectre=true&player=twitchweb&nauth=%7B%22user_id%22%3Anull%2C%22vod_id%22%3A6528877%2C%22expires%22%3A1492887874%2C%22chansub%22%3A%7B%22restricted_bitrates%22%3A%5B%5D%7D%2C%22privileged%22%3Afalse%2C%22https_required%22%3Afalse%7D&nauthsig=3e29296a6824a0f48f9e731383f77a614fc79bee',
                     'ext': 'mp4',
                     'format_id': 'Low',
                     'protocol': 'm3u8',
@@ -406,7 +410,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 360,
                 }, {
                     'url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/medium/index-muted-HM49I092CC.m3u8',
-                    'manifest_url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/medium/index-muted-HM49I092CC.m3u8',
+                    'manifest_url': 'https://usher.ttvnw.net/vod/6528877?allow_source=true&allow_audio_only=true&allow_spectre=true&player=twitchweb&nauth=%7B%22user_id%22%3Anull%2C%22vod_id%22%3A6528877%2C%22expires%22%3A1492887874%2C%22chansub%22%3A%7B%22restricted_bitrates%22%3A%5B%5D%7D%2C%22privileged%22%3Afalse%2C%22https_required%22%3Afalse%7D&nauthsig=3e29296a6824a0f48f9e731383f77a614fc79bee',
                     'ext': 'mp4',
                     'format_id': 'Medium',
                     'protocol': 'm3u8',
@@ -417,7 +421,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 480,
                 }, {
                     'url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/high/index-muted-HM49I092CC.m3u8',
-                    'manifest_url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/high/index-muted-HM49I092CC.m3u8',
+                    'manifest_url': 'https://usher.ttvnw.net/vod/6528877?allow_source=true&allow_audio_only=true&allow_spectre=true&player=twitchweb&nauth=%7B%22user_id%22%3Anull%2C%22vod_id%22%3A6528877%2C%22expires%22%3A1492887874%2C%22chansub%22%3A%7B%22restricted_bitrates%22%3A%5B%5D%7D%2C%22privileged%22%3Afalse%2C%22https_required%22%3Afalse%7D&nauthsig=3e29296a6824a0f48f9e731383f77a614fc79bee',
                     'ext': 'mp4',
                     'format_id': 'High',
                     'protocol': 'm3u8',
@@ -428,7 +432,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 720,
                 }, {
                     'url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/chunked/index-muted-HM49I092CC.m3u8',
-                    'manifest_url': 'https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/chunked/index-muted-HM49I092CC.m3u8',
+                    'manifest_url': 'https://usher.ttvnw.net/vod/6528877?allow_source=true&allow_audio_only=true&allow_spectre=true&player=twitchweb&nauth=%7B%22user_id%22%3Anull%2C%22vod_id%22%3A6528877%2C%22expires%22%3A1492887874%2C%22chansub%22%3A%7B%22restricted_bitrates%22%3A%5B%5D%7D%2C%22privileged%22%3Afalse%2C%22https_required%22%3Afalse%7D&nauthsig=3e29296a6824a0f48f9e731383f77a614fc79bee',
                     'ext': 'mp4',
                     'format_id': 'Source',
                     'protocol': 'm3u8',
@@ -447,7 +451,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 'https://www.vidio.com/videos/165683/playlist.m3u8',
                 [{
                     'url': 'https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b300.mp4.m3u8',
-                    'manifest_url': 'https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b300.mp4.m3u8',
+                    'manifest_url': 'https://www.vidio.com/videos/165683/playlist.m3u8',
                     'ext': 'mp4',
                     'format_id': '270p 3G',
                     'protocol': 'm3u8',
@@ -456,7 +460,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 270,
                 }, {
                     'url': 'https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b600.mp4.m3u8',
-                    'manifest_url': 'https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b600.mp4.m3u8',
+                    'manifest_url': 'https://www.vidio.com/videos/165683/playlist.m3u8',
                     'ext': 'mp4',
                     'format_id': '360p SD',
                     'protocol': 'm3u8',
@@ -465,7 +469,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 360,
                 }, {
                     'url': 'https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b1200.mp4.m3u8',
-                    'manifest_url': 'https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b1200.mp4.m3u8',
+                    'manifest_url': 'https://www.vidio.com/videos/165683/playlist.m3u8',
                     'ext': 'mp4',
                     'format_id': '720p HD',
                     'protocol': 'm3u8',

From dde97ea8da64aae2fc328ea5e34645fad8d9b29d Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 27 Apr 2017 22:07:42 +0200
Subject: [PATCH 0731/1696] [canalc2] Update test

---
 youtube_dl/extractor/canalc2.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/canalc2.py b/youtube_dl/extractor/canalc2.py
index f1f128c45..acd87e371 100644
--- a/youtube_dl/extractor/canalc2.py
+++ b/youtube_dl/extractor/canalc2.py
@@ -16,13 +16,10 @@ class Canalc2IE(InfoExtractor):
         'md5': '060158428b650f896c542dfbb3d6487f',
         'info_dict': {
             'id': '12163',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Terrasses du Numérique',
             'duration': 122,
         },
-        'params': {
-            'skip_download': True,  # Requires rtmpdump
-        }
     }, {
         'url': 'http://archives-canalc2.u-strasbg.fr/video.asp?idVideo=11427&voir=oui',
         'only_matching': True,

From 0f63dc2402325481f087f862adefc86cb44cc888 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 27 Apr 2017 22:13:12 +0200
Subject: [PATCH 0732/1696] [bandcamp] Update test

---
 youtube_dl/extractor/bandcamp.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index 056e06376..df2972f26 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -34,12 +34,12 @@ class BandcampIE(InfoExtractor):
         '_skip': 'There is a limit of 200 free downloads / month for the test song'
     }, {
         'url': 'http://benprunty.bandcamp.com/track/lanius-battle',
-        'md5': '73d0b3171568232574e45652f8720b5c',
+        'md5': '0369ace6b939f0927e62c67a1a8d9fa7',
         'info_dict': {
             'id': '2650410135',
-            'ext': 'mp3',
-            'title': 'Lanius (Battle)',
-            'uploader': 'Ben Prunty Music',
+            'ext': 'aiff',
+            'title': 'Ben Prunty - Lanius (Battle)',
+            'uploader': 'Ben Prunty',
         },
     }]
 

From 8814ae42bc756b97aa0b341792d70ddcbab1de3b Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 27 Apr 2017 22:14:11 +0200
Subject: [PATCH 0733/1696] [beeg] Update test

---
 youtube_dl/extractor/beeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index b0b7914d8..d5c5822f2 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -16,7 +16,7 @@ class BeegIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?beeg\.com/(?P<id>\d+)'
     _TEST = {
         'url': 'http://beeg.com/5416503',
-        'md5': '46c384def73b33dbc581262e5ee67cef',
+        'md5': 'a1a1b1a8bc70a89e49ccfd113aed0820',
         'info_dict': {
             'id': '5416503',
             'ext': 'mp4',

From c0fa4245cea1dc493e5a72e36644114a2743039c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 28 Apr 2017 03:24:59 +0700
Subject: [PATCH 0734/1696] [downloader/fragment] Remove assert for resume_len
 when no fragments downloaded This may be incorrect due some header (e.g. flv
 header in f4m downloader)

---
 youtube_dl/downloader/fragment.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index bb9e82578..d529ae09a 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -155,8 +155,6 @@ class FragmentFD(FileDownloader):
                 self._write_ytdl_file(ctx)
             if ctx['fragment_index'] > 0:
                 assert resume_len > 0
-            else:
-                assert resume_len == 0
 
         dest_stream, tmpfilename = sanitize_open(tmpfilename, open_mode)
 

From d8571dd6bfd62274b553e2dbc1acb67d7c0b636b Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 27 Apr 2017 22:28:27 +0200
Subject: [PATCH 0735/1696] [bleacherreport] Update tests

---
 youtube_dl/extractor/bleacherreport.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/bleacherreport.py b/youtube_dl/extractor/bleacherreport.py
index 7a8e1f60b..e829974ff 100644
--- a/youtube_dl/extractor/bleacherreport.py
+++ b/youtube_dl/extractor/bleacherreport.py
@@ -35,7 +35,7 @@ class BleacherReportIE(InfoExtractor):
             'title': 'Aussie Golfers Get Fright of Their Lives After Being Chased by Angry Kangaroo',
             'timestamp': 1446839961,
             'uploader': 'Sean Fay',
-            'description': 'md5:825e94e0f3521df52fa83b2ed198fa20',
+            'description': 'md5:b1601e2314c4d8eec23b6eafe086a757',
             'uploader_id': 6466954,
             'upload_date': '20151011',
         },
@@ -90,17 +90,13 @@ class BleacherReportCMSIE(AMPIE):
     _VALID_URL = r'https?://(?:www\.)?bleacherreport\.com/video_embed\?id=(?P<id>[0-9a-f-]{36})'
     _TESTS = [{
         'url': 'http://bleacherreport.com/video_embed?id=8fd44c2f-3dc5-4821-9118-2c825a98c0e1',
-        'md5': '8c2c12e3af7805152675446c905d159b',
+        'md5': '2e4b0a997f9228ffa31fada5c53d1ed1',
         'info_dict': {
             'id': '8fd44c2f-3dc5-4821-9118-2c825a98c0e1',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Cena vs. Rollins Would Expose the Heavyweight Division',
             'description': 'md5:984afb4ade2f9c0db35f3267ed88b36e',
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
     }]
 
     def _real_extract(self, url):

From a15777491ad6b3bff56d8f6532ce703e0f05356a Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 27 Apr 2017 22:32:25 +0200
Subject: [PATCH 0736/1696] [atresplayer] Update test

---
 youtube_dl/extractor/atresplayer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 99af6dc5a..01fa308ff 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -36,7 +36,7 @@ class AtresPlayerIE(InfoExtractor):
         },
         {
             'url': 'http://www.atresplayer.com/television/especial/videoencuentros/temporada-1/capitulo-112-david-bustamante_2014121600375.html',
-            'md5': '0d0e918533bbd4b263f2de4d197d4aac',
+            'md5': '6e52cbb513c405e403dbacb7aacf8747',
             'info_dict': {
                 'id': 'capitulo-112-david-bustamante',
                 'ext': 'flv',

From ac1a5b9a12f339fbed2c5b54298d8919238816cd Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 27 Apr 2017 22:36:28 +0200
Subject: [PATCH 0737/1696] [audioboom] Update test

---
 youtube_dl/extractor/audioboom.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/audioboom.py b/youtube_dl/extractor/audioboom.py
index 8fc5f65c6..e48bb8972 100644
--- a/youtube_dl/extractor/audioboom.py
+++ b/youtube_dl/extractor/audioboom.py
@@ -16,7 +16,7 @@ class AudioBoomIE(InfoExtractor):
             'title': '3/09/2016 Czaban Hour 3',
             'description': 'Guest:   Nate Davis - NFL free agency,   Guest:   Stan Gans',
             'duration': 2245.72,
-            'uploader': 'Steve Czaban',
+            'uploader': 'SB Nation A.M.',
             'uploader_url': r're:https?://(?:www\.)?audioboom\.com/channel/steveczabanyahoosportsradio',
         }
     }, {

From 064fafe932c499c43416485c6a1fa0cc0422f818 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 27 Apr 2017 22:47:25 +0200
Subject: [PATCH 0738/1696] [appleconnect] Update test

---
 youtube_dl/extractor/appleconnect.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/appleconnect.py b/youtube_dl/extractor/appleconnect.py
index ea7a70393..a84b8b1eb 100644
--- a/youtube_dl/extractor/appleconnect.py
+++ b/youtube_dl/extractor/appleconnect.py
@@ -12,13 +12,13 @@ class AppleConnectIE(InfoExtractor):
     _VALID_URL = r'https?://itunes\.apple\.com/\w{0,2}/?post/idsa\.(?P<id>[\w-]+)'
     _TEST = {
         'url': 'https://itunes.apple.com/us/post/idsa.4ab17a39-2720-11e5-96c5-a5b38f6c42d3',
-        'md5': '10d0f2799111df4cb1c924520ca78f98',
+        'md5': 'e7c38568a01ea45402570e6029206723',
         'info_dict': {
             'id': '4ab17a39-2720-11e5-96c5-a5b38f6c42d3',
             'ext': 'm4v',
             'title': 'Energy',
             'uploader': 'Drake',
-            'thumbnail': 'http://is5.mzstatic.com/image/thumb/Video5/v4/78/61/c5/7861c5fa-ad6d-294b-1464-cf7605b911d6/source/1920x1080sr.jpg',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'upload_date': '20150710',
             'timestamp': 1436545535,
         },

From c12b4b80f8371fd8554589ab777f5b9294a88c4a Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 27 Apr 2017 22:48:32 +0200
Subject: [PATCH 0739/1696] [archiveorg] Update test

---
 youtube_dl/extractor/archiveorg.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/archiveorg.py b/youtube_dl/extractor/archiveorg.py
index e21045bed..3c7d7250b 100644
--- a/youtube_dl/extractor/archiveorg.py
+++ b/youtube_dl/extractor/archiveorg.py
@@ -24,12 +24,12 @@ class ArchiveOrgIE(InfoExtractor):
         }
     }, {
         'url': 'https://archive.org/details/Cops1922',
-        'md5': 'bc73c8ab3838b5a8fc6c6651fa7b58ba',
+        'md5': '0869000b4ce265e8ca62738b336b268a',
         'info_dict': {
             'id': 'Cops1922',
             'ext': 'mp4',
             'title': 'Buster Keaton\'s "Cops" (1922)',
-            'description': 'md5:b4544662605877edd99df22f9620d858',
+            'description': 'md5:89e7c77bf5d965dd5c0372cfb49470f6',
         }
     }, {
         'url': 'http://archive.org/embed/XD300-23_68HighlightsAResearchCntAugHumanIntellect',

From 6b4ddd336cf50f2ecc00b2addf4950d73b1b3087 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 27 Apr 2017 23:00:15 +0200
Subject: [PATCH 0740/1696] [afreecatv] Fix title extraction

---
 youtube_dl/extractor/afreecatv.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py
index 78d29c861..c8cb91dcb 100644
--- a/youtube_dl/extractor/afreecatv.py
+++ b/youtube_dl/extractor/afreecatv.py
@@ -207,11 +207,10 @@ class AfreecaTVIE(InfoExtractor):
                     file_url, video_id, 'mp4', entry_protocol='m3u8_native',
                     m3u8_id='hls',
                     note='Downloading part %d m3u8 information' % file_num)
-                title = title if one else '%s (part %d)' % (title, file_num)
                 file_info = common_entry.copy()
                 file_info.update({
                     'id': format_id,
-                    'title': title,
+                    'title': title if one else '%s (part %d)' % (title, file_num),
                     'upload_date': upload_date,
                     'duration': file_duration,
                     'formats': formats,

From 33c62efc32be0566ec21ecba817c1b2f7a6be1ce Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 27 Apr 2017 23:00:49 +0200
Subject: [PATCH 0741/1696] [collegerama] Update tests

---
 youtube_dl/extractor/collegerama.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/collegerama.py b/youtube_dl/extractor/collegerama.py
index 18c734766..6a41db87c 100644
--- a/youtube_dl/extractor/collegerama.py
+++ b/youtube_dl/extractor/collegerama.py
@@ -21,7 +21,7 @@ class CollegeRamaIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Een nieuwe wereld: waarden, bewustzijn en techniek van de mensheid 2.0.',
                 'description': '',
-                'thumbnail': r're:^https?://.*\.jpg$',
+                'thumbnail': r're:^https?://.*\.jpg(?:\?.*?)?$',
                 'duration': 7713.088,
                 'timestamp': 1413309600,
                 'upload_date': '20141014',
@@ -35,6 +35,7 @@ class CollegeRamaIE(InfoExtractor):
                 'ext': 'wmv',
                 'title': '64ste Vakantiecursus: Afvalwater',
                 'description': 'md5:7fd774865cc69d972f542b157c328305',
+                'thumbnail': r're:^https?://.*\.jpg(?:\?.*?)?$',
                 'duration': 10853,
                 'timestamp': 1326446400,
                 'upload_date': '20120113',

From 3fd0f70f6a1107a217f5c4c860ae180a6daf1ab6 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 27 Apr 2017 23:26:59 +0200
Subject: [PATCH 0742/1696] [cbslocal] Update test

---
 youtube_dl/extractor/cbslocal.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/cbslocal.py b/youtube_dl/extractor/cbslocal.py
index 8d5f11dd1..7d78e3aae 100644
--- a/youtube_dl/extractor/cbslocal.py
+++ b/youtube_dl/extractor/cbslocal.py
@@ -60,8 +60,8 @@ class CBSLocalIE(AnvatoIE):
             'title': 'A Very Blue Anniversary',
             'description': 'CBS2’s Cindy Hsu has more.',
             'thumbnail': 're:^https?://.*',
-            'timestamp': 1479962220,
-            'upload_date': '20161124',
+            'timestamp': int,
+            'upload_date': r're:^\d{8}$',
             'uploader': 'CBS',
             'subtitles': {
                 'en': 'mincount:5',

From e31fed95b4b3026c34cec85c38c89b9b5c98b3d6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 28 Apr 2017 15:47:26 +0800
Subject: [PATCH 0743/1696] [youtube] Recognize new locale-based player URLs
 (fixes #12885)

---
 ChangeLog                       | 6 ++++++
 youtube_dl/extractor/youtube.py | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index b23aa10d5..a31f99853 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractor
+* [youtube] Recognize new locale-based player URLs (#12885)
+
+
 version 2017.04.26
 
 Core
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 9e2b9115c..b59f746af 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -963,7 +963,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
     def _extract_signature_function(self, video_id, player_url, example_sig):
         id_m = re.match(
-            r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|/base)?\.(?P<ext>[a-z]+)$',
+            r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|(?:/[a-z]{2}_[A-Z]{2})?/base)?\.(?P<ext>[a-z]+)$',
             player_url)
         if not id_m:
             raise ExtractorError('Cannot identify player %r' % player_url)

From b62985a9a5d25643f1b1d6db8711013b1f123698 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 28 Apr 2017 16:25:04 +0800
Subject: [PATCH 0744/1696] [youtube] Recognize another HTML5 player URL
 (#12885)

---
 youtube_dl/extractor/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index b59f746af..480f403da 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1629,7 +1629,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                                 player_desc = 'flash player %s' % player_version
                             else:
                                 player_version = self._search_regex(
-                                    [r'html5player-([^/]+?)(?:/html5player(?:-new)?)?\.js', r'(?:www|player)-([^/]+)/base\.js'],
+                                    [r'html5player-([^/]+?)(?:/html5player(?:-new)?)?\.js',
+                                     r'(?:www|player)-([^/]+)(?:/[a-z]{2}_[A-Z]{2})?/base\.js'],
                                     player_url,
                                     'html5 player', fatal=False)
                                 player_desc = 'html5 player %s' % player_version

From fb8e8b2d1684e66c1d3f3587010d8ef51ebff7ce Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 28 Apr 2017 09:47:30 +0100
Subject: [PATCH 0745/1696] [adobepass] use geo verification headers for all
 requests

---
 youtube_dl/extractor/adobepass.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
index 100cf997f..7da96c65c 100644
--- a/youtube_dl/extractor/adobepass.py
+++ b/youtube_dl/extractor/adobepass.py
@@ -1308,6 +1308,12 @@ class AdobePassIE(InfoExtractor):
     _USER_AGENT = 'Mozilla/5.0 (X11; Linux i686; rv:47.0) Gecko/20100101 Firefox/47.0'
     _MVPD_CACHE = 'ap-mvpd'
 
+    def _download_webpage_handle(self, *args, **kwargs):
+        headers = kwargs.get('headers', {})
+        headers.update(self.geo_verification_headers())
+        kwargs['headers'] = headers
+        return super(AdobePassIE, self)._download_webpage_handle(*args, **kwargs)
+
     @staticmethod
     def _get_mvpd_resource(provider_id, title, guid, rating):
         channel = etree.Element('channel')

From bf2a5555c0d56579484df4d2a5d65546aeb1bf39 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 28 Apr 2017 09:48:22 +0100
Subject: [PATCH 0746/1696] [go] add support for Disney, DisneyJunior and
 DisneyXD show pages

---
 youtube_dl/extractor/go.py | 49 +++++++++++++++++++++++++++-----------
 1 file changed, 35 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/go.py b/youtube_dl/extractor/go.py
index 4c9be47b4..9c7b1bd37 100644
--- a/youtube_dl/extractor/go.py
+++ b/youtube_dl/extractor/go.py
@@ -36,22 +36,26 @@ class GoIE(AdobePassIE):
             'requestor_id': 'DisneyXD',
         }
     }
-    _VALID_URL = r'https?://(?:(?P<sub_domain>%s)\.)?go\.com/(?:[^/]+/)*(?:vdka(?P<id>\w+)|(?:[^/]+/)*(?P<display_id>[^/?#]+))' % '|'.join(_SITE_INFO.keys())
+    _VALID_URL = r'https?://(?:(?P<sub_domain>%s)\.)?go\.com/(?:(?:[^/]+/)*(?P<id>vdka\w+)|(?:[^/]+/)*(?P<display_id>[^/?#]+))' % '|'.join(_SITE_INFO.keys())
     _TESTS = [{
-        'url': 'http://abc.go.com/shows/castle/video/most-recent/vdka0_g86w5onx',
+        'url': 'http://abc.go.com/shows/designated-survivor/video/most-recent/VDKA3807643',
         'info_dict': {
-            'id': '0_g86w5onx',
+            'id': 'VDKA3807643',
             'ext': 'mp4',
-            'title': 'Sneak Peek: Language Arts',
-            'description': 'md5:7dcdab3b2d17e5217c953256af964e9c',
+            'title': 'The Traitor in the White House',
+            'description': 'md5:05b009d2d145a1e85d25111bd37222e8',
         },
         'params': {
             # m3u8 download
             'skip_download': True,
         },
     }, {
-        'url': 'http://abc.go.com/shows/after-paradise/video/most-recent/vdka3335601',
-        'only_matching': True,
+        'url': 'http://watchdisneyxd.go.com/doraemon',
+        'info_dict': {
+            'title': 'Doraemon',
+            'id': 'SH55574025',
+        },
+        'playlist_mincount': 51,
     }, {
         'url': 'http://abc.go.com/shows/the-catch/episode-guide/season-01/10-the-wedding',
         'only_matching': True,
@@ -60,19 +64,36 @@ class GoIE(AdobePassIE):
         'only_matching': True,
     }]
 
+    def _extract_videos(self, brand, video_id='-1', show_id='-1'):
+        display_id = video_id if video_id != '-1' else show_id
+        return self._download_json(
+            'http://api.contents.watchabc.go.com/vp2/ws/contents/3000/videos/%s/001/-1/%s/-1/%s/-1/-1.json' % (brand, show_id, video_id),
+            display_id)['video']
+
     def _real_extract(self, url):
         sub_domain, video_id, display_id = re.match(self._VALID_URL, url).groups()
+        site_info = self._SITE_INFO[sub_domain]
+        brand = site_info['brand']
         if not video_id:
             webpage = self._download_webpage(url, display_id)
             video_id = self._search_regex(
                 # There may be inner quotes, e.g. data-video-id="'VDKA3609139'"
                 # from http://freeform.go.com/shows/shadowhunters/episodes/season-2/1-this-guilty-blood
-                r'data-video-id=["\']*VDKA(\w+)', webpage, 'video id')
-        site_info = self._SITE_INFO[sub_domain]
-        brand = site_info['brand']
-        video_data = self._download_json(
-            'http://api.contents.watchabc.go.com/vp2/ws/contents/3000/videos/%s/001/-1/-1/-1/%s/-1/-1.json' % (brand, video_id),
-            video_id)['video'][0]
+                r'data-video-id=["\']*(VDKA\w+)', webpage, 'video id', default=None)
+            if not video_id:
+                # show extraction works for Disney, DisneyJunior and DisneyXD
+                # ABC and Freeform has different layout
+                show_id = self._search_regex(r'data-show-id=["\']*(SH\d+)', webpage, 'show id')
+                videos = self._extract_videos(brand, show_id=show_id)
+                show_title = self._search_regex(r'data-show-title="([^"]+)"', webpage, 'show title', fatal=False)
+                entries = []
+                for video in videos:
+                    entries.append(self.url_result(
+                        video['url'], 'Go', video.get('id'), video.get('title')))
+                entries.reverse()
+                return self.playlist_result(entries, show_id, show_title)
+        video_data = self._extract_videos(brand, video_id)[0]
+        video_id = video_data['id']
         title = video_data['title']
 
         formats = []
@@ -105,7 +126,7 @@ class GoIE(AdobePassIE):
                     self._initialize_geo_bypass(['US'])
                 entitlement = self._download_json(
                     'https://api.entitlement.watchabc.go.com/vp2/ws-secure/entitlement/2020/authorize.json',
-                    video_id, data=urlencode_postdata(data), headers=self.geo_verification_headers())
+                    video_id, data=urlencode_postdata(data))
                 errors = entitlement.get('errors', {}).get('errors', [])
                 if errors:
                     for error in errors:

From 20783b8b50e068180c62b973f6a589bd762be839 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 28 Apr 2017 12:04:56 +0100
Subject: [PATCH 0747/1696] [aenetworks] fix extraction for shows with single
 season

---
 youtube_dl/extractor/aenetworks.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index c01c67303..2dcdba9d2 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -101,10 +101,14 @@ class AENetworksIE(AENetworksBaseIE):
                 for season_url_path in re.findall(r'(?s)<li[^>]+data-href="(/shows/%s/season-\d+)"' % url_parts[0], webpage):
                     entries.append(self.url_result(
                         compat_urlparse.urljoin(url, season_url_path), 'AENetworks'))
-                return self.playlist_result(
-                    entries, self._html_search_meta('aetn:SeriesId', webpage),
-                    self._html_search_meta('aetn:SeriesTitle', webpage))
-            elif url_parts_len == 2:
+                if entries:
+                    return self.playlist_result(
+                        entries, self._html_search_meta('aetn:SeriesId', webpage),
+                        self._html_search_meta('aetn:SeriesTitle', webpage))
+                else:
+                    # single season
+                    url_parts_len = 2
+            if url_parts_len == 2:
                 entries = []
                 for episode_item in re.findall(r'(?s)<[^>]+class="[^"]*(?:episode|program)-item[^"]*"[^>]*>', webpage):
                     episode_attributes = extract_attributes(episode_item)
@@ -112,7 +116,7 @@ class AENetworksIE(AENetworksBaseIE):
                         url, episode_attributes['data-canonical'])
                     entries.append(self.url_result(
                         episode_url, 'AENetworks',
-                        episode_attributes['data-videoid']))
+                        episode_attributes.get('data-videoid') or episode_attributes.get('data-video-id')))
                 return self.playlist_result(
                     entries, self._html_search_meta('aetn:SeasonId', webpage))
 

From 7527923371ed5dea83ab102d882fed52ebc4672e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 28 Apr 2017 18:27:29 +0700
Subject: [PATCH 0748/1696] [ChangeLog] Actualize

---
 ChangeLog | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index a31f99853..0af1c6188 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,19 @@
 version <unreleased>
 
+Core
++ [adobepass] Use geo verification headers for all requests
+- [downloader/fragment] Remove assert for resume_len when no fragments
+  downloaded
++ [extractor/common] Add manifest_url for explicit group rendition formats
+* [extractor/common] Fix manifest_url for m3u8 formats
+- [extractor/common] Don't list master m3u8 playlists in format list (#12832)
+
 Extractor
+* [aenetworks] Fix extraction for shows with single season
++ [go] Add support for Disney, DisneyJunior and DisneyXD show pages
 * [youtube] Recognize new locale-based player URLs (#12885)
++ [streamable] Add support for new embedded URL schema (#12844)
+* [arte:+7] Relax URL regular expression (#12837)
 
 
 version 2017.04.26

From bc8a2ea07107c65e3561826b066888d450b98391 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 28 Apr 2017 18:30:03 +0700
Subject: [PATCH 0749/1696] release 2017.04.28

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index dedc07604..a84cbc801 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.26*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.26**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.28*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.28**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.04.26
+[debug] youtube-dl version 2017.04.28
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 0af1c6188..cd49ac42d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.04.28
 
 Core
 + [adobepass] Use geo verification headers for all requests
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e206501e1..8c77f1905 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.04.26'
+__version__ = '2017.04.28'

From edd9221cd22614b9d06f52b16bad278201fabd01 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Fri, 28 Apr 2017 17:34:27 +0200
Subject: [PATCH 0750/1696] [utils] Fix inconsistent output of clean_html

`\s` in Python 2.x doesn't match unicode whitespace characters by
default
---
 youtube_dl/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 91e235ff2..41bc20544 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -421,8 +421,8 @@ def clean_html(html):
 
     # Newline vs <br />
     html = html.replace('\n', ' ')
-    html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
-    html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
+    html = re.sub(r'(?u)\s*<\s*br\s*/?\s*>\s*', '\n', html)
+    html = re.sub(r'(?u)<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
     # Strip html tags
     html = re.sub('<.*?>', '', html)
     # Replace html entities

From 9222d945107411ba657dab965fab63de8e8781b9 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Fri, 28 Apr 2017 18:05:14 +0200
Subject: [PATCH 0751/1696] [test_utils] Add one more clean_html test

---
 test/test_utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 4cd818850..9e1808e6b 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -899,6 +899,7 @@ class TestUtil(unittest.TestCase):
     def test_clean_html(self):
         self.assertEqual(clean_html('a:\nb'), 'a: b')
         self.assertEqual(clean_html('a:\n   "b"'), 'a:    "b"')
+        self.assertEqual(clean_html('a<br>\xa0b'), 'a\nb')
 
     def test_intlist_to_bytes(self):
         self.assertEqual(

From b364c87c423cbc031cd64566a1b283097b560d59 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Apr 2017 03:39:39 +0700
Subject: [PATCH 0752/1696] [tvplayer] Fix extraction (closes #12908)

---
 youtube_dl/extractor/tvplayer.py | 35 +++++++++++++++++++++-----------
 1 file changed, 23 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/tvplayer.py b/youtube_dl/extractor/tvplayer.py
index b6537141a..ebde6053f 100644
--- a/youtube_dl/extractor/tvplayer.py
+++ b/youtube_dl/extractor/tvplayer.py
@@ -2,9 +2,13 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..compat import (
+    compat_HTTPError,
+    compat_str,
+)
 from ..utils import (
     extract_attributes,
+    try_get,
     urlencode_postdata,
     ExtractorError,
 )
@@ -34,25 +38,32 @@ class TVPlayerIE(InfoExtractor):
             webpage, 'channel element'))
         title = current_channel['data-name']
 
-        resource_id = self._search_regex(
-            r'resourceId\s*=\s*"(\d+)"', webpage, 'resource id')
-        platform = self._search_regex(
-            r'platform\s*=\s*"([^"]+)"', webpage, 'platform')
+        resource_id = current_channel['data-id']
+
         token = self._search_regex(
-            r'token\s*=\s*"([^"]+)"', webpage, 'token', default='null')
-        validate = self._search_regex(
-            r'validate\s*=\s*"([^"]+)"', webpage, 'validate', default='null')
+            r'data-token=(["\'])(?P<token>(?!\1).+)\1', webpage,
+            'token', group='token')
+
+        context = self._download_json(
+            'https://tvplayer.com/watch/context', display_id,
+            'Downloading JSON context', query={
+                'resource': resource_id,
+                'nonce': token,
+            })
+
+        validate = context['validate']
+        platform = try_get(
+            context, lambda x: x['platform']['key'], compat_str) or 'firefox'
 
         try:
             response = self._download_json(
                 'http://api.tvplayer.com/api/v2/stream/live',
-                resource_id, headers={
+                display_id, 'Downloading JSON stream', headers={
                     'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
                 }, data=urlencode_postdata({
+                    'id': resource_id,
                     'service': 1,
                     'platform': platform,
-                    'id': resource_id,
-                    'token': token,
                     'validate': validate,
                 }))['tvplayer']['response']
         except ExtractorError as e:
@@ -63,7 +74,7 @@ class TVPlayerIE(InfoExtractor):
                     '%s said: %s' % (self.IE_NAME, response['error']), expected=True)
             raise
 
-        formats = self._extract_m3u8_formats(response['stream'], resource_id, 'mp4')
+        formats = self._extract_m3u8_formats(response['stream'], display_id, 'mp4')
         self._sort_formats(formats)
 
         return {

From e7db6759e4c4ca5099a997285642e60b84eee2c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Apr 2017 04:33:35 +0700
Subject: [PATCH 0753/1696] [downloader/external] Properly handle live stream
 downloading cancellation (closes #8932)

---
 youtube_dl/downloader/external.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index e13cf547d..e78169a0d 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -29,7 +29,17 @@ class ExternalFD(FileDownloader):
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
 
-        retval = self._call_downloader(tmpfilename, info_dict)
+        try:
+            retval = self._call_downloader(tmpfilename, info_dict)
+        except KeyboardInterrupt:
+            if not info_dict.get('is_live'):
+                raise
+            # Live stream downloading cancellation should be considered as
+            # correct and expected termination thus all postprocessing
+            # should take place
+            retval = 0
+            self.to_screen('[%s] Interrupted by user' % self.get_basename())
+
         if retval == 0:
             fsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen('\r[%s] Downloaded %s bytes' % (self.get_basename(), fsize))

From 24eb7c257807aceef713f6d01824cf600a9d6232 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 29 Apr 2017 15:55:08 +0800
Subject: [PATCH 0754/1696] [xtube] Fix extraction with non-standard JSON
 'sources'

Closes #12734

Thanks @paulguy for the fix!
---
 ChangeLog                     |  6 ++++++
 youtube_dl/extractor/xtube.py | 23 +++++++++++++++++++++--
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index cd49ac42d..aa7543325 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [xtube] Fix extraction for older FLV videos (#12734)
+
+
 version 2017.04.28
 
 Core
diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index 5584674a0..bea9b87ad 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -6,6 +6,7 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
+    js_to_json,
     orderedSet,
     parse_duration,
     sanitized_Request,
@@ -37,6 +38,22 @@ class XTubeIE(InfoExtractor):
             'comment_count': int,
             'age_limit': 18,
         }
+    }, {
+        # FLV videos with duplicated formats
+        'url': 'http://www.xtube.com/video-watch/A-Super-Run-Part-1-YT-9299752',
+        'md5': 'a406963eb349dd43692ec54631efd88b',
+        'info_dict': {
+            'id': '9299752',
+            'display_id': 'A-Super-Run-Part-1-YT',
+            'ext': 'flv',
+            'title': 'A Super Run - Part 1 (YT)',
+            'description': 'md5:ca0d47afff4a9b2942e4b41aa970fd93',
+            'uploader': 'tshirtguy59',
+            'duration': 579,
+            'view_count': int,
+            'comment_count': int,
+            'age_limit': 18,
+        },
     }, {
         # new URL schema
         'url': 'http://www.xtube.com/video-watch/strange-erotica-625837',
@@ -68,8 +85,9 @@ class XTubeIE(InfoExtractor):
             })
 
         sources = self._parse_json(self._search_regex(
-            r'(["\'])sources\1\s*:\s*(?P<sources>{.+?}),',
-            webpage, 'sources', group='sources'), video_id)
+            r'(["\'])?sources\1?\s*:\s*(?P<sources>{.+?}),',
+            webpage, 'sources', group='sources'), video_id,
+            transform_source=js_to_json)
 
         formats = []
         for format_id, format_url in sources.items():
@@ -78,6 +96,7 @@ class XTubeIE(InfoExtractor):
                 'format_id': format_id,
                 'height': int_or_none(format_id),
             })
+        self._remove_duplicate_formats(formats)
         self._sort_formats(formats)
 
         title = self._search_regex(

From fcace2d1adac5d1f306b22219fde3a4542bcd719 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Sat, 29 Apr 2017 10:30:45 +0200
Subject: [PATCH 0755/1696] [openload] raise `not found` before executing js

---
 youtube_dl/extractor/openload.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index ac5e0bb08..0adf17765 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -74,12 +74,14 @@ class OpenloadIE(InfoExtractor):
             'User-Agent': self._USER_AGENT,
         }
 
-        phantom = PhantomJSwrapper(self)
-        webpage, _ = phantom.get(url, video_id=video_id, headers=headers)
+        webpage = self._download_webpage(url, video_id, headers=headers)
 
         if 'File not found' in webpage or 'deleted by the owner' in webpage:
             raise ExtractorError('File not found', expected=True, video_id=video_id)
 
+        phantom = PhantomJSwrapper(self)
+        webpage, _ = phantom.get(url, html=webpage, video_id=video_id, headers=headers)
+
         decoded_id = get_element_by_id('streamurl', webpage)
 
         video_url = 'https://openload.co/stream/%s?mime=true' % decoded_id

From 98f9d873814da2a8584cc30c0e197c15ed249db3 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Sat, 29 Apr 2017 12:41:42 +0200
Subject: [PATCH 0756/1696] [phantomjs] Add required version checking

---
 youtube_dl/utils.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 9c94b7ec9..84aaac664 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3734,13 +3734,22 @@ class PhantomJSwrapper(object):
 
     _TMP_FILE_NAMES = ['script', 'html', 'cookies']
 
-    def __init__(self, extractor, timeout=10000):
+    def __init__(self, extractor, required_version=None, timeout=10000):
         self.exe = check_executable('phantomjs', ['-v'])
         if not self.exe:
             raise ExtractorError('PhantomJS executable not found in PATH, '
                                  'download it from http://phantomjs.org',
                                  expected=True)
+
         self.extractor = extractor
+
+        if required_version:
+            version = get_exe_version(self.exe, version_re=r'([0-9.]+)')
+            if is_outdated_version(version, required_version):
+                self.extractor._downloader.report_warning(
+                    'Your copy of PhantomJS is outdated, update it to version '
+                    '%s or newer if you encounter any errors.' % required_version)
+
         self.options = {
             'timeout': timeout,
         }

From 7552f96352f35cd877e52fd0770b77ba1856fc62 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Sat, 29 Apr 2017 12:41:57 +0200
Subject: [PATCH 0757/1696] [openload] Add required version

---
 youtube_dl/extractor/openload.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 0adf17765..292476ef8 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -79,7 +79,7 @@ class OpenloadIE(InfoExtractor):
         if 'File not found' in webpage or 'deleted by the owner' in webpage:
             raise ExtractorError('File not found', expected=True, video_id=video_id)
 
-        phantom = PhantomJSwrapper(self)
+        phantom = PhantomJSwrapper(self, required_version='2.0')
         webpage, _ = phantom.get(url, html=webpage, video_id=video_id, headers=headers)
 
         decoded_id = get_element_by_id('streamurl', webpage)

From 7986c3abcdad819b61bdf0fb7111759f9fc1fc32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Apr 2017 19:49:04 +0700
Subject: [PATCH 0758/1696] [anvato] Improve extraction (closes #12913) *
 Promote to regular shortcut based extractor * Add mcp to access key mapping
 table * Add support for embeds extraction * Add support for anvato embeds in
 generic extractor

---
 youtube_dl/extractor/anvato.py     | 59 ++++++++++++++++++++++++++++--
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/generic.py    | 16 ++++++++
 3 files changed, 73 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/anvato.py b/youtube_dl/extractor/anvato.py
index 9fd91c2f6..ab8c237e9 100644
--- a/youtube_dl/extractor/anvato.py
+++ b/youtube_dl/extractor/anvato.py
@@ -5,6 +5,7 @@ import base64
 import hashlib
 import json
 import random
+import re
 import time
 
 from .common import InfoExtractor
@@ -16,6 +17,7 @@ from ..utils import (
     intlist_to_bytes,
     int_or_none,
     strip_jsonp,
+    unescapeHTML,
 )
 
 
@@ -26,6 +28,8 @@ def md5_text(s):
 
 
 class AnvatoIE(InfoExtractor):
+    _VALID_URL = r'anvato:(?P<access_key_or_mcp>[^:]+):(?P<id>\d+)'
+    
     # Copied from anvplayer.min.js
     _ANVACK_TABLE = {
         'nbcu_nbcd_desktop_web_prod_93d8ead38ce2024f8f544b78306fbd15895ae5e6': 'NNemUkySjxLyPTKvZRiGntBIjEyK8uqicjMakIaQ',
@@ -114,6 +118,22 @@ class AnvatoIE(InfoExtractor):
         'nbcu_nbcd_desktop_web_prod_93d8ead38ce2024f8f544b78306fbd15895ae5e6_secure': 'NNemUkySjxLyPTKvZRiGntBIjEyK8uqicjMakIaQ'
     }
 
+    _MCP_TO_ACCESS_KEY_TABLE = {
+        'qa': 'anvato_mcpqa_demo_web_stage_18b55e00db5a13faa8d03ae6e41f6f5bcb15b922',
+        'lin': 'anvato_mcp_lin_web_prod_4c36fbfd4d8d8ecae6488656e21ac6d1ac972749',
+        'univison': 'anvato_mcp_univision_web_prod_37fe34850c99a3b5cdb71dab10a417dd5cdecafa',
+        'uni': 'anvato_mcp_univision_web_prod_37fe34850c99a3b5cdb71dab10a417dd5cdecafa',
+        'dev': 'anvato_mcp_fs2go_web_prod_c7b90a93e171469cdca00a931211a2f556370d0a',
+        'sps': 'anvato_mcp_sps_web_prod_54bdc90dd6ba21710e9f7074338365bba28da336',
+        'spsstg': 'anvato_mcp_sps_web_prod_54bdc90dd6ba21710e9f7074338365bba28da336',
+        'anv': 'anvato_mcp_anv_web_prod_791407490f4c1ef2a4bcb21103e0cb1bcb3352b3',
+        'gray': 'anvato_mcp_gray_web_prod_4c10f067c393ed8fc453d3930f8ab2b159973900',
+        'hearst': 'anvato_mcp_hearst_web_prod_5356c3de0fc7c90a3727b4863ca7fec3a4524a99',
+        'cbs': 'anvato_mcp_cbs_web_prod_02f26581ff80e5bda7aad28226a8d369037f2cbe',
+        'telemundo': 'anvato_mcp_telemundo_web_prod_c5278d51ad46fda4b6ca3d0ea44a7846a054f582'
+    }
+
+    _ANVP_RE = r'<script[^>]+\bdata-anvp\s*=\s*(["\'])(?P<anvp>(?:(?!\1).)+)\1'
     _AUTH_KEY = b'\x31\xc2\x42\x84\x9e\x73\xa0\xce'
 
     def __init__(self, *args, **kwargs):
@@ -217,9 +237,42 @@ class AnvatoIE(InfoExtractor):
             'subtitles': subtitles,
         }
 
+    @staticmethod
+    def _extract_urls(ie, webpage, video_id):
+        entries = []
+        for mobj in re.finditer(AnvatoIE._ANVP_RE, webpage):
+            anvplayer_data = ie._parse_json(
+                mobj.group('anvp'), video_id, transform_source=unescapeHTML,
+                fatal=False)
+            if not anvplayer_data:
+                continue
+            video = anvplayer_data.get('video')
+            if not isinstance(video, compat_str) or not video.isdigit():
+                continue
+            access_key = anvplayer_data.get('accessKey')
+            if not access_key:
+                mcp = anvplayer_data.get('mcp')
+                if mcp:
+                    access_key = AnvatoIE._MCP_TO_ACCESS_KEY_TABLE.get(
+                        mcp.lower())
+            if not access_key:
+                continue
+            entries.append(ie.url_result(
+                'anvato:%s:%s' % (access_key, video), ie=AnvatoIE.ie_key(),
+                video_id=video))
+        return entries
+
     def _extract_anvato_videos(self, webpage, video_id):
-        anvplayer_data = self._parse_json(self._html_search_regex(
-            r'<script[^>]+data-anvp=\'([^\']+)\'', webpage,
-            'Anvato player data'), video_id)
+        anvplayer_data = self._parse_json(
+            self._html_search_regex(
+                self._ANVP_RE, webpage, 'Anvato player data', group='anvp'),
+            video_id)
         return self._get_anvato_videos(
             anvplayer_data['accessKey'], anvplayer_data['video'])
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        access_key, video_id = mobj.group('access_key_or_mcp', 'id')
+        if access_key not in self._ANVACK_TABLE:
+            access_key = self._MCP_TO_ACCESS_KEY_TABLE[access_key]
+        return self._get_anvato_videos(access_key, video_id)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 355a4e56f..39e5380b8 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -41,6 +41,7 @@ from .alphaporno import AlphaPornoIE
 from .amcnetworks import AMCNetworksIE
 from .animeondemand import AnimeOnDemandIE
 from .anitube import AnitubeIE
+from .anvato import AnvatoIE
 from .anysex import AnySexIE
 from .aol import AolIE
 from .allocine import AllocineIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 67184bc5d..7f7c1ba29 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -86,6 +86,7 @@ from .openload import OpenloadIE
 from .videopress import VideoPressIE
 from .rutube import RutubeIE
 from .limelight import LimelightBaseIE
+from .anvato import AnvatoIE
 
 
 class GenericIE(InfoExtractor):
@@ -1677,6 +1678,15 @@ class GenericIE(InfoExtractor):
             },
             'playlist_mincount': 5,
         },
+        {
+            'url': 'http://kron4.com/2017/04/28/standoff-with-walnut-creek-murder-suspect-ends-with-arrest/',
+            'info_dict': {
+                'id': 'standoff-with-walnut-creek-murder-suspect-ends-with-arrest',
+                'title': 'Standoff with Walnut Creek murder suspect ends',
+                'description': 'md5:3ccc48a60fc9441eeccfc9c469ebf788',
+            },
+            'playlist_mincount': 4,
+        },
         # {
         #     # TODO: find another test
         #     # http://schema.org/VideoObject
@@ -2537,6 +2547,12 @@ class GenericIE(InfoExtractor):
                 'limelight:media:%s' % mobj.group('id'),
                 {'source_url': url}), 'LimelightMedia', mobj.group('id'))
 
+        # Look for Anvato embeds
+        anvato_urls = AnvatoIE._extract_urls(self, webpage, video_id)
+        if anvato_urls:
+            return self.playlist_result(
+                anvato_urls, video_id, video_title, video_description)
+
         # Look for AdobeTVVideo embeds
         mobj = re.search(
             r'<iframe[^>]+src=[\'"]((?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]',

From a4c81e49688f081834656c6bbf3afc8bab9820cd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Apr 2017 20:23:26 +0700
Subject: [PATCH 0759/1696] [yandexmusic:playlist] Fix extraction for python 3
 (closes #12888)

---
 youtube_dl/extractor/yandexmusic.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index fd6268ba4..eb1062142 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -234,7 +234,8 @@ class YandexMusicPlaylistIE(YandexMusicPlaylistBaseIE):
                 'overembed': 'false',
             })['playlist']
 
-        tracks, track_ids = playlist['tracks'], map(compat_str, playlist['trackIds'])
+        tracks = playlist['tracks']
+        track_ids = [compat_str(track_id) for track_id in playlist['trackIds']]
 
         # tracks dictionary shipped with playlist.jsx API is limited to 150 tracks,
         # missing tracks should be retrieved manually.

From 557194591a3afcf16f2d554aa0af34ce83ca163f Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Mon, 10 Apr 2017 01:09:10 -0400
Subject: [PATCH 0760/1696] [washingtonpost] Add support for embeds (closes
 #12699)

---
 youtube_dl/extractor/generic.py        | 21 +++++++++++++++++++++
 youtube_dl/extractor/washingtonpost.py |  6 ++++++
 2 files changed, 27 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 7f7c1ba29..bd9d9aa13 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -87,6 +87,7 @@ from .videopress import VideoPressIE
 from .rutube import RutubeIE
 from .limelight import LimelightBaseIE
 from .anvato import AnvatoIE
+from .washingtonpost import WashingtonPostIE
 
 
 class GenericIE(InfoExtractor):
@@ -1687,6 +1688,20 @@ class GenericIE(InfoExtractor):
             },
             'playlist_mincount': 4,
         },
+        {
+            # WashingtonPost embed
+            'url': 'http://www.vanityfair.com/hollywood/2017/04/donald-trump-tv-pitches',
+            'info_dict': {
+                'id': '8caf6e88-d0ec-11e5-90d3-34c2c42653ac',
+                'ext': 'mp4',
+                'title': "No one has seen the drama series based on Trump's life \u2014 until now",
+                'description': 'Donald Trump wanted a weekly TV drama based on his life. It never aired. But The Washington Post recently obtained a scene from the pilot script — and enlisted actors.',
+                'timestamp': 1455216756,
+                'uploader': 'The Washington Post',
+                'upload_date': '20160211',
+            },
+            'add_ie': [WashingtonPostIE.ie_key()],
+        },
         # {
         #     # TODO: find another test
         #     # http://schema.org/VideoObject
@@ -2670,6 +2685,12 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 rutube_urls, ie=RutubeIE.ie_key())
 
+        # Look for WashingtonPost embeds
+        wapo_urls = WashingtonPostIE._extract_urls(webpage)
+        if wapo_urls:
+            return self.playlist_from_matches(
+                wapo_urls, video_id, video_title, ie=WashingtonPostIE.ie_key())
+
         # Looking for http://schema.org/VideoObject
         json_ld = self._search_json_ld(
             webpage, video_id, default={}, expected_type='VideoObject')
diff --git a/youtube_dl/extractor/washingtonpost.py b/youtube_dl/extractor/washingtonpost.py
index 839cad986..7e5cf0e1d 100644
--- a/youtube_dl/extractor/washingtonpost.py
+++ b/youtube_dl/extractor/washingtonpost.py
@@ -13,6 +13,7 @@ from ..utils import (
 class WashingtonPostIE(InfoExtractor):
     IE_NAME = 'washingtonpost'
     _VALID_URL = r'(?:washingtonpost:|https?://(?:www\.)?washingtonpost\.com/video/(?:[^/]+/)*)(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
+    _EMBED_URL = 'https?://(?:www\.)?washingtonpost\.com/video/c/embed/[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}'
     _TEST = {
         'url': 'https://www.washingtonpost.com/video/c/video/480ba4ee-1ec7-11e6-82c2-a7dcb313287d',
         'md5': '6f537e1334b714eb15f9563bd4b9cdfa',
@@ -27,6 +28,11 @@ class WashingtonPostIE(InfoExtractor):
         },
     }
 
+    @classmethod
+    def _extract_urls(cls, webpage):
+        return re.findall(
+            r'<iframe[^>]+\bsrc=["\'](%s)' % cls._EMBED_URL, webpage)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         video_data = self._download_json(

From 1c7c76e4fb1f5d5c05cfa22483a46e7dd870ac0f Mon Sep 17 00:00:00 2001
From: Frederic Bournival <fredbournival@gmail.com>
Date: Wed, 19 Apr 2017 17:43:32 -0400
Subject: [PATCH 0761/1696] [noovo] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/noovo.py      | 52 ++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+)
 create mode 100644 youtube_dl/extractor/noovo.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 39e5380b8..612007a46 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -663,6 +663,7 @@ from .nintendo import NintendoIE
 from .njpwworld import NJPWWorldIE
 from .nobelprize import NobelPrizeIE
 from .noco import NocoIE
+from .noovo import NoovoIE
 from .normalboots import NormalbootsIE
 from .nosvideo import NosVideoIE
 from .nova import NovaIE
diff --git a/youtube_dl/extractor/noovo.py b/youtube_dl/extractor/noovo.py
new file mode 100644
index 000000000..9f441c2fc
--- /dev/null
+++ b/youtube_dl/extractor/noovo.py
@@ -0,0 +1,52 @@
+# coding: utf-8
+from __future__ import unicode_literals
+from .common import InfoExtractor
+
+
+class NoovoIE(InfoExtractor):
+    IE_NAME = 'Noovo'
+    IE_DESC = 'VTele, Max, MusiquePlus'
+    _VALID_URL = r'https?://(?:[a-z0-9-]+\.)?noovo\.ca/videos/(?P<id>[a-z0-9-]+/[a-z0-9-]+)'
+    _TESTS = [{
+        'url': 'http://noovo.ca/videos/rpm-plus/chrysler-imperial',
+        'md5': '2fcc04d0a8f4a853fad91233c2fdd121',
+        'info_dict': {
+            'id': '5386045029001',
+            'description': 'Antoine présente des véhicules qu\'il aperçoit sur la rue.',
+            'ext': 'mp4',
+            'timestamp': 1491399228,
+            'title': 'Chrysler Imperial',
+            'upload_date': '20170405',
+            'uploader_id': '618566855001'
+        }
+    }, {
+        'url': 'http://noovo.ca/videos/l-amour-est-dans-le-pre/episode-13-8',
+        'md5': '1199e96fbb93f2d42717115f72097b6b',
+        'info_dict': {
+            'id': '5395865725001',
+            'description': 'md5:336d5ebc5436534e61d16e63ddfca327',
+            'ext': 'mp4',
+            'timestamp': 1492019320,
+            'title': 'md5:2895fdc124639be0ef64ea0d06f5e493',
+            'upload_date': '20170412',
+            'uploader_id': '618566855001'
+        }
+    }, {
+        'url': 'http://interventions.noovo.ca/911/video/intoxication-aux-drogues-dures/?autoplay=1',
+        'only_matching': True
+    }]
+    API_URL_TEMPLATE = 'http://api.noovo.ca/api/v1/pages/single-episode/%s'
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/618566855001/default_default/index.html?videoId=%s'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        api_url = self.API_URL_TEMPLATE % video_id
+        api_content = self._download_json(api_url, video_id)
+
+        brightcove_id = api_content.get('data').get('brightcoveId')
+        if not brightcove_id:
+            brightcove_id = api_content.get('data').get('contents')[0].get('brightcoveId')
+
+        return self.url_result(
+            self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id
+        )

From b5c39537be8b7c3405af0df30dfaea3c5ed9d649 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Apr 2017 00:23:50 +0700
Subject: [PATCH 0762/1696] [noovo] Improve extraction (closes #12792)

---
 youtube_dl/extractor/noovo.py | 93 ++++++++++++++++++++++++++---------
 1 file changed, 69 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/noovo.py b/youtube_dl/extractor/noovo.py
index 9f441c2fc..f7fa098a5 100644
--- a/youtube_dl/extractor/noovo.py
+++ b/youtube_dl/extractor/noovo.py
@@ -1,52 +1,97 @@
 # coding: utf-8
 from __future__ import unicode_literals
+
+from .brightcove import BrightcoveNewIE
 from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    int_or_none,
+    smuggle_url,
+    try_get,
+)
 
 
 class NoovoIE(InfoExtractor):
-    IE_NAME = 'Noovo'
-    IE_DESC = 'VTele, Max, MusiquePlus'
-    _VALID_URL = r'https?://(?:[a-z0-9-]+\.)?noovo\.ca/videos/(?P<id>[a-z0-9-]+/[a-z0-9-]+)'
+    _VALID_URL = r'https?://(?:[^/]+\.)?noovo\.ca/videos/(?P<id>[^/]+/[^/?#&]+)'
     _TESTS = [{
+        # clip
         'url': 'http://noovo.ca/videos/rpm-plus/chrysler-imperial',
-        'md5': '2fcc04d0a8f4a853fad91233c2fdd121',
         'info_dict': {
             'id': '5386045029001',
-            'description': 'Antoine présente des véhicules qu\'il aperçoit sur la rue.',
             'ext': 'mp4',
-            'timestamp': 1491399228,
             'title': 'Chrysler Imperial',
+            'description': 'md5:de3c898d1eb810f3e6243e08c8b4a056',
+            'timestamp': 1491399228,
             'upload_date': '20170405',
-            'uploader_id': '618566855001'
-        }
+            'uploader_id': '618566855001',
+            'creator': 'vtele',
+            'view_count': int,
+            'series': 'RPM+',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
+        # episode
         'url': 'http://noovo.ca/videos/l-amour-est-dans-le-pre/episode-13-8',
-        'md5': '1199e96fbb93f2d42717115f72097b6b',
         'info_dict': {
             'id': '5395865725001',
+            'title': 'Épisode 13 : Les retrouvailles',
             'description': 'md5:336d5ebc5436534e61d16e63ddfca327',
             'ext': 'mp4',
             'timestamp': 1492019320,
-            'title': 'md5:2895fdc124639be0ef64ea0d06f5e493',
             'upload_date': '20170412',
-            'uploader_id': '618566855001'
-        }
-    }, {
-        'url': 'http://interventions.noovo.ca/911/video/intoxication-aux-drogues-dures/?autoplay=1',
-        'only_matching': True
+            'uploader_id': '618566855001',
+            'creator': 'vtele',
+            'view_count': int,
+            'series': "L'amour est dans le pré",
+            'season_number': 5,
+            'episode': 'Épisode 13',
+            'episode_number': 13,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
-    API_URL_TEMPLATE = 'http://api.noovo.ca/api/v1/pages/single-episode/%s'
     BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/618566855001/default_default/index.html?videoId=%s'
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        api_url = self.API_URL_TEMPLATE % video_id
-        api_content = self._download_json(api_url, video_id)
 
-        brightcove_id = api_content.get('data').get('brightcoveId')
-        if not brightcove_id:
-            brightcove_id = api_content.get('data').get('contents')[0].get('brightcoveId')
+        data = self._download_json(
+            'http://api.noovo.ca/api/v1/pages/single-episode/%s' % video_id,
+            video_id)['data']
 
-        return self.url_result(
-            self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id
-        )
+        content = try_get(data, lambda x: x['contents'][0])
+
+        brightcove_id = data.get('brightcoveId') or content['brightcoveId']
+
+        series = try_get(
+            data, (
+                lambda x: x['show']['title'],
+                lambda x: x['season']['show']['title']),
+            compat_str)
+
+        episode = None
+        og = data.get('og')
+        if isinstance(og, dict) and og.get('type') == 'video.episode':
+            episode = og.get('title')
+
+        video = content or data
+
+        return {
+            '_type': 'url_transparent',
+            'ie_key': BrightcoveNewIE.ie_key(),
+            'url': smuggle_url(
+                self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id,
+                {'geo_countries': ['CA']}),
+            'id': brightcove_id,
+            'title': video.get('title'),
+            'creator': video.get('source'),
+            'view_count': int_or_none(video.get('viewsCount')),
+            'series': series,
+            'season_number': int_or_none(try_get(
+                data, lambda x: x['season']['seasonNumber'])),
+            'episode': episode,
+            'episode_number': int_or_none(data.get('episodeNumber')),
+        }

From ba5c3caf88417d81fda5d3b629336c9c2d6fa6e9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 30 Apr 2017 02:15:28 +0800
Subject: [PATCH 0763/1696] [washingtonpost] Fix invalid escape sequence on
 Python 3.6

---
 youtube_dl/extractor/washingtonpost.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/washingtonpost.py b/youtube_dl/extractor/washingtonpost.py
index 7e5cf0e1d..625d0a1cc 100644
--- a/youtube_dl/extractor/washingtonpost.py
+++ b/youtube_dl/extractor/washingtonpost.py
@@ -13,7 +13,7 @@ from ..utils import (
 class WashingtonPostIE(InfoExtractor):
     IE_NAME = 'washingtonpost'
     _VALID_URL = r'(?:washingtonpost:|https?://(?:www\.)?washingtonpost\.com/video/(?:[^/]+/)*)(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
-    _EMBED_URL = 'https?://(?:www\.)?washingtonpost\.com/video/c/embed/[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}'
+    _EMBED_URL = r'https?://(?:www\.)?washingtonpost\.com/video/c/embed/[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}'
     _TEST = {
         'url': 'https://www.washingtonpost.com/video/c/video/480ba4ee-1ec7-11e6-82c2-a7dcb313287d',
         'md5': '6f537e1334b714eb15f9563bd4b9cdfa',

From 5ff1bc0cc10bf3006834c4b49fc36d733c83ce5c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 28 Apr 2017 22:25:20 +0100
Subject: [PATCH 0764/1696] [YoutubeDL] write raw subtitle files

---
 youtube_dl/YoutubeDL.py | 43 +++++++++++++++++++++--------------------
 1 file changed, 22 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index eb465c425..c7100bb91 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1696,29 +1696,30 @@ class YoutubeDL(object):
             ie = self.get_info_extractor(info_dict['extractor_key'])
             for sub_lang, sub_info in subtitles.items():
                 sub_format = sub_info['ext']
-                if sub_info.get('data') is not None:
-                    sub_data = sub_info['data']
+                sub_filename = subtitles_filename(filename, sub_lang, sub_format)
+                if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
+                    self.to_screen('[info] Video subtitle %s.%s is already present' % (sub_lang, sub_format))
                 else:
-                    try:
-                        sub_data = ie._download_webpage(
-                            sub_info['url'], info_dict['id'], note=False)
-                    except ExtractorError as err:
-                        self.report_warning('Unable to download subtitle for "%s": %s' %
-                                            (sub_lang, error_to_compat_str(err.cause)))
-                        continue
-                try:
-                    sub_filename = subtitles_filename(filename, sub_lang, sub_format)
-                    if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
-                        self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
+                    self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
+                    if sub_info.get('data') is not None:
+                        try:
+                            # Use newline='' to prevent conversion of newline characters
+                            # See https://github.com/rg3/youtube-dl/issues/10268
+                            with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8', newline='') as subfile:
+                                subfile.write(sub_info['data'])
+                        except (OSError, IOError):
+                            self.report_error('Cannot write subtitles file ' + sub_filename)
+                            return
                     else:
-                        self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
-                        # Use newline='' to prevent conversion of newline characters
-                        # See https://github.com/rg3/youtube-dl/issues/10268
-                        with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8', newline='') as subfile:
-                            subfile.write(sub_data)
-                except (OSError, IOError):
-                    self.report_error('Cannot write subtitles file ' + sub_filename)
-                    return
+                        try:
+                            sub_data = ie._request_webpage(
+                                sub_info['url'], info_dict['id'], note=False).read()
+                            with io.open(encodeFilename(sub_filename), 'wb') as subfile:
+                                subfile.write(sub_data)
+                        except (ExtractorError, IOError, OSError, ValueError) as err:
+                            self.report_warning('Unable to download subtitle for "%s": %s' %
+                                                (sub_lang, error_to_compat_str(err)))
+                            continue
 
         if self.params.get('writeinfojson', False):
             infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))

From 98ce1a3fd31b11119d031ceb7ad032bea22d373a Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 30 Apr 2017 09:01:05 +0100
Subject: [PATCH 0765/1696] [utils] add video/mp2t to mimetype2ext

---
 youtube_dl/utils.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 41bc20544..9d7798ee8 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2273,10 +2273,8 @@ def mimetype2ext(mt):
     return {
         '3gpp': '3gp',
         'smptett+xml': 'tt',
-        'srt': 'srt',
         'ttaf+xml': 'dfxp',
         'ttml+xml': 'ttml',
-        'vtt': 'vtt',
         'x-flv': 'flv',
         'x-mp4-fragmented': 'mp4',
         'x-ms-wmv': 'wmv',
@@ -2284,11 +2282,11 @@ def mimetype2ext(mt):
         'x-mpegurl': 'm3u8',
         'vnd.apple.mpegurl': 'm3u8',
         'dash+xml': 'mpd',
-        'f4m': 'f4m',
         'f4m+xml': 'f4m',
         'hds+xml': 'f4m',
         'vnd.ms-sstr+xml': 'ism',
         'quicktime': 'mov',
+        'mp2t': 'ts',
     }.get(res, res)
 
 

From 559925300959669a08456475650a42f395385372 Mon Sep 17 00:00:00 2001
From: gritstub <gritstub@users.noreply.github.com>
Date: Thu, 27 Apr 2017 11:00:02 -0700
Subject: [PATCH 0766/1696] [vevo] Fix extraction (config.token.key)

---
 youtube_dl/extractor/vevo.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 9aa38bc5a..9434cd585 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 import re
+import json
 
 from .common import InfoExtractor
 from ..compat import (
@@ -154,8 +155,15 @@ class VevoIE(VevoBaseIE):
     }
 
     def _initialize_api(self, video_id):
+        post_data = json.dumps({
+            'client_id': 'SPupX1tvqFEopQ1YS6SS',
+            'grant_type': 'urn:vevo:params:oauth:grant-type:anonymous',
+        }).encode('utf-8')
+        headers = {
+            'Content-Type': 'application/json',
+        }
         req = sanitized_Request(
-            'http://www.vevo.com/auth', data=b'')
+            'https://accounts.vevo.com/token', post_data, headers)
         webpage = self._download_webpage(
             req, None,
             note='Retrieving oauth token',
@@ -166,7 +174,7 @@ class VevoIE(VevoBaseIE):
                 '%s said: This page is currently unavailable in your region' % self.IE_NAME)
 
         auth_info = self._parse_json(webpage, video_id)
-        self._api_url_template = self.http_scheme() + '//apiv2.vevo.com/%s?token=' + auth_info['access_token']
+        self._api_url_template = self.http_scheme() + '//apiv2.vevo.com/%s?token=' + auth_info['legacy_token']
 
     def _call_api(self, path, *args, **kwargs):
         try:

From b07ea5eaecd6e3ab3a31c996ec815682efaf8edb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Apr 2017 17:58:22 +0700
Subject: [PATCH 0767/1696] [vevo] Modernize

---
 youtube_dl/extractor/vevo.py | 21 +++++++++------------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 9434cd585..890a149ea 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -12,7 +12,6 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     int_or_none,
-    sanitized_Request,
     parse_iso8601,
 )
 
@@ -155,19 +154,17 @@ class VevoIE(VevoBaseIE):
     }
 
     def _initialize_api(self, video_id):
-        post_data = json.dumps({
-            'client_id': 'SPupX1tvqFEopQ1YS6SS',
-            'grant_type': 'urn:vevo:params:oauth:grant-type:anonymous',
-        }).encode('utf-8')
-        headers = {
-            'Content-Type': 'application/json',
-        }
-        req = sanitized_Request(
-            'https://accounts.vevo.com/token', post_data, headers)
         webpage = self._download_webpage(
-            req, None,
+            'https://accounts.vevo.com/token', None,
             note='Retrieving oauth token',
-            errnote='Unable to retrieve oauth token')
+            errnote='Unable to retrieve oauth token',
+            data=json.dumps({
+                'client_id': 'SPupX1tvqFEopQ1YS6SS',
+                'grant_type': 'urn:vevo:params:oauth:grant-type:anonymous',
+            }).encode('utf-8'),
+            headers={
+                'Content-Type': 'application/json',
+            })
 
         if re.search(r'(?i)THIS PAGE IS CURRENTLY UNAVAILABLE IN YOUR REGION', webpage):
             self.raise_geo_restricted(

From 13081db1f5e026e8571e4f46758dc947702228f5 Mon Sep 17 00:00:00 2001
From: Sander <sander@plex.tv>
Date: Sun, 23 Apr 2017 23:49:41 +0200
Subject: [PATCH 0768/1696] [xvideos] Add video duration

---
 youtube_dl/extractor/xvideos.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/xvideos.py b/youtube_dl/extractor/xvideos.py
index 30825daae..ce3a05e41 100644
--- a/youtube_dl/extractor/xvideos.py
+++ b/youtube_dl/extractor/xvideos.py
@@ -8,6 +8,7 @@ from ..utils import (
     clean_html,
     ExtractorError,
     determine_ext,
+    parse_duration,
 )
 
 
@@ -20,6 +21,7 @@ class XVideosIE(InfoExtractor):
             'id': '4588838',
             'ext': 'mp4',
             'title': 'Biker Takes his Girl',
+            'duration': 120,
             'age_limit': 18,
         }
     }
@@ -36,6 +38,8 @@ class XVideosIE(InfoExtractor):
             r'<title>(.*?)\s+-\s+XVID', webpage, 'title')
         video_thumbnail = self._search_regex(
             r'url_bigthumb=(.+?)&amp', webpage, 'thumbnail', fatal=False)
+        video_duration = parse_duration(self._search_regex(
+            r'<span class="duration">.*?(\d[^<]+)', webpage, 'duration', fatal=False))
 
         formats = []
 
@@ -67,6 +71,7 @@ class XVideosIE(InfoExtractor):
             'id': video_id,
             'formats': formats,
             'title': video_title,
+            'duration': video_duration,
             'thumbnail': video_thumbnail,
             'age_limit': 18,
         }

From 6ec371cd9e49eb3acf352e9211f05c3d1de72161 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Apr 2017 18:11:29 +0700
Subject: [PATCH 0769/1696] [xvideos] Extract og:duration (closes #12828)

---
 youtube_dl/extractor/xvideos.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/xvideos.py b/youtube_dl/extractor/xvideos.py
index ce3a05e41..eca603028 100644
--- a/youtube_dl/extractor/xvideos.py
+++ b/youtube_dl/extractor/xvideos.py
@@ -6,8 +6,9 @@ from .common import InfoExtractor
 from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     clean_html,
-    ExtractorError,
     determine_ext,
+    ExtractorError,
+    int_or_none,
     parse_duration,
 )
 
@@ -21,7 +22,7 @@ class XVideosIE(InfoExtractor):
             'id': '4588838',
             'ext': 'mp4',
             'title': 'Biker Takes his Girl',
-            'duration': 120,
+            'duration': 108,
             'age_limit': 18,
         }
     }
@@ -38,8 +39,11 @@ class XVideosIE(InfoExtractor):
             r'<title>(.*?)\s+-\s+XVID', webpage, 'title')
         video_thumbnail = self._search_regex(
             r'url_bigthumb=(.+?)&amp', webpage, 'thumbnail', fatal=False)
-        video_duration = parse_duration(self._search_regex(
-            r'<span class="duration">.*?(\d[^<]+)', webpage, 'duration', fatal=False))
+        video_duration = int_or_none(self._og_search_property(
+            'duration', webpage, default=None)) or parse_duration(
+            self._search_regex(
+                r'<span[^>]+class=["\']duration["\'][^>]*>.*?(\d[^<]+)',
+                webpage, 'duration', fatal=False))
 
         formats = []
 

From 9dac2cec2d7e31b65cf063164b3a99f257a86a63 Mon Sep 17 00:00:00 2001
From: slocum <piotr5151@interia.pl>
Date: Sun, 9 Apr 2017 12:02:44 +0200
Subject: [PATCH 0770/1696] [zaq1] Add new extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/zaq1.py       | 67 ++++++++++++++++++++++++++++++
 2 files changed, 68 insertions(+)
 create mode 100644 youtube_dl/extractor/zaq1.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 612007a46..c0020dd7d 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1300,5 +1300,6 @@ from .youtube import (
     YoutubeWatchLaterIE,
 )
 from .zapiks import ZapiksIE
+from .zaq1 import Zaq1IE
 from .zdf import ZDFIE, ZDFChannelIE
 from .zingmp3 import ZingMp3IE
diff --git a/youtube_dl/extractor/zaq1.py b/youtube_dl/extractor/zaq1.py
new file mode 100644
index 000000000..33068f6b5
--- /dev/null
+++ b/youtube_dl/extractor/zaq1.py
@@ -0,0 +1,67 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    unified_strdate,
+    int_or_none
+)
+
+
+class Zaq1IE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?zaq1\.pl/video/(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'http://zaq1.pl/video/xev0e',
+        'md5': '24a5eb3f052e604ae597c4d0d19b351e',
+        'info_dict': {
+            'id': 'xev0e',
+            'title': 'DJ NA WESELE. TANIEC Z FIGURAMI.węgrów/sokołów podlaski/siedlce/mińsk mazowiecki/warszawa',
+            'ext': 'mp4',
+            'duration': 511,
+            'uploader': 'Anonim',
+            'upload_date': '20170330',
+        }
+    }, {
+        'url': 'http://zaq1.pl/video/x80nc',
+        'md5': '1245973520adc78139928a820959d9c5',
+        'info_dict': {
+            'id': 'x80nc',
+            'title': 'DIY Inspiration Challenge #86 | koraliki | gwiazdka na choinkę z koralików i drutu',
+            'ext': 'mp4',
+            'duration': 438,
+            'uploader': 'Anonim',
+            'upload_date': '20170404',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(
+            r'(?s)<h1>\s*<span.+class="watch-title".+title="([^"]+)">\1\s*</span>\s*</h1>', webpage, 'title')
+
+        div = self._search_regex(r'(?s)(?P<div><div.+id=(["\'])video_player\2.+</div>)', webpage, 'video url', group='div')
+        video_url = self._search_regex(r'data-video-url="(http[^"]+)"', div, 'video url')
+
+        ext = self._search_regex(r'data-file-extension="([^"]+)"', div, 'ext', None, False)
+        duration = int_or_none(self._search_regex(r'data-duration="([^"]+)"', div, 'duration', None, False))
+        thumbnail = self._search_regex(r'data-photo-url="([^"]+)"', div, 'thumbnail', None, False)
+
+        upload_date = unified_strdate(self._search_regex(r'<strong\s+class="watch-time-text">\s*Opublikowany\s+([0-9]{4}-[0-9]{2}-[0-9]{2})', webpage, 'upload date'))
+        uploader = self._search_regex(r'<div\s+id="watch7-user-header">.*Wideo dodał:\s*<a[^>]*>\s*([^<]+)\s*</a>', webpage, 'uploader')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': [{
+                'url': video_url,
+                'ext': ext,
+                'http_headers': {'Referer': url},
+            }],
+            'thumbnail': thumbnail,
+            'uploader': uploader,
+            'upload_date': upload_date,
+            'duration': duration,
+        }

From deef31955bb6106939606cd2a8b677db700055e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Apr 2017 21:07:30 +0700
Subject: [PATCH 0771/1696] [utils] Improve unified_timestamp Seen at
 http://zaq1.pl/video/xev0e

---
 test/test_utils.py  | 1 +
 youtube_dl/utils.py | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 9e1808e6b..05fdc0e95 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -338,6 +338,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(unified_timestamp('UNKNOWN DATE FORMAT'), None)
         self.assertEqual(unified_timestamp('May 16, 2016 11:15 PM'), 1463440500)
         self.assertEqual(unified_timestamp('Feb 7, 2016 at 6:35 pm'), 1454870100)
+        self.assertEqual(unified_timestamp('2017-03-30T17:52:41Q'), 1490896361)
 
     def test_determine_ext(self):
         self.assertEqual(determine_ext('http://example.com/foo/bar.mp4/?download'), 'mp4')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 9d7798ee8..86fc5ccac 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1194,6 +1194,11 @@ def unified_timestamp(date_str, day_first=True):
     # Remove AM/PM + timezone
     date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
 
+    # Remove unrecognized timezones from ISO 8601 alike timestamps
+    m = re.search(r'\d{1,2}:\d{1,2}(?:\.\d+)?(?P<tz>\s*[A-Z]+)$', date_str)
+    if m:
+        date_str = date_str[:-len(m.group('tz'))]
+
     for expression in date_formats(day_first):
         try:
             dt = datetime.datetime.strptime(date_str, expression) - timezone + datetime.timedelta(hours=pm_delta)

From 33a81c2c6f4bd180ef69d5631862637ae0c8ec8e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Apr 2017 21:11:55 +0700
Subject: [PATCH 0772/1696] [extractor/common] Extract view count from JSON-LD

---
 youtube_dl/extractor/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 2cb55d6af..fba15d446 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -990,6 +990,7 @@ class InfoExtractor(object):
                 'tbr': int_or_none(e.get('bitrate')),
                 'width': int_or_none(e.get('width')),
                 'height': int_or_none(e.get('height')),
+                'view_count': int_or_none(e.get('interactionCount')),
             })
 
         for e in json_ld:

From 9cd4209724c60efe636efdec43d7df6c385d0e2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Apr 2017 21:45:00 +0700
Subject: [PATCH 0773/1696] [zaq1] Improve extraction (closes #12693)

---
 youtube_dl/extractor/zaq1.py | 88 +++++++++++++++++++++++++-----------
 1 file changed, 61 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/zaq1.py b/youtube_dl/extractor/zaq1.py
index 33068f6b5..889aff5d8 100644
--- a/youtube_dl/extractor/zaq1.py
+++ b/youtube_dl/extractor/zaq1.py
@@ -3,35 +3,44 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
-    unified_strdate,
-    int_or_none
+    int_or_none,
+    unified_timestamp,
 )
 
 
 class Zaq1IE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?zaq1\.pl/video/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?zaq1\.pl/video/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://zaq1.pl/video/xev0e',
         'md5': '24a5eb3f052e604ae597c4d0d19b351e',
         'info_dict': {
             'id': 'xev0e',
             'title': 'DJ NA WESELE. TANIEC Z FIGURAMI.węgrów/sokołów podlaski/siedlce/mińsk mazowiecki/warszawa',
+            'description': 'www.facebook.com/weseledjKontakt: 728 448 199 / 505 419 147',
             'ext': 'mp4',
             'duration': 511,
+            'timestamp': 1490896361,
             'uploader': 'Anonim',
             'upload_date': '20170330',
+            'view_count': int,
         }
     }, {
-        'url': 'http://zaq1.pl/video/x80nc',
-        'md5': '1245973520adc78139928a820959d9c5',
+        # malformed JSON-LD
+        'url': 'http://zaq1.pl/video/x81vn',
         'info_dict': {
-            'id': 'x80nc',
-            'title': 'DIY Inspiration Challenge #86 | koraliki | gwiazdka na choinkę z koralików i drutu',
+            'id': 'x81vn',
+            'title': 'SEKRETNE ŻYCIE WALTERA MITTY',
             'ext': 'mp4',
-            'duration': 438,
+            'duration': 6234,
+            'timestamp': 1493494860,
             'uploader': 'Anonim',
-            'upload_date': '20170404',
-        }
+            'upload_date': '20170429',
+            'view_count': int,
+        },
+        'params': {
+            'skip_download': True,
+        },
+        'expected_warnings': ['Failed to parse JSON'],
     }]
 
     def _real_extract(self, url):
@@ -39,29 +48,54 @@ class Zaq1IE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(
-            r'(?s)<h1>\s*<span.+class="watch-title".+title="([^"]+)">\1\s*</span>\s*</h1>', webpage, 'title')
+        video_url = self._search_regex(
+            r'data-video-url=(["\'])(?P<url>(?:(?!\1).)+)\1', webpage,
+            'video url', group='url')
 
-        div = self._search_regex(r'(?s)(?P<div><div.+id=(["\'])video_player\2.+</div>)', webpage, 'video url', group='div')
-        video_url = self._search_regex(r'data-video-url="(http[^"]+)"', div, 'video url')
+        info = self._search_json_ld(webpage, video_id, fatal=False)
 
-        ext = self._search_regex(r'data-file-extension="([^"]+)"', div, 'ext', None, False)
-        duration = int_or_none(self._search_regex(r'data-duration="([^"]+)"', div, 'duration', None, False))
-        thumbnail = self._search_regex(r'data-photo-url="([^"]+)"', div, 'thumbnail', None, False)
+        def extract_data(field, name, fatal=False):
+            return self._search_regex(
+                r'data-%s=(["\'])(?P<field>(?:(?!\1).)+)\1' % field,
+                webpage, field, fatal=fatal, group='field')
 
-        upload_date = unified_strdate(self._search_regex(r'<strong\s+class="watch-time-text">\s*Opublikowany\s+([0-9]{4}-[0-9]{2}-[0-9]{2})', webpage, 'upload date'))
-        uploader = self._search_regex(r'<div\s+id="watch7-user-header">.*Wideo dodał:\s*<a[^>]*>\s*([^<]+)\s*</a>', webpage, 'uploader')
+        if not info.get('title'):
+            info['title'] = extract_data('file-name', 'title', fatal=True)
 
-        return {
+        if not info.get('duration'):
+            info['duration'] = int_or_none(extract_data('duration', 'duration'))
+
+        if not info.get('thumbnail'):
+            info['thumbnail'] = extract_data('photo-url', 'thumbnail')
+
+        if not info.get('timestamp'):
+            info['timestamp'] = unified_timestamp(self._html_search_meta(
+                'uploadDate', webpage, 'timestamp'))
+
+        if not info.get('interactionCount'):
+            info['view_count'] = int_or_none(self._html_search_meta(
+                'interactionCount', webpage, 'view count'))
+
+        uploader = self._html_search_regex(
+            r'Wideo dodał:\s*<a[^>]*>([^<]+)</a>', webpage, 'uploader',
+            fatal=False)
+
+        width = int_or_none(self._html_search_meta(
+            'width', webpage, fatal=False))
+        height = int_or_none(self._html_search_meta(
+            'height', webpage, fatal=False))
+
+        info.update({
             'id': video_id,
-            'title': title,
             'formats': [{
                 'url': video_url,
-                'ext': ext,
-                'http_headers': {'Referer': url},
+                'width': width,
+                'height': height,
+                'http_headers': {
+                    'Referer': url,
+                },
             }],
-            'thumbnail': thumbnail,
             'uploader': uploader,
-            'upload_date': upload_date,
-            'duration': duration,
-        }
+        })
+
+        return info

From 50534b7158e0e1c842d05eae39fd6687a04a76e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Apr 2017 22:04:01 +0700
Subject: [PATCH 0774/1696] [downloader/fragment] PEP 8

---
 youtube_dl/downloader/fragment.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index d529ae09a..bccc8ecc1 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -49,7 +49,7 @@ class FragmentFD(FileDownloader):
                 index:  0-based index of current fragment among all fragments
             fragment_count:
                 Total count of fragments
-                
+
     This feature is experimental and file format may change in future.
     """
 

From 238cec17ae51c0f624588095d3a370a3b6964bdd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Apr 2017 22:04:21 +0700
Subject: [PATCH 0775/1696] [extractor/anvato] PEP 8

---
 youtube_dl/extractor/anvato.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/anvato.py b/youtube_dl/extractor/anvato.py
index ab8c237e9..8023da702 100644
--- a/youtube_dl/extractor/anvato.py
+++ b/youtube_dl/extractor/anvato.py
@@ -29,7 +29,7 @@ def md5_text(s):
 
 class AnvatoIE(InfoExtractor):
     _VALID_URL = r'anvato:(?P<access_key_or_mcp>[^:]+):(?P<id>\d+)'
-    
+
     # Copied from anvplayer.min.js
     _ANVACK_TABLE = {
         'nbcu_nbcd_desktop_web_prod_93d8ead38ce2024f8f544b78306fbd15895ae5e6': 'NNemUkySjxLyPTKvZRiGntBIjEyK8uqicjMakIaQ',

From 9edf47df7bfbcdd67bce68ea04865aa1f51df2f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 1 May 2017 01:03:47 +0700
Subject: [PATCH 0776/1696] [brightcove] Allow whitespace around attribute
 names in embedded code

---
 youtube_dl/extractor/brightcove.py |  2 +-
 youtube_dl/extractor/generic.py    | 16 ++++++++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 97602ca30..3f017a2b1 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -522,7 +522,7 @@ class BrightcoveNewIE(InfoExtractor):
         # [2] looks like:
         for video, script_tag, account_id, player_id, embed in re.findall(
                 r'''(?isx)
-                    (<video\s+[^>]*data-video-id=['"]?[^>]+>)
+                    (<video\s+[^>]*\bdata-video-id\s*=\s*['"]?[^>]+>)
                     (?:.*?
                         (<script[^>]+
                             src=["\'](?:https?:)?//players\.brightcove\.net/
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index bd9d9aa13..b06f43446 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1429,6 +1429,22 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            # Brightcove embed with whitespace around attribute names
+            'url': 'http://www.stack.com/video/3167554373001/learn-to-hit-open-three-pointers-with-damian-lillard-s-baseline-drift-drill',
+            'info_dict': {
+                'id': '3167554373001',
+                'ext': 'mp4',
+                'title': "Learn to Hit Open Three-Pointers With Damian Lillard's Baseline Drift Drill",
+                'description': 'md5:57bacb0e0f29349de4972bfda3191713',
+                'uploader_id': '1079349493',
+                'upload_date': '20140207',
+                'timestamp': 1391810548,
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
         # Another form of arte.tv embed
         {
             'url': 'http://www.tv-replay.fr/redirection/09-04-16/arte-reportage-arte-11508975.html',

From 74da856544177c15d95ffa78c5eda4d7089f8efd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 1 May 2017 01:23:05 +0700
Subject: [PATCH 0777/1696] [infoq] Make audio format extraction non fatal
 (closes #12938)

---
 youtube_dl/extractor/infoq.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/infoq.py b/youtube_dl/extractor/infoq.py
index 9fb71e8ef..fe425e786 100644
--- a/youtube_dl/extractor/infoq.py
+++ b/youtube_dl/extractor/infoq.py
@@ -87,8 +87,8 @@ class InfoQIE(BokeCCBaseIE):
 
     def _extract_http_audio(self, webpage, video_id):
         fields = self._hidden_inputs(webpage)
-        http_audio_url = fields['filename']
-        if http_audio_url is None:
+        http_audio_url = fields.get('filename')
+        if not http_audio_url:
             return []
 
         cookies_header = {'Cookie': self._extract_cookies(webpage)}

From 086041e2f8711ab579faf5cba681eba933be1021 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 1 May 2017 01:34:51 +0700
Subject: [PATCH 0778/1696] [ChangeLog] Actualize

---
 ChangeLog | 40 +++++++++++++++++++++++++++++++---------
 1 file changed, 31 insertions(+), 9 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index aa7543325..a4a8eac8e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,29 @@
 version <unreleased>
 
+Core
++ [extractor/common] Extract view count from JSON-LD
+* [utils] Improve unified_timestamp
++ [utils] Add video/mp2t to mimetype2ext
+* [downloader/external] Properly handle live stream downloading cancellation
+  (#8932)
++ [utils] Add support for unicode whitespace in clean_html on python 2 (#12906)
+
 Extractors
+* [infoq] Make audio format extraction non fatal (#12938)
+* [brightcove] Allow whitespace around attribute names in embedded code
++ [zaq1] Add support for zaq1.pl (#12693)
++ [xvideos] Extract duration (#12828)
+* [vevo] Fix extraction (#12879)
++ [noovo] Add support for noovo.ca (#12792)
++ [washingtonpost] Add support for embeds (#12699)
+* [yandexmusic:playlist] Fix extraction for python 3 (#12888)
+* [anvato] Improve extraction (#12913)
+    * Promote to regular shortcut based extractor
+    * Add mcp to access key mapping table
+    * Add support for embeds extraction
+    * Add support for anvato embeds in generic extractor
 * [xtube] Fix extraction for older FLV videos (#12734)
+* [tvplayer] Fix extraction (#12908)
 
 
 version 2017.04.28
@@ -30,19 +52,19 @@ Core
 * [YoutubeDL] Fix output template for missing timestamp (#12796)
 * [socks] Handle cases where credentials are required but missing
 * [extractor/common] Improve HLS extraction (#12211)
-    - Extract m3u8 parsing to separate method
-    - Improve rendition groups extraction
-    - Build stream name according stream GROUP-ID
-    - Ignore reference to AUDIO group without URI when stream has no CODECS
-    - Use float for scaled tbr in _parse_m3u8_formats
+    * Extract m3u8 parsing to separate method
+    * Improve rendition groups extraction
+    * Build stream name according stream GROUP-ID
+    * Ignore reference to AUDIO group without URI when stream has no CODECS
+    * Use float for scaled tbr in _parse_m3u8_formats
 * [utils] Add support for TTML styles in dfxp2srt
 * [downloader/hls] No need to download keys for fragments that have been
   already downloaded
 * [downloader/fragment] Improve fragment downloading
-    - Resume immediately
-    - Don't concatenate fragments and decrypt them on every resume
-    - Optimize disk storage usage, don't store intermediate fragments on disk
-    - Store bookkeeping download state file
+    * Resume immediately
+    * Don't concatenate fragments and decrypt them on every resume
+    * Optimize disk storage usage, don't store intermediate fragments on disk
+    * Store bookkeeping download state file
 + [extractor/common] Add support for multiple getters in try_get
 + [extractor/common] Add support for video of WebPage context in _json_ld
   (#12778)

From e0c1e9a98c3a22c0ae559a30cff1d1ab35105beb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 1 May 2017 01:39:52 +0700
Subject: [PATCH 0779/1696] release 2017.05.01

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 3 +++
 youtube_dl/version.py     | 2 +-
 4 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index a84cbc801..fc221594f 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.04.28*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.04.28**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.01*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.01**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.04.28
+[debug] youtube-dl version 2017.05.01
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index a4a8eac8e..4b27f3eff 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.05.01
 
 Core
 + [extractor/common] Extract view count from JSON-LD
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 0be7af8c4..e3c038c48 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -45,6 +45,7 @@
  - **anderetijden**: npo.nl and ntr.nl
  - **AnimeOnDemand**
  - **anitube.se**
+ - **Anvato**
  - **AnySex**
  - **Aparat**
  - **AppleConnect**
@@ -529,6 +530,7 @@
  - **NJPWWorld**: 新日本プロレスワールド
  - **NobelPrize**
  - **Noco**
+ - **Noovo**
  - **Normalboots**
  - **NosVideo**
  - **Nova**: TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz
@@ -1013,6 +1015,7 @@
  - **youtube:user**: YouTube.com user videos (URL or "ytuser" keyword)
  - **youtube:watchlater**: Youtube watch later list, ":ytwatchlater" for short (requires authentication)
  - **Zapiks**
+ - **Zaq1**
  - **ZDF**
  - **ZDFChannel**
  - **zingmp3**: mp3.zing.vn
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 8c77f1905..c19ac49b0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.04.28'
+__version__ = '2017.05.01'

From a146fa1c68dade0fde7ef81cdb8debbec6a6fc04 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Mon, 1 May 2017 00:54:44 +0200
Subject: [PATCH 0780/1696] [coub] Update test and remove comment count
 extraction

---
 youtube_dl/extractor/coub.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/coub.py b/youtube_dl/extractor/coub.py
index 5fa1f006b..6ea03e65c 100644
--- a/youtube_dl/extractor/coub.py
+++ b/youtube_dl/extractor/coub.py
@@ -24,12 +24,11 @@ class CoubIE(InfoExtractor):
             'duration': 4.6,
             'timestamp': 1428527772,
             'upload_date': '20150408',
-            'uploader': 'Артём Лоскутников',
+            'uploader': 'Artyom Loskutnikov',
             'uploader_id': 'artyom.loskutnikov',
             'view_count': int,
             'like_count': int,
             'repost_count': int,
-            'comment_count': int,
             'age_limit': 0,
         },
     }, {
@@ -118,7 +117,6 @@ class CoubIE(InfoExtractor):
         view_count = int_or_none(coub.get('views_count') or coub.get('views_increase_count'))
         like_count = int_or_none(coub.get('likes_count'))
         repost_count = int_or_none(coub.get('recoubs_count'))
-        comment_count = int_or_none(coub.get('comments_count'))
 
         age_restricted = coub.get('age_restricted', coub.get('age_restricted_by_admin'))
         if age_restricted is not None:
@@ -137,7 +135,6 @@ class CoubIE(InfoExtractor):
             'view_count': view_count,
             'like_count': like_count,
             'repost_count': repost_count,
-            'comment_count': comment_count,
             'age_limit': age_limit,
             'formats': formats,
         }

From ab39a25c75be0f27f5b8ffad08e3a2d3ac51007f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 1 May 2017 09:02:15 +0100
Subject: [PATCH 0781/1696] [foxsports] fix extraction(closes #12945)

---
 youtube_dl/extractor/foxsports.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/foxsports.py b/youtube_dl/extractor/foxsports.py
index a3bb98377..985542727 100644
--- a/youtube_dl/extractor/foxsports.py
+++ b/youtube_dl/extractor/foxsports.py
@@ -11,10 +11,10 @@ class FoxSportsIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?foxsports\.com/(?:[^/]+/)*(?P<id>[^/]+)'
 
     _TEST = {
-        'url': 'http://www.foxsports.com/video?vid=432609859715',
+        'url': 'http://www.foxsports.com/tennessee/video/432609859715',
         'md5': 'b49050e955bebe32c301972e4012ac17',
         'info_dict': {
-            'id': 'i0qKWsk3qJaM',
+            'id': 'bwduI3X_TgUB',
             'ext': 'mp4',
             'title': 'Courtney Lee on going up 2-0 in series vs. Blazers',
             'description': 'Courtney Lee talks about Memphis being focused.',
@@ -31,8 +31,9 @@ class FoxSportsIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         config = self._parse_json(
-            self._search_regex(
-                r"data-player-config='([^']+)'", webpage, 'data player config'),
+            self._html_search_regex(
+                r"""class="[^"]*(?:fs-player|platformPlayer-wrapper)[^"]*".+?data-player-config='([^']+)'""",
+                webpage, 'data player config'),
             video_id)
 
         return self.url_result(smuggle_url(update_url_query(

From b9f9f361fa93ffaff0146a1f0ce9920770d1891c Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Mon, 1 May 2017 19:56:51 +0200
Subject: [PATCH 0782/1696] [crunchyroll] Update test

---
 youtube_dl/extractor/crunchyroll.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 2ed8b30bb..2ffa4a7f8 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -171,7 +171,7 @@ class CrunchyrollIE(CrunchyrollBaseIE):
         'info_dict': {
             'id': '727589',
             'ext': 'mp4',
-            'title': "KONOSUBA -God's blessing on this wonderful world! 2 Episode 1 – Give Me Deliverance from this Judicial Injustice!",
+            'title': "KONOSUBA -God's blessing on this wonderful world! 2 Episode 1 – Give Me Deliverance From This Judicial Injustice!",
             'description': 'md5:cbcf05e528124b0f3a0a419fc805ea7d',
             'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': 'Kadokawa Pictures Inc.',
@@ -179,7 +179,7 @@ class CrunchyrollIE(CrunchyrollBaseIE):
             'series': "KONOSUBA -God's blessing on this wonderful world!",
             'season': "KONOSUBA -God's blessing on this wonderful world! 2",
             'season_number': 2,
-            'episode': 'Give Me Deliverance from this Judicial Injustice!',
+            'episode': 'Give Me Deliverance From This Judicial Injustice!',
             'episode_number': 1,
         },
         'params': {

From d86d169dd5bcefc35e13aa8274ac3b30c701cd87 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Mon, 1 May 2017 20:37:23 +0200
Subject: [PATCH 0783/1696] [dailymotion] Add working test

---
 youtube_dl/extractor/dailymotion.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 246efde43..441114d19 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -50,6 +50,24 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
     ]
 
     _TESTS = [
+        {
+            'url': 'http://www.dailymotion.com/video/x5kesuj_office-christmas-party-review-jason-bateman-olivia-munn-t-j-miller_news',
+            'md5': '074b95bdee76b9e3654137aee9c79dfe',
+            'info_dict': {
+                'id': 'x5kesuj',
+                'ext': 'mp4',
+                'title': 'Office Christmas Party Review –  Jason Bateman, Olivia Munn, T.J. Miller',
+                'description': 'Office Christmas Party Review -  Jason Bateman, Olivia Munn, T.J. Miller',
+                'thumbnail': r're:^https?:.*\.(?:jpg|png)$',
+                'duration': 187,
+                'timestamp': 1493651285,
+                'upload_date': '20170501',
+                'uploader': 'Deadline',
+                'uploader_id': 'x1xm8ri',
+                'age_limit': 0,
+                'view_count': int,
+            },
+        },
         {
             'url': 'https://www.dailymotion.com/video/x2iuewm_steam-machine-models-pricing-listed-on-steam-store-ign-news_videogames',
             'md5': '2137c41a8e78554bb09225b8eb322406',
@@ -66,7 +84,8 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                 'uploader_id': 'xijv66',
                 'age_limit': 0,
                 'view_count': int,
-            }
+            },
+            'skip': 'video gone',
         },
         # Vevo video
         {

From 77481f138670f13eada366f2bb1c54333084d72a Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Mon, 1 May 2017 20:25:09 +0200
Subject: [PATCH 0784/1696] [democracynow] Update test

---
 youtube_dl/extractor/democracynow.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py
index bdfe638b4..5c9c0ecdc 100644
--- a/youtube_dl/extractor/democracynow.py
+++ b/youtube_dl/extractor/democracynow.py
@@ -21,7 +21,8 @@ class DemocracynowIE(InfoExtractor):
         'info_dict': {
             'id': '2015-0703-001',
             'ext': 'mp4',
-            'title': 'Daily Show',
+            'title': 'Daily Show for July 03, 2015',
+            'description': 'md5:80eb927244d6749900de6072c7cc2c86',
         },
     }, {
         'url': 'http://www.democracynow.org/2015/7/3/this_flag_comes_down_today_bree',

From ae21d2fd940b6aa2e2d619c037d701848b011f5a Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Mon, 1 May 2017 20:51:51 +0200
Subject: [PATCH 0785/1696] [dotsub] Update test

---
 youtube_dl/extractor/dotsub.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dotsub.py b/youtube_dl/extractor/dotsub.py
index 1f75352ca..148605c0b 100644
--- a/youtube_dl/extractor/dotsub.py
+++ b/youtube_dl/extractor/dotsub.py
@@ -35,7 +35,7 @@ class DotsubIE(InfoExtractor):
             'thumbnail': 're:^https?://dotsub.com/media/747bcf58-bd59-45b7-8c8c-ac312d084ee6/p',
             'duration': 290,
             'timestamp': 1476767794.2809999,
-            'upload_date': '20160525',
+            'upload_date': '20161018',
             'uploader': 'parthivi001',
             'uploader_id': 'user52596202',
             'view_count': int,

From c6fe5a7e12a5ebbe137b889afb8aefc206d132af Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Mon, 1 May 2017 21:29:34 +0200
Subject: [PATCH 0786/1696] [douyutv] Update test

---
 youtube_dl/extractor/douyutv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/douyutv.py b/youtube_dl/extractor/douyutv.py
index 82d8a042f..d22133d24 100644
--- a/youtube_dl/extractor/douyutv.py
+++ b/youtube_dl/extractor/douyutv.py
@@ -20,7 +20,7 @@ class DouyuTVIE(InfoExtractor):
             'id': '17732',
             'display_id': 'iseven',
             'ext': 'flv',
-            'title': 're:^清晨醒脑！T-ARA根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'title': 're:^清晨醒脑！根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
             'description': r're:.*m7show@163\.com.*',
             'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': '7师傅',
@@ -51,7 +51,7 @@ class DouyuTVIE(InfoExtractor):
             'id': '17732',
             'display_id': '17732',
             'ext': 'flv',
-            'title': 're:^清晨醒脑！T-ARA根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'title': 're:^清晨醒脑！根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
             'description': r're:.*m7show@163\.com.*',
             'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': '7师傅',

From 7fc875195fd74c235d6ca86afc642f683a485a4b Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 2 May 2017 00:06:19 +0100
Subject: [PATCH 0787/1696] [amp] imporove thumbnail and subtitle extraction

---
 youtube_dl/extractor/amp.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/amp.py b/youtube_dl/extractor/amp.py
index 98f8e69cd..fde1a8ff7 100644
--- a/youtube_dl/extractor/amp.py
+++ b/youtube_dl/extractor/amp.py
@@ -34,9 +34,12 @@ class AMPIE(InfoExtractor):
             if isinstance(media_thumbnail, dict):
                 media_thumbnail = [media_thumbnail]
             for thumbnail_data in media_thumbnail:
-                thumbnail = thumbnail_data['@attributes']
+                thumbnail = thumbnail_data.get('@attributes', {})
+                thumbnail_url = thumbnail.get('url')
+                if not thumbnail_url:
+                    continue
                 thumbnails.append({
-                    'url': self._proto_relative_url(thumbnail['url'], 'http:'),
+                    'url': self._proto_relative_url(thumbnail_url, 'http:'),
                     'width': int_or_none(thumbnail.get('width')),
                     'height': int_or_none(thumbnail.get('height')),
                 })
@@ -47,9 +50,14 @@ class AMPIE(InfoExtractor):
             if isinstance(media_subtitle, dict):
                 media_subtitle = [media_subtitle]
             for subtitle_data in media_subtitle:
-                subtitle = subtitle_data['@attributes']
-                lang = subtitle.get('lang') or 'en'
-                subtitles[lang] = [{'url': subtitle['href']}]
+                subtitle = subtitle_data.get('@attributes', {})
+                subtitle_href = subtitle.get('href')
+                if not subtitle_href:
+                    continue
+                subtitles.setdefault(subtitle.get('lang') or 'en', []).append({
+                    'url': subtitle_href,
+                    'ext': mimetype2ext(subtitle.get('type')) or determine_ext(subtitle_href),
+                })
 
         formats = []
         media_content = get_media_node('content')

From 55949fede6da7c2d612f56863196eadcbc583de9 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 5 May 2016 21:40:19 +0100
Subject: [PATCH 0788/1696] [common] introduce chapters field

---
 youtube_dl/extractor/common.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index fba15d446..9541e5b42 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -245,6 +245,10 @@ class InfoExtractor(object):
                     specified in the URL.
     end_time:       Time in seconds where the reproduction should end, as
                     specified in the URL.
+    chapters:       A list of dictionaries, with the following entries:
+                        * "start_time" - The start time of the chapter in seconds
+                        * "end_time" - The end time of the chapter in seconds
+                        * "title" (optional, string)
 
     The following fields should only be used when the video belongs to some logical
     chapter or section:

From fa2a36d9bcaddf793e8bac281f5fa7859af16a46 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 5 May 2016 21:41:30 +0100
Subject: [PATCH 0789/1696] [ffmpeg] add support for chapters field
 postprocessing

---
 youtube_dl/postprocessor/ffmpeg.py | 31 +++++++++++++++++++++++++++---
 1 file changed, 28 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 665109558..c91ec8588 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -4,6 +4,7 @@ import io
 import os
 import subprocess
 import time
+import re
 
 
 from .common import AudioConversionError, PostProcessor
@@ -22,6 +23,7 @@ from ..utils import (
     subtitles_filename,
     dfxp2srt,
     ISO639Utils,
+    replace_extension,
 )
 
 
@@ -429,17 +431,40 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
 
         filename = info['filepath']
         temp_filename = prepend_extension(filename, 'temp')
+        in_filenames = [filename]
+        options = []
 
         if info['ext'] == 'm4a':
-            options = ['-vn', '-acodec', 'copy']
+            options.extend(['-vn', '-acodec', 'copy'])
         else:
-            options = ['-c', 'copy']
+            options.extend(['-c', 'copy'])
 
         for (name, value) in metadata.items():
             options.extend(['-metadata', '%s=%s' % (name, value)])
 
+        chapters = info.get('chapters', [])
+        if chapters:
+            metadata_filename = encodeFilename(replace_extension(filename, 'meta'))
+            with io.open(metadata_filename, 'wt', encoding='utf-8') as f:
+                def ffmpeg_escape(text):
+                    return re.sub(r'(=|;|#|\\|\n)', r'\\\1', text)
+
+                metadata_file_content = ';FFMETADATA1\n'
+                for chapter in chapters:
+                    metadata_file_content += '[CHAPTER]\nTIMEBASE=1/1000\n'
+                    metadata_file_content += 'START=%d\n' % (chapter['start_time'] * 1000)
+                    metadata_file_content += 'END=%d\n' % (chapter['end_time'] * 1000)
+                    chapter_title = chapter.get('title')
+                    if chapter_title:
+                        metadata_file_content += 'title=%s\n' % ffmpeg_escape(chapter_title)
+                f.write(metadata_file_content)
+                in_filenames.append(metadata_filename)
+                options.extend(['-map_metadata', '1'])
+
         self._downloader.to_screen('[ffmpeg] Adding metadata to \'%s\'' % filename)
-        self.run_ffmpeg(filename, temp_filename, options)
+        self.run_ffmpeg_multiple_files(in_filenames, temp_filename, options)
+        if chapters:
+            os.remove(metadata_filename)
         os.remove(encodeFilename(filename))
         os.rename(encodeFilename(temp_filename), encodeFilename(filename))
         return [], info

From 7a6d33a9a5390d0dab7e9162d6b2552cb0fe23a5 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 5 May 2016 21:42:37 +0100
Subject: [PATCH 0790/1696] [pbs] extract chapters information

---
 youtube_dl/extractor/pbs.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 6166dc2ad..0727e381b 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -8,6 +8,7 @@ from ..utils import (
     ExtractorError,
     determine_ext,
     int_or_none,
+    float_or_none,
     js_to_json,
     strip_jsonp,
     strip_or_none,
@@ -464,6 +465,7 @@ class PBSIE(InfoExtractor):
                     redirects.append(redirect)
                     redirect_urls.add(redirect_url)
 
+        chapters = []
         # Player pages may also serve different qualities
         for page in ('widget/partnerplayer', 'portalplayer'):
             player = self._download_webpage(
@@ -479,6 +481,20 @@ class PBSIE(InfoExtractor):
                     extract_redirect_urls(video_info)
                     if not info:
                         info = video_info
+                if not chapters:
+                    for chapter_data in re.findall(r'(?s)chapters\.push\(({.*?})\)', player):
+                        chapter = self._parse_json(chapter_data, video_id, js_to_json, fatal=False)
+                        if not chapter:
+                            continue
+                        start_time = float_or_none(chapter.get('start_time'), 1000)
+                        duration = float_or_none(chapter.get('duration'), 1000)
+                        if start_time is None or duration is None:
+                            continue
+                        chapters.append({
+                            'start_time': start_time,
+                            'end_time': start_time + duration,
+                            'title': chapter.get('title'),
+                        })
 
         formats = []
         http_url = None
@@ -588,4 +604,5 @@ class PBSIE(InfoExtractor):
             'upload_date': upload_date,
             'formats': formats,
             'subtitles': subtitles,
+            'chapters': chapters,
         }

From 5401bea27fd6bcdd030e50f3af85145807eeab27 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 4 May 2017 03:18:56 +0800
Subject: [PATCH 0791/1696] [leeco] Fix extraction (closes #12974)

Seems on mobile devices a similar API is used, but I always get an AD
with mimicking that API.
---
 ChangeLog                     |   6 ++
 youtube_dl/extractor/leeco.py | 103 +++++++++++-----------------------
 2 files changed, 39 insertions(+), 70 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 4b27f3eff..e31f54304 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [leeco] Fix extraction (#12974)
+
+
 version 2017.05.01
 
 Core
diff --git a/youtube_dl/extractor/leeco.py b/youtube_dl/extractor/leeco.py
index 9eda956d2..0a07c1320 100644
--- a/youtube_dl/extractor/leeco.py
+++ b/youtube_dl/extractor/leeco.py
@@ -23,7 +23,6 @@ from ..utils import (
     str_or_none,
     url_basename,
     urshift,
-    update_url_query,
 )
 
 
@@ -51,7 +50,7 @@ class LeIE(InfoExtractor):
             'id': '1415246',
             'ext': 'mp4',
             'title': '美人天下01',
-            'description': 'md5:f88573d9d7225ada1359eaf0dbf8bcda',
+            'description': 'md5:28942e650e82ed4fcc8e4de919ee854d',
         },
         'params': {
             'hls_prefer_native': True,
@@ -69,7 +68,6 @@ class LeIE(InfoExtractor):
         'params': {
             'hls_prefer_native': True,
         },
-        'skip': 'Only available in China',
     }, {
         'url': 'http://sports.le.com/video/25737697.html',
         'only_matching': True,
@@ -81,7 +79,7 @@ class LeIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    # ror() and calc_time_key() are reversed from a embedded swf file in KLetvPlayer.swf
+    # ror() and calc_time_key() are reversed from a embedded swf file in LetvPlayer.swf
     def ror(self, param1, param2):
         _loc3_ = 0
         while _loc3_ < param2:
@@ -90,15 +88,8 @@ class LeIE(InfoExtractor):
         return param1
 
     def calc_time_key(self, param1):
-        _loc2_ = 773625421
-        _loc3_ = self.ror(param1, _loc2_ % 13)
-        _loc3_ = _loc3_ ^ _loc2_
-        _loc3_ = self.ror(_loc3_, _loc2_ % 17)
-        return _loc3_
-
-    # reversed from http://jstatic.letvcdn.com/sdk/player.js
-    def get_mms_key(self, time):
-        return self.ror(time, 8) ^ 185025305
+        _loc2_ = 185025305
+        return self.ror(param1, _loc2_ % 17) ^ _loc2_
 
     # see M3U8Encryption class in KLetvPlayer.swf
     @staticmethod
@@ -122,7 +113,7 @@ class LeIE(InfoExtractor):
 
     def _check_errors(self, play_json):
         # Check for errors
-        playstatus = play_json['playstatus']
+        playstatus = play_json['msgs']['playstatus']
         if playstatus['status'] == 0:
             flag = playstatus['flag']
             if flag == 1:
@@ -134,58 +125,31 @@ class LeIE(InfoExtractor):
         media_id = self._match_id(url)
         page = self._download_webpage(url, media_id)
 
-        play_json_h5 = self._download_json(
-            'http://api.le.com/mms/out/video/playJsonH5',
-            media_id, 'Downloading html5 playJson data', query={
-                'id': media_id,
-                'platid': 3,
-                'splatid': 304,
-                'format': 1,
-                'tkey': self.get_mms_key(int(time.time())),
-                'domain': 'www.le.com',
-                'tss': 'no',
-            },
-            headers=self.geo_verification_headers())
-        self._check_errors(play_json_h5)
-
         play_json_flash = self._download_json(
-            'http://api.le.com/mms/out/video/playJson',
+            'http://player-pc.le.com/mms/out/video/playJson',
             media_id, 'Downloading flash playJson data', query={
                 'id': media_id,
                 'platid': 1,
                 'splatid': 101,
                 'format': 1,
+                'source': 1000,
                 'tkey': self.calc_time_key(int(time.time())),
                 'domain': 'www.le.com',
+                'region': 'cn',
             },
             headers=self.geo_verification_headers())
         self._check_errors(play_json_flash)
 
-        def get_h5_urls(media_url, format_id):
-            location = self._download_json(
-                media_url, media_id,
-                'Download JSON metadata for format %s' % format_id, query={
-                    'format': 1,
-                    'expect': 3,
-                    'tss': 'no',
-                })['location']
-
-            return {
-                'http': update_url_query(location, {'tss': 'no'}),
-                'hls': update_url_query(location, {'tss': 'ios'}),
-            }
-
         def get_flash_urls(media_url, format_id):
-            media_url += '&' + compat_urllib_parse_urlencode({
-                'm3v': 1,
-                'format': 1,
-                'expect': 3,
-                'rateid': format_id,
-            })
-
             nodes_data = self._download_json(
                 media_url, media_id,
-                'Download JSON metadata for format %s' % format_id)
+                'Download JSON metadata for format %s' % format_id,
+                query={
+                    'm3v': 1,
+                    'format': 1,
+                    'expect': 3,
+                    'tss': 'ios',
+                })
 
             req = self._request_webpage(
                 nodes_data['nodelist'][0]['location'], media_id,
@@ -199,29 +163,28 @@ class LeIE(InfoExtractor):
 
         extracted_formats = []
         formats = []
-        for play_json, get_urls in ((play_json_h5, get_h5_urls), (play_json_flash, get_flash_urls)):
-            playurl = play_json['playurl']
-            play_domain = playurl['domain'][0]
+        playurl = play_json_flash['msgs']['playurl']
+        play_domain = playurl['domain'][0]
 
-            for format_id, format_data in playurl.get('dispatch', []).items():
-                if format_id in extracted_formats:
-                    continue
-                extracted_formats.append(format_id)
+        for format_id, format_data in playurl.get('dispatch', []).items():
+            if format_id in extracted_formats:
+                continue
+            extracted_formats.append(format_id)
 
-                media_url = play_domain + format_data[0]
-                for protocol, format_url in get_urls(media_url, format_id).items():
-                    f = {
-                        'url': format_url,
-                        'ext': determine_ext(format_data[1]),
-                        'format_id': '%s-%s' % (protocol, format_id),
-                        'protocol': 'm3u8_native' if protocol == 'hls' else 'http',
-                        'quality': int_or_none(format_id),
-                    }
+            media_url = play_domain + format_data[0]
+            for protocol, format_url in get_flash_urls(media_url, format_id).items():
+                f = {
+                    'url': format_url,
+                    'ext': determine_ext(format_data[1]),
+                    'format_id': '%s-%s' % (protocol, format_id),
+                    'protocol': 'm3u8_native' if protocol == 'hls' else 'http',
+                    'quality': int_or_none(format_id),
+                }
 
-                    if format_id[-1:] == 'p':
-                        f['height'] = int_or_none(format_id[:-1])
+                if format_id[-1:] == 'p':
+                    f['height'] = int_or_none(format_id[:-1])
 
-                    formats.append(f)
+                formats.append(f)
         self._sort_formats(formats, ('height', 'quality', 'format_id'))
 
         publish_time = parse_iso8601(self._html_search_regex(

From 0c265486016b06342fb257966474ce591667aaff Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 May 2017 23:09:18 +0800
Subject: [PATCH 0792/1696] [cda] Implement birthday verification (closes
 #12789)

---
 ChangeLog                          |  1 +
 test/test_utils.py                 | 11 ++++++
 youtube_dl/extractor/cda.py        | 52 +++++++++++++++++++++++--
 youtube_dl/extractor/videopress.py |  9 ++---
 youtube_dl/utils.py                | 61 ++++++++++++++++++++++++++++++
 5 files changed, 125 insertions(+), 9 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index e31f54304..7324a9dd9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
++ [cda] Support birthday verification (#12789)
 * [leeco] Fix extraction (#12974)
 
 
diff --git a/test/test_utils.py b/test/test_utils.py
index 05fdc0e95..f31559e71 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -44,6 +44,7 @@ from youtube_dl.utils import (
     limit_length,
     mimetype2ext,
     month_by_name,
+    multipart_encode,
     ohdave_rsa_encrypt,
     OnDemandPagedList,
     orderedSet,
@@ -620,6 +621,16 @@ class TestUtil(unittest.TestCase):
             'http://example.com/path', {'test': '第二行тест'})),
             query_dict('http://example.com/path?test=%E7%AC%AC%E4%BA%8C%E8%A1%8C%D1%82%D0%B5%D1%81%D1%82'))
 
+    def test_multipart_encode(self):
+        self.assertEqual(
+            multipart_encode({b'field': b'value'}, boundary='AAAAAA')[0],
+            b'--AAAAAA\r\nContent-Disposition: form-data; name="field"\r\n\r\nvalue\r\n--AAAAAA--\r\n')
+        self.assertEqual(
+            multipart_encode({'欄位'.encode('utf-8'): '值'.encode('utf-8')}, boundary='AAAAAA')[0],
+            b'--AAAAAA\r\nContent-Disposition: form-data; name="\xe6\xac\x84\xe4\xbd\x8d"\r\n\r\n\xe5\x80\xbc\r\n--AAAAAA--\r\n')
+        self.assertRaises(
+            ValueError, multipart_encode, {b'field': b'value'}, boundary='value')
+
     def test_dict_get(self):
         FALSE_VALUES = {
             'none': None,
diff --git a/youtube_dl/extractor/cda.py b/youtube_dl/extractor/cda.py
index 1ee35b501..78b7a923c 100755
--- a/youtube_dl/extractor/cda.py
+++ b/youtube_dl/extractor/cda.py
@@ -9,7 +9,10 @@ from ..utils import (
     ExtractorError,
     float_or_none,
     int_or_none,
+    multipart_encode,
     parse_duration,
+    random_birthday,
+    urljoin,
 )
 
 
@@ -27,7 +30,8 @@ class CDAIE(InfoExtractor):
             'description': 'md5:269ccd135d550da90d1662651fcb9772',
             'thumbnail': r're:^https?://.*\.jpg$',
             'average_rating': float,
-            'duration': 39
+            'duration': 39,
+            'age_limit': 0,
         }
     }, {
         'url': 'http://www.cda.pl/video/57413289',
@@ -41,13 +45,41 @@ class CDAIE(InfoExtractor):
             'uploader': 'crash404',
             'view_count': int,
             'average_rating': float,
-            'duration': 137
+            'duration': 137,
+            'age_limit': 0,
         }
+    }, {
+        # Age-restricted
+        'url': 'http://www.cda.pl/video/1273454c4',
+        'info_dict': {
+            'id': '1273454c4',
+            'ext': 'mp4',
+            'title': 'Bronson (2008) napisy HD 1080p',
+            'description': 'md5:1b6cb18508daf2dc4e0fa4db77fec24c',
+            'height': 1080,
+            'uploader': 'boniek61',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 5554,
+            'age_limit': 18,
+            'view_count': int,
+            'average_rating': float,
+        },
     }, {
         'url': 'http://ebd.cda.pl/0x0/5749950c',
         'only_matching': True,
     }]
 
+    def _download_age_confirm_page(self, url, video_id, *args, **kwargs):
+        form_data = random_birthday('rok', 'miesiac', 'dzien')
+        form_data.update({'return': url, 'module': 'video', 'module_id': video_id})
+        data, content_type = multipart_encode(form_data)
+        return self._download_webpage(
+            urljoin(url, '/a/validatebirth'), video_id, *args,
+            data=data, headers={
+                'Referer': url,
+                'Content-Type': content_type,
+            }, **kwargs)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         self._set_cookie('cda.pl', 'cda.player', 'html5')
@@ -57,6 +89,13 @@ class CDAIE(InfoExtractor):
         if 'Ten film jest dostępny dla użytkowników premium' in webpage:
             raise ExtractorError('This video is only available for premium users.', expected=True)
 
+        need_confirm_age = False
+        if self._html_search_regex(r'(<form[^>]+action="/a/validatebirth")',
+                                   webpage, 'birthday validate form', default=None):
+            webpage = self._download_age_confirm_page(
+                url, video_id, note='Confirming age')
+            need_confirm_age = True
+
         formats = []
 
         uploader = self._search_regex(r'''(?x)
@@ -81,6 +120,7 @@ class CDAIE(InfoExtractor):
             'thumbnail': self._og_search_thumbnail(webpage),
             'formats': formats,
             'duration': None,
+            'age_limit': 18 if need_confirm_age else 0,
         }
 
         def extract_format(page, version):
@@ -121,7 +161,12 @@ class CDAIE(InfoExtractor):
         for href, resolution in re.findall(
                 r'<a[^>]+data-quality="[^"]+"[^>]+href="([^"]+)"[^>]+class="quality-btn"[^>]*>([0-9]+p)',
                 webpage):
-            webpage = self._download_webpage(
+            if need_confirm_age:
+                handler = self._download_age_confirm_page
+            else:
+                handler = self._download_webpage
+
+            webpage = handler(
                 self._BASE_URL + href, video_id,
                 'Downloading %s version information' % resolution, fatal=False)
             if not webpage:
@@ -129,6 +174,7 @@ class CDAIE(InfoExtractor):
                 # invalid version is requested.
                 self.report_warning('Unable to download %s version information' % resolution)
                 continue
+
             extract_format(webpage, resolution)
 
         self._sort_formats(formats)
diff --git a/youtube_dl/extractor/videopress.py b/youtube_dl/extractor/videopress.py
index 049db25a5..e5f964d39 100644
--- a/youtube_dl/extractor/videopress.py
+++ b/youtube_dl/extractor/videopress.py
@@ -1,7 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import random
 import re
 
 from .common import InfoExtractor
@@ -11,6 +10,7 @@ from ..utils import (
     float_or_none,
     parse_age_limit,
     qualities,
+    random_birthday,
     try_get,
     unified_timestamp,
     urljoin,
@@ -47,13 +47,10 @@ class VideoPressIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
+        query = random_birthday('birth_year', 'birth_month', 'birth_day')
         video = self._download_json(
             'https://public-api.wordpress.com/rest/v1.1/videos/%s' % video_id,
-            video_id, query={
-                'birth_month': random.randint(1, 12),
-                'birth_day': random.randint(1, 31),
-                'birth_year': random.randint(1950, 1995),
-            })
+            video_id, query=query)
 
         title = video['title']
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 86fc5ccac..25bd228ab 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -11,6 +11,7 @@ import contextlib
 import ctypes
 import datetime
 import email.utils
+import email.header
 import errno
 import functools
 import gzip
@@ -2097,6 +2098,58 @@ def update_Request(req, url=None, data=None, headers={}, query={}):
     return new_req
 
 
+def try_multipart_encode(data, boundary):
+    content_type = 'multipart/form-data; boundary=%s' % boundary
+
+    out = b''
+    for k, v in data.items():
+        out += b'--' + boundary.encode('ascii') + b'\r\n'
+        if isinstance(k, compat_str):
+            k = k.encode('utf-8')
+        if isinstance(v, compat_str):
+            v = v.encode('utf-8')
+        # RFC 2047 requires non-ASCII field names to be encoded, while RFC 7578
+        # suggests sending UTF-8 directly. Firefox sends UTF-8, too
+        content = b'Content-Disposition: form-data; name="%s"\r\n\r\n' % k + v + b'\r\n'
+        if boundary.encode('ascii') in content:
+            raise ValueError('Boundary overlaps with data')
+        out += content
+
+    out += b'--' + boundary.encode('ascii') + b'--\r\n'
+
+    return out, content_type
+
+
+def multipart_encode(data, boundary=None):
+    '''
+    Encode a dict to RFC 7578-compliant form-data
+
+    data:
+        A dict where keys and values can be either Unicode or bytes-like
+        objects.
+    boundary:
+        If specified a Unicode object, it's used as the boundary. Otherwise
+        a random boundary is generated.
+
+    Reference: https://tools.ietf.org/html/rfc7578
+    '''
+    has_specified_boundary = boundary is not None
+
+    while True:
+        if boundary is None:
+            boundary = '---------------' + str(random.randrange(0x0fffffff, 0xffffffff))
+
+        try:
+            out, content_type = try_multipart_encode(data, boundary)
+            break
+        except ValueError:
+            if has_specified_boundary:
+                raise
+            boundary = None
+
+    return out, content_type
+
+
 def dict_get(d, key_or_keys, default=None, skip_false_values=True):
     if isinstance(key_or_keys, (list, tuple)):
         for key in key_or_keys:
@@ -3760,3 +3813,11 @@ def write_xattr(path, key, value):
                         "Couldn't find a tool to set the xattrs. "
                         "Install either the python 'xattr' module, "
                         "or the 'xattr' binary.")
+
+
+def random_birthday(year_field, month_field, day_field):
+    return {
+        year_field: str(random.randint(1950, 1995)),
+        month_field: str(random.randint(1, 12)),
+        day_field: str(random.randint(1, 31)),
+    }

From ff6f9a67040c47ea1645e3f91f3153c212ebc7ca Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 4 May 2017 16:04:25 +0100
Subject: [PATCH 0793/1696] [extractor/common] fix typo in
 _extract_akamai_formats

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9541e5b42..b9ad8461a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2174,7 +2174,7 @@ class InfoExtractor(object):
     def _extract_akamai_formats(self, manifest_url, video_id, hosts={}):
         formats = []
         hdcore_sign = 'hdcore=3.7.0'
-        f4m_url = re.sub(r'(https?://[^/+])/i/', r'\1/z/', manifest_url).replace('/master.m3u8', '/manifest.f4m')
+        f4m_url = re.sub(r'(https?://[^/]+)/i/', r'\1/z/', manifest_url).replace('/master.m3u8', '/manifest.f4m')
         hds_host = hosts.get('hds')
         if hds_host:
             f4m_url = re.sub(r'(https?://)[^/]+', r'\1' + hds_host, f4m_url)

From 4fe14732a24214d030cdbdfc25e5906244be2dc2 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 4 May 2017 16:06:21 +0100
Subject: [PATCH 0794/1696] [laola1tv] fix extraction(closes #12880)

---
 youtube_dl/extractor/laola1tv.py | 92 +++++++++++++++++++++++---------
 1 file changed, 67 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py
index 3190b187c..e8853fb77 100644
--- a/youtube_dl/extractor/laola1tv.py
+++ b/youtube_dl/extractor/laola1tv.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import json
+
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
@@ -10,6 +12,7 @@ from ..utils import (
     xpath_text,
     urljoin,
     update_url_query,
+    js_to_json,
 )
 
 
@@ -28,6 +31,29 @@ class Laola1TvEmbedIE(InfoExtractor):
         },
     }
 
+    def _extract_token_url(self, stream_access_url, video_id, data):
+        return self._download_json(
+            stream_access_url, video_id, headers={
+                'Content-Type': 'application/json',
+            }, data=json.dumps(data).encode())['data']['stream-access'][0]
+
+    def _extract_formats(self, token_url, video_id):
+        token_doc = self._download_xml(
+            token_url, video_id, 'Downloading token',
+            headers=self.geo_verification_headers())
+
+        token_attrib = xpath_element(token_doc, './/token').attrib
+
+        if token_attrib['status'] != '0':
+            raise ExtractorError(
+                'Token error: %s' % token_attrib['comment'], expected=True)
+
+        formats = self._extract_akamai_formats(
+            '%s?hdnea=%s' % (token_attrib['url'], token_attrib['auth']),
+            video_id)
+        self._sort_formats(formats)
+        return formats
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
@@ -68,29 +94,16 @@ class Laola1TvEmbedIE(InfoExtractor):
         else:
             data_abo = urlencode_postdata(
                 dict((i, v) for i, v in enumerate(_v('req_liga_abos').split(','))))
-            token_url = self._download_json(
-                'https://club.laola1.tv/sp/laola1/api/v3/user/session/premium/player/stream-access',
-                video_id, query={
+            stream_access_url = update_url_query(
+                'https://club.laola1.tv/sp/laola1/api/v3/user/session/premium/player/stream-access', {
                     'videoId': _v('id'),
                     'target': self._search_regex(r'vs_target = (\d+);', webpage, 'vs target'),
                     'label': _v('label'),
                     'area': _v('area'),
-                }, data=data_abo)['data']['stream-access'][0]
+                })
+            token_url = self._extract_token_url(stream_access_url, video_id, data_abo)
 
-        token_doc = self._download_xml(
-            token_url, video_id, 'Downloading token',
-            headers=self.geo_verification_headers())
-
-        token_attrib = xpath_element(token_doc, './/token').attrib
-
-        if token_attrib['status'] != '0':
-            raise ExtractorError(
-                'Token error: %s' % token_attrib['comment'], expected=True)
-
-        formats = self._extract_akamai_formats(
-            '%s?hdnea=%s' % (token_attrib['url'], token_attrib['auth']),
-            video_id)
-        self._sort_formats(formats)
+        formats = self._extract_formats(token_url, video_id)
 
         categories_str = _v('meta_sports')
         categories = categories_str.split(',') if categories_str else []
@@ -107,7 +120,7 @@ class Laola1TvEmbedIE(InfoExtractor):
         }
 
 
-class Laola1TvIE(InfoExtractor):
+class Laola1TvIE(Laola1TvEmbedIE):
     IE_NAME = 'laola1tv'
     _VALID_URL = r'https?://(?:www\.)?laola1\.tv/[a-z]+-[a-z]+/[^/]+/(?P<id>[^/?#&]+)'
     _TESTS = [{
@@ -164,13 +177,42 @@ class Laola1TvIE(InfoExtractor):
         if 'Dieser Livestream ist bereits beendet.' in webpage:
             raise ExtractorError('This live stream has already finished.', expected=True)
 
-        iframe_url = urljoin(url, self._search_regex(
-            r'<iframe[^>]*?id="videoplayer"[^>]*?src="([^"]+)"',
-            webpage, 'iframe url'))
+        conf = self._parse_json(self._search_regex(
+            r'(?s)conf\s*=\s*({.+?});', webpage, 'conf'),
+            display_id, js_to_json)
+
+        video_id = conf['videoid']
+
+        config = self._download_json(conf['configUrl'], video_id, query={
+            'videoid': video_id,
+            'partnerid': conf['partnerid'],
+            'language': conf.get('language', ''),
+            'portal': conf.get('portalid', ''),
+        })
+        error = config.get('error')
+        if error:
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, error), expected=True)
+
+        video_data = config['video']
+        title = video_data['title']
+        is_live = video_data.get('isLivestream') and video_data.get('isLive')
+        meta = video_data.get('metaInformation')
+        sports = meta.get('sports')
+        categories = sports.split(',') if sports else []
+
+        token_url = self._extract_token_url(
+            video_data['streamAccess'], video_id,
+            video_data['abo']['required'])
+
+        formats = self._extract_formats(token_url, video_id)
 
         return {
-            '_type': 'url',
+            'id': video_id,
             'display_id': display_id,
-            'url': iframe_url,
-            'ie_key': 'Laola1TvEmbed',
+            'title': self._live_title(title) if is_live else title,
+            'description': video_data.get('description'),
+            'thumbnail': video_data.get('image'),
+            'categories': categories,
+            'formats': formats,
+            'is_live': is_live,
         }

From 7f09e523e87ab6fcf5713fbcd6767d00d98039e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 May 2017 22:41:47 +0700
Subject: [PATCH 0795/1696] [laola1tv:embed] Fix tests

---
 youtube_dl/extractor/laola1tv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py
index e8853fb77..d9f53c356 100644
--- a/youtube_dl/extractor/laola1tv.py
+++ b/youtube_dl/extractor/laola1tv.py
@@ -19,7 +19,7 @@ from ..utils import (
 class Laola1TvEmbedIE(InfoExtractor):
     IE_NAME = 'laola1tv:embed'
     _VALID_URL = r'https?://(?:www\.)?laola1\.tv/titanplayer\.php\?.*?\bvideoid=(?P<id>\d+)'
-    _TEST = {
+    _TESTS = [{
         # flashvars.premium = "false";
         'url': 'https://www.laola1.tv/titanplayer.php?videoid=708065&type=V&lang=en&portal=int&customer=1024',
         'info_dict': {
@@ -29,7 +29,7 @@ class Laola1TvEmbedIE(InfoExtractor):
             'uploader': 'ITTF - International Table Tennis Federation',
             'upload_date': '20161211',
         },
-    }
+    }]
 
     def _extract_token_url(self, stream_access_url, video_id, data):
         return self._download_json(

From 4947f13cd0e541bd2223187df19389c8b721c665 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 May 2017 22:42:49 +0700
Subject: [PATCH 0796/1696] [pbs] Improve multipart video support (closes
 #12981)

---
 youtube_dl/extractor/pbs.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 0727e381b..16cc667d0 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -10,6 +10,7 @@ from ..utils import (
     int_or_none,
     float_or_none,
     js_to_json,
+    orderedSet,
     strip_jsonp,
     strip_or_none,
     unified_strdate,
@@ -264,6 +265,13 @@ class PBSIE(InfoExtractor):
             },
             'playlist_count': 2,
         },
+        {
+            'url': 'http://www.pbs.org/wgbh/americanexperience/films/great-war/',
+            'info_dict': {
+                'id': 'great-war',
+            },
+            'playlist_count': 3,
+        },
         {
             'url': 'http://www.pbs.org/wgbh/americanexperience/films/death/player/',
             'info_dict': {
@@ -382,10 +390,10 @@ class PBSIE(InfoExtractor):
             # tabbed frontline videos
             MULTI_PART_REGEXES = (
                 r'<div[^>]+class="videotab[^"]*"[^>]+vid="(\d+)"',
-                r'<a[^>]+href=["\']#video-\d+["\'][^>]+data-coveid=["\'](\d+)',
+                r'<a[^>]+href=["\']#(?:video-|part)\d+["\'][^>]+data-cove[Ii]d=["\'](\d+)',
             )
             for p in MULTI_PART_REGEXES:
-                tabbed_videos = re.findall(p, webpage)
+                tabbed_videos = orderedSet(re.findall(p, webpage))
                 if tabbed_videos:
                     return tabbed_videos, presumptive_id, upload_date, description
 

From 50ad078b7bca28ec4a85caf7689e3a260def2189 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 5 May 2017 15:13:40 +0800
Subject: [PATCH 0797/1696] [gdcvault] Fix extraction for videos with
 gdc-player.html

Closes #12733
---
 ChangeLog                        |  1 +
 youtube_dl/extractor/gdcvault.py | 15 ++++++++++++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 7324a9dd9..930fdded4 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [gdcvault] Fix extraction for some videos (#12733)
 + [cda] Support birthday verification (#12789)
 * [leeco] Fix extraction (#12974)
 
diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index 3136427db..f71d9092e 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -75,6 +75,19 @@ class GDCVaultIE(InfoExtractor):
                 'format': 'jp',  # The japanese audio
             }
         },
+        {
+            # gdc-player.html
+            'url': 'http://www.gdcvault.com/play/1435/An-American-engine-in-Tokyo',
+            'info_dict': {
+                'id': '1435',
+                'display_id': 'An-American-engine-in-Tokyo',
+                'ext': 'flv',
+                'title': 'An American Engine in Tokyo:/nThe collaboration of Epic Games and Square Enix/nFor THE LAST REMINANT',
+            },
+            'params': {
+                'skip_download': True,  # Requires rtmpdump
+            },
+        },
     ]
 
     def _login(self, webpage_url, display_id):
@@ -128,7 +141,7 @@ class GDCVaultIE(InfoExtractor):
                 'title': title,
             }
 
-        PLAYER_REGEX = r'<iframe src="(?P<xml_root>.+?)/player.*?\.html.*?".*?</iframe>'
+        PLAYER_REGEX = r'<iframe src="(?P<xml_root>.+?)/(?:gdc-)?player.*?\.html.*?".*?</iframe>'
 
         xml_root = self._html_search_regex(
             PLAYER_REGEX, start_page, 'xml root', default=None)

From cc7bda4fffa3913178e5ad345a8c9c6d1d8a2626 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 5 May 2017 20:01:02 +0800
Subject: [PATCH 0798/1696] [vice] Fix extraction for non en_us videos (closes
 #12967)

---
 ChangeLog                        |  1 +
 youtube_dl/extractor/vice.py     | 34 ++++++++++++++++++++++++++------
 youtube_dl/extractor/viceland.py | 11 ++++++++---
 3 files changed, 37 insertions(+), 9 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 930fdded4..23f83cc56 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [vice] Fix extraction for non en_us videos (#12967)
 * [gdcvault] Fix extraction for some videos (#12733)
 + [cda] Support birthday verification (#12789)
 * [leeco] Fix extraction (#12974)
diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index f0a7fd739..275fc1395 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -20,7 +20,7 @@ from ..utils import (
 
 
 class ViceBaseIE(AdobePassIE):
-    def _extract_preplay_video(self, url, webpage):
+    def _extract_preplay_video(self, url, locale, webpage):
         watch_hub_data = extract_attributes(self._search_regex(
             r'(?s)(<watch-hub\s*.+?</watch-hub>)', webpage, 'watch hub'))
         video_id = watch_hub_data['vms-id']
@@ -45,7 +45,7 @@ class ViceBaseIE(AdobePassIE):
 
         try:
             host = 'www.viceland' if is_locked else self._PREPLAY_HOST
-            preplay = self._download_json('https://%s.com/en_us/preplay/%s' % (host, video_id), video_id, query=query)
+            preplay = self._download_json('https://%s.com/%s/preplay/%s' % (host, locale, video_id), video_id, query=query)
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
                 error = json.loads(e.cause.read().decode())
@@ -88,7 +88,7 @@ class ViceBaseIE(AdobePassIE):
 
 
 class ViceIE(ViceBaseIE):
-    _VALID_URL = r'https?://(?:.+?\.)?vice\.com/(?:[^/]+/)?videos?/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:.+?\.)?vice\.com/(?P<locale>[^/]+/)(?:[^/]+/)?videos?/(?P<id>[^/?#&]+)'
 
     _TESTS = [{
         'url': 'http://www.vice.com/video/cowboy-capitalists-part-1',
@@ -115,20 +115,39 @@ class ViceIE(ViceBaseIE):
         'add_ie': ['Youtube'],
     }, {
         'url': 'https://video.vice.com/en_us/video/the-signal-from-tolva/5816510690b70e6c5fd39a56',
-        'md5': '',
         'info_dict': {
             'id': '5816510690b70e6c5fd39a56',
             'ext': 'mp4',
             'uploader': 'Waypoint',
             'title': 'The Signal From Tölva',
+            'description': 'md5:3927e3c79f9e8094606a2b3c5b5e55d5',
             'uploader_id': '57f7d621e05ca860fa9ccaf9',
-            'timestamp': 1477941983938,
+            'timestamp': 1477941983,
+            'upload_date': '20161031',
         },
         'params': {
             # m3u8 download
             'skip_download': True,
         },
         'add_ie': ['UplynkPreplay'],
+    }, {
+        'url': 'https://video.vice.com/alps/video/ulfs-wien-beruchtigste-grafitti-crew-part-1/581b12b60a0e1f4c0fb6ea2f',
+        'info_dict': {
+            'id': '581b12b60a0e1f4c0fb6ea2f',
+            'ext': 'mp4',
+            'title': 'ULFs - Wien berüchtigste Grafitti Crew - Part 1',
+            'description': '<p>Zwischen Hinterzimmer-Tattoos und U-Bahnschächten erzählen uns die Ulfs, wie es ist, "süchtig nach Sachbeschädigung" zu sein.</p>',
+            'uploader': 'VICE',
+            'uploader_id': '57a204088cb727dec794c67b',
+            'timestamp': 1485368119,
+            'upload_date': '20170125',
+            'age_limit': 14,
+        },
+        'params': {
+            # AES-encrypted m3u8
+            'skip_download': True,
+        },
+        'add_ie': ['UplynkPreplay'],
     }, {
         'url': 'https://news.vice.com/video/experimenting-on-animals-inside-the-monkey-lab',
         'only_matching': True,
@@ -142,6 +161,9 @@ class ViceIE(ViceBaseIE):
     _PREPLAY_HOST = 'video.vice'
 
     def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        locale = mobj.group('locale')
         video_id = self._match_id(url)
         webpage, urlh = self._download_webpage_handle(url, video_id)
         embed_code = self._search_regex(
@@ -153,7 +175,7 @@ class ViceIE(ViceBaseIE):
             r'data-youtube-id="([^"]+)"', webpage, 'youtube id', default=None)
         if youtube_id:
             return self.url_result(youtube_id, 'Youtube')
-        return self._extract_preplay_video(urlh.geturl(), webpage)
+        return self._extract_preplay_video(urlh.geturl(), locale, webpage)
 
 
 class ViceShowIE(InfoExtractor):
diff --git a/youtube_dl/extractor/viceland.py b/youtube_dl/extractor/viceland.py
index 87f9216b5..bd60235c8 100644
--- a/youtube_dl/extractor/viceland.py
+++ b/youtube_dl/extractor/viceland.py
@@ -1,11 +1,13 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .vice import ViceBaseIE
 
 
 class VicelandIE(ViceBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?viceland\.com/[^/]+/video/[^/]+/(?P<id>[a-f0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?viceland\.com/(?P<locale>[^/]+)/video/[^/]+/(?P<id>[a-f0-9]+)'
     _TEST = {
         'url': 'https://www.viceland.com/en_us/video/trapped/588a70d0dba8a16007de7316',
         'info_dict': {
@@ -24,10 +26,13 @@ class VicelandIE(ViceBaseIE):
             'skip_download': True,
         },
         'add_ie': ['UplynkPreplay'],
+        'skip': '404',
     }
     _PREPLAY_HOST = 'www.viceland'
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        locale = mobj.group('locale')
         webpage = self._download_webpage(url, video_id)
-        return self._extract_preplay_video(url, webpage)
+        return self._extract_preplay_video(url, locale, webpage)

From 4ac6dc3732492839b8c5e9a144ac8ef8fd1539aa Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 5 May 2017 20:26:51 +0800
Subject: [PATCH 0799/1696] [vice] Support Vice articles (closes #12968)

---
 ChangeLog                          |  1 +
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/vice.py       | 79 ++++++++++++++++++++++++------
 3 files changed, 67 insertions(+), 14 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 23f83cc56..0908e2e93 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
++ [vice] Support vice articles (#12968)
 * [vice] Fix extraction for non en_us videos (#12967)
 * [gdcvault] Fix extraction for some videos (#12733)
 + [cda] Support birthday verification (#12789)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index c0020dd7d..1d7495910 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1125,6 +1125,7 @@ from .vgtv import (
 from .vh1 import VH1IE
 from .vice import (
     ViceIE,
+    ViceArticleIE,
     ViceShowIE,
 )
 from .viceland import VicelandIE
diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index 275fc1395..b2e95734b 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -88,7 +88,7 @@ class ViceBaseIE(AdobePassIE):
 
 
 class ViceIE(ViceBaseIE):
-    _VALID_URL = r'https?://(?:.+?\.)?vice\.com/(?P<locale>[^/]+/)(?:[^/]+/)?videos?/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:.+?\.)?vice\.com/(?P<locale>[^/]+)/(?:[^/]+/)?videos?/(?P<id>[^/?#&]+)'
 
     _TESTS = [{
         'url': 'http://www.vice.com/video/cowboy-capitalists-part-1',
@@ -100,19 +100,6 @@ class ViceIE(ViceBaseIE):
             'duration': 725.983,
         },
         'add_ie': ['Ooyala'],
-    }, {
-        'url': 'http://www.vice.com/video/how-to-hack-a-car',
-        'md5': 'a7ecf64ee4fa19b916c16f4b56184ae2',
-        'info_dict': {
-            'id': '3jstaBeXgAs',
-            'ext': 'mp4',
-            'title': 'How to Hack a Car: Phreaked Out (Episode 2)',
-            'description': 'md5:ee95453f7ff495db8efe14ae8bf56f30',
-            'uploader_id': 'MotherboardTV',
-            'uploader': 'Motherboard',
-            'upload_date': '20140529',
-        },
-        'add_ie': ['Youtube'],
     }, {
         'url': 'https://video.vice.com/en_us/video/the-signal-from-tolva/5816510690b70e6c5fd39a56',
         'info_dict': {
@@ -208,3 +195,67 @@ class ViceShowIE(InfoExtractor):
         description = self._html_search_meta('description', webpage, 'description')
 
         return self.playlist_result(entries, show_id, title, description)
+
+
+class ViceArticleIE(InfoExtractor):
+    _VALID_URL = r'https://www.vice.com/[^/]+/article/(?P<id>[^?#]+)'
+
+    _TESTS = [{
+        'url': 'https://www.vice.com/en_us/article/on-set-with-the-woman-making-mormon-porn-in-utah',
+        'info_dict': {
+            'id': '58dc0a3dee202d2a0ccfcbd8',
+            'ext': 'mp4',
+            'title': 'Mormon War on Porn ',
+            'description': 'md5:ad396a2481e7f8afb5ed486878421090',
+            'uploader': 'VICE',
+            'uploader_id': '57a204088cb727dec794c693',
+            'timestamp': 1489160690,
+            'upload_date': '20170310',
+        },
+        'params': {
+            # AES-encrypted m3u8
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://www.vice.com/video/how-to-hack-a-car',
+        'md5': 'a7ecf64ee4fa19b916c16f4b56184ae2',
+        'info_dict': {
+            'id': '3jstaBeXgAs',
+            'ext': 'mp4',
+            'title': 'How to Hack a Car: Phreaked Out (Episode 2)',
+            'description': 'md5:ee95453f7ff495db8efe14ae8bf56f30',
+            'uploader_id': 'MotherboardTV',
+            'uploader': 'Motherboard',
+            'upload_date': '20140529',
+        },
+        'add_ie': ['Youtube'],
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        prefetch_data = self._parse_json(self._search_regex(
+            r'window\.__PREFETCH_DATA\s*=\s*({.*});',
+            webpage, 'prefetch data'), display_id)
+        body = prefetch_data['body']
+        youtube_url = self._html_search_regex(
+            r'<iframe[^>]+src="(.*youtube\.com/.*)"', body, 'YouTube URL', default=None)
+        if youtube_url:
+            return {
+                '_type': 'url_transparent',
+                'url': youtube_url,
+                'display_id': display_id,
+                'ie_key': 'Youtube',
+            }
+
+        video_url = self._html_search_regex(
+            r'data-video-url="([^"]+)"', prefetch_data['embed_code'], 'video URL')
+
+        return {
+            '_type': 'url_transparent',
+            'url': video_url,
+            'display_id': display_id,
+            'ie_key': ViceIE.ie_key(),
+        }

From b2ad479d17c435b02fe4f8a02f21bb713c2833f1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 5 May 2017 20:51:59 +0800
Subject: [PATCH 0800/1696] [utils] Fix multipart_encode for Python < 3.5

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 25bd228ab..911cacd29 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2110,7 +2110,7 @@ def try_multipart_encode(data, boundary):
             v = v.encode('utf-8')
         # RFC 2047 requires non-ASCII field names to be encoded, while RFC 7578
         # suggests sending UTF-8 directly. Firefox sends UTF-8, too
-        content = b'Content-Disposition: form-data; name="%s"\r\n\r\n' % k + v + b'\r\n'
+        content = b'Content-Disposition: form-data; name="' + k + b'"\r\n\r\n' + v + b'\r\n'
         if boundary.encode('ascii') in content:
             raise ValueError('Boundary overlaps with data')
         out += content

From 7ad53cb7ff9afdcffb0b2d027f06f4120dbde9a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 5 May 2017 21:59:23 +0700
Subject: [PATCH 0801/1696] [laola1tv] PEP 8

---
 youtube_dl/extractor/laola1tv.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py
index d9f53c356..1f91ba017 100644
--- a/youtube_dl/extractor/laola1tv.py
+++ b/youtube_dl/extractor/laola1tv.py
@@ -10,7 +10,6 @@ from ..utils import (
     urlencode_postdata,
     xpath_element,
     xpath_text,
-    urljoin,
     update_url_query,
     js_to_json,
 )

From 1d9e0a4f40deaeb2f722cf964e6bf13b3835e617 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 5 May 2017 16:12:40 +0100
Subject: [PATCH 0802/1696] [vice] update tests and add support for ooyala
 embeds in article pages

---
 youtube_dl/extractor/vice.py | 83 +++++++++++++++++++++---------------
 1 file changed, 48 insertions(+), 35 deletions(-)

diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index b2e95734b..54e207b39 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -32,7 +32,8 @@ class ViceBaseIE(AdobePassIE):
             resource = self._get_mvpd_resource(
                 'VICELAND', title, video_id,
                 watch_hub_data.get('video-rating'))
-            query['tvetoken'] = self._extract_mvpd_auth(url, video_id, 'VICELAND', resource)
+            query['tvetoken'] = self._extract_mvpd_auth(
+                url, video_id, 'VICELAND', resource)
 
         # signature generation algorithm is reverse engineered from signatureGenerator in
         # webpack:///../shared/~/vice-player/dist/js/vice-player.js in
@@ -45,11 +46,14 @@ class ViceBaseIE(AdobePassIE):
 
         try:
             host = 'www.viceland' if is_locked else self._PREPLAY_HOST
-            preplay = self._download_json('https://%s.com/%s/preplay/%s' % (host, locale, video_id), video_id, query=query)
+            preplay = self._download_json(
+                'https://%s.com/%s/preplay/%s' % (host, locale, video_id),
+                video_id, query=query)
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
                 error = json.loads(e.cause.read().decode())
-                raise ExtractorError('%s said: %s' % (self.IE_NAME, error['details']), expected=True)
+                raise ExtractorError('%s said: %s' % (
+                    self.IE_NAME, error['details']), expected=True)
             raise
 
         video_data = preplay['video']
@@ -88,16 +92,17 @@ class ViceBaseIE(AdobePassIE):
 
 
 class ViceIE(ViceBaseIE):
-    _VALID_URL = r'https?://(?:.+?\.)?vice\.com/(?P<locale>[^/]+)/(?:[^/]+/)?videos?/(?P<id>[^/?#&]+)'
+    IE_NAME = 'vice'
+    _VALID_URL = r'https?://(?:.+?\.)?vice\.com/(?:(?P<locale>[^/]+)/)?videos?/(?P<id>[^/?#&]+)'
 
     _TESTS = [{
-        'url': 'http://www.vice.com/video/cowboy-capitalists-part-1',
-        'md5': 'e9d77741f9e42ba583e683cd170660f7',
+        'url': 'https://news.vice.com/video/experimenting-on-animals-inside-the-monkey-lab',
+        'md5': '7d3ae2f9ba5f196cdd9f9efd43657ac2',
         'info_dict': {
-            'id': '43cW1mYzpia9IlestBjVpd23Yu3afAfp',
+            'id': 'N2bzkydjraWDGwnt8jAttCF6Y0PDv4Zj',
             'ext': 'flv',
-            'title': 'VICE_COWBOYCAPITALISTS_PART01_v1_VICE_WM_1080p.mov',
-            'duration': 725.983,
+            'title': 'Monkey Labs of Holland',
+            'description': 'md5:92b3c7dcbfe477f772dd4afa496c9149',
         },
         'add_ie': ['Ooyala'],
     }, {
@@ -136,22 +141,13 @@ class ViceIE(ViceBaseIE):
         },
         'add_ie': ['UplynkPreplay'],
     }, {
-        'url': 'https://news.vice.com/video/experimenting-on-animals-inside-the-monkey-lab',
-        'only_matching': True,
-    }, {
-        'url': 'http://www.vice.com/ru/video/big-night-out-ibiza-clive-martin-229',
-        'only_matching': True,
-    }, {
-        'url': 'https://munchies.vice.com/en/videos/watch-the-trailer-for-our-new-series-the-pizza-show',
+        'url': 'https://video.vice.com/en_us/video/pizza-show-trailer/56d8c9a54d286ed92f7f30e4',
         'only_matching': True,
     }]
     _PREPLAY_HOST = 'video.vice'
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        locale = mobj.group('locale')
-        video_id = self._match_id(url)
+        locale, video_id = re.match(self._VALID_URL, url).groups()
         webpage, urlh = self._download_webpage_handle(url, video_id)
         embed_code = self._search_regex(
             r'embedCode=([^&\'"]+)', webpage,
@@ -166,6 +162,7 @@ class ViceIE(ViceBaseIE):
 
 
 class ViceShowIE(InfoExtractor):
+    IE_NAME = 'vice:show'
     _VALID_URL = r'https?://(?:.+?\.)?vice\.com/(?:[^/]+/)?show/(?P<id>[^/?#&]+)'
 
     _TEST = {
@@ -192,12 +189,14 @@ class ViceShowIE(InfoExtractor):
             r'<title>(.+?)</title>', webpage, 'title', default=None)
         if title:
             title = re.sub(r'(.+)\s*\|\s*.+$', r'\1', title).strip()
-        description = self._html_search_meta('description', webpage, 'description')
+        description = self._html_search_meta(
+            'description', webpage, 'description')
 
         return self.playlist_result(entries, show_id, title, description)
 
 
 class ViceArticleIE(InfoExtractor):
+    IE_NAME = 'vice:article'
     _VALID_URL = r'https://www.vice.com/[^/]+/article/(?P<id>[^?#]+)'
 
     _TESTS = [{
@@ -216,8 +215,9 @@ class ViceArticleIE(InfoExtractor):
             # AES-encrypted m3u8
             'skip_download': True,
         },
+        'add_ie': ['UplynkPreplay'],
     }, {
-        'url': 'http://www.vice.com/video/how-to-hack-a-car',
+        'url': 'https://www.vice.com/en_us/article/how-to-hack-a-car',
         'md5': 'a7ecf64ee4fa19b916c16f4b56184ae2',
         'info_dict': {
             'id': '3jstaBeXgAs',
@@ -229,6 +229,12 @@ class ViceArticleIE(InfoExtractor):
             'upload_date': '20140529',
         },
         'add_ie': ['Youtube'],
+    }, {
+        'url': 'https://www.vice.com/en_us/article/cowboy-capitalists-part-1',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.vice.com/ru/article/big-night-out-ibiza-clive-martin-229',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -240,22 +246,29 @@ class ViceArticleIE(InfoExtractor):
             r'window\.__PREFETCH_DATA\s*=\s*({.*});',
             webpage, 'prefetch data'), display_id)
         body = prefetch_data['body']
-        youtube_url = self._html_search_regex(
-            r'<iframe[^>]+src="(.*youtube\.com/.*)"', body, 'YouTube URL', default=None)
-        if youtube_url:
+
+        def _url_res(video_url, ie_key):
             return {
                 '_type': 'url_transparent',
-                'url': youtube_url,
+                'url': video_url,
                 'display_id': display_id,
-                'ie_key': 'Youtube',
+                'ie_key': ie_key,
             }
 
-        video_url = self._html_search_regex(
-            r'data-video-url="([^"]+)"', prefetch_data['embed_code'], 'video URL')
+        embed_code = self._search_regex(
+            r'embedCode=([^&\'"]+)', body,
+            'ooyala embed code', default=None)
+        if embed_code:
+            return _url_res('ooyala:%s' % embed_code, 'Ooyala')
 
-        return {
-            '_type': 'url_transparent',
-            'url': video_url,
-            'display_id': display_id,
-            'ie_key': ViceIE.ie_key(),
-        }
+        youtube_url = self._html_search_regex(
+            r'<iframe[^>]+src="(.*youtube\.com/.*)"',
+            body, 'YouTube URL', default=None)
+        if youtube_url:
+            return _url_res(youtube_url, 'Youtube')
+
+        video_url = self._html_search_regex(
+            r'data-video-url="([^"]+)"',
+            prefetch_data['embed_code'], 'video URL')
+
+        return _url_res(video_url, ViceIE.ie_key())

From 329e3dd5adf52520c87ba31395d090455114783b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 5 May 2017 22:59:15 +0700
Subject: [PATCH 0803/1696] [nrk] Extract chapters

---
 youtube_dl/extractor/nrk.py | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 7fe79cb53..3b4f51f61 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -148,13 +148,34 @@ class NRKBaseIE(InfoExtractor):
 
         vcodec = 'none' if data.get('mediaType') == 'Audio' else None
 
-        # TODO: extract chapters when https://github.com/rg3/youtube-dl/pull/9409 is merged
-
         for entry in entries:
             entry.update(common_info)
             for f in entry['formats']:
                 f['vcodec'] = vcodec
 
+        points = data.get('shortIndexPoints')
+        if isinstance(points, list):
+            chapters = []
+            for next_num, point in enumerate(points, start=1):
+                if not isinstance(point, dict):
+                    continue
+                start_time = parse_duration(point.get('startPoint'))
+                if start_time is None:
+                    continue
+                end_time = parse_duration(
+                    data.get('duration')
+                    if next_num == len(points)
+                    else points[next_num].get('startPoint'))
+                if end_time is None:
+                    continue
+                chapters.append({
+                    'start_time': start_time,
+                    'end_time': end_time,
+                    'title': point.get('title'),
+                })
+            if chapters and len(entries) == 1:
+                entries[0]['chapters'] = chapters
+
         return self.playlist_result(entries, video_id, title, description)
 
 

From 9cafc3fd8b54b9b91a145cddf9e4db0bd59e1b5f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 May 2017 02:27:06 +0700
Subject: [PATCH 0804/1696] [youtube] Extract chapters

---
 test/test_youtube_chapters.py   | 267 ++++++++++++++++++++++++++++++++
 youtube_dl/extractor/youtube.py |  36 ++++-
 2 files changed, 301 insertions(+), 2 deletions(-)
 create mode 100644 test/test_youtube_chapters.py

diff --git a/test/test_youtube_chapters.py b/test/test_youtube_chapters.py
new file mode 100644
index 000000000..a5d05355f
--- /dev/null
+++ b/test/test_youtube_chapters.py
@@ -0,0 +1,267 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from test.helper import expect_value
+from youtube_dl.extractor import YoutubeIE
+
+
+class TestYoutubeChapters(unittest.TestCase):
+
+    _TEST_CASES = [
+        (
+            # https://www.youtube.com/watch?v=A22oy8dFjqc
+            # pattern: 00:00 - <title>
+            '''This is the absolute ULTIMATE experience of Queen's set at LIVE AID, this is the best video mixed to the absolutely superior stereo radio broadcast. This vastly superior audio mix takes a huge dump on all of the official mixes. Best viewed in 1080p. ENJOY! ***MAKE SURE TO READ THE DESCRIPTION***<br /><a href="#" onclick="yt.www.watch.player.seekTo(00*60+36);return false;">00:36</a> - Bohemian Rhapsody<br /><a href="#" onclick="yt.www.watch.player.seekTo(02*60+42);return false;">02:42</a> - Radio Ga Ga<br /><a href="#" onclick="yt.www.watch.player.seekTo(06*60+53);return false;">06:53</a> - Ay Oh!<br /><a href="#" onclick="yt.www.watch.player.seekTo(07*60+34);return false;">07:34</a> - Hammer To Fall<br /><a href="#" onclick="yt.www.watch.player.seekTo(12*60+08);return false;">12:08</a> - Crazy Little Thing Called Love<br /><a href="#" onclick="yt.www.watch.player.seekTo(16*60+03);return false;">16:03</a> - We Will Rock You<br /><a href="#" onclick="yt.www.watch.player.seekTo(17*60+18);return false;">17:18</a> - We Are The Champions<br /><a href="#" onclick="yt.www.watch.player.seekTo(21*60+12);return false;">21:12</a> - Is This The World We Created...?<br /><br />Short song analysis:<br /><br />- "Bohemian Rhapsody": Although it's a short medley version, it's one of the best performances of the ballad section, with Freddie nailing the Bb4s with the correct studio phrasing (for the first time ever!).<br /><br />- "Radio Ga Ga": Although it's missing one chorus, this is one of - if not the best - the best versions ever, Freddie nails all the Bb4s and sounds very clean! Spike Edney's Roland Jupiter 8 also really shines through on this mix, compared to the DVD releases!<br /><br />- "Audience Improv": A great improv, Freddie sounds strong and confident. You gotta love when he sustains that A4 for 4 seconds!<br /><br />- "Hammer To Fall": Despite missing a verse and a chorus, it's a strong version (possibly the best ever). Freddie sings the song amazingly, and even ad-libs a C#5 and a C5! Also notice how heavy Brian's guitar sounds compared to the thin DVD mixes - it roars!<br /><br />- "Crazy Little Thing Called Love": A great version, the crowd loves the song, the jam is great as well! Only downside to this is the slight feedback issues.<br /><br />- "We Will Rock You": Although cut down to the 1st verse and chorus, Freddie sounds strong. He nails the A4, and the solo from Dr. May is brilliant!<br /><br />- "We Are the Champions": Perhaps the high-light of the performance - Freddie is very daring on this version, he sustains the pre-chorus Bb4s, nails the 1st C5, belts great A4s, but most importantly: He nails the chorus Bb4s, in all 3 choruses! This is the only time he has ever done so! It has to be said though, the last one sounds a bit rough, but that's a side effect of belting high notes for the past 18 minutes, with nodules AND laryngitis!<br /><br />- "Is This The World We Created... ?": Freddie and Brian perform a beautiful version of this, and it is one of the best versions ever. It's both sad and hilarious that a couple of BBC engineers are talking over the song, one of them being completely oblivious of the fact that he is interrupting the performance, on live television... Which was being televised to almost 2 billion homes.<br /><br /><br />All rights go to their respective owners!<br />-----Copyright Disclaimer Under Section 107 of the Copyright Act 1976, allowance is made for fair use for purposes such as criticism, comment, news reporting, teaching, scholarship, and research. Fair use is a use permitted by copyright statute that might otherwise be infringing. Non-profit, educational or personal use tips the balance in favor of fair use''',
+            1477,
+            [{
+                'start_time': 36,
+                'end_time': 162,
+                'title': 'Bohemian Rhapsody',
+            }, {
+                'start_time': 162,
+                'end_time': 413,
+                'title': 'Radio Ga Ga',
+            }, {
+                'start_time': 413,
+                'end_time': 454,
+                'title': 'Ay Oh!',
+            }, {
+                'start_time': 454,
+                'end_time': 728,
+                'title': 'Hammer To Fall',
+            }, {
+                'start_time': 728,
+                'end_time': 963,
+                'title': 'Crazy Little Thing Called Love',
+            }, {
+                'start_time': 963,
+                'end_time': 1038,
+                'title': 'We Will Rock You',
+            }, {
+                'start_time': 1038,
+                'end_time': 1272,
+                'title': 'We Are The Champions',
+            }, {
+                'start_time': 1272,
+                'end_time': 1477,
+                'title': 'Is This The World We Created...?',
+            }]
+        ),
+        (
+            # https://www.youtube.com/watch?v=ekYlRhALiRQ
+            # pattern: <num>. <title> 0:00
+            '1.  Those Beaten Paths of Confusion <a href="#" onclick="yt.www.watch.player.seekTo(0*60+00);return false;">0:00</a><br />2.  Beyond the Shadows of Emptiness & Nothingness <a href="#" onclick="yt.www.watch.player.seekTo(11*60+47);return false;">11:47</a><br />3.  Poison Yourself...With Thought <a href="#" onclick="yt.www.watch.player.seekTo(26*60+30);return false;">26:30</a><br />4.  The Agents of Transformation <a href="#" onclick="yt.www.watch.player.seekTo(35*60+57);return false;">35:57</a><br />5.  Drowning in the Pain of Consciousness <a href="#" onclick="yt.www.watch.player.seekTo(44*60+32);return false;">44:32</a><br />6.  Deny the Disease of Life <a href="#" onclick="yt.www.watch.player.seekTo(53*60+07);return false;">53:07</a><br /><br />More info/Buy: http://crepusculonegro.storenvy.com/products/257645-cn-03-arizmenda-within-the-vacuum-of-infinity<br /><br />No copyright is intended. The rights to this video are assumed by the owner and its affiliates.',
+            4009,
+            [{
+                'start_time': 0,
+                'end_time': 707,
+                'title': '1. Those Beaten Paths of Confusion',
+            }, {
+                'start_time': 707,
+                'end_time': 1590,
+                'title': '2. Beyond the Shadows of Emptiness & Nothingness',
+            }, {
+                'start_time': 1590,
+                'end_time': 2157,
+                'title': '3. Poison Yourself...With Thought',
+            }, {
+                'start_time': 2157,
+                'end_time': 2672,
+                'title': '4. The Agents of Transformation',
+            }, {
+                'start_time': 2672,
+                'end_time': 3187,
+                'title': '5. Drowning in the Pain of Consciousness',
+            }, {
+                'start_time': 3187,
+                'end_time': 4009,
+                'title': '6. Deny the Disease of Life',
+            }]
+        ),
+        (
+            # https://www.youtube.com/watch?v=WjL4pSzog9w
+            # pattern: 00:00 <title>
+            '<a href="https://arizmenda.bandcamp.com/merch/despairs-depths-descended-cd" class="yt-uix-servicelink  " data-target-new-window="True" data-servicelink="CDAQ6TgYACITCNf1raqT2dMCFdRjGAod_o0CBSj4HQ" data-url="https://arizmenda.bandcamp.com/merch/despairs-depths-descended-cd" rel="nofollow noopener" target="_blank">https://arizmenda.bandcamp.com/merch/...</a><br /><br /><a href="#" onclick="yt.www.watch.player.seekTo(00*60+00);return false;">00:00</a> Christening Unborn Deformities <br /><a href="#" onclick="yt.www.watch.player.seekTo(07*60+08);return false;">07:08</a> Taste of Purity<br /><a href="#" onclick="yt.www.watch.player.seekTo(16*60+16);return false;">16:16</a> Sculpting Sins of a Universal Tongue<br /><a href="#" onclick="yt.www.watch.player.seekTo(24*60+45);return false;">24:45</a> Birth<br /><a href="#" onclick="yt.www.watch.player.seekTo(31*60+24);return false;">31:24</a> Neves<br /><a href="#" onclick="yt.www.watch.player.seekTo(37*60+55);return false;">37:55</a> Libations in Limbo',
+            2705,
+            [{
+                'start_time': 0,
+                'end_time': 428,
+                'title': 'Christening Unborn Deformities',
+            }, {
+                'start_time': 428,
+                'end_time': 976,
+                'title': 'Taste of Purity',
+            }, {
+                'start_time': 976,
+                'end_time': 1485,
+                'title': 'Sculpting Sins of a Universal Tongue',
+            }, {
+                'start_time': 1485,
+                'end_time': 1884,
+                'title': 'Birth',
+            }, {
+                'start_time': 1884,
+                'end_time': 2275,
+                'title': 'Neves',
+            }, {
+                'start_time': 2275,
+                'end_time': 2705,
+                'title': 'Libations in Limbo',
+            }]
+        ),
+        (
+            # https://www.youtube.com/watch?v=o3r1sn-t3is
+            # pattern: <title> 00:00 <note>
+            'Download this show in MP3: <a href="http://sh.st/njZKK" class="yt-uix-servicelink  " data-url="http://sh.st/njZKK" data-target-new-window="True" data-servicelink="CDAQ6TgYACITCK3j8_6o2dMCFVDCGAoduVAKKij4HQ" rel="nofollow noopener" target="_blank">http://sh.st/njZKK</a><br /><br />Setlist:<br />I-E-A-I-A-I-O <a href="#" onclick="yt.www.watch.player.seekTo(00*60+45);return false;">00:45</a><br />Suite-Pee <a href="#" onclick="yt.www.watch.player.seekTo(4*60+26);return false;">4:26</a>  (Incomplete)<br />Attack <a href="#" onclick="yt.www.watch.player.seekTo(5*60+31);return false;">5:31</a> (First live performance since 2011)<br />Prison Song <a href="#" onclick="yt.www.watch.player.seekTo(8*60+42);return false;">8:42</a><br />Know <a href="#" onclick="yt.www.watch.player.seekTo(12*60+32);return false;">12:32</a> (First live performance since 2011)<br />Aerials <a href="#" onclick="yt.www.watch.player.seekTo(15*60+32);return false;">15:32</a><br />Soldier Side - Intro <a href="#" onclick="yt.www.watch.player.seekTo(19*60+13);return false;">19:13</a><br />B.Y.O.B. <a href="#" onclick="yt.www.watch.player.seekTo(20*60+09);return false;">20:09</a><br />Soil <a href="#" onclick="yt.www.watch.player.seekTo(24*60+32);return false;">24:32</a><br />Darts <a href="#" onclick="yt.www.watch.player.seekTo(27*60+48);return false;">27:48</a><br />Radio/Video <a href="#" onclick="yt.www.watch.player.seekTo(30*60+38);return false;">30:38</a><br />Hypnotize <a href="#" onclick="yt.www.watch.player.seekTo(35*60+05);return false;">35:05</a><br />Temper <a href="#" onclick="yt.www.watch.player.seekTo(38*60+08);return false;">38:08</a> (First live performance since 1999)<br />CUBErt <a href="#" onclick="yt.www.watch.player.seekTo(41*60+00);return false;">41:00</a><br />Needles <a href="#" onclick="yt.www.watch.player.seekTo(42*60+57);return false;">42:57</a><br />Deer Dance <a href="#" onclick="yt.www.watch.player.seekTo(46*60+27);return false;">46:27</a><br />Bounce <a href="#" onclick="yt.www.watch.player.seekTo(49*60+38);return false;">49:38</a><br />Suggestions <a href="#" onclick="yt.www.watch.player.seekTo(51*60+25);return false;">51:25</a><br />Psycho <a href="#" onclick="yt.www.watch.player.seekTo(53*60+52);return false;">53:52</a><br />Chop Suey! <a href="#" onclick="yt.www.watch.player.seekTo(58*60+13);return false;">58:13</a><br />Lonely Day <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+01*60+15);return false;">1:01:15</a><br />Question! <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+04*60+14);return false;">1:04:14</a><br />Lost in Hollywood <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+08*60+10);return false;">1:08:10</a><br />Vicinity of Obscenity  <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+13*60+40);return false;">1:13:40</a>(First live performance since 2012)<br />Forest <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+16*60+17);return false;">1:16:17</a><br />Cigaro <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+20*60+02);return false;">1:20:02</a><br />Toxicity <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+23*60+57);return false;">1:23:57</a>(with Chino Moreno)<br />Sugar <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+27*60+53);return false;">1:27:53</a>',
+            5640,
+            [{
+                'start_time': 45,
+                'end_time': 266,
+                'title': 'I-E-A-I-A-I-O',
+            }, {
+                'start_time': 266,
+                'end_time': 331,
+                'title': 'Suite-Pee (Incomplete)',
+            }, {
+                'start_time': 331,
+                'end_time': 522,
+                'title': 'Attack (First live performance since 2011)',
+            }, {
+                'start_time': 522,
+                'end_time': 752,
+                'title': 'Prison Song',
+            }, {
+                'start_time': 752,
+                'end_time': 932,
+                'title': 'Know (First live performance since 2011)',
+            }, {
+                'start_time': 932,
+                'end_time': 1153,
+                'title': 'Aerials',
+            }, {
+                'start_time': 1153,
+                'end_time': 1209,
+                'title': 'Soldier Side - Intro',
+            }, {
+                'start_time': 1209,
+                'end_time': 1472,
+                'title': 'B.Y.O.B.',
+            }, {
+                'start_time': 1472,
+                'end_time': 1668,
+                'title': 'Soil',
+            }, {
+                'start_time': 1668,
+                'end_time': 1838,
+                'title': 'Darts',
+            }, {
+                'start_time': 1838,
+                'end_time': 2105,
+                'title': 'Radio/Video',
+            }, {
+                'start_time': 2105,
+                'end_time': 2288,
+                'title': 'Hypnotize',
+            }, {
+                'start_time': 2288,
+                'end_time': 2460,
+                'title': 'Temper (First live performance since 1999)',
+            }, {
+                'start_time': 2460,
+                'end_time': 2577,
+                'title': 'CUBErt',
+            }, {
+                'start_time': 2577,
+                'end_time': 2787,
+                'title': 'Needles',
+            }, {
+                'start_time': 2787,
+                'end_time': 2978,
+                'title': 'Deer Dance',
+            }, {
+                'start_time': 2978,
+                'end_time': 3085,
+                'title': 'Bounce',
+            }, {
+                'start_time': 3085,
+                'end_time': 3232,
+                'title': 'Suggestions',
+            }, {
+                'start_time': 3232,
+                'end_time': 3493,
+                'title': 'Psycho',
+            }, {
+                'start_time': 3493,
+                'end_time': 3675,
+                'title': 'Chop Suey!',
+            }, {
+                'start_time': 3675,
+                'end_time': 3854,
+                'title': 'Lonely Day',
+            }, {
+                'start_time': 3854,
+                'end_time': 4090,
+                'title': 'Question!',
+            }, {
+                'start_time': 4090,
+                'end_time': 4420,
+                'title': 'Lost in Hollywood',
+            }, {
+                'start_time': 4420,
+                'end_time': 4577,
+                'title': 'Vicinity of Obscenity (First live performance since 2012)',
+            }, {
+                'start_time': 4577,
+                'end_time': 4802,
+                'title': 'Forest',
+            }, {
+                'start_time': 4802,
+                'end_time': 5037,
+                'title': 'Cigaro',
+            }, {
+                'start_time': 5037,
+                'end_time': 5273,
+                'title': 'Toxicity (with Chino Moreno)',
+            }, {
+                'start_time': 5273,
+                'end_time': 5640,
+                'title': 'Sugar',
+            }]
+        ),
+        (
+            # https://www.youtube.com/watch?v=PkYLQbsqCE8
+            # pattern: <num> - <title> [<latinized title>] 0:00:00
+            '''Затемно (Zatemno) is an Obscure Black Metal Band from Russia.<br /><br />"Во прах (Vo prakh)'' Into The Ashes", Debut mini-album released may 6, 2016, by Death Knell Productions<br />Released on 6 panel digipak CD, limited to 100 copies only<br />And digital format on Bandcamp<br /><br />Tracklist<br /><br />1 - Во прах [Vo prakh] <a href="#" onclick="yt.www.watch.player.seekTo(0*3600+00*60+00);return false;">0:00:00</a><br />2 - Искупление [Iskupleniye] <a href="#" onclick="yt.www.watch.player.seekTo(0*3600+08*60+10);return false;">0:08:10</a><br />3 - Из серпов луны...[Iz serpov luny] <a href="#" onclick="yt.www.watch.player.seekTo(0*3600+14*60+30);return false;">0:14:30</a><br /><br />Links:<br /><a href="https://deathknellprod.bandcamp.com/album/--2" class="yt-uix-servicelink  " data-target-new-window="True" data-url="https://deathknellprod.bandcamp.com/album/--2" data-servicelink="CC8Q6TgYACITCNP234Kr2dMCFcNxGAodQqsIwSj4HQ" target="_blank" rel="nofollow noopener">https://deathknellprod.bandcamp.com/a...</a><br /><a href="https://www.facebook.com/DeathKnellProd/" class="yt-uix-servicelink  " data-target-new-window="True" data-url="https://www.facebook.com/DeathKnellProd/" data-servicelink="CC8Q6TgYACITCNP234Kr2dMCFcNxGAodQqsIwSj4HQ" target="_blank" rel="nofollow noopener">https://www.facebook.com/DeathKnellProd/</a><br /><br /><br />I don't have any right about this artifact, my only intention is to spread the music of the band, all rights are reserved to the Затемно (Zatemno) and his producers, Death Knell Productions.<br /><br />------------------------------------------------------------------<br /><br />Subscribe for more videos like this.<br />My link: <a href="https://web.facebook.com/AttackOfTheDragons" class="yt-uix-servicelink  " data-target-new-window="True" data-url="https://web.facebook.com/AttackOfTheDragons" data-servicelink="CC8Q6TgYACITCNP234Kr2dMCFcNxGAodQqsIwSj4HQ" target="_blank" rel="nofollow noopener">https://web.facebook.com/AttackOfTheD...</a>''',
+            1138,
+            [{
+                'start_time': 0,
+                'end_time': 490,
+                'title': '1 - Во прах [Vo prakh]',
+            }, {
+                'start_time': 490,
+                'end_time': 870,
+                'title': '2 - Искупление [Iskupleniye]',
+            }, {
+                'start_time': 870,
+                'end_time': 1138,
+                'title': '3 - Из серпов луны...[Iz serpov luny]',
+            }]
+        ),
+    ]
+
+    def test_youtube_chapters(self):
+        for description, duration, expected_chapters in self._TEST_CASES:
+            ie = YoutubeIE()
+            expect_value(
+                self, ie._extract_chapters(description, duration),
+                expected_chapters, None)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 480f403da..ec12e313c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1257,6 +1257,35 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         url = 'https://www.youtube.com/annotations_invideo?features=1&legacy=1&video_id=%s' % video_id
         return self._download_webpage(url, video_id, note='Searching for annotations.', errnote='Unable to download video annotations.')
 
+    @staticmethod
+    def _extract_chapters(description, duration):
+        if not description:
+            return None
+        chapter_lines = re.findall(
+            r'(?:^|<br\s*/>)([^<]*<a[^>]+onclick=["\']yt\.www\.watch\.player\.seekTo[^>]+>(\d{1,2}:\d{1,2}(?::\d{1,2})?)</a>[^>]*)(?=$|<br\s*/>)',
+            description)
+        if not chapter_lines:
+            return None
+        chapters = []
+        for next_num, (chapter_line, time_point) in enumerate(
+                chapter_lines, start=1):
+            start_time = parse_duration(time_point)
+            if start_time is None:
+                continue
+            end_time = (duration if next_num == len(chapter_lines)
+                        else parse_duration(chapter_lines[next_num][1]))
+            if end_time is None:
+                continue
+            chapter_title = re.sub(
+                r'<a[^>]+>[^<]+</a>', '', chapter_line).strip(' \t-')
+            chapter_title = re.sub(r'\s+', ' ', chapter_title)
+            chapters.append({
+                'start_time': start_time,
+                'end_time': end_time,
+                'title': chapter_title,
+            })
+        return chapters
+
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})
 
@@ -1399,9 +1428,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             video_title = '_'
 
         # description
-        video_description = get_element_by_id("eow-description", video_webpage)
+        description_original = video_description = get_element_by_id("eow-description", video_webpage)
         if video_description:
-            video_description = re.sub(r'''(?x)
+            description_original = video_description = re.sub(r'''(?x)
                 <a\s+
                     (?:[a-zA-Z-]+="[^"]*"\s+)*?
                     (?:title|href)="([^"]+)"\s+
@@ -1558,6 +1587,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         if self._downloader.params.get('writeannotations', False):
             video_annotations = self._extract_annotations(video_id)
 
+        chapters = self._extract_chapters(description_original, video_duration)
+
         if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
             self.report_rtmp_download()
             formats = [{
@@ -1790,6 +1821,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'duration': video_duration,
             'age_limit': 18 if age_gate else 0,
             'annotations': video_annotations,
+            'chapters': chapters,
             'webpage_url': proto + '://www.youtube.com/watch?v=%s' % video_id,
             'view_count': view_count,
             'like_count': like_count,

From a99cc4ca167e3244f068737110168716352dbbc1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 May 2017 02:46:37 +0700
Subject: [PATCH 0805/1696] [pornhub] Extend _VALID_URL (closes #12996)

---
 youtube_dl/extractor/pornhub.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index b25f1f193..1dcc8df00 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -33,7 +33,7 @@ class PornHubIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                     https?://
                         (?:
-                            (?:[a-z]+\.)?pornhub\.com/(?:view_video\.php\?viewkey=|embed/)|
+                            (?:[a-z]+\.)?pornhub\.com/(?:(?:view_video\.php|video/show)\?viewkey=|embed/)|
                             (?:www\.)?thumbzilla\.com/video/
                         )
                         (?P<id>[\da-z]+)
@@ -97,6 +97,9 @@ class PornHubIE(InfoExtractor):
     }, {
         'url': 'https://www.thumbzilla.com/video/ph56c6114abd99a/horny-girlfriend-sex',
         'only_matching': True,
+    }, {
+        'url': 'http://www.pornhub.com/video/show?viewkey=648719015',
+        'only_matching': True,
     }]
 
     @staticmethod

From 8b4774dcac4cb82b47213126f3430ac9dc25ec28 Mon Sep 17 00:00:00 2001
From: Luca Steeb <contact@luca-steeb.com>
Date: Fri, 5 May 2017 22:35:42 +0200
Subject: [PATCH 0806/1696] [bandcamp] Fix thumbnail extraction

---
 youtube_dl/extractor/bandcamp.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index df2972f26..489d0ba53 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -47,6 +47,7 @@ class BandcampIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         title = mobj.group('title')
         webpage = self._download_webpage(url, title)
+        thumbnail = self._html_search_meta('og:image', webpage, default=None)
         m_download = re.search(r'freeDownloadPage: "(.*?)"', webpage)
         if not m_download:
             m_trackinfo = re.search(r'trackinfo: (.+),\s*?\n', webpage)
@@ -75,6 +76,7 @@ class BandcampIE(InfoExtractor):
                 return {
                     'id': track_id,
                     'title': data['title'],
+                    'thumbnail': thumbnail,
                     'formats': formats,
                     'duration': float_or_none(data.get('duration')),
                 }
@@ -143,7 +145,7 @@ class BandcampIE(InfoExtractor):
         return {
             'id': video_id,
             'title': title,
-            'thumbnail': info.get('thumb_url'),
+            'thumbnail': info.get('thumb_url') or thumbnail,
             'uploader': info.get('artist'),
             'artist': artist,
             'track': track,

From 1f9fefe7f578169d41d75324decbd46ec8594311 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Fri, 5 May 2017 22:39:14 +0200
Subject: [PATCH 0807/1696] [crackle] Update test

---
 youtube_dl/extractor/crackle.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/crackle.py b/youtube_dl/extractor/crackle.py
index f919ed208..13f425b2b 100644
--- a/youtube_dl/extractor/crackle.py
+++ b/youtube_dl/extractor/crackle.py
@@ -21,9 +21,10 @@ class CrackleIE(InfoExtractor):
             'season_number': 8,
             'episode_number': 4,
             'subtitles': {
-                'en-US': [{
-                    'ext': 'ttml',
-                }]
+                'en-US': [
+                    {'ext': 'vtt'},
+                    {'ext': 'tt'},
+                ]
             },
         },
         'params': {

From a57a8e991860fb96e6111769ab0a22455c8854fc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 May 2017 05:30:56 +0700
Subject: [PATCH 0808/1696] [test_youtube_chapters] Add coding cookie

---
 test/test_youtube_chapters.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/test_youtube_chapters.py b/test/test_youtube_chapters.py
index a5d05355f..5435ee4a8 100644
--- a/test/test_youtube_chapters.py
+++ b/test/test_youtube_chapters.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 #!/usr/bin/env python
 from __future__ import unicode_literals
 

From fd178b8748979ab19f1cb3d7d689fe19a447ca2f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 6 May 2017 07:19:07 +0100
Subject: [PATCH 0809/1696] [theplatform] extract chapters

---
 youtube_dl/extractor/theplatform.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 9a424b1c6..de236bbba 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -80,14 +80,33 @@ class ThePlatformBaseIE(OnceIE):
                     'url': src,
                 })
 
+        duration = info.get('duration')
+        tp_chapters = info.get('chapters', [])
+        chapters = []
+        if tp_chapters:
+            def _add_chapter(start_time, end_time):
+                start_time = float_or_none(start_time, 1000)
+                end_time = float_or_none(end_time, 1000)
+                if start_time is None or end_time is None:
+                    return
+                chapters.append({
+                    'start_time': start_time,
+                    'end_time': end_time,
+                })
+
+            for chapter in tp_chapters[:-1]:
+                _add_chapter(chapter.get('startTime'), chapter.get('endTime'))
+            _add_chapter(tp_chapters[-1].get('startTime'), tp_chapters[-1].get('endTime') or duration)
+
         return {
             'title': info['title'],
             'subtitles': subtitles,
             'description': info['description'],
             'thumbnail': info['defaultThumbnailUrl'],
-            'duration': int_or_none(info.get('duration'), 1000),
+            'duration': float_or_none(duration, 1000),
             'timestamp': int_or_none(info.get('pubDate'), 1000) or None,
             'uploader': info.get('billingCode'),
+            'chapters': chapters,
         }
 
     def _extract_theplatform_metadata(self, path, video_id):

From 74c09c852a183813174803306176a86d449da889 Mon Sep 17 00:00:00 2001
From: midas02 <midas02@users.noreply.github.com>
Date: Sun, 30 Apr 2017 19:36:44 +0200
Subject: [PATCH 0810/1696] [rmcdecouverte] Fix extraction

---
 youtube_dl/extractor/rmcdecouverte.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/rmcdecouverte.py b/youtube_dl/extractor/rmcdecouverte.py
index 2340dae53..f6ef6d747 100644
--- a/youtube_dl/extractor/rmcdecouverte.py
+++ b/youtube_dl/extractor/rmcdecouverte.py
@@ -13,15 +13,15 @@ class RMCDecouverteIE(InfoExtractor):
     _VALID_URL = r'https?://rmcdecouverte\.bfmtv\.com/mediaplayer-replay.*?\bid=(?P<id>\d+)'
 
     _TEST = {
-        'url': 'http://rmcdecouverte.bfmtv.com/mediaplayer-replay/?id=1430&title=LES%20HEROS%20DU%2088e%20ETAGE',
+        'url': 'http://rmcdecouverte.bfmtv.com/mediaplayer-replay/?id=16548',
         'info_dict': {
-            'id': '5111223049001',
+            'id': '5411254766001',
             'ext': 'mp4',
-            'title': ': LES HEROS DU 88e ETAGE',
-            'description': 'Découvrez comment la bravoure de deux hommes dans la Tour Nord du World Trade Center a sauvé  la vie d\'innombrables personnes le 11 septembre 2001.',
+            'title': '39/45:LE RESEAU DES FAUX BILLETS',
+            'description': 'ic Brunet propose un nouvel \u00e9pisode des Grains de sable de l\'Histoire sur la plus grosse affaire de contrefa\u00e7on de la Seconde Guerre mondiale.',
             'uploader_id': '1969646226001',
-            'upload_date': '20160904',
-            'timestamp': 1472951103,
+            'upload_date': '20170426',
+            'timestamp': 1493166610,
         },
         'params': {
             # rtmp download
@@ -35,5 +35,8 @@ class RMCDecouverteIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         brightcove_legacy_url = BrightcoveLegacyIE._extract_brightcove_url(webpage)
-        brightcove_id = compat_parse_qs(compat_urlparse.urlparse(brightcove_legacy_url).query)['@videoPlayer'][0]
+        if brightcove_legacy_url:
+            brightcove_id = compat_parse_qs(compat_urlparse.urlparse(brightcove_legacy_url).query)['@videoPlayer'][0]
+        else:
+            brightcove_id = self._search_regex(r'data-video-id="(.*?)"', webpage, 'brightcove_id')
         return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)

From 566fbbaefd459df46ebce512cd916248e069e4f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 May 2017 17:55:32 +0700
Subject: [PATCH 0811/1696] [rmcdecouverte] Improve (closes #12937)

---
 youtube_dl/extractor/rmcdecouverte.py | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/rmcdecouverte.py b/youtube_dl/extractor/rmcdecouverte.py
index f6ef6d747..e921ca3e6 100644
--- a/youtube_dl/extractor/rmcdecouverte.py
+++ b/youtube_dl/extractor/rmcdecouverte.py
@@ -13,21 +13,20 @@ class RMCDecouverteIE(InfoExtractor):
     _VALID_URL = r'https?://rmcdecouverte\.bfmtv\.com/mediaplayer-replay.*?\bid=(?P<id>\d+)'
 
     _TEST = {
-        'url': 'http://rmcdecouverte.bfmtv.com/mediaplayer-replay/?id=16548',
+        'url': 'http://rmcdecouverte.bfmtv.com/mediaplayer-replay/?id=13502&title=AQUAMEN:LES%20ROIS%20DES%20AQUARIUMS%20:UN%20DELICIEUX%20PROJET',
         'info_dict': {
-            'id': '5411254766001',
+            'id': '5419055995001',
             'ext': 'mp4',
-            'title': '39/45:LE RESEAU DES FAUX BILLETS',
-            'description': 'ic Brunet propose un nouvel \u00e9pisode des Grains de sable de l\'Histoire sur la plus grosse affaire de contrefa\u00e7on de la Seconde Guerre mondiale.',
+            'title': 'UN DELICIEUX PROJET',
+            'description': 'md5:63610df7c8b1fc1698acd4d0d90ba8b5',
             'uploader_id': '1969646226001',
-            'upload_date': '20170426',
-            'timestamp': 1493166610,
+            'upload_date': '20170502',
+            'timestamp': 1493745308,
         },
         'params': {
-            # rtmp download
             'skip_download': True,
         },
-        'skip': 'Only works from France',
+        'skip': 'only available for a week',
     }
     BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/1969646226001/default_default/index.html?videoId=%s'
 
@@ -36,7 +35,11 @@ class RMCDecouverteIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
         brightcove_legacy_url = BrightcoveLegacyIE._extract_brightcove_url(webpage)
         if brightcove_legacy_url:
-            brightcove_id = compat_parse_qs(compat_urlparse.urlparse(brightcove_legacy_url).query)['@videoPlayer'][0]
+            brightcove_id = compat_parse_qs(compat_urlparse.urlparse(
+                brightcove_legacy_url).query)['@videoPlayer'][0]
         else:
-            brightcove_id = self._search_regex(r'data-video-id="(.*?)"', webpage, 'brightcove_id')
-        return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)
+            brightcove_id = self._search_regex(
+                r'data-video-id=["\'](\d+)', webpage, 'brightcove id')
+        return self.url_result(
+            self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew',
+            brightcove_id)

From 228cd9bb906eb1dae476a05dbff0796a2d9f9be5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 6 May 2017 18:57:24 +0800
Subject: [PATCH 0812/1696] [bilibili] Fix video downloading (closes #13001)

---
 ChangeLog                        | 1 +
 youtube_dl/extractor/bilibili.py | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 0908e2e93..7cceec10f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [bilibili] Fix video downloading (#13001)
 + [vice] Support vice articles (#12968)
 * [vice] Fix extraction for non en_us videos (#12967)
 * [gdcvault] Fix extraction for some videos (#12733)
diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 80dd8382e..1e3f25515 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -122,6 +122,11 @@ class BiliBiliIE(InfoExtractor):
                     'preference': -2 if 'hd.mp4' in backup_url else -3,
                 })
 
+            for a_format in formats:
+                a_format.setdefault('http_headers', {}).update({
+                    'Referer': url,
+                })
+
             self._sort_formats(formats)
 
             entries.append({

From 10c87c151b6f6a2f45b1a3ad1ac2d38fcf0fb0a6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 6 May 2017 19:06:18 +0800
Subject: [PATCH 0813/1696] [utils] Rename try_multipart_encode to
 _multipart_encode_impl

To state that this is an internal function and people should be careful
when using it outside youtube-dl.
---
 youtube_dl/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 911cacd29..aa1900436 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2098,7 +2098,7 @@ def update_Request(req, url=None, data=None, headers={}, query={}):
     return new_req
 
 
-def try_multipart_encode(data, boundary):
+def _multipart_encode_impl(data, boundary):
     content_type = 'multipart/form-data; boundary=%s' % boundary
 
     out = b''
@@ -2140,7 +2140,7 @@ def multipart_encode(data, boundary=None):
             boundary = '---------------' + str(random.randrange(0x0fffffff, 0xffffffff))
 
         try:
-            out, content_type = try_multipart_encode(data, boundary)
+            out, content_type = _multipart_encode_impl(data, boundary)
             break
         except ValueError:
             if has_specified_boundary:

From e00eb564e921b609c94f87d7db7ad7f709b704f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 May 2017 23:58:47 +0700
Subject: [PATCH 0814/1696] [youtube] Fix authentication (closes #12927)

---
 youtube_dl/extractor/youtube.py | 142 +++++++++++++++++---------------
 1 file changed, 77 insertions(+), 65 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index ec12e313c..c2e06c3a6 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -37,8 +37,7 @@ from ..utils import (
     parse_codecs,
     parse_duration,
     remove_quotes,
-    remove_start,
-    sanitized_Request,
+    # remove_start,
     smuggle_url,
     str_to_int,
     try_get,
@@ -54,7 +53,16 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
     """Provide base functions for Youtube extractors"""
     _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
     _TWOFACTOR_URL = 'https://accounts.google.com/signin/challenge'
-    _PASSWORD_CHALLENGE_URL = 'https://accounts.google.com/signin/challenge/sl/password'
+
+    _LOOKUP_URL = 'https://accounts.google.com/_/signin/sl/lookup'
+    _LOOKUP_REQ_TEMPLATE = '["{0}",null,[],null,"US",null,null,2,false,true,[null,null,[2,1,null,1,"https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn",null,[],4],1,[null,null,[]],null,null,null,true],"{0}"]'
+
+    _PASSWORD_CHALLENGE_URL = 'https://accounts.google.com/_/signin/sl/challenge'
+    _PASSWORD_CHALLENGE_REQ_TEMPLATE = '["{0}",null,1,null,[1,null,null,null,["{1}",null,true]],[null,null,[2,1,null,1,"https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn",null,[],4],1,[null,null,[]],null,null,null,true]]'
+
+    _TFA_URL = 'https://accounts.google.com/_/signin/challenge'
+    _TFA_REQ_TEMPLATE = '["{0}",null,2,null,[9,null,null,null,null,null,null,null,[null,"{1}",false,2]]]'
+
     _NETRC_MACHINE = 'youtube'
     # If True it will raise an error if no login info is provided
     _LOGIN_REQUIRED = False
@@ -96,72 +104,76 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
 
         login_form = self._hidden_inputs(login_page)
 
-        login_form.update({
-            'checkConnection': 'youtube',
-            'Email': username,
-            'Passwd': password,
-        })
-
-        login_results = self._download_webpage(
-            self._PASSWORD_CHALLENGE_URL, None,
-            note='Logging in', errnote='unable to log in', fatal=False,
-            data=urlencode_postdata(login_form))
-        if login_results is False:
-            return False
-
-        error_msg = self._html_search_regex(
-            r'<[^>]+id="errormsg_0_Passwd"[^>]*>([^<]+)<',
-            login_results, 'error message', default=None)
-        if error_msg:
-            raise ExtractorError('Unable to login: %s' % error_msg, expected=True)
-
-        if re.search(r'id="errormsg_0_Passwd"', login_results) is not None:
-            raise ExtractorError('Please use your account password and a two-factor code instead of an application-specific password.', expected=True)
-
-        # Two-Factor
-        # TODO add SMS and phone call support - these require making a request and then prompting the user
-
-        if re.search(r'(?i)<form[^>]+id="challenge"', login_results) is not None:
-            tfa_code = self._get_tfa_info('2-step verification code')
-
-            if not tfa_code:
-                self._downloader.report_warning(
-                    'Two-factor authentication required. Provide it either interactively or with --twofactor <code>'
-                    '(Note that only TOTP (Google Authenticator App) codes work at this time.)')
-                return False
-
-            tfa_code = remove_start(tfa_code, 'G-')
-
-            tfa_form_strs = self._form_hidden_inputs('challenge', login_results)
-
-            tfa_form_strs.update({
-                'Pin': tfa_code,
-                'TrustDevice': 'on',
+        def req(url, f_req, note, errnote):
+            data = login_form.copy()
+            data.update({
+                'pstMsg': 1,
+                'checkConnection': 'youtube',
+                'checkedDomains': 'youtube',
+                'hl': 'en',
+                'deviceinfo': '[null,null,null,[],null,"US",null,null,[],"GlifWebSignIn",null,[null,null,[]]]',
+                'f.req': f_req,
+                'flowName': 'GlifWebSignIn',
+                'flowEntry': 'ServiceLogin',
             })
+            return self._download_json(
+                url, None, note=note, errnote=errnote,
+                transform_source=lambda s: re.sub(r'^[^[]*', '', s),
+                fatal=False,
+                data=urlencode_postdata(data), headers={
+                    'Content-Type': 'application/x-www-form-urlencoded;charset=utf-8',
+                    'Google-Accounts-XSRF': 1,
+                })
 
-            tfa_data = urlencode_postdata(tfa_form_strs)
+        lookup_results = req(
+            self._LOOKUP_URL, self._LOOKUP_REQ_TEMPLATE.format(username),
+            'Looking up account info', 'Unable to look up account info')
 
-            tfa_req = sanitized_Request(self._TWOFACTOR_URL, tfa_data)
-            tfa_results = self._download_webpage(
-                tfa_req, None,
-                note='Submitting TFA code', errnote='unable to submit tfa', fatal=False)
-
-            if tfa_results is False:
-                return False
-
-            if re.search(r'(?i)<form[^>]+id="challenge"', tfa_results) is not None:
-                self._downloader.report_warning('Two-factor code expired or invalid. Please try again, or use a one-use backup code instead.')
-                return False
-            if re.search(r'(?i)<form[^>]+id="gaia_loginform"', tfa_results) is not None:
-                self._downloader.report_warning('unable to log in - did the page structure change?')
-                return False
-            if re.search(r'smsauth-interstitial-reviewsettings', tfa_results) is not None:
-                self._downloader.report_warning('Your Google account has a security notice. Please log in on your web browser, resolve the notice, and try again.')
-                return False
-
-        if re.search(r'(?i)<form[^>]+id="gaia_loginform"', login_results) is not None:
-            self._downloader.report_warning('unable to log in: bad username or password')
+        if lookup_results is False:
             return False
+
+        user_hash = lookup_results[0][2]
+
+        password_challenge_results = req(
+            self._PASSWORD_CHALLENGE_URL,
+            self._PASSWORD_CHALLENGE_REQ_TEMPLATE.format(user_hash, password),
+            'Logging in', 'Unable to log in')[0]
+
+        if password_challenge_results is False:
+            return
+
+        msg = password_challenge_results[5]
+        if msg is not None and isinstance(msg, list):
+            raise ExtractorError('Unable to login: %s' % msg[5], expected=True)
+
+        password_challenge_results = password_challenge_results[-1]
+
+        # tfa = password_challenge_results[0]
+        # if isinstance(tfa, list) and tfa[0][2] == 'TWO_STEP_VERIFICATION':
+        #     tfa_code = self._get_tfa_info('2-step verification code')
+        #
+        #     if not tfa_code:
+        #         self._downloader.report_warning(
+        #             'Two-factor authentication required. Provide it either interactively or with --twofactor <code>'
+        #             '(Note that only TOTP (Google Authenticator App) codes work at this time.)')
+        #         return False
+        #
+        #     tfa_code = remove_start(tfa_code, 'G-')
+        #     print('tfa', tfa_code)
+        #     tfa_results = req(
+        #         self._TFA_URL,
+        #         self._TFA_REQ_TEMPLATE.format(user_hash, tfa_code),
+        #         'Submitting TFA code', 'Unable to submit TFA code')
+        #
+        #     TODO
+
+        check_cookie_results = self._download_webpage(
+            password_challenge_results[2], None, 'Checking cookie')
+
+        if '>Sign out<' not in check_cookie_results:
+            self._downloader.report_warning('Unable to log in')
+            return False
+
         return True
 
     def _real_initialize(self):

From e4a75d7932013f49133178d5d84e5f33634a4879 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 May 2017 00:00:11 +0700
Subject: [PATCH 0815/1696] [test_youtube_chapters] PEP 8

---
 test/test_youtube_chapters.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_youtube_chapters.py b/test/test_youtube_chapters.py
index 5435ee4a8..cb12f8384 100644
--- a/test/test_youtube_chapters.py
+++ b/test/test_youtube_chapters.py
@@ -1,5 +1,5 @@
-# coding: utf-8
 #!/usr/bin/env python
+# coding: utf-8
 from __future__ import unicode_literals
 
 # Allow direct execution

From 3995d37da58ed071b54b7f81757cff4d534f5b9b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 May 2017 04:19:11 +0700
Subject: [PATCH 0816/1696] [youtube] Fix TFA (#12927)

---
 youtube_dl/extractor/youtube.py | 151 +++++++++++++++++++++++---------
 1 file changed, 110 insertions(+), 41 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c2e06c3a6..44a39282f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -37,7 +37,7 @@ from ..utils import (
     parse_codecs,
     parse_duration,
     remove_quotes,
-    # remove_start,
+    remove_start,
     smuggle_url,
     str_to_int,
     try_get,
@@ -55,13 +55,8 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
     _TWOFACTOR_URL = 'https://accounts.google.com/signin/challenge'
 
     _LOOKUP_URL = 'https://accounts.google.com/_/signin/sl/lookup'
-    _LOOKUP_REQ_TEMPLATE = '["{0}",null,[],null,"US",null,null,2,false,true,[null,null,[2,1,null,1,"https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn",null,[],4],1,[null,null,[]],null,null,null,true],"{0}"]'
-
-    _PASSWORD_CHALLENGE_URL = 'https://accounts.google.com/_/signin/sl/challenge'
-    _PASSWORD_CHALLENGE_REQ_TEMPLATE = '["{0}",null,1,null,[1,null,null,null,["{1}",null,true]],[null,null,[2,1,null,1,"https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn",null,[],4],1,[null,null,[]],null,null,null,true]]'
-
-    _TFA_URL = 'https://accounts.google.com/_/signin/challenge'
-    _TFA_REQ_TEMPLATE = '["{0}",null,2,null,[9,null,null,null,null,null,null,null,[null,"{1}",false,2]]]'
+    _CHALLENGE_URL = 'https://accounts.google.com/_/signin/sl/challenge'
+    _TFA_URL = 'https://accounts.google.com/_/signin/challenge?hl=en&TL={0}'
 
     _NETRC_MACHINE = 'youtube'
     # If True it will raise an error if no login info is provided
@@ -112,7 +107,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
                 'checkedDomains': 'youtube',
                 'hl': 'en',
                 'deviceinfo': '[null,null,null,[],null,"US",null,null,[],"GlifWebSignIn",null,[null,null,[]]]',
-                'f.req': f_req,
+                'f.req': json.dumps(f_req),
                 'flowName': 'GlifWebSignIn',
                 'flowEntry': 'ServiceLogin',
             })
@@ -125,53 +120,127 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
                     'Google-Accounts-XSRF': 1,
                 })
 
+        def warn(message):
+            self._downloader.report_warning(message)
+
+        lookup_req = [
+            username,
+            None, [], None, 'US', None, None, 2, False, True,
+            [
+                None, None,
+                [2, 1, None, 1,
+                 'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn',
+                 None, [], 4],
+                1, [None, None, []], None, None, None, True
+            ],
+            username,
+        ]
+
         lookup_results = req(
-            self._LOOKUP_URL, self._LOOKUP_REQ_TEMPLATE.format(username),
+            self._LOOKUP_URL, lookup_req,
             'Looking up account info', 'Unable to look up account info')
 
         if lookup_results is False:
             return False
 
-        user_hash = lookup_results[0][2]
+        user_hash = try_get(lookup_results, lambda x: x[0][2], compat_str)
+        if not user_hash:
+            warn('Unable to extract user hash')
+            return False
 
-        password_challenge_results = req(
-            self._PASSWORD_CHALLENGE_URL,
-            self._PASSWORD_CHALLENGE_REQ_TEMPLATE.format(user_hash, password),
-            'Logging in', 'Unable to log in')[0]
+        challenge_req = [
+            user_hash,
+            None, 1, None, [1, None, None, None, [password, None, True]],
+            [
+                None, None, [2, 1, None, 1, 'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn', None, [], 4],
+                1, [None, None, []], None, None, None, True
+            ]]
 
-        if password_challenge_results is False:
+        challenge_results = req(
+            self._CHALLENGE_URL, challenge_req,
+            'Logging in', 'Unable to log in')
+
+        if challenge_results is False:
             return
 
-        msg = password_challenge_results[5]
-        if msg is not None and isinstance(msg, list):
-            raise ExtractorError('Unable to login: %s' % msg[5], expected=True)
+        login_res = try_get(challenge_results, lambda x: x[0][5], list)
+        if login_res:
+            login_msg = try_get(login_res, lambda x: x[5], compat_str)
+            warn(
+                'Unable to login: %s' % 'Invalid password'
+                if login_msg == 'INCORRECT_ANSWER_ENTERED' else login_msg)
+            return False
 
-        password_challenge_results = password_challenge_results[-1]
+        res = try_get(challenge_results, lambda x: x[0][-1], list)
+        if not res:
+            warn('Unable to extract result entry')
+            return False
 
-        # tfa = password_challenge_results[0]
-        # if isinstance(tfa, list) and tfa[0][2] == 'TWO_STEP_VERIFICATION':
-        #     tfa_code = self._get_tfa_info('2-step verification code')
-        #
-        #     if not tfa_code:
-        #         self._downloader.report_warning(
-        #             'Two-factor authentication required. Provide it either interactively or with --twofactor <code>'
-        #             '(Note that only TOTP (Google Authenticator App) codes work at this time.)')
-        #         return False
-        #
-        #     tfa_code = remove_start(tfa_code, 'G-')
-        #     print('tfa', tfa_code)
-        #     tfa_results = req(
-        #         self._TFA_URL,
-        #         self._TFA_REQ_TEMPLATE.format(user_hash, tfa_code),
-        #         'Submitting TFA code', 'Unable to submit TFA code')
-        #
-        #     TODO
+        tfa = try_get(res, lambda x: x[0][0], list)
+        if tfa:
+            tfa_str = try_get(tfa, lambda x: x[2], compat_str)
+            if tfa_str == 'TWO_STEP_VERIFICATION':
+                # SEND_SUCCESS - TFA code has been successfully sent to phone
+                # QUOTA_EXCEEDED - reached the limit of TFA codes
+                status = try_get(tfa, lambda x: x[5], compat_str)
+                if status == 'QUOTA_EXCEEDED':
+                    warn('Exceeded the limit of TFA codes, try later')
+                    return False
+
+                tl = try_get(challenge_results, lambda x: x[1][2], compat_str)
+                if not tl:
+                    warn('Unable to extract TL')
+                    return False
+
+                tfa_code = self._get_tfa_info('2-step verification code')
+
+                if not tfa_code:
+                    warn(
+                        'Two-factor authentication required. Provide it either interactively or with --twofactor <code>'
+                        '(Note that only TOTP (Google Authenticator App) codes work at this time.)')
+                    return False
+
+                tfa_code = remove_start(tfa_code, 'G-')
+
+                tfa_req = [
+                    user_hash, None, 2, None,
+                    [
+                        9, None, None, None, None, None, None, None,
+                        [None, tfa_code, True, 2]
+                    ]]
+
+                tfa_results = req(
+                    self._TFA_URL.format(tl), tfa_req,
+                    'Submitting TFA code', 'Unable to submit TFA code')
+
+                if tfa_results is False:
+                    return False
+
+                tfa_res = try_get(tfa_results, lambda x: x[0][5], list)
+                if tfa_res:
+                    tfa_msg = try_get(tfa_res, lambda x: x[5], compat_str)
+                    warn(
+                        'Unable to finish TFA: %s' % 'Invalid TFA code'
+                        if tfa_msg == 'INCORRECT_ANSWER_ENTERED' else tfa_msg)
+                    return False
+
+                check_cookie_url = try_get(
+                    tfa_results, lambda x: x[0][-1][2], compat_str)
+        else:
+            check_cookie_url = try_get(res, lambda x: x[2], compat_str)
+
+        if not check_cookie_url:
+            warn('Unable to extract CheckCookie URL')
+            return False
 
         check_cookie_results = self._download_webpage(
-            password_challenge_results[2], None, 'Checking cookie')
+            check_cookie_url, None, 'Checking cookie', fatal=False)
 
-        if '>Sign out<' not in check_cookie_results:
-            self._downloader.report_warning('Unable to log in')
+        if check_cookie_results is False:
+            return False
+
+        if 'https://myaccount.google.com/' not in check_cookie_results:
+            warn('Unable to log in')
             return False
 
         return True

From 3892a9f4ab54a0da5386b6ff612dd5ef9ff70267 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 May 2017 04:44:54 +0700
Subject: [PATCH 0817/1696] [ChangeLog] Actualize

---
 ChangeLog | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 7cceec10f..7a344c40f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,12 +1,32 @@
 version <unreleased>
 
+Common
+* [extractor/common] Fix typo in _extract_akamai_formats
++ [postprocessor/ffmpeg] Embed chapters into media file with --add-metadata
++ [extractor/common] Introduce chapters meta field
+
 Extractors
+* [youtube] Fix authentication (#12820, #12927, #12973, #12992, #12993, #12995,
+  #13003)
 * [bilibili] Fix video downloading (#13001)
+* [rmcdecouverte] Fix extraction (#12937)
+* [theplatform] Extract chapters
+* [bandcamp] Fix thumbnail extraction (#12980)
+* [pornhub] Extend URL regular expression (#12996)
++ [youtube] Extract chapters
++ [nrk] Extract chapters
++ [vice] Add support for ooyala embeds in article pages
 + [vice] Support vice articles (#12968)
 * [vice] Fix extraction for non en_us videos (#12967)
 * [gdcvault] Fix extraction for some videos (#12733)
+* [pbs] Improve multipart video support (#12981)
+* [laola1tv] Fix extraction (#12880)
 + [cda] Support birthday verification (#12789)
 * [leeco] Fix extraction (#12974)
++ [pbs] Extract chapters
+* [amp] Imporove thumbnail and subtitles extraction
+* [foxsports] Fix extraction (#12945)
+- [coub] Remove comment count extraction (#12941)
 
 
 version 2017.05.01

From 4ac0f573ef10d4e2bd883b63936e3fe1a41c8abb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 May 2017 04:51:34 +0700
Subject: [PATCH 0818/1696] release 2017.05.07

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 5 +++--
 youtube_dl/version.py     | 2 +-
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index fc221594f..86d11e142 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.01*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.01**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.07*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.07**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.05.01
+[debug] youtube-dl version 2017.05.07
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 7a344c40f..5418bb95f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.05.07
 
 Common
 * [extractor/common] Fix typo in _extract_akamai_formats
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index e3c038c48..d867dfe65 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -879,9 +879,10 @@
  - **VGTV**: VGTV, BTTV, FTV, Aftenposten and Aftonbladet
  - **vh1.com**
  - **Viafree**
- - **Vice**
+ - **vice**
+ - **vice:article**
+ - **vice:show**
  - **Viceland**
- - **ViceShow**
  - **Vidbit**
  - **Viddler**
  - **Videa**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index c19ac49b0..dc707071a 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.05.01'
+__version__ = '2017.05.07'

From 2eeb588efe9a7df4b2dcd90de9e461e8ff4a40fa Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 7 May 2017 08:58:34 +0100
Subject: [PATCH 0819/1696] [nbc] improve extraction(closes #12364)

---
 youtube_dl/extractor/nbc.py | 98 ++++++++++++-------------------------
 1 file changed, 31 insertions(+), 67 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index d2a44d05d..3b31ca3ef 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -17,7 +17,7 @@ from ..utils import (
 
 
 class NBCIE(AdobePassIE):
-    _VALID_URL = r'https?://(?:www\.)?nbc\.com/(?:[^/]+/)+(?P<id>n?\d+)'
+    _VALID_URL = r'https?://(?:www\.)?nbc\.com/[^/]+/video/[^/]+/(?P<id>n?\d+)'
 
     _TESTS = [
         {
@@ -36,16 +36,6 @@ class NBCIE(AdobePassIE):
                 'skip_download': True,
             },
         },
-        {
-            'url': 'http://www.nbc.com/the-tonight-show/episodes/176',
-            'info_dict': {
-                'id': '176',
-                'ext': 'flv',
-                'title': 'Ricky Gervais, Steven Van Zandt, ILoveMakonnen',
-                'description': 'A brand new episode of The Tonight Show welcomes Ricky Gervais, Steven Van Zandt and ILoveMakonnen.',
-            },
-            'skip': '404 Not Found',
-        },
         {
             'url': 'http://www.nbc.com/saturday-night-live/video/star-wars-teaser/2832821',
             'info_dict': {
@@ -63,11 +53,6 @@ class NBCIE(AdobePassIE):
             },
             'skip': 'Only works from US',
         },
-        {
-            # This video has expired but with an escaped embedURL
-            'url': 'http://www.nbc.com/parenthood/episode-guide/season-5/just-like-at-home/515',
-            'only_matching': True,
-        },
         {
             # HLS streams requires the 'hdnea3' cookie
             'url': 'http://www.nbc.com/Kings/video/goliath/n1806',
@@ -89,58 +74,37 @@ class NBCIE(AdobePassIE):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        info = {
-            '_type': 'url_transparent',
-            'ie_key': 'ThePlatform',
-            'id': video_id,
+        video_data = self._download_json(
+            'https://api.nbc.com/v3/videos', video_id, query={
+                'filter[permalink]': url,
+            })['data'][0]['attributes']
+        query = {
+            'mbr': 'true',
+            'manifest': 'm3u',
+        }
+        video_id = video_data['guid']
+        title = video_data['title']
+        if video_data.get('entitlement') == 'auth':
+            resource = self._get_mvpd_resource(
+                'nbcentertainment', title, video_id,
+                video_data.get('vChipRating'))
+            query['auth'] = self._extract_mvpd_auth(
+                url, video_id, 'nbcentertainment', resource)
+        theplatform_url = smuggle_url(update_url_query(
+            'http://link.theplatform.com/s/NnzsPC/media/guid/2410887629/' + video_id,
+            query), {'force_smil_url': True})
+        return {
+            '_type': 'url_transparent',
+            'id': video_id,
+            'title': title,
+            'url': theplatform_url,
+            'description': video_data.get('description'),
+            'keywords': video_data.get('keywords'),
+            'season_number': int_or_none(video_data.get('seasonNumber')),
+            'episode_number': int_or_none(video_data.get('episodeNumber')),
+            'series': video_data.get('showName'),
+            'ie_key': 'ThePlatform',
         }
-        video_data = None
-        preload = self._search_regex(
-            r'PRELOAD\s*=\s*({.+})', webpage, 'preload data', default=None)
-        if preload:
-            preload_data = self._parse_json(preload, video_id)
-            path = compat_urllib_parse_urlparse(url).path.rstrip('/')
-            entity_id = preload_data.get('xref', {}).get(path)
-            video_data = preload_data.get('entities', {}).get(entity_id)
-        if video_data:
-            query = {
-                'mbr': 'true',
-                'manifest': 'm3u',
-            }
-            video_id = video_data['guid']
-            title = video_data['title']
-            if video_data.get('entitlement') == 'auth':
-                resource = self._get_mvpd_resource(
-                    'nbcentertainment', title, video_id,
-                    video_data.get('vChipRating'))
-                query['auth'] = self._extract_mvpd_auth(
-                    url, video_id, 'nbcentertainment', resource)
-            theplatform_url = smuggle_url(update_url_query(
-                'http://link.theplatform.com/s/NnzsPC/media/guid/2410887629/' + video_id,
-                query), {'force_smil_url': True})
-            info.update({
-                'id': video_id,
-                'title': title,
-                'url': theplatform_url,
-                'description': video_data.get('description'),
-                'keywords': video_data.get('keywords'),
-                'season_number': int_or_none(video_data.get('seasonNumber')),
-                'episode_number': int_or_none(video_data.get('episodeNumber')),
-                'series': video_data.get('showName'),
-            })
-        else:
-            theplatform_url = unescapeHTML(lowercase_escape(self._html_search_regex(
-                [
-                    r'(?:class="video-player video-player-full" data-mpx-url|class="player" src)="(.*?)"',
-                    r'<iframe[^>]+src="((?:https?:)?//player\.theplatform\.com/[^"]+)"',
-                    r'"embedURL"\s*:\s*"([^"]+)"'
-                ],
-                webpage, 'theplatform url').replace('_no_endcard', '').replace('\\/', '/')))
-            if theplatform_url.startswith('//'):
-                theplatform_url = 'http:' + theplatform_url
-            info['url'] = smuggle_url(theplatform_url, {'source_url': url})
-        return info
 
 
 class NBCSportsVPlayerIE(InfoExtractor):

From 52294cdda761ad08785e7118ae8e121ceef257ec Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 7 May 2017 09:31:14 +0100
Subject: [PATCH 0820/1696] [nbc] remove unused imports and extract permalink
 from modified urls

---
 youtube_dl/extractor/nbc.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 3b31ca3ef..62db70b43 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -5,10 +5,8 @@ import re
 from .common import InfoExtractor
 from .theplatform import ThePlatformIE
 from .adobepass import AdobePassIE
-from ..compat import compat_urllib_parse_urlparse
 from ..utils import (
     find_xpath_attr,
-    lowercase_escape,
     smuggle_url,
     unescapeHTML,
     update_url_query,
@@ -17,7 +15,7 @@ from ..utils import (
 
 
 class NBCIE(AdobePassIE):
-    _VALID_URL = r'https?://(?:www\.)?nbc\.com/[^/]+/video/[^/]+/(?P<id>n?\d+)'
+    _VALID_URL = r'(?P<permalink>https?://(?:www\.)?nbc\.com/[^/]+/video/[^/]+/(?P<id>n?\d+))'
 
     _TESTS = [
         {
@@ -73,10 +71,10 @@ class NBCIE(AdobePassIE):
     ]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        permalink, video_id = re.match(self._VALID_URL, url).groups()
         video_data = self._download_json(
             'https://api.nbc.com/v3/videos', video_id, query={
-                'filter[permalink]': url,
+                'filter[permalink]': permalink,
             })['data'][0]['attributes']
         query = {
             'mbr': 'true',

From 3d40084b835afe49adff7eca67dec26bc9b95c0a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 8 May 2017 20:03:38 +0700
Subject: [PATCH 0821/1696] [nuevo] Pass headers to _extract_nuevo

---
 youtube_dl/extractor/nuevo.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nuevo.py b/youtube_dl/extractor/nuevo.py
index 87fb94d1f..be1e09d37 100644
--- a/youtube_dl/extractor/nuevo.py
+++ b/youtube_dl/extractor/nuevo.py
@@ -10,9 +10,10 @@ from ..utils import (
 
 
 class NuevoBaseIE(InfoExtractor):
-    def _extract_nuevo(self, config_url, video_id):
+    def _extract_nuevo(self, config_url, video_id, headers={}):
         config = self._download_xml(
-            config_url, video_id, transform_source=lambda s: s.strip())
+            config_url, video_id, transform_source=lambda s: s.strip(),
+            headers=headers)
 
         title = xpath_text(config, './title', 'title', fatal=True).strip()
         video_id = xpath_text(config, './mediaid', default=video_id)

From b6eb74e340fd523fb0c710b3141d6f9c696069f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 8 May 2017 20:08:58 +0700
Subject: [PATCH 0822/1696] [nonktube] Add extractor (closes #8647, closes
 #13024)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/nonktube.py   | 37 ++++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+)
 create mode 100644 youtube_dl/extractor/nonktube.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 1d7495910..968cca9d2 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -663,6 +663,7 @@ from .nintendo import NintendoIE
 from .njpwworld import NJPWWorldIE
 from .nobelprize import NobelPrizeIE
 from .noco import NocoIE
+from .nonktube import NonkTubeIE
 from .noovo import NoovoIE
 from .normalboots import NormalbootsIE
 from .nosvideo import NosVideoIE
diff --git a/youtube_dl/extractor/nonktube.py b/youtube_dl/extractor/nonktube.py
new file mode 100644
index 000000000..ba6007cd1
--- /dev/null
+++ b/youtube_dl/extractor/nonktube.py
@@ -0,0 +1,37 @@
+from __future__ import unicode_literals
+
+from .nuevo import NuevoBaseIE
+
+
+class NonkTubeIE(NuevoBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?nonktube\.com/(?:(?:video|embed)/|media/nuevo/embed\.php\?.*?\bid=)(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.nonktube.com/video/118636/sensual-wife-uncensored-fucked-in-hairy-pussy-and-facialized',
+        'info_dict': {
+            'id': '118636',
+            'ext': 'mp4',
+            'title': 'Sensual Wife Uncensored Fucked In Hairy Pussy And Facialized',
+            'age_limit': 18,
+            'duration': 1150.98,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }, {
+        'url': 'https://www.nonktube.com/embed/118636',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        nuevo_url = 'https://www.nonktube.com/media/nuevo/config.php?key=%s' % video_id
+
+        info = self._extract_nuevo(
+            nuevo_url, video_id, headers={
+                'Referer': 'https://www.nonktube.com/media/nuevo/player.swf?config=%s' % nuevo_url
+            })
+        info.update({
+            'age_limit': 18
+        })
+        return info

From bf82b8732353f45582cfefc6cced01721fbf4154 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 8 May 2017 20:13:22 +0700
Subject: [PATCH 0823/1696] [nonktube] Use econfig nuevo URL

---
 youtube_dl/extractor/nonktube.py | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/nonktube.py b/youtube_dl/extractor/nonktube.py
index ba6007cd1..63e58aae2 100644
--- a/youtube_dl/extractor/nonktube.py
+++ b/youtube_dl/extractor/nonktube.py
@@ -25,13 +25,9 @@ class NonkTubeIE(NuevoBaseIE):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        nuevo_url = 'https://www.nonktube.com/media/nuevo/config.php?key=%s' % video_id
-
         info = self._extract_nuevo(
-            nuevo_url, video_id, headers={
-                'Referer': 'https://www.nonktube.com/media/nuevo/player.swf?config=%s' % nuevo_url
-            })
-        info.update({
-            'age_limit': 18
-        })
+            'https://www.nonktube.com/media/nuevo/econfig.php?key=%s'
+            % video_id, video_id)
+
+        info['age_limit'] = 18
         return info

From 04c09f19618345203a787c300e4ecf99f1ff9b02 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 8 May 2017 15:00:57 +0100
Subject: [PATCH 0824/1696] [turner] extract thumbnail and is_live and strip
 description

---
 youtube_dl/extractor/turner.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/turner.py b/youtube_dl/extractor/turner.py
index 1c0be9fc6..efeb677ee 100644
--- a/youtube_dl/extractor/turner.py
+++ b/youtube_dl/extractor/turner.py
@@ -13,6 +13,7 @@ from ..utils import (
     xpath_attr,
     update_url_query,
     ExtractorError,
+    strip_or_none,
 )
 
 
@@ -163,17 +164,21 @@ class TurnerBaseIE(AdobePassIE):
             'height': int_or_none(image.get('height')),
         } for image in video_data.findall('images/image')]
 
+        is_live = xpath_text(video_data, 'isLive') == 'true'
+
         return {
             'id': video_id,
-            'title': title,
+            'title': self._live_title(title) if is_live else title,
             'formats': formats,
             'subtitles': subtitles,
             'thumbnails': thumbnails,
-            'description': xpath_text(video_data, 'description'),
+            'thumbnail': xpath_text(video_data, 'poster'),
+            'description': strip_or_none(xpath_text(video_data, 'description')),
             'duration': parse_duration(xpath_text(video_data, 'length') or xpath_text(video_data, 'trt')),
             'timestamp': self._extract_timestamp(video_data),
             'upload_date': xpath_attr(video_data, 'metas', 'version'),
             'series': xpath_text(video_data, 'showTitle'),
             'season_number': int_or_none(xpath_text(video_data, 'seasonNumber')),
             'episode_number': int_or_none(xpath_text(video_data, 'episodeNumber')),
+            'is_live': is_live,
         }

From 3b859145c2ad4efd5250fc0199174f1ec2068b34 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 8 May 2017 15:01:10 +0100
Subject: [PATCH 0825/1696] [adultswim] Fix Extraction(closes #8640)(closes
 #10950)(closes closes #11042)(closes #12121)

- add support for adobe pass authentication
- add support for live streams
- add support for show pages
---
 youtube_dl/extractor/adultswim.py | 285 +++++++++++-------------------
 1 file changed, 106 insertions(+), 179 deletions(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 989505c82..9c37ea33c 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -5,91 +5,52 @@ import re
 
 from .turner import TurnerBaseIE
 from ..utils import (
-    ExtractorError,
     int_or_none,
+    strip_or_none,
 )
 
 
 class AdultSwimIE(TurnerBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?adultswim\.com/videos/(?P<is_playlist>playlists/)?(?P<show_path>[^/]+)/(?P<episode_path>[^/?#]+)/?'
+    _VALID_URL = r'https?://(?:www\.)?adultswim\.com/videos/(?P<show_path>[^/?#]+)(?:/(?P<episode_path>[^/?#]+))?'
 
     _TESTS = [{
         'url': 'http://adultswim.com/videos/rick-and-morty/pilot',
-        'playlist': [
-            {
-                'md5': '247572debc75c7652f253c8daa51a14d',
-                'info_dict': {
-                    'id': 'rQxZvXQ4ROaSOqq-or2Mow-0',
-                    'ext': 'flv',
-                    'title': 'Rick and Morty - Pilot Part 1',
-                    'description': "Rick moves in with his daughter's family and establishes himself as a bad influence on his grandson, Morty. "
-                },
-            },
-            {
-                'md5': '77b0e037a4b20ec6b98671c4c379f48d',
-                'info_dict': {
-                    'id': 'rQxZvXQ4ROaSOqq-or2Mow-3',
-                    'ext': 'flv',
-                    'title': 'Rick and Morty - Pilot Part 4',
-                    'description': "Rick moves in with his daughter's family and establishes himself as a bad influence on his grandson, Morty. "
-                },
-            },
-        ],
         'info_dict': {
             'id': 'rQxZvXQ4ROaSOqq-or2Mow',
+            'ext': 'mp4',
             'title': 'Rick and Morty - Pilot',
-            'description': "Rick moves in with his daughter's family and establishes himself as a bad influence on his grandson, Morty. "
-        },
-        'skip': 'This video is only available for registered users',
-    }, {
-        'url': 'http://www.adultswim.com/videos/playlists/american-parenting/putting-francine-out-of-business/',
-        'playlist': [
-            {
-                'md5': '2eb5c06d0f9a1539da3718d897f13ec5',
-                'info_dict': {
-                    'id': '-t8CamQlQ2aYZ49ItZCFog-0',
-                    'ext': 'flv',
-                    'title': 'American Dad - Putting Francine Out of Business',
-                    'description': 'Stan hatches a plan to get Francine out of the real estate business.Watch more American Dad on [adult swim].'
-                },
-            }
-        ],
-        'info_dict': {
-            'id': '-t8CamQlQ2aYZ49ItZCFog',
-            'title': 'American Dad - Putting Francine Out of Business',
-            'description': 'Stan hatches a plan to get Francine out of the real estate business.Watch more American Dad on [adult swim].'
-        },
-    }, {
-        'url': 'http://www.adultswim.com/videos/tim-and-eric-awesome-show-great-job/dr-steve-brule-for-your-wine/',
-        'playlist': [
-            {
-                'md5': '3e346a2ab0087d687a05e1e7f3b3e529',
-                'info_dict': {
-                    'id': 'sY3cMUR_TbuE4YmdjzbIcQ-0',
-                    'ext': 'mp4',
-                    'title': 'Tim and Eric Awesome Show Great Job! - Dr. Steve Brule, For Your Wine',
-                    'description': 'Dr. Brule reports live from Wine Country with a special report on wines.  \r\nWatch Tim and Eric Awesome Show Great Job! episode #20, "Embarrassed" on Adult Swim.\r\n\r\n',
-                },
-            }
-        ],
-        'info_dict': {
-            'id': 'sY3cMUR_TbuE4YmdjzbIcQ',
-            'title': 'Tim and Eric Awesome Show Great Job! - Dr. Steve Brule, For Your Wine',
-            'description': 'Dr. Brule reports live from Wine Country with a special report on wines.  \r\nWatch Tim and Eric Awesome Show Great Job! episode #20, "Embarrassed" on Adult Swim.\r\n\r\n',
+            'description': 'Rick moves in with his daughter\'s family and establishes himself as a bad influence on his grandson, Morty.',
+            'timestamp': 1493267400,
+            'upload_date': '20170427',
         },
         'params': {
             # m3u8 download
             'skip_download': True,
-        }
+        },
+        'expected_warnings': ['Unable to download f4m manifest'],
+    }, {
+        'url': 'http://www.adultswim.com/videos/tim-and-eric-awesome-show-great-job/dr-steve-brule-for-your-wine/',
+        'info_dict': {
+            'id': 'sY3cMUR_TbuE4YmdjzbIcQ',
+            'ext': 'mp4',
+            'title': 'Tim and Eric Awesome Show Great Job! - Dr. Steve Brule, For Your Wine',
+            'description': 'Dr. Brule reports live from Wine Country with a special report on wines.  \nWatch Tim and Eric Awesome Show Great Job! episode #20, "Embarrassed" on Adult Swim.',
+            'upload_date': '20080124',
+            'timestamp': 1201150800,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
-        # heroMetadata.trailer
         'url': 'http://www.adultswim.com/videos/decker/inside-decker-a-new-hero/',
         'info_dict': {
             'id': 'I0LQFQkaSUaFp8PnAWHhoQ',
             'ext': 'mp4',
             'title': 'Decker - Inside Decker: A New Hero',
-            'description': 'md5:c916df071d425d62d70c86d4399d3ee0',
-            'duration': 249.008,
+            'description': 'The guys recap the conclusion of the season. They announce a new hero, take a peek into the Victorville Film Archive and welcome back the talented James Dean.',
+            'timestamp': 1469480460,
+            'upload_date': '20160725',
         },
         'params': {
             # m3u8 download
@@ -97,136 +58,102 @@ class AdultSwimIE(TurnerBaseIE):
         },
         'expected_warnings': ['Unable to download f4m manifest'],
     }, {
-        'url': 'http://www.adultswim.com/videos/toonami/friday-october-14th-2016/',
+        'url': 'http://www.adultswim.com/videos/attack-on-titan',
         'info_dict': {
-            'id': 'eYiLsKVgQ6qTC6agD67Sig',
-            'title': 'Toonami - Friday, October 14th, 2016',
-            'description': 'md5:99892c96ffc85e159a428de85c30acde',
+            'id': 'b7A69dzfRzuaXIECdxW8XQ',
+            'title': 'Attack on Titan',
+            'description': 'md5:6c8e003ea0777b47013e894767f5e114',
+        },
+        'playlist_mincount': 12,
+    }, {
+        'url': 'http://www.adultswim.com/videos/streams/williams-stream',
+        'info_dict': {
+            'id': 'd8DEBj7QRfetLsRgFnGEyg',
+            'ext': 'mp4',
+            'title': r're:^Williams Stream \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
+            'description': 'original programming',
         },
-        'playlist': [{
-            'md5': '',
-            'info_dict': {
-                'id': 'eYiLsKVgQ6qTC6agD67Sig',
-                'ext': 'mp4',
-                'title': 'Toonami - Friday, October 14th, 2016',
-                'description': 'md5:99892c96ffc85e159a428de85c30acde',
-            },
-        }],
         'params': {
             # m3u8 download
             'skip_download': True,
         },
-        'expected_warnings': ['Unable to download f4m manifest'],
     }]
 
-    @staticmethod
-    def find_video_info(collection, slug):
-        for video in collection.get('videos'):
-            if video.get('slug') == slug:
-                return video
-
-    @staticmethod
-    def find_collection_by_linkURL(collections, linkURL):
-        for collection in collections:
-            if collection.get('linkURL') == linkURL:
-                return collection
-
-    @staticmethod
-    def find_collection_containing_video(collections, slug):
-        for collection in collections:
-            for video in collection.get('videos'):
-                if video.get('slug') == slug:
-                    return collection, video
-        return None, None
-
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        show_path = mobj.group('show_path')
-        episode_path = mobj.group('episode_path')
-        is_playlist = True if mobj.group('is_playlist') else False
+        show_path, episode_path = re.match(self._VALID_URL, url).groups()
+        display_id = episode_path or show_path
+        webpage = self._download_webpage(url, display_id)
+        initial_data = self._parse_json(self._search_regex(
+            r'AS_INITIAL_DATA(?:__)?\s*=\s*({.+?});',
+            webpage, 'initial data'), display_id)
 
-        webpage = self._download_webpage(url, episode_path)
+        is_stream = show_path == 'streams'
+        if is_stream:
+            if not episode_path:
+                episode_path = 'live-stream'
 
-        # Extract the value of `bootstrappedData` from the Javascript in the page.
-        bootstrapped_data = self._parse_json(self._search_regex(
-            r'var bootstrappedData = ({.*});', webpage, 'bootstraped data'), episode_path)
+            video_data = next(stream for stream_path, stream in initial_data['streams'].items() if stream_path == episode_path)
+            video_id = video_data.get('stream')
 
-        # Downloading videos from a /videos/playlist/ URL needs to be handled differently.
-        # NOTE: We are only downloading one video (the current one) not the playlist
-        if is_playlist:
-            collections = bootstrapped_data['playlists']['collections']
-            collection = self.find_collection_by_linkURL(collections, show_path)
-            video_info = self.find_video_info(collection, episode_path)
-
-            show_title = video_info['showTitle']
-            segment_ids = [video_info['videoPlaybackID']]
+            if not video_id:
+                entries = []
+                for episode in video_data.get('archiveEpisodes', []):
+                    episode_url = episode.get('url')
+                    if not episode_url:
+                        continue
+                    entries.append(self.url_result(
+                        episode_url, 'AdultSwim', episode.get('id')))
+                return self.playlist_result(
+                    entries, video_data.get('id'), video_data.get('title'),
+                    strip_or_none(video_data.get('description')))
         else:
-            collections = bootstrapped_data['show']['collections']
-            collection, video_info = self.find_collection_containing_video(collections, episode_path)
-            # Video wasn't found in the collections, let's try `slugged_video`.
-            if video_info is None:
-                if bootstrapped_data.get('slugged_video', {}).get('slug') == episode_path:
-                    video_info = bootstrapped_data['slugged_video']
-            if not video_info:
-                video_info = bootstrapped_data.get(
-                    'heroMetadata', {}).get('trailer', {}).get('video')
-            if not video_info:
-                video_info = bootstrapped_data.get('onlineOriginals', [None])[0]
-            if not video_info:
-                raise ExtractorError('Unable to find video info')
+            show_data = initial_data['show']
 
-            show = bootstrapped_data['show']
-            show_title = show['title']
-            stream = video_info.get('stream')
-            if stream and stream.get('videoPlaybackID'):
-                segment_ids = [stream['videoPlaybackID']]
-            elif video_info.get('clips'):
-                segment_ids = [clip['videoPlaybackID'] for clip in video_info['clips']]
-            elif video_info.get('videoPlaybackID'):
-                segment_ids = [video_info['videoPlaybackID']]
-            elif video_info.get('id'):
-                segment_ids = [video_info['id']]
-            else:
-                if video_info.get('auth') is True:
-                    raise ExtractorError(
-                        'This video is only available via cable service provider subscription that'
-                        ' is not currently supported. You may want to use --cookies.', expected=True)
-                else:
-                    raise ExtractorError('Unable to find stream or clips')
+            if not episode_path:
+                entries = []
+                for video in show_data.get('videos', []):
+                    slug = video.get('slug')
+                    if not slug:
+                        continue
+                    entries.append(self.url_result(
+                        'http://adultswim.com/videos/%s/%s' % (show_path, slug),
+                        'AdultSwim', video.get('id')))
+                return self.playlist_result(
+                    entries, show_data.get('id'), show_data.get('title'),
+                    strip_or_none(show_data.get('metadata', {}).get('description')))
 
-        episode_id = video_info['id']
-        episode_title = video_info['title']
-        episode_description = video_info.get('description')
-        episode_duration = int_or_none(video_info.get('duration'))
-        view_count = int_or_none(video_info.get('views'))
+            video_data = show_data['sluggedVideo']
+            video_id = video_data['id']
 
-        entries = []
-        for part_num, segment_id in enumerate(segment_ids):
-            segement_info = self._extract_cvp_info(
-                'http://www.adultswim.com/videos/api/v0/assets?id=%s&platform=desktop' % segment_id,
-                segment_id, {
-                    'secure': {
-                        'media_src': 'http://androidhls-secure.cdn.turner.com/adultswim/big',
-                        'tokenizer_src': 'http://www.adultswim.com/astv/mvpd/processors/services/token_ipadAdobe.do',
-                    },
-                })
-            segment_title = '%s - %s' % (show_title, episode_title)
-            if len(segment_ids) > 1:
-                segment_title += ' Part %d' % (part_num + 1)
-            segement_info.update({
-                'id': segment_id,
-                'title': segment_title,
-                'description': episode_description,
+        info = self._extract_cvp_info(
+            'http://www.adultswim.com/videos/api/v0/assets?id=' + video_id,
+            video_id, {
+                'secure': {
+                    'media_src': 'http://androidhls-secure.cdn.turner.com/adultswim/big',
+                    'tokenizer_src': 'http://www.adultswim.com/astv/mvpd/processors/services/token_ipadAdobe.do',
+                },
+            }, {
+                'url': url,
+                'site_name': 'AdultSwim',
+                'auth_required': video_data.get('auth'),
             })
-            entries.append(segement_info)
 
-        return {
-            '_type': 'playlist',
-            'id': episode_id,
-            'display_id': episode_path,
-            'entries': entries,
-            'title': '%s - %s' % (show_title, episode_title),
-            'description': episode_description,
-            'duration': episode_duration,
-            'view_count': view_count,
-        }
+        info.update({
+            'id': video_id,
+            'display_id': display_id,
+            'description': info.get('description') or strip_or_none(video_data.get('description')),
+        })
+        if not is_stream:
+            info.update({
+                'duration': info.get('duration') or int_or_none(video_data.get('duration')),
+                'timestamp': info.get('timestamp') or int_or_none(video_data.get('launch_date')),
+                'season_number': info.get('season_number') or int_or_none(video_data.get('season_number')),
+                'episode': info['title'],
+                'episode_number': info.get('episode_number') or int_or_none(video_data.get('episode_number')),
+            })
+
+            info['series'] = video_data.get('collection_title') or info.get('series')
+            if info['series'] and info['series'] != info['title']:
+                info['title'] = '%s - %s' % (info['series'], info['title'])
+
+        return info

From 8fa17117df723ab0bb4b0105142ea861df663ba5 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 8 May 2017 18:13:58 +0100
Subject: [PATCH 0826/1696] [funimation] fix authentication(closes #13021)

---
 youtube_dl/extractor/funimation.py | 40 ++++++++++++++----------------
 1 file changed, 18 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index e44a2a87f..c1559be0a 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -20,6 +20,7 @@ class FunimationIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?funimation(?:\.com|now\.uk)/shows/[^/]+/(?P<id>[^/?#&]+)'
 
     _NETRC_MACHINE = 'funimation'
+    _TOKEN = None
 
     _TESTS = [{
         'url': 'https://www.funimation.com/shows/hacksign/role-play/',
@@ -67,27 +68,19 @@ class FunimationIE(InfoExtractor):
         (username, password) = self._get_login_info()
         if username is None:
             return
-        data = urlencode_postdata({
-            'email_field': username,
-            'password_field': password,
-        })
-        user_agent = self._extract_cloudflare_session_ua(self._LOGIN_URL)
-        if not user_agent:
-            user_agent = 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0'
-        login_request = sanitized_Request(self._LOGIN_URL, data, headers={
-            'User-Agent': user_agent,
-            'Content-Type': 'application/x-www-form-urlencoded'
-        })
-        login_page = self._download_webpage(
-            login_request, None, 'Logging in as %s' % username)
-        if any(p in login_page for p in ('funimation.com/logout', '>Log Out<')):
-            return
-        error = self._html_search_regex(
-            r'(?s)<div[^>]+id=["\']errorMessages["\'][^>]*>(.+?)</div>',
-            login_page, 'error messages', default=None)
-        if error:
-            raise ExtractorError('Unable to login: %s' % error, expected=True)
-        raise ExtractorError('Unable to log in')
+        try:
+            data = self._download_json(
+                'https://prod-api-funimationnow.dadcdigital.com/api/auth/login/',
+                None, 'Logging in as %s' % username, data=urlencode_postdata({
+                    'username': username,
+                    'password': password,
+                }))
+            self._TOKEN = data['token']
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+                error = self._parse_json(e.cause.read().decode(), None)['error']
+                raise ExtractorError(error, expected=True)
+            raise
 
     def _real_initialize(self):
         self._login()
@@ -125,9 +118,12 @@ class FunimationIE(InfoExtractor):
         description = self._html_search_meta(['description', 'og:description'], webpage, fatal=True)
 
         try:
+            headers = {}
+            if self._TOKEN:
+                headers['Authorization'] = 'Token %s' % self._TOKEN
             sources = self._download_json(
                 'https://prod-api-funimationnow.dadcdigital.com/api/source/catalog/video/%s/signed/' % video_id,
-                video_id)['items']
+                video_id, headers=headers)['items']
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
                 error = self._parse_json(e.cause.read(), video_id)['errors'][0]

From 804181dda9cbb7f2cbb8305bb4097908312a83f6 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 8 May 2017 18:50:25 +0100
Subject: [PATCH 0827/1696] [funimation] remove codes related to old login
 method and update test

---
 youtube_dl/extractor/funimation.py | 26 ++++++--------------------
 1 file changed, 6 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index c1559be0a..8c37509ec 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -2,15 +2,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_HTTPError,
-    compat_urllib_parse_unquote_plus,
-)
+from ..compat import compat_HTTPError
 from ..utils import (
     determine_ext,
     int_or_none,
     js_to_json,
-    sanitized_Request,
     ExtractorError,
     urlencode_postdata
 )
@@ -39,31 +35,21 @@ class FunimationIE(InfoExtractor):
     }, {
         'url': 'https://www.funimation.com/shows/attack-on-titan-junior-high/broadcast-dub-preview/',
         'info_dict': {
-            'id': '9635',
+            'id': '210051',
             'display_id': 'broadcast-dub-preview',
             'ext': 'mp4',
             'title': 'Attack on Titan: Junior High - Broadcast Dub Preview',
-            'description': 'md5:f8ec49c0aff702a7832cd81b8a44f803',
             'thumbnail': r're:https?://.*\.(?:jpg|png)',
         },
-        'skip': 'Access without user interaction is forbidden by CloudFlare',
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
         'url': 'https://www.funimationnow.uk/shows/puzzle-dragons-x/drop-impact/simulcast/',
         'only_matching': True,
     }]
 
-    _LOGIN_URL = 'http://www.funimation.com/login'
-
-    def _extract_cloudflare_session_ua(self, url):
-        ci_session_cookie = self._get_cookies(url).get('ci_session')
-        if ci_session_cookie:
-            ci_session = compat_urllib_parse_unquote_plus(ci_session_cookie.value)
-            # ci_session is a string serialized by PHP function serialize()
-            # This case is simple enough to use regular expressions only
-            return self._search_regex(
-                r'"user_agent";s:\d+:"([^"]+)"', ci_session, 'user agent',
-                default=None)
-
     def _login(self):
         (username, password) = self._get_login_info()
         if username is None:

From e9137224b3ef638cce3dac024814397fd020f3b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 May 2017 01:14:02 +0700
Subject: [PATCH 0828/1696] [YoutubeDL] Force restrict filenames when no locale
 is set for python 2 as well (#13027)

---
 youtube_dl/YoutubeDL.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index eb465c425..4c33d494a 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -370,10 +370,10 @@ class YoutubeDL(object):
                 else:
                     raise
 
-        if (sys.version_info >= (3,) and sys.platform != 'win32' and
+        if (sys.platform != 'win32' and
                 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
                 not params.get('restrictfilenames', False)):
-            # On Python 3, the Unicode filesystem API will throw errors (#1474)
+            # Unicode filesystem API will throw errors (#1474, #13027)
             self.report_warning(
                 'Assuming --restrict-filenames since file system encoding '
                 'cannot encode all characters. '

From afa0200bf05671be5a380a79206309bdd4f886d6 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 8 May 2017 20:03:26 +0100
Subject: [PATCH 0829/1696] [vrv] extract dash formats and subtitles

---
 youtube_dl/extractor/vrv.py | 47 +++++++++++++++++++++++++++----------
 1 file changed, 34 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/vrv.py b/youtube_dl/extractor/vrv.py
index 487047fd7..9959627c0 100644
--- a/youtube_dl/extractor/vrv.py
+++ b/youtube_dl/extractor/vrv.py
@@ -112,21 +112,41 @@ class VRVIE(VRVBaseIE):
 
         audio_locale = streams_json.get('audio_locale')
         formats = []
-        for stream_id, stream in streams_json.get('streams', {}).get('adaptive_hls', {}).items():
-            stream_url = stream.get('url')
-            if not stream_url:
-                continue
-            stream_id = stream_id or audio_locale
-            m3u8_formats = self._extract_m3u8_formats(
-                stream_url, video_id, 'mp4', m3u8_id=stream_id,
-                note='Downloading %s m3u8 information' % stream_id,
-                fatal=False)
-            if audio_locale:
-                for f in m3u8_formats:
-                    f['language'] = audio_locale
-            formats.extend(m3u8_formats)
+        for stream_type, streams in streams_json.get('streams', {}).items():
+            if stream_type in ('adaptive_hls', 'adaptive_dash'):
+                for stream in streams.values():
+                    stream_url = stream.get('url')
+                    if not stream_url:
+                        continue
+                    stream_id = stream.get('hardsub_locale') or audio_locale
+                    format_id = '%s-%s' % (stream_type.split('_')[1], stream_id)
+                    if stream_type == 'adaptive_hls':
+                        adaptive_formats = self._extract_m3u8_formats(
+                            stream_url, video_id, 'mp4', m3u8_id=format_id,
+                            note='Downloading %s m3u8 information' % stream_id,
+                            fatal=False)
+                    else:
+                        adaptive_formats = self._extract_mpd_formats(
+                            stream_url, video_id, mpd_id=format_id,
+                            note='Downloading %s MPD information' % stream_id,
+                            fatal=False)
+                    if audio_locale:
+                        for f in adaptive_formats:
+                            if f.get('acodec') != 'none':
+                                f['language'] = audio_locale
+                    formats.extend(adaptive_formats)
         self._sort_formats(formats)
 
+        subtitles = {}
+        for subtitle in streams_json.get('subtitles', {}).values():
+            subtitle_url = subtitle.get('url')
+            if not subtitle_url:
+                continue
+            subtitles.setdefault(subtitle.get('locale', 'en-US'), []).append({
+                'url': subtitle_url,
+                'ext': subtitle.get('format', 'ass'),
+            })
+
         thumbnails = []
         for thumbnail in video_data.get('images', {}).get('thumbnails', []):
             thumbnail_url = thumbnail.get('source')
@@ -142,6 +162,7 @@ class VRVIE(VRVBaseIE):
             'id': video_id,
             'title': title,
             'formats': formats,
+            'subtitles': subtitles,
             'thumbnails': thumbnails,
             'description': video_data.get('description'),
             'duration': float_or_none(video_data.get('duration_ms'), 1000),

From 5996d21aeac73509eee060bd26dbec95ede5e2eb Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 9 May 2017 00:47:37 +0100
Subject: [PATCH 0830/1696] [cspan] add support for brightcove live
 embeds(closes #13028)

---
 youtube_dl/extractor/brightcove.py | 20 +++++++++++++++++---
 youtube_dl/extractor/cspan.py      | 15 +++++++++++++++
 2 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 3f017a2b1..0ed59bcbc 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -5,6 +5,7 @@ import re
 import json
 
 from .common import InfoExtractor
+from .adobepass import AdobePassIE
 from ..compat import (
     compat_etree_fromstring,
     compat_parse_qs,
@@ -448,7 +449,7 @@ class BrightcoveLegacyIE(InfoExtractor):
         return info
 
 
-class BrightcoveNewIE(InfoExtractor):
+class BrightcoveNewIE(AdobePassIE):
     IE_NAME = 'brightcove:new'
     _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[^/]+)_(?P<embed>[^/]+)/index\.html\?.*videoId=(?P<video_id>\d+|ref:[^&]+)'
     _TESTS = [{
@@ -602,6 +603,20 @@ class BrightcoveNewIE(InfoExtractor):
                 raise ExtractorError(message, expected=True)
             raise
 
+        errors = json_data.get('errors')
+        if errors and errors[0].get('error_subcode') == 'TVE_AUTH':
+            custom_fields = json_data['custom_fields']
+            tve_token = self._extract_mvpd_auth(
+                smuggled_data['source_url'], video_id,
+                custom_fields['bcadobepassrequestorid'],
+                custom_fields['bcadobepassresourceid'])
+            json_data = self._download_json(
+                api_url, video_id, headers={
+                    'Accept': 'application/json;pk=%s' % policy_key
+                }, query={
+                    'tveToken': tve_token,
+                })
+
         title = json_data['name'].strip()
 
         formats = []
@@ -667,7 +682,6 @@ class BrightcoveNewIE(InfoExtractor):
                     })
                 formats.append(f)
 
-        errors = json_data.get('errors')
         if not formats and errors:
             error = errors[0]
             raise ExtractorError(
@@ -684,7 +698,7 @@ class BrightcoveNewIE(InfoExtractor):
 
         is_live = False
         duration = float_or_none(json_data.get('duration'), 1000)
-        if duration and duration < 0:
+        if duration is not None and duration <= 0:
             is_live = True
 
         return {
diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index d4576160b..171820e27 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -10,6 +10,7 @@ from ..utils import (
     smuggle_url,
     determine_ext,
     ExtractorError,
+    extract_attributes,
 )
 from .senateisvp import SenateISVPIE
 from .ustream import UstreamIE
@@ -68,6 +69,7 @@ class CSpanIE(InfoExtractor):
             'uploader_id': '12987475',
         },
     }]
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -78,6 +80,19 @@ class CSpanIE(InfoExtractor):
         if ustream_url:
             return self.url_result(ustream_url, UstreamIE.ie_key())
 
+        if '&vod' not in url:
+            bc = self._search_regex(
+                r"(<[^>]+id='brightcove-player-embed'[^>]+>)",
+                webpage, 'brightcove embed', default=None)
+            if bc:
+                bc_attr = extract_attributes(bc)
+                bc_url = self.BRIGHTCOVE_URL_TEMPLATE % (
+                    bc_attr.get('data-bcaccountid', '3162030207001'),
+                    bc_attr.get('data-noprebcplayerid', 'SyGGpuJy3g'),
+                    bc_attr.get('data-newbcplayerid', 'default'),
+                    bc_attr['data-bcid'])
+                return self.url_result(smuggle_url(bc_url, {'source_url': url}))
+
         # We first look for clipid, because clipprog always appears before
         patterns = [r'id=\'clip(%s)\'\s*value=\'([0-9]+)\'' % t for t in ('id', 'prog')]
         results = list(filter(None, (re.search(p, webpage) for p in patterns)))

From b972fb037bc4a07694df6c010a25f12aedb9f731 Mon Sep 17 00:00:00 2001
From: Rasmus Rendal <rasmusrendal@gmail.com>
Date: Sun, 7 May 2017 13:04:23 +0200
Subject: [PATCH 0831/1696] [drtv] Lower preference for SignLanguage formats
 (closes #13013)

---
 youtube_dl/extractor/drtv.py | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index e4917014a..e6f72ac50 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -45,6 +45,18 @@ class DRTVIE(InfoExtractor):
             'upload_date': '20160902',
             'duration': 131.4,
         },
+    }, {
+        'url': 'https://www.dr.dk/tv/se/historien-om-danmark/-/historien-om-danmark-stenalder',
+        'md5': '7c8ca12e6c3d3e3edd59ba5a9b7ca10a',
+        'info_dict': {
+            'id': 'historien-om-danmark-stenalder',
+            'ext': 'mp4',
+            'title': 'Historien om Danmark: Stenalder (1)',
+            'description': 'Én fascinerende historie om tusindvis af år, hvor vores land bliver skabt ud af is og vand, og hvor de første danskere ankommer til vores egn. Det bliver en rejse ind i urtiden og det liv, som urtidsjægerne har levet i skovene og ved havet og helt frem til bondestenalderen. Gennem skeletfund afslører eksperter, hvordan vores forfædre har set ud i stenalderen og hvorfor stenaldermennesket byggede de imponerende jættestuer, som ligger overalt i det danske.',
+            'timestamp': 1490401996,
+            'upload_date': '20170325',
+            'duration': 3502.04,
+        },
     }]
 
     def _real_extract(self, url):
@@ -85,7 +97,11 @@ class DRTVIE(InfoExtractor):
             kind = asset.get('Kind')
             if kind == 'Image':
                 thumbnail = asset.get('Uri')
-            elif kind in ('VideoResource', 'AudioResource'):
+            preference = 0
+
+            sign_language = asset.get('Target') == 'SignLanguage'
+
+            if kind in ('VideoResource', 'AudioResource'):
                 duration = float_or_none(asset.get('DurationInMilliseconds'), 1000)
                 restricted_to_denmark = asset.get('RestrictedToDenmark')
                 spoken_subtitles = asset.get('Target') == 'SpokenSubtitles'
@@ -95,10 +111,12 @@ class DRTVIE(InfoExtractor):
                         continue
                     target = link.get('Target')
                     format_id = target or ''
-                    preference = None
                     if spoken_subtitles:
                         preference = -1
                         format_id += '-spoken-subtitles'
+                    if sign_language:
+                        preference = -1
+                        format_id += "-sign-language"
                     if target == 'HDS':
                         f4m_formats = self._extract_f4m_formats(
                             uri + '?hdcore=3.3.0&plugin=aasp-3.3.0.99.43',

From 8d65880e247d32e47dfb42616b0734639d4c070f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 May 2017 15:37:09 +0700
Subject: [PATCH 0832/1696] [drtv] Improve extraction and update tests (closes
 #13013, closes #13016)

---
 youtube_dl/extractor/drtv.py | 36 +++++++++++++++++-------------------
 1 file changed, 17 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index e6f72ac50..c84624f1e 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -20,7 +20,7 @@ class DRTVIE(InfoExtractor):
     IE_NAME = 'drtv'
     _TESTS = [{
         'url': 'https://www.dr.dk/tv/se/boern/ultra/klassen-ultra/klassen-darlig-taber-10',
-        'md5': '25e659cccc9a2ed956110a299fdf5983',
+        'md5': '7ae17b4e18eb5d29212f424a7511c184',
         'info_dict': {
             'id': 'klassen-darlig-taber-10',
             'ext': 'mp4',
@@ -30,32 +30,36 @@ class DRTVIE(InfoExtractor):
             'upload_date': '20160823',
             'duration': 606.84,
         },
-        'params': {
-            'skip_download': True,
-        },
     }, {
+        # embed
         'url': 'https://www.dr.dk/nyheder/indland/live-christianias-rydning-af-pusher-street-er-i-gang',
-        'md5': '2c37175c718155930f939ef59952474a',
         'info_dict': {
             'id': 'christiania-pusher-street-ryddes-drdkrjpo',
             'ext': 'mp4',
             'title': 'LIVE Christianias rydning af Pusher Street er i gang',
-            'description': '- Det er det fedeste, der er sket i 20 år, fortæller christianit til DR Nyheder.',
+            'description': 'md5:2a71898b15057e9b97334f61d04e6eb5',
             'timestamp': 1472800279,
             'upload_date': '20160902',
             'duration': 131.4,
         },
+        'params': {
+            'skip_download': True,
+        },
     }, {
+        # with SignLanguage formats
         'url': 'https://www.dr.dk/tv/se/historien-om-danmark/-/historien-om-danmark-stenalder',
-        'md5': '7c8ca12e6c3d3e3edd59ba5a9b7ca10a',
         'info_dict': {
             'id': 'historien-om-danmark-stenalder',
             'ext': 'mp4',
             'title': 'Historien om Danmark: Stenalder (1)',
-            'description': 'Én fascinerende historie om tusindvis af år, hvor vores land bliver skabt ud af is og vand, og hvor de første danskere ankommer til vores egn. Det bliver en rejse ind i urtiden og det liv, som urtidsjægerne har levet i skovene og ved havet og helt frem til bondestenalderen. Gennem skeletfund afslører eksperter, hvordan vores forfædre har set ud i stenalderen og hvorfor stenaldermennesket byggede de imponerende jættestuer, som ligger overalt i det danske.',
+            'description': 'md5:8c66dcbc1669bbc6f873879880f37f2a',
             'timestamp': 1490401996,
             'upload_date': '20170325',
             'duration': 3502.04,
+            'formats': 'mincount:20',
+        },
+        'params': {
+            'skip_download': True,
         },
     }]
 
@@ -97,26 +101,20 @@ class DRTVIE(InfoExtractor):
             kind = asset.get('Kind')
             if kind == 'Image':
                 thumbnail = asset.get('Uri')
-            preference = 0
-
-            sign_language = asset.get('Target') == 'SignLanguage'
-
-            if kind in ('VideoResource', 'AudioResource'):
+            elif kind in ('VideoResource', 'AudioResource'):
                 duration = float_or_none(asset.get('DurationInMilliseconds'), 1000)
                 restricted_to_denmark = asset.get('RestrictedToDenmark')
-                spoken_subtitles = asset.get('Target') == 'SpokenSubtitles'
+                asset_target = asset.get('Target')
                 for link in asset.get('Links', []):
                     uri = link.get('Uri')
                     if not uri:
                         continue
                     target = link.get('Target')
                     format_id = target or ''
-                    if spoken_subtitles:
+                    preference = None
+                    if asset_target in ('SpokenSubtitles', 'SignLanguage'):
                         preference = -1
-                        format_id += '-spoken-subtitles'
-                    if sign_language:
-                        preference = -1
-                        format_id += "-sign-language"
+                        format_id += '-%s' % asset_target
                     if target == 'HDS':
                         f4m_formats = self._extract_f4m_formats(
                             uri + '?hdcore=3.3.0&plugin=aasp-3.3.0.99.43',

From 5d0968f0af3ce2a7da9a5f3098c6436f07c661aa Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 9 May 2017 11:14:29 +0100
Subject: [PATCH 0833/1696] [packtpub] add support for authentication(closes
 #12622)

---
 youtube_dl/extractor/packtpub.py | 39 +++++++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/packtpub.py b/youtube_dl/extractor/packtpub.py
index 881f3bcc7..bb668c999 100644
--- a/youtube_dl/extractor/packtpub.py
+++ b/youtube_dl/extractor/packtpub.py
@@ -3,7 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_str
+from ..compat import (
+    compat_str,
+    compat_HTTPError,
+)
 from ..utils import (
     clean_html,
     ExtractorError,
@@ -11,6 +14,7 @@ from ..utils import (
     strip_or_none,
     unified_timestamp,
     urljoin,
+    urlencode_postdata,
 )
 
 
@@ -34,6 +38,32 @@ class PacktPubIE(PacktPubBaseIE):
             'upload_date': '20170331',
         },
     }
+    _NETRC_MACHINE = 'packtpub'
+    _TOKEN = None
+
+    def _real_initialize(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+        webpage = self._download_webpage(self._PACKT_BASE, None)
+        login_form = self._form_hidden_inputs(
+            'packt-user-login-form', webpage)
+        login_form.update({
+            'email': username,
+            'password': password,
+        })
+        self._download_webpage(
+            self._PACKT_BASE, None, 'Logging in as %s' % username,
+            data=urlencode_postdata(login_form))
+        try:
+            self._TOKEN = self._download_json(
+                '%s/users/tokens/sessions' % self._MAPT_REST, None,
+                'Downloading Authorization Token')['data']['token']
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code in (401, 404):
+                message = self._parse_json(e.cause.read().decode(), None)['message']
+                raise ExtractorError(message, expected=True)
+            raise
 
     def _handle_error(self, response):
         if response.get('status') != 'success':
@@ -51,14 +81,17 @@ class PacktPubIE(PacktPubBaseIE):
         course_id, chapter_id, video_id = mobj.group(
             'course_id', 'chapter_id', 'id')
 
+        headers = {}
+        if self._TOKEN:
+            headers['Authorization'] = self._TOKEN
         video = self._download_json(
             '%s/users/me/products/%s/chapters/%s/sections/%s'
             % (self._MAPT_REST, course_id, chapter_id, video_id), video_id,
-            'Downloading JSON video')['data']
+            'Downloading JSON video', headers=headers)['data']
 
         content = video.get('content')
         if not content:
-            raise ExtractorError('This video is locked', expected=True)
+            self.raise_login_required('This video is locked')
 
         video_url = content['file']
 

From 6d1ded75021ca76f14c4fe8d3aa698704bae1dd9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 May 2017 23:07:01 +0700
Subject: [PATCH 0834/1696] [francetv] Adapt to site redesign (closes #13034)

---
 youtube_dl/extractor/extractors.py |   4 +-
 youtube_dl/extractor/francetv.py   | 217 +++++++++++------------------
 2 files changed, 86 insertions(+), 135 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 968cca9d2..61a6f1013 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -350,9 +350,9 @@ from .foxsports import FoxSportsIE
 from .franceculture import FranceCultureIE
 from .franceinter import FranceInterIE
 from .francetv import (
-    PluzzIE,
-    FranceTvInfoIE,
     FranceTVIE,
+    FranceTVEmbedIE,
+    FranceTVInfoIE,
     GenerationQuoiIE,
     CultureboxIE,
 )
diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 48d43ae58..6e1a537a0 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -21,11 +21,13 @@ from .dailymotion import (
 
 
 class FranceTVBaseInfoExtractor(InfoExtractor):
-    def _extract_video(self, video_id, catalogue):
+    def _extract_video(self, video_id, catalogue=None):
         info = self._download_json(
-            'http://webservices.francetelevisions.fr/tools/getInfosOeuvre/v2/?idDiffusion=%s&catalogue=%s'
-            % (video_id, catalogue),
-            video_id, 'Downloading video JSON')
+            'https://sivideo.webservices.francetelevisions.fr/tools/getInfosOeuvre/v2/',
+            video_id, 'Downloading video JSON', query={
+                'idDiffusion': video_id,
+                'catalogue': catalogue or '',
+            })
 
         if info.get('status') == 'NOK':
             raise ExtractorError(
@@ -109,27 +111,94 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
         }
 
 
-class PluzzIE(FranceTVBaseInfoExtractor):
-    IE_NAME = 'pluzz.francetv.fr'
-    _VALID_URL = r'https?://(?:m\.)?pluzz\.francetv\.fr/videos/(?P<id>.+?)\.html'
+class FranceTVIE(FranceTVBaseInfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?france\.tv/(?:[^/]+/)+(?P<id>[^/]+)\.html'
 
-    # Can't use tests, videos expire in 7 days
+    _TESTS = [{
+        'url': 'https://www.france.tv/france-2/13h15-le-dimanche/140921-les-mysteres-de-jesus.html',
+        'info_dict': {
+            'id': '157550144',
+            'ext': 'mp4',
+            'title': '13h15, le dimanche... - Les mystères de Jésus',
+            'description': 'md5:75efe8d4c0a8205e5904498ffe1e1a42',
+            'timestamp': 1494156300,
+            'upload_date': '20170507',
+        },
+        'params': {
+            # m3u8 downloads
+            'skip_download': True,
+        },
+    }, {
+        # france3
+        'url': 'https://www.france.tv/france-3/des-chiffres-et-des-lettres/139063-emission-du-mardi-9-mai-2017.html',
+        'only_matching': True,
+    }, {
+        # france4
+        'url': 'https://www.france.tv/france-4/hero-corp/saison-1/134151-apres-le-calme.html',
+        'only_matching': True,
+    }, {
+        # france5
+        'url': 'https://www.france.tv/france-5/c-a-dire/saison-10/137013-c-a-dire.html',
+        'only_matching': True,
+    }, {
+        # franceo
+        'url': 'https://www.france.tv/france-o/archipels/132249-mon-ancetre-l-esclave.html',
+        'only_matching': True,
+    }, {
+        # france2 live
+        'url': 'https://www.france.tv/france-2/direct.html',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.france.tv/documentaires/histoire/136517-argentine-les-500-bebes-voles-de-la-dictature.html',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.france.tv/jeux-et-divertissements/divertissements/133965-le-web-contre-attaque.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
 
         webpage = self._download_webpage(url, display_id)
 
-        video_id = self._html_search_meta(
-            'id_video', webpage, 'video id', default=None)
+        catalogue = None
+        video_id = self._search_regex(
+            r'data-main-video=(["\'])(?P<id>(?:(?!\1).)+)\1',
+            webpage, 'video id', default=None, group='id')
+
         if not video_id:
-            video_id = self._search_regex(
-                r'data-diffusion=["\'](\d+)', webpage, 'video id')
-
-        return self._extract_video(video_id, 'Pluzz')
+            video_id, catalogue = self._html_search_regex(
+                r'(?:href=|player\.setVideo\(\s*)"http://videos?\.francetv\.fr/video/([^@]+@[^"]+)"',
+                webpage, 'video ID').split('@')
+        return self._extract_video(video_id, catalogue)
 
 
-class FranceTvInfoIE(FranceTVBaseInfoExtractor):
+class FranceTVEmbedIE(FranceTVBaseInfoExtractor):
+    _VALID_URL = r'https?://embed\.francetv\.fr/*\?.*?\bue=(?P<id>[^&]+)'
+
+    _TEST = {
+        'url': 'http://embed.francetv.fr/?ue=7fd581a2ccf59d2fc5719c5c13cf6961',
+        'info_dict': {
+            'id': 'NI_983319',
+            'ext': 'mp4',
+            'title': 'Le Pen Reims',
+            'upload_date': '20170505',
+            'timestamp': 1493981780,
+            'duration': 16,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        video = self._download_json(
+            'http://api-embed.webservices.francetelevisions.fr/key/%s' % video_id,
+            video_id)
+
+        return self._extract_video(video['video_id'], video.get('catalog'))
+
+
+class FranceTVInfoIE(FranceTVBaseInfoExtractor):
     IE_NAME = 'francetvinfo.fr'
     _VALID_URL = r'https?://(?:www|mobile|france3-regions)\.francetvinfo\.fr/(?:[^/]+/)*(?P<title>[^/?#&.]+)'
 
@@ -233,124 +302,6 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):
         return self._extract_video(video_id, catalogue)
 
 
-class FranceTVIE(FranceTVBaseInfoExtractor):
-    IE_NAME = 'francetv'
-    IE_DESC = 'France 2, 3, 4, 5 and Ô'
-    _VALID_URL = r'''(?x)
-                    https?://
-                        (?:
-                            (?:www\.)?france[2345o]\.fr/
-                                (?:
-                                    emissions/[^/]+/(?:videos|diffusions)|
-                                    emission/[^/]+|
-                                    videos|
-                                    jt
-                                )
-                            /|
-                            embed\.francetv\.fr/\?ue=
-                        )
-                        (?P<id>[^/?]+)
-                    '''
-
-    _TESTS = [
-        # france2
-        {
-            'url': 'http://www.france2.fr/emissions/13h15-le-samedi-le-dimanche/videos/75540104',
-            'md5': 'c03fc87cb85429ffd55df32b9fc05523',
-            'info_dict': {
-                'id': '109169362',
-                'ext': 'flv',
-                'title': '13h15, le dimanche...',
-                'description': 'md5:9a0932bb465f22d377a449be9d1a0ff7',
-                'upload_date': '20140914',
-                'timestamp': 1410693600,
-            },
-        },
-        # france3
-        {
-            'url': 'http://www.france3.fr/emissions/pieces-a-conviction/diffusions/13-11-2013_145575',
-            'md5': '679bb8f8921f8623bd658fa2f8364da0',
-            'info_dict': {
-                'id': '000702326_CAPP_PicesconvictionExtrait313022013_120220131722_Au',
-                'ext': 'mp4',
-                'title': 'Le scandale du prix des médicaments',
-                'description': 'md5:1384089fbee2f04fc6c9de025ee2e9ce',
-                'upload_date': '20131113',
-                'timestamp': 1384380000,
-            },
-        },
-        # france4
-        {
-            'url': 'http://www.france4.fr/emissions/hero-corp/videos/rhozet_herocorp_bonus_1_20131106_1923_06112013172108_F4',
-            'md5': 'a182bf8d2c43d88d46ec48fbdd260c1c',
-            'info_dict': {
-                'id': 'rhozet_herocorp_bonus_1_20131106_1923_06112013172108_F4',
-                'ext': 'mp4',
-                'title': 'Hero Corp Making of - Extrait 1',
-                'description': 'md5:c87d54871b1790679aec1197e73d650a',
-                'upload_date': '20131106',
-                'timestamp': 1383766500,
-            },
-        },
-        # france5
-        {
-            'url': 'http://www.france5.fr/emissions/c-a-dire/videos/quels_sont_les_enjeux_de_cette_rentree_politique__31-08-2015_908948?onglet=tous&page=1',
-            'md5': 'f6c577df3806e26471b3d21631241fd0',
-            'info_dict': {
-                'id': '123327454',
-                'ext': 'flv',
-                'title': 'C à dire ?! - Quels sont les enjeux de cette rentrée politique ?',
-                'description': 'md5:4a0d5cb5dce89d353522a84462bae5a4',
-                'upload_date': '20150831',
-                'timestamp': 1441035120,
-            },
-        },
-        # franceo
-        {
-            'url': 'http://www.franceo.fr/jt/info-soir/18-07-2015',
-            'md5': '47d5816d3b24351cdce512ad7ab31da8',
-            'info_dict': {
-                'id': '125377621',
-                'ext': 'flv',
-                'title': 'Infô soir',
-                'description': 'md5:01b8c6915a3d93d8bbbd692651714309',
-                'upload_date': '20150718',
-                'timestamp': 1437241200,
-                'duration': 414,
-            },
-        },
-        {
-            # francetv embed
-            'url': 'http://embed.francetv.fr/?ue=8d7d3da1e3047c42ade5a5d7dfd3fc87',
-            'info_dict': {
-                'id': 'EV_30231',
-                'ext': 'flv',
-                'title': 'Alcaline, le concert avec Calogero',
-                'description': 'md5:61f08036dcc8f47e9cfc33aed08ffaff',
-                'upload_date': '20150226',
-                'timestamp': 1424989860,
-                'duration': 5400,
-            },
-        },
-        {
-            'url': 'http://www.france4.fr/emission/highlander/diffusion-du-17-07-2015-04h05',
-            'only_matching': True,
-        },
-        {
-            'url': 'http://www.franceo.fr/videos/125377617',
-            'only_matching': True,
-        }
-    ]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        video_id, catalogue = self._html_search_regex(
-            r'(?:href=|player\.setVideo\(\s*)"http://videos?\.francetv\.fr/video/([^@]+@[^"]+)"',
-            webpage, 'video ID').split('@')
-        return self._extract_video(video_id, catalogue)
-
-
 class GenerationQuoiIE(InfoExtractor):
     IE_NAME = 'france2.fr:generation-quoi'
     _VALID_URL = r'https?://generation-quoi\.france2\.fr/portrait/(?P<id>[^/?#]+)'

From cbd84b58175144fa898c5dfd32c58d37ffaee2c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 May 2017 23:17:22 +0700
Subject: [PATCH 0835/1696] [ChangeLog] Actualize

---
 ChangeLog | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 5418bb95f..5137266f6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,26 @@
+version <unreleased>
+
+Core
+* [YoutubeDL] Force --restrict-filenames when no locale is set on all python
+  versions (#13027)
+
+Extractors
+* [francetv] Adapt to site redesign (#13034)
++ [packtpub] Add support for authentication (#12622)
+* [drtv] Lower preference for SignLanguage formats (#13013, #13016)
++ [cspan] Add support for brightcove live embeds (#13028)
+* [vrv] Extract DASH formats and subtitles
+* [funimation] Fix authentication (#13021)
+* [adultswim] Fix extraction (#8640, #10950, #11042, #12121)
+    + Add support for Adobe Pass authentication
+    + Add support for live streams
+    + Add support for show pages
+* [turner] Extract thumbnail, is_live and strip description
++ [nonktube] Add support for nonktube.com (#8647, #13024)
++ [nuevo] Pass headers to _extract_nuevo
+* [nbc] Improve extraction (#12364)
+
+
 version 2017.05.07
 
 Common

From a7ed6b341cce27b6b5731969d39711b15dca8d25 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 May 2017 04:20:13 +0700
Subject: [PATCH 0836/1696] release 2017.05.09

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 5 +++--
 youtube_dl/version.py     | 2 +-
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 86d11e142..f7742ddcf 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.07*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.07**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.09*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.09**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.05.07
+[debug] youtube-dl version 2017.05.09
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 5137266f6..4775f8695 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.05.09
 
 Core
 * [YoutubeDL] Force --restrict-filenames when no locale is set on all python
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index d867dfe65..2f27686b3 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -281,7 +281,8 @@
  - **france2.fr:generation-quoi**
  - **FranceCulture**
  - **FranceInter**
- - **francetv**: France 2, 3, 4, 5 and Ô
+ - **FranceTV**
+ - **FranceTVEmbed**
  - **francetvinfo.fr**
  - **Freesound**
  - **freespeech.org**
@@ -530,6 +531,7 @@
  - **NJPWWorld**: 新日本プロレスワールド
  - **NobelPrize**
  - **Noco**
+ - **NonkTube**
  - **Noovo**
  - **Normalboots**
  - **NosVideo**
@@ -602,7 +604,6 @@
  - **pluralsight**
  - **pluralsight:course**
  - **plus.google**: Google Plus
- - **pluzz.francetv.fr**
  - **podomatic**
  - **Pokemon**
  - **PolskieRadio**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index dc707071a..33a0c2130 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.05.07'
+__version__ = '2017.05.09'

From 39ee2638194cb40bfc6652ecb2fca2455e87e327 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 10 May 2017 08:50:30 +0100
Subject: [PATCH 0837/1696] use platform=desktop in assets url(closes #13041)

---
 youtube_dl/extractor/adultswim.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 9c37ea33c..acc4ce38d 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -126,7 +126,7 @@ class AdultSwimIE(TurnerBaseIE):
             video_id = video_data['id']
 
         info = self._extract_cvp_info(
-            'http://www.adultswim.com/videos/api/v0/assets?id=' + video_id,
+            'http://www.adultswim.com/videos/api/v0/assets?platform=desktop&id=' + video_id,
             video_id, {
                 'secure': {
                     'media_src': 'http://androidhls-secure.cdn.turner.com/adultswim/big',

From 3166b1f0ac16769190f6ab1e7e5b0e45bb63d482 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 May 2017 22:35:10 +0700
Subject: [PATCH 0838/1696] [myspace] Improve _VALID_URL (closes #13040)

---
 youtube_dl/extractor/myspace.py | 100 ++++++++++++++++----------------
 1 file changed, 49 insertions(+), 51 deletions(-)

diff --git a/youtube_dl/extractor/myspace.py b/youtube_dl/extractor/myspace.py
index f281238c9..e164d5940 100644
--- a/youtube_dl/extractor/myspace.py
+++ b/youtube_dl/extractor/myspace.py
@@ -12,64 +12,62 @@ from ..utils import (
 
 
 class MySpaceIE(InfoExtractor):
-    _VALID_URL = r'https?://myspace\.com/([^/]+)/(?P<mediatype>video/[^/]+/|music/song/.*?)(?P<id>\d+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        myspace\.com/[^/]+/
+                        (?P<mediatype>
+                            video/[^/]+/(?P<video_id>\d+)|
+                            music/song/[^/?#&]+-(?P<song_id>\d+)-\d+(?:[/?#&]|$)
+                        )
+                    '''
 
-    _TESTS = [
-        {
-            'url': 'https://myspace.com/fiveminutestothestage/video/little-big-town/109594919',
-            'md5': '9c1483c106f4a695c47d2911feed50a7',
-            'info_dict': {
-                'id': '109594919',
-                'ext': 'mp4',
-                'title': 'Little Big Town',
-                'description': 'This country quartet was all smiles while playing a sold out show at the Pacific Amphitheatre in Orange County, California.',
-                'uploader': 'Five Minutes to the Stage',
-                'uploader_id': 'fiveminutestothestage',
-                'timestamp': 1414108751,
-                'upload_date': '20141023',
-            },
+    _TESTS = [{
+        'url': 'https://myspace.com/fiveminutestothestage/video/little-big-town/109594919',
+        'md5': '9c1483c106f4a695c47d2911feed50a7',
+        'info_dict': {
+            'id': '109594919',
+            'ext': 'mp4',
+            'title': 'Little Big Town',
+            'description': 'This country quartet was all smiles while playing a sold out show at the Pacific Amphitheatre in Orange County, California.',
+            'uploader': 'Five Minutes to the Stage',
+            'uploader_id': 'fiveminutestothestage',
+            'timestamp': 1414108751,
+            'upload_date': '20141023',
         },
+    }, {
         # songs
-        {
-            'url': 'https://myspace.com/killsorrow/music/song/of-weakened-soul...-93388656-103880681',
-            'md5': '1d7ee4604a3da226dd69a123f748b262',
-            'info_dict': {
-                'id': '93388656',
-                'ext': 'm4a',
-                'title': 'Of weakened soul...',
-                'uploader': 'Killsorrow',
-                'uploader_id': 'killsorrow',
-            },
-        }, {
-            'add_ie': ['Youtube'],
-            'url': 'https://myspace.com/threedaysgrace/music/song/animal-i-have-become-28400208-28218041',
-            'info_dict': {
-                'id': 'xqds0B_meys',
-                'ext': 'webm',
-                'title': 'Three Days Grace - Animal I Have Become',
-                'description': 'md5:8bd86b3693e72a077cf863a8530c54bb',
-                'uploader': 'ThreeDaysGraceVEVO',
-                'uploader_id': 'ThreeDaysGraceVEVO',
-                'upload_date': '20091002',
-            },
-        }, {
-            'add_ie': ['Youtube'],
-            'url': 'https://myspace.com/starset2/music/song/first-light-95799905-106964426',
-            'info_dict': {
-                'id': 'ypWvQgnJrSU',
-                'ext': 'mp4',
-                'title': 'Starset - First Light',
-                'description': 'md5:2d5db6c9d11d527683bcda818d332414',
-                'uploader': 'Yumi K',
-                'uploader_id': 'SorenPromotions',
-                'upload_date': '20140725',
-            }
+        'url': 'https://myspace.com/killsorrow/music/song/of-weakened-soul...-93388656-103880681',
+        'md5': '1d7ee4604a3da226dd69a123f748b262',
+        'info_dict': {
+            'id': '93388656',
+            'ext': 'm4a',
+            'title': 'Of weakened soul...',
+            'uploader': 'Killsorrow',
+            'uploader_id': 'killsorrow',
         },
-    ]
+    }, {
+        'add_ie': ['Youtube'],
+        'url': 'https://myspace.com/threedaysgrace/music/song/animal-i-have-become-28400208-28218041',
+        'info_dict': {
+            'id': 'xqds0B_meys',
+            'ext': 'webm',
+            'title': 'Three Days Grace - Animal I Have Become',
+            'description': 'md5:8bd86b3693e72a077cf863a8530c54bb',
+            'uploader': 'ThreeDaysGraceVEVO',
+            'uploader_id': 'ThreeDaysGraceVEVO',
+            'upload_date': '20091002',
+        },
+    }, {
+        'url': 'https://myspace.com/starset2/music/song/first-light-95799905-106964426',
+        'only_matching': True,
+    }, {
+        'url': 'https://myspace.com/thelargemouthbassband/music/song/02-pure-eyes.mp3-94422330-105113388',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = mobj.group('video_id') or mobj.group('song_id')
         is_song = mobj.group('mediatype').startswith('music/song')
         webpage = self._download_webpage(url, video_id)
         player_url = self._search_regex(

From 7274f3d0e9e7d76d00fa3d438df1e1dc43143435 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 12 May 2017 18:44:10 +0800
Subject: [PATCH 0839/1696] [douyu] Support Douyu shows (closes #12228)

---
 ChangeLog                          |  6 +++
 youtube_dl/extractor/douyutv.py    | 82 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  5 +-
 3 files changed, 92 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 4775f8695..43431ac93 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractor
++ [douyu] Support Douyu shows (#12228)
+
+
 version 2017.05.09
 
 Core
diff --git a/youtube_dl/extractor/douyutv.py b/youtube_dl/extractor/douyutv.py
index d22133d24..9757f4422 100644
--- a/youtube_dl/extractor/douyutv.py
+++ b/youtube_dl/extractor/douyutv.py
@@ -3,11 +3,14 @@ from __future__ import unicode_literals
 
 import time
 import hashlib
+import re
 
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     unescapeHTML,
+    unified_strdate,
+    urljoin,
 )
 
 
@@ -117,3 +120,82 @@ class DouyuTVIE(InfoExtractor):
             'uploader': uploader,
             'is_live': True,
         }
+
+
+class DouyuShowIE(InfoExtractor):
+    _VALID_URL = r'https?://v(?:mobile)?\.douyu\.com/show/(?P<id>[0-9a-zA-Z]+)'
+
+    _TESTS = [{
+        'url': 'https://v.douyu.com/show/rjNBdvnVXNzvE2yw',
+        'md5': '0c2cfd068ee2afe657801269b2d86214',
+        'info_dict': {
+            'id': 'rjNBdvnVXNzvE2yw',
+            'ext': 'mp4',
+            'title': '陈一发儿：砒霜 我有个室友系列！04-01 22点场',
+            'duration': 7150.08,
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'uploader': '陈一发儿',
+            'uploader_id': 'XrZwYelr5wbK',
+            'uploader_url': 'https://v.douyu.com/author/XrZwYelr5wbK',
+            'upload_date': '20170402',
+        },
+    }, {
+        'url': 'https://vmobile.douyu.com/show/rjNBdvnVXNzvE2yw',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        url = url.replace('vmobile.', 'v.')
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        room_info = self._parse_json(self._search_regex(
+            r'var\s+\$ROOM\s*=\s*({.+});', webpage, 'room info'), video_id)
+
+        video_info = None
+
+        for trial in range(5):
+            # Sometimes Douyu rejects our request. Let's try it more times
+            try:
+                video_info = self._download_json(
+                    'https://vmobile.douyu.com/video/getInfo', video_id,
+                    query={'vid': video_id},
+                    headers={
+                        'Referer': url,
+                        'x-requested-with': 'XMLHttpRequest',
+                    })
+                break
+            except ExtractorError:
+                self._sleep(1, video_id)
+
+        if not video_info:
+            raise ExtractorError('Can\'t fetch video info')
+
+        formats = self._extract_m3u8_formats(
+            video_info['data']['video_url'], video_id,
+            entry_protocol='m3u8_native', ext='mp4')
+
+        upload_date = unified_strdate(self._html_search_regex(
+            r'<em>上传时间：</em><span>([^<]+)</span>', webpage,
+            'upload date', fatal=False))
+
+        uploader = uploader_id = uploader_url = None
+        mobj = re.search(
+            r'(?m)<a[^>]+href="/author/([0-9a-zA-Z]+)".+?<strong[^>]+title="([^"]+)"',
+            webpage)
+        if mobj:
+            uploader_id, uploader = mobj.groups()
+            uploader_url = urljoin(url, '/author/' + uploader_id)
+
+        return {
+            'id': video_id,
+            'title': room_info['name'],
+            'formats': formats,
+            'duration': room_info.get('duration'),
+            'thumbnail': room_info.get('pic'),
+            'upload_date': upload_date,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'uploader_url': uploader_url,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 61a6f1013..510c35fcb 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -251,7 +251,10 @@ from .democracynow import DemocracynowIE
 from .dfb import DFBIE
 from .dhm import DHMIE
 from .dotsub import DotsubIE
-from .douyutv import DouyuTVIE
+from .douyutv import (
+    DouyuShowIE,
+    DouyuTVIE,
+)
 from .dplay import (
     DPlayIE,
     DPlayItIE,

From 116283ff640ec77de7359ddaffaa8dfbb50f6e17 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 12 May 2017 19:15:33 +0800
Subject: [PATCH 0840/1696] [liveleak] Fix extraction (#12053)

---
 ChangeLog                        |  1 +
 youtube_dl/extractor/liveleak.py | 83 +++++++++++++-------------------
 2 files changed, 34 insertions(+), 50 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 43431ac93..96e4ce8a1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractor
+* [liveleak] Fix extraction (#12053)
 + [douyu] Support Douyu shows (#12228)
 
 
diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index c7de65353..c54519636 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import json
 import re
 
 from .common import InfoExtractor
@@ -11,10 +10,10 @@ class LiveLeakIE(InfoExtractor):
     _VALID_URL = r'https?://(?:\w+\.)?liveleak\.com/view\?(?:.*?)i=(?P<id>[\w_]+)(?:.*)'
     _TESTS = [{
         'url': 'http://www.liveleak.com/view?i=757_1364311680',
-        'md5': '50f79e05ba149149c1b4ea961223d5b3',
+        'md5': '0813c2430bea7a46bf13acf3406992f4',
         'info_dict': {
             'id': '757_1364311680',
-            'ext': 'flv',
+            'ext': 'mp4',
             'description': 'extremely bad day for this guy..!',
             'uploader': 'ljfriel2',
             'title': 'Most unlucky car accident',
@@ -22,7 +21,7 @@ class LiveLeakIE(InfoExtractor):
         }
     }, {
         'url': 'http://www.liveleak.com/view?i=f93_1390833151',
-        'md5': 'b13a29626183c9d33944e6a04f41aafc',
+        'md5': 'd3f1367d14cc3c15bf24fbfbe04b9abf',
         'info_dict': {
             'id': 'f93_1390833151',
             'ext': 'mp4',
@@ -32,6 +31,7 @@ class LiveLeakIE(InfoExtractor):
             'thumbnail': r're:^https?://.*\.jpg$'
         }
     }, {
+        # Prochan embed
         'url': 'http://www.liveleak.com/view?i=4f7_1392687779',
         'md5': '42c6d97d54f1db107958760788c5f48f',
         'info_dict': {
@@ -41,11 +41,13 @@ class LiveLeakIE(InfoExtractor):
             'uploader': 'CapObveus',
             'title': 'Man is Fatally Struck by Reckless Car While Packing up a Moving Truck',
             'age_limit': 18,
-        }
+        },
+        'skip': 'Video is dead',
     }, {
         # Covers https://github.com/rg3/youtube-dl/pull/5983
+        # Multiple resolutions
         'url': 'http://www.liveleak.com/view?i=801_1409392012',
-        'md5': '0b3bec2d888c20728ca2ad3642f0ef15',
+        'md5': 'c3a449dbaca5c0d1825caecd52a57d7b',
         'info_dict': {
             'id': '801_1409392012',
             'ext': 'mp4',
@@ -93,57 +95,38 @@ class LiveLeakIE(InfoExtractor):
             webpage, 'age limit', default=None))
         video_thumbnail = self._og_search_thumbnail(webpage)
 
-        sources_raw = self._search_regex(
-            r'(?s)sources:\s*(\[.*?\]),', webpage, 'video URLs', default=None)
-        if sources_raw is None:
-            alt_source = self._search_regex(
-                r'(file: ".*?"),', webpage, 'video URL', default=None)
-            if alt_source:
-                sources_raw = '[{ %s}]' % alt_source
-            else:
-                # Maybe an embed?
-                embed_url = self._search_regex(
-                    r'<iframe[^>]+src="(https?://(?:www\.)?(?:prochan|youtube)\.com/embed[^"]+)"',
-                    webpage, 'embed URL')
-                return {
-                    '_type': 'url_transparent',
-                    'url': embed_url,
-                    'id': video_id,
-                    'title': video_title,
-                    'description': video_description,
-                    'uploader': video_uploader,
-                    'age_limit': age_limit,
-                }
+        entries = self._parse_html5_media_entries(url, webpage, video_id)
+        if not entries:
+            # Maybe an embed?
+            embed_url = self._search_regex(
+                r'<iframe[^>]+src="((?:https?:)?//(?:www\.)?(?:prochan|youtube)\.com/embed[^"]+)"',
+                webpage, 'embed URL')
+            return {
+                '_type': 'url_transparent',
+                'url': embed_url,
+                'id': video_id,
+                'title': video_title,
+                'description': video_description,
+                'uploader': video_uploader,
+                'age_limit': age_limit,
+            }
 
-        sources_json = re.sub(r'\s([a-z]+):\s', r'"\1": ', sources_raw)
-        sources = json.loads(sources_json)
+        info_dict = entries[0]
 
-        formats = [{
-            'format_id': '%s' % i,
-            'format_note': s.get('label'),
-            'url': s['file'],
-        } for i, s in enumerate(sources)]
+        for a_format in info_dict['formats']:
+            if not a_format.get('height'):
+                a_format['height'] = self._search_regex(
+                    r'([0-9]+)p\.mp4', a_format['url'], 'height label', default=None)
 
-        for i, s in enumerate(sources):
-            # Removing '.h264_*.mp4' gives the raw video, which is essentially
-            # the same video without the LiveLeak logo at the top (see
-            # https://github.com/rg3/youtube-dl/pull/4768)
-            orig_url = re.sub(r'\.h264_.+?\.mp4', '', s['file'])
-            if s['file'] != orig_url:
-                formats.append({
-                    'format_id': 'original-%s' % i,
-                    'format_note': s.get('label'),
-                    'url': orig_url,
-                    'preference': 1,
-                })
-        self._sort_formats(formats)
+        self._sort_formats(info_dict['formats'])
 
-        return {
+        info_dict.update({
             'id': video_id,
             'title': video_title,
             'description': video_description,
             'uploader': video_uploader,
-            'formats': formats,
             'age_limit': age_limit,
             'thumbnail': video_thumbnail,
-        }
+        })
+
+        return info_dict

From 6c52477f595176d16f063407bcf44439b9c26022 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 12 May 2017 16:17:32 +0100
Subject: [PATCH 0841/1696] [condenast] improve embed support

---
 youtube_dl/extractor/condenast.py | 81 ++++++++++++++++++++-----------
 youtube_dl/extractor/generic.py   | 16 ------
 2 files changed, 54 insertions(+), 43 deletions(-)

diff --git a/youtube_dl/extractor/condenast.py b/youtube_dl/extractor/condenast.py
index d3463b874..0c3f0c0e4 100644
--- a/youtube_dl/extractor/condenast.py
+++ b/youtube_dl/extractor/condenast.py
@@ -16,7 +16,6 @@ from ..utils import (
     mimetype2ext,
     orderedSet,
     parse_iso8601,
-    remove_end,
 )
 
 
@@ -50,10 +49,17 @@ class CondeNastIE(InfoExtractor):
         'wmagazine': 'W Magazine',
     }
 
-    _VALID_URL = r'https?://(?:video|www|player)\.(?P<site>%s)\.com/(?P<type>watch|series|video|embed(?:js)?)/(?P<id>[^/?#]+)' % '|'.join(_SITES.keys())
+    _VALID_URL = r'''(?x)https?://(?:video|www|player(?:-backend)?)\.(?:%s)\.com/
+        (?:
+            (?:
+                embed(?:js)?|
+                (?:script|inline)/video
+            )/(?P<id>[0-9a-f]{24})(?:/(?P<player_id>[0-9a-f]{24}))?(?:.+?\btarget=(?P<target>[^&]+))?|
+            (?P<type>watch|series|video)/(?P<display_id>[^/?#]+)
+        )''' % '|'.join(_SITES.keys())
     IE_DESC = 'Condé Nast media group: %s' % ', '.join(sorted(_SITES.values()))
 
-    EMBED_URL = r'(?:https?:)?//player\.(?P<site>%s)\.com/(?P<type>embed(?:js)?)/.+?' % '|'.join(_SITES.keys())
+    EMBED_URL = r'(?:https?:)?//player(?:-backend)?\.(?:%s)\.com/(?:embed(?:js)?|(?:script|inline)/video)/.+?' % '|'.join(_SITES.keys())
 
     _TESTS = [{
         'url': 'http://video.wired.com/watch/3d-printed-speakers-lit-with-led',
@@ -89,6 +95,12 @@ class CondeNastIE(InfoExtractor):
             'upload_date': '20150916',
             'timestamp': 1442434955,
         }
+    }, {
+        'url': 'https://player.cnevids.com/inline/video/59138decb57ac36b83000005.js?target=js-cne-player',
+        'only_matching': True,
+    }, {
+        'url': 'http://player-backend.cnevids.com/script/video/59138decb57ac36b83000005.js',
+        'only_matching': True,
     }]
 
     def _extract_series(self, url, webpage):
@@ -104,7 +116,7 @@ class CondeNastIE(InfoExtractor):
         entries = [self.url_result(build_url(path), 'CondeNast') for path in paths]
         return self.playlist_result(entries, playlist_title=title)
 
-    def _extract_video(self, webpage, url_type):
+    def _extract_video_params(self, webpage):
         query = {}
         params = self._search_regex(
             r'(?s)var params = {(.+?)}[;,]', webpage, 'player params', default=None)
@@ -123,17 +135,30 @@ class CondeNastIE(InfoExtractor):
                 'playerId': params['data-player'],
                 'target': params['id'],
             })
-        video_id = query['videoId']
+        return query
+
+    def _extract_video(self, params):
+        video_id = params['videoId']
+
         video_info = None
-        info_page = self._download_json(
-            'http://player.cnevids.com/player/video.js',
-            video_id, 'Downloading video info', fatal=False, query=query)
-        if info_page:
-            video_info = info_page.get('video')
-        if not video_info:
+        if params.get('playerId'):
+            info_page = self._download_json(
+                'http://player.cnevids.com/player/video.js',
+                video_id, 'Downloading video info', fatal=False, query=params)
+            if info_page:
+                video_info = info_page.get('video')
+            if not video_info:
+                info_page = self._download_webpage(
+                    'http://player.cnevids.com/player/loader.js',
+                    video_id, 'Downloading loader info', query=params)
+        else:
             info_page = self._download_webpage(
-                'http://player.cnevids.com/player/loader.js',
-                video_id, 'Downloading loader info', query=query)
+                'https://player.cnevids.com/inline/video/%s.js' % video_id,
+                video_id, 'Downloading inline info', query={
+                    'target': params.get('target', 'embedplayer')
+                })
+
+        if not video_info:
             video_info = self._parse_json(
                 self._search_regex(
                     r'(?s)var\s+config\s*=\s*({.+?});', info_page, 'config'),
@@ -161,9 +186,7 @@ class CondeNastIE(InfoExtractor):
             })
         self._sort_formats(formats)
 
-        info = self._search_json_ld(
-            webpage, video_id, fatal=False) if url_type != 'embed' else {}
-        info.update({
+        return {
             'id': video_id,
             'formats': formats,
             'title': title,
@@ -174,22 +197,26 @@ class CondeNastIE(InfoExtractor):
             'series': video_info.get('series_title'),
             'season': video_info.get('season_title'),
             'timestamp': parse_iso8601(video_info.get('premiere_date')),
-        })
-        return info
+            'categories': video_info.get('categories'),
+        }
 
     def _real_extract(self, url):
-        site, url_type, item_id = re.match(self._VALID_URL, url).groups()
+        video_id, player_id, target, url_type, display_id = re.match(self._VALID_URL, url).groups()
 
-        # Convert JS embed to regular embed
-        if url_type == 'embedjs':
-            parsed_url = compat_urlparse.urlparse(url)
-            url = compat_urlparse.urlunparse(parsed_url._replace(
-                path=remove_end(parsed_url.path, '.js').replace('/embedjs/', '/embed/')))
-            url_type = 'embed'
+        if video_id:
+            return self._extract_video({
+                'videoId': video_id,
+                'playerId': player_id,
+                'target': target,
+            })
 
-        webpage = self._download_webpage(url, item_id)
+        webpage = self._download_webpage(url, display_id)
 
         if url_type == 'series':
             return self._extract_series(url, webpage)
         else:
-            return self._extract_video(webpage, url_type)
+            params = self._extract_video_params(webpage)
+            info = self._search_json_ld(
+                webpage, display_id, fatal=False)
+            info.update(self._extract_video(params))
+            return info
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index b06f43446..635043c9e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2146,22 +2146,6 @@ class GenericIE(InfoExtractor):
         if svt_url:
             return self.url_result(svt_url, 'SVT')
 
-        # Look for embedded condenast player
-        matches = re.findall(
-            r'<iframe\s+(?:[a-zA-Z-]+="[^"]+"\s+)*?src="(https?://player\.cnevids\.com/embed/[^"]+")',
-            webpage)
-        if matches:
-            return {
-                '_type': 'playlist',
-                'entries': [{
-                    '_type': 'url',
-                    'ie_key': 'CondeNast',
-                    'url': ma,
-                } for ma in matches],
-                'title': video_title,
-                'id': video_id,
-            }
-
         # Look for Bandcamp pages with custom domain
         mobj = re.search(r'<meta property="og:url"[^>]*?content="(.*?bandcamp\.com.*?)"', webpage)
         if mobj is not None:

From 7ad4362357efe2d1b769c8cfd758af81e3131261 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 12 May 2017 16:32:38 +0100
Subject: [PATCH 0842/1696] [thescene] fix extraction(closes #13061)

---
 youtube_dl/extractor/thescene.py | 36 ++++++--------------------------
 1 file changed, 6 insertions(+), 30 deletions(-)

diff --git a/youtube_dl/extractor/thescene.py b/youtube_dl/extractor/thescene.py
index b8504f0eb..cd642355c 100644
--- a/youtube_dl/extractor/thescene.py
+++ b/youtube_dl/extractor/thescene.py
@@ -3,10 +3,6 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 
 from ..compat import compat_urlparse
-from ..utils import (
-    int_or_none,
-    qualities,
-)
 
 
 class TheSceneIE(InfoExtractor):
@@ -24,6 +20,9 @@ class TheSceneIE(InfoExtractor):
             'season': 'Ready To Wear Spring 2013',
             'tags': list,
             'categories': list,
+            'upload_date': '20120913',
+            'timestamp': 1347512400,
+            'uploader': 'vogue',
         },
     }
 
@@ -37,32 +36,9 @@ class TheSceneIE(InfoExtractor):
             self._html_search_regex(
                 r'id=\'js-player-script\'[^>]+src=\'(.+?)\'', webpage, 'player url'))
 
-        player = self._download_webpage(player_url, display_id)
-        info = self._parse_json(
-            self._search_regex(
-                r'(?m)video\s*:\s*({.+?}),$', player, 'info json'),
-            display_id)
-
-        video_id = info['id']
-        title = info['title']
-
-        qualities_order = qualities(('low', 'high'))
-        formats = [{
-            'format_id': '{0}-{1}'.format(f['type'].split('/')[0], f['quality']),
-            'url': f['src'],
-            'quality': qualities_order(f['quality']),
-        } for f in info['sources']]
-        self._sort_formats(formats)
-
         return {
-            'id': video_id,
+            '_type': 'url_transparent',
             'display_id': display_id,
-            'title': title,
-            'formats': formats,
-            'thumbnail': info.get('poster_frame'),
-            'duration': int_or_none(info.get('duration')),
-            'series': info.get('series_title'),
-            'season': info.get('season_title'),
-            'tags': info.get('tags'),
-            'categories': info.get('categories'),
+            'url': player_url,
+            'ie_key': 'CondeNast',
         }

From 58bb4402834bc3ed5ec0046ee5a8d71e56532134 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 May 2017 21:51:58 +0700
Subject: [PATCH 0843/1696] [extractor/generic] Extract wistia embed code into
 separate method

---
 youtube_dl/extractor/generic.py | 30 +++++-------------------------
 youtube_dl/extractor/wistia.py  | 22 ++++++++++++++++++++++
 2 files changed, 27 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 635043c9e..8f9097945 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -88,6 +88,7 @@ from .rutube import RutubeIE
 from .limelight import LimelightBaseIE
 from .anvato import AnvatoIE
 from .washingtonpost import WashingtonPostIE
+from .wistia import WistiaIE
 
 
 class GenericIE(InfoExtractor):
@@ -2111,36 +2112,15 @@ class GenericIE(InfoExtractor):
                     playlists, video_id, video_title, lambda p: '//dailymotion.com/playlist/%s' % p)
 
         # Look for embedded Wistia player
-        match = re.search(
-            r'<(?:meta[^>]+?content|iframe[^>]+?src)=(["\'])(?P<url>(?:https?:)?//(?:fast\.)?wistia\.net/embed/iframe/.+?)\1', webpage)
-        if match:
-            embed_url = self._proto_relative_url(
-                unescapeHTML(match.group('url')))
+        wistia_url = WistiaIE._extract_url(webpage)
+        if wistia_url:
             return {
                 '_type': 'url_transparent',
-                'url': embed_url,
-                'ie_key': 'Wistia',
+                'url': self._proto_relative_url(wistia_url),
+                'ie_key': WistiaIE.ie_key(),
                 'uploader': video_uploader,
             }
 
-        match = re.search(r'(?:id=["\']wistia_|data-wistia-?id=["\']|Wistia\.embed\(["\'])(?P<id>[^"\']+)', webpage)
-        if match:
-            return {
-                '_type': 'url_transparent',
-                'url': 'wistia:%s' % match.group('id'),
-                'ie_key': 'Wistia',
-                'uploader': video_uploader,
-            }
-
-        match = re.search(
-            r'''(?sx)
-                <script[^>]+src=(["'])(?:https?:)?//fast\.wistia\.com/assets/external/E-v1\.js\1[^>]*>.*?
-                <div[^>]+class=(["']).*?\bwistia_async_(?P<id>[a-z0-9]+)\b.*?\2
-            ''', webpage)
-        if match:
-            return self.url_result(self._proto_relative_url(
-                'wistia:%s' % match.group('id')), 'Wistia')
-
         # Look for SVT player
         svt_url = SVTIE._extract_url(webpage)
         if svt_url:
diff --git a/youtube_dl/extractor/wistia.py b/youtube_dl/extractor/wistia.py
index c634b8dec..2182d6fd4 100644
--- a/youtube_dl/extractor/wistia.py
+++ b/youtube_dl/extractor/wistia.py
@@ -1,10 +1,13 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     int_or_none,
     float_or_none,
+    unescapeHTML,
 )
 
 
@@ -34,6 +37,25 @@ class WistiaIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_url(webpage):
+        match = re.search(
+            r'<(?:meta[^>]+?content|iframe[^>]+?src)=(["\'])(?P<url>(?:https?:)?//(?:fast\.)?wistia\.net/embed/iframe/.+?)\1', webpage)
+        if match:
+            return unescapeHTML(match.group('url'))
+
+        match = re.search(r'(?:id=["\']wistia_|data-wistia-?id=["\']|Wistia\.embed\(["\'])(?P<id>[^"\']+)', webpage)
+        if match:
+            return 'wistia:%s' % match.group('id')
+
+        match = re.search(
+            r'''(?sx)
+                <script[^>]+src=(["'])(?:https?:)?//fast\.wistia\.com/assets/external/E-v1\.js\1[^>]*>.*?
+                <div[^>]+class=(["']).*?\bwistia_async_(?P<id>[a-z0-9]+)\b.*?\2
+            ''', webpage)
+        if match:
+            return 'wistia:%s' % match.group('id')
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From 7fc60f4ee9129c67aecacd6f9164e98dbf7601f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 May 2017 21:52:59 +0700
Subject: [PATCH 0844/1696] [upskill] Add extractor (closes #13043)

---
 youtube_dl/extractor/extractors.py |   4 +
 youtube_dl/extractor/upskill.py    | 176 +++++++++++++++++++++++++++++
 2 files changed, 180 insertions(+)
 create mode 100644 youtube_dl/extractor/upskill.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 510c35fcb..811519925 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1102,6 +1102,10 @@ from .uplynk import (
     UplynkIE,
     UplynkPreplayIE,
 )
+from .upskill import (
+    UpskillIE,
+    UpskillCourseIE,
+)
 from .urort import UrortIE
 from .urplay import URPlayIE
 from .usanetwork import USANetworkIE
diff --git a/youtube_dl/extractor/upskill.py b/youtube_dl/extractor/upskill.py
new file mode 100644
index 000000000..30297b4dd
--- /dev/null
+++ b/youtube_dl/extractor/upskill.py
@@ -0,0 +1,176 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from .wistia import WistiaIE
+from ..compat import compat_str
+from ..utils import (
+    clean_html,
+    ExtractorError,
+    get_element_by_class,
+    urlencode_postdata,
+    urljoin,
+)
+
+
+class UpskillBaseIE(InfoExtractor):
+    _LOGIN_URL = 'http://upskillcourses.com/sign_in'
+    _NETRC_MACHINE = 'upskill'
+
+    def _real_initialize(self):
+        self._login()
+
+    def _login(self):
+        username, password = self._get_login_info()
+        if username is None:
+            return
+
+        login_page, urlh = self._download_webpage_handle(
+            self._LOGIN_URL, None, 'Downloading login page')
+
+        login_url = compat_str(urlh.geturl())
+
+        login_form = self._hidden_inputs(login_page)
+
+        login_form.update({
+            'user[email]': username,
+            'user[password]': password,
+        })
+
+        post_url = self._search_regex(
+            r'<form[^>]+action=(["\'])(?P<url>(?:(?!\1).)+)\1', login_page,
+            'post url', default=login_url, group='url')
+
+        if not post_url.startswith('http'):
+            post_url = urljoin(login_url, post_url)
+
+        response = self._download_webpage(
+            post_url, None, 'Logging in',
+            data=urlencode_postdata(login_form),
+            headers={
+                'Content-Type': 'application/x-www-form-urlencoded',
+                'Referer': login_url,
+            })
+
+        # Successful login
+        if any(re.search(p, response) for p in (
+                r'class=["\']user-signout',
+                r'<a[^>]+\bhref=["\']/sign_out',
+                r'>\s*Log out\s*<')):
+            return
+
+        message = get_element_by_class('alert', response)
+        if message is not None:
+            raise ExtractorError(
+                'Unable to login: %s' % clean_html(message), expected=True)
+
+        raise ExtractorError('Unable to log in')
+
+
+class UpskillIE(UpskillBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?upskillcourses\.com/courses/[^/]+/lectures/(?P<id>\d+)'
+
+    _TESTS = [{
+        'url': 'http://upskillcourses.com/courses/essential-web-developer-course/lectures/1747100',
+        'info_dict': {
+            'id': 'uzw6zw58or',
+            'ext': 'mp4',
+            'title': 'Welcome to the Course!',
+            'description': 'md5:8d66c13403783370af62ca97a7357bdd',
+            'duration': 138.763,
+            'timestamp': 1479846621,
+            'upload_date': '20161122',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://upskillcourses.com/courses/119763/lectures/1747100',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        wistia_url = WistiaIE._extract_url(webpage)
+        if not wistia_url:
+            if any(re.search(p, webpage) for p in (
+                    r'class=["\']lecture-contents-locked',
+                    r'>\s*Lecture contents locked',
+                    r'id=["\']lecture-locked')):
+                self.raise_login_required('Lecture contents locked')
+
+        title = self._og_search_title(webpage, default=None)
+
+        return {
+            '_type': 'url_transparent',
+            'url': wistia_url,
+            'ie_key': WistiaIE.ie_key(),
+            'title': title,
+        }
+
+
+class UpskillCourseIE(UpskillBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?upskillcourses\.com/courses/(?:enrolled/)?(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'http://upskillcourses.com/courses/essential-web-developer-course/',
+        'info_dict': {
+            'id': '119763',
+            'title': 'The Essential Web Developer Course (Free)',
+        },
+        'playlist_count': 192,
+    }, {
+        'url': 'http://upskillcourses.com/courses/119763/',
+        'only_matching': True,
+    }, {
+        'url': 'http://upskillcourses.com/courses/enrolled/119763',
+        'only_matching': True,
+    }]
+
+    @classmethod
+    def suitable(cls, url):
+        return False if UpskillIE.suitable(url) else super(
+            UpskillCourseIE, cls).suitable(url)
+
+    def _real_extract(self, url):
+        course_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, course_id)
+
+        course_id = self._search_regex(
+            r'data-course-id=["\'](\d+)', webpage, 'course id',
+            default=course_id)
+
+        entries = []
+
+        for mobj in re.finditer(
+                r'(?s)(?P<li><li[^>]+class=(["\'])(?:(?!\2).)*?section-item[^>]+>.+?</li>)',
+                webpage):
+            li = mobj.group('li')
+            if 'fa-youtube-play' not in li:
+                continue
+            lecture_url = self._search_regex(
+                r'<a[^>]+href=(["\'])(?P<url>(?:(?!\1).)+)\1', li,
+                'lecture url', default=None, group='url')
+            if not lecture_url:
+                continue
+            lecture_id = self._search_regex(
+                r'/lectures/(\d+)', lecture_url, 'lecture id', default=None)
+            title = self._html_search_regex(
+                r'<span[^>]+class=["\']lecture-name[^>]+>([^<]+)', li,
+                'title', default=None)
+            entries.append(
+                self.url_result(
+                    urljoin('http://upskillcourses.com/', lecture_url),
+                    ie=UpskillIE.ie_key(), video_id=lecture_id,
+                    video_title=clean_html(title)))
+
+        course_title = self._html_search_regex(
+            (r'(?s)<img[^>]+class=["\']course-image[^>]+>\s*<h\d>(.+?)</h',
+             r'(?s)<h\d[^>]+class=["\']course-title[^>]+>(.+?)</h'),
+            webpage, 'course title', fatal=False)
+
+        return self.playlist_result(entries, course_id, course_title)

From 12f01118b0bd621e3e3df813a774fa0a9c1cfd73 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 May 2017 21:57:00 +0700
Subject: [PATCH 0845/1696] [francetv] Add support for mobile.france.tv (closes
 #13068)

---
 youtube_dl/extractor/francetv.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 6e1a537a0..546d5caa0 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -112,7 +112,7 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
 
 
 class FranceTVIE(FranceTVBaseInfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?france\.tv/(?:[^/]+/)+(?P<id>[^/]+)\.html'
+    _VALID_URL = r'https?://(?:(?:www\.)?france\.tv|mobile\.france\.tv)/(?:[^/]+/)+(?P<id>[^/]+)\.html'
 
     _TESTS = [{
         'url': 'https://www.france.tv/france-2/13h15-le-dimanche/140921-les-mysteres-de-jesus.html',
@@ -154,6 +154,9 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
     }, {
         'url': 'https://www.france.tv/jeux-et-divertissements/divertissements/133965-le-web-contre-attaque.html',
         'only_matching': True,
+    }, {
+        'url': 'https://mobile.france.tv/france-5/c-dans-l-air/137347-emission-du-vendredi-12-mai-2017.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From fa26734e0760549b7b727017106e9d649bb695ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 00:03:15 +0700
Subject: [PATCH 0846/1696] [postprocessor/metadatafromtitle] Add support regex
 syntax for --metadata-from-title (closes #13065)

---
 youtube_dl/options.py                         | 11 ++++++-----
 youtube_dl/postprocessor/metadatafromtitle.py |  4 +++-
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 52309fb84..57f2e2c1e 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -814,11 +814,12 @@ def parseOpts(overrideArguments=None):
         '--metadata-from-title',
         metavar='FORMAT', dest='metafromtitle',
         help='Parse additional metadata like song title / artist from the video title. '
-             'The format syntax is the same as --output, '
-             'the parsed parameters replace existing values. '
-             'Additional templates: %(album)s, %(artist)s. '
-             'Example: --metadata-from-title "%(artist)s - %(title)s" matches a title like '
-             '"Coldplay - Paradise"')
+             'The format syntax is the same as --output. Regular expression with '
+             'named capture groups may also be used. '
+             'The parsed parameters replace existing values. '
+             'Example: --metadata-from-title "%(artist)s - %(title)s" matches a title like ' 
+             '"Coldplay - Paradise". '
+             'Example (regex): --metadata-from-title "(?P<artist>.+?) - (?P<title>.+)"')
     postproc.add_option(
         '--xattrs',
         action='store_true', dest='xattrs', default=False,
diff --git a/youtube_dl/postprocessor/metadatafromtitle.py b/youtube_dl/postprocessor/metadatafromtitle.py
index a7d637a3c..c73f02447 100644
--- a/youtube_dl/postprocessor/metadatafromtitle.py
+++ b/youtube_dl/postprocessor/metadatafromtitle.py
@@ -9,7 +9,9 @@ class MetadataFromTitlePP(PostProcessor):
     def __init__(self, downloader, titleformat):
         super(MetadataFromTitlePP, self).__init__(downloader)
         self._titleformat = titleformat
-        self._titleregex = self.format_to_regex(titleformat)
+        self._titleregex = (self.format_to_regex(titleformat)
+                            if re.search(r'%\(\w+\)s', titleformat)
+                            else titleformat)
 
     def format_to_regex(self, fmt):
         r"""

From b845766597ea360ea474fac6764694ab34e8ef60 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 00:32:50 +0700
Subject: [PATCH 0847/1696] [imdb] Relax _VALID_URL (closes #13056)

---
 youtube_dl/extractor/imdb.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py
index f95c00c73..3ff672a89 100644
--- a/youtube_dl/extractor/imdb.py
+++ b/youtube_dl/extractor/imdb.py
@@ -13,7 +13,7 @@ from ..utils import (
 class ImdbIE(InfoExtractor):
     IE_NAME = 'imdb'
     IE_DESC = 'Internet Movie Database trailers'
-    _VALID_URL = r'https?://(?:www|m)\.imdb\.com/(?:video/[^/]+/|title/tt\d+.*?#lb-|videoplayer/)vi(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www|m)\.imdb\.com/(?:video|title).+?[/-]vi(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.imdb.com/video/imdb/vi2524815897',
@@ -35,6 +35,9 @@ class ImdbIE(InfoExtractor):
     }, {
         'url': 'http://www.imdb.com/videoplayer/vi1562949145',
         'only_matching': True,
+    }, {
+        'url': 'http://www.imdb.com/title/tt4218696/videoplayer/vi2608641561',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 851a01aed6c8fd2d7fa0bccb9f7fca7b95f99f16 Mon Sep 17 00:00:00 2001
From: Luca Steeb <contact@luca-steeb.com>
Date: Sat, 13 May 2017 19:57:02 +0200
Subject: [PATCH 0848/1696] [aljazeera] Extend _VALID_URL

---
 youtube_dl/extractor/aljazeera.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/aljazeera.py b/youtube_dl/extractor/aljazeera.py
index 388e578d5..c68be3134 100644
--- a/youtube_dl/extractor/aljazeera.py
+++ b/youtube_dl/extractor/aljazeera.py
@@ -4,9 +4,9 @@ from .common import InfoExtractor
 
 
 class AlJazeeraIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?aljazeera\.com/programmes/.*?/(?P<id>[^/]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?aljazeera\.com/(?:programmes|video)/.*?/(?P<id>[^/]+)\.html'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.aljazeera.com/programmes/the-slum/2014/08/deliverance-201482883754237240.html',
         'info_dict': {
             'id': '3792260579001',
@@ -19,7 +19,10 @@ class AlJazeeraIE(InfoExtractor):
         },
         'add_ie': ['BrightcoveNew'],
         'skip': 'Not accessible from Travis CI server',
-    }
+    }, {
+        'url': 'http://www.aljazeera.com/video/news/2017/05/sierra-leone-709-carat-diamond-auctioned-170511100111930.html',
+        'only_matching': True,
+    }]
     BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/665003303001/default_default/index.html?videoId=%s'
 
     def _real_extract(self, url):

From efe9316703efa5dc1561dc1c5159a77c838f5ab0 Mon Sep 17 00:00:00 2001
From: phaer <phaer@phaer.org>
Date: Sun, 8 Jan 2017 15:49:54 +0100
Subject: [PATCH 0849/1696] [orf:radio] Fix extraction

Since oe1.orf.at has been updated, both ORF radios supported by youtube_dl
use the same API. This commit honors this fact by merging both extractors
into one.
---
 youtube_dl/extractor/extractors.py |   2 +-
 youtube_dl/extractor/orf.py        | 113 +++++++++++++----------------
 2 files changed, 53 insertions(+), 62 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 811519925..dc4f9f24b 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -736,8 +736,8 @@ from .openload import OpenloadIE
 from .ora import OraTVIE
 from .orf import (
     ORFTVthekIE,
-    ORFOE1IE,
     ORFFM4IE,
+    ORFOE1IE,
     ORFIPTVIE,
 )
 from .packtpub import (
diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index 1e2c54e68..10e4a2e1c 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -2,8 +2,6 @@
 from __future__ import unicode_literals
 
 import re
-import calendar
-import datetime
 
 from .common import InfoExtractor
 from ..compat import compat_str
@@ -144,77 +142,25 @@ class ORFTVthekIE(InfoExtractor):
         }
 
 
-class ORFOE1IE(InfoExtractor):
-    IE_NAME = 'orf:oe1'
-    IE_DESC = 'Radio Österreich 1'
-    _VALID_URL = r'https?://oe1\.orf\.at/(?:programm/|konsole\?.*?\btrack_id=)(?P<id>[0-9]+)'
-
-    # Audios on ORF radio are only available for 7 days, so we can't add tests.
-    _TESTS = [{
-        'url': 'http://oe1.orf.at/konsole?show=on_demand#?track_id=394211',
-        'only_matching': True,
-    }, {
-        'url': 'http://oe1.orf.at/konsole?show=ondemand&track_id=443608&load_day=/programm/konsole/tag/20160726',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        show_id = self._match_id(url)
-        data = self._download_json(
-            'http://oe1.orf.at/programm/%s/konsole' % show_id,
-            show_id
-        )
-
-        timestamp = datetime.datetime.strptime('%s %s' % (
-            data['item']['day_label'],
-            data['item']['time']
-        ), '%d.%m.%Y %H:%M')
-        unix_timestamp = calendar.timegm(timestamp.utctimetuple())
-
-        return {
-            'id': show_id,
-            'title': data['item']['title'],
-            'url': data['item']['url_stream'],
-            'ext': 'mp3',
-            'description': data['item'].get('info'),
-            'timestamp': unix_timestamp
-        }
-
-
-class ORFFM4IE(InfoExtractor):
-    IE_NAME = 'orf:fm4'
-    IE_DESC = 'radio FM4'
-    _VALID_URL = r'https?://fm4\.orf\.at/(?:7tage/?#|player/)(?P<date>[0-9]+)/(?P<show>\w+)'
-
-    _TEST = {
-        'url': 'http://fm4.orf.at/player/20160110/IS/',
-        'md5': '01e736e8f1cef7e13246e880a59ad298',
-        'info_dict': {
-            'id': '2016-01-10_2100_tl_54_7DaysSun13_11244',
-            'ext': 'mp3',
-            'title': 'Im Sumpf',
-            'description': 'md5:384c543f866c4e422a55f66a62d669cd',
-            'duration': 7173,
-            'timestamp': 1452456073,
-            'upload_date': '20160110',
-        },
-        'skip': 'Live streams on FM4 got deleted soon',
-    }
-
+class ORFRadioIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
+        station = mobj.group('station')
         show_date = mobj.group('date')
         show_id = mobj.group('show')
 
+        if station == 'fm4':
+            show_id = '4%s' % show_id
+
         data = self._download_json(
-            'http://audioapi.orf.at/fm4/json/2.0/broadcasts/%s/4%s' % (show_date, show_id),
+            'http://audioapi.orf.at/%s/api/json/current/broadcast/%s/%s' % (station, show_id, show_date),
             show_id
         )
 
         def extract_entry_dict(info, title, subtitle):
             return {
                 'id': info['loopStreamId'].replace('.mp3', ''),
-                'url': 'http://loopstream01.apa.at/?channel=fm4&id=%s' % info['loopStreamId'],
+                'url': 'http://loopstream01.apa.at/?channel=%s&id=%s' % (station, info['loopStreamId']),
                 'title': title,
                 'description': subtitle,
                 'duration': (info['end'] - info['start']) / 1000,
@@ -233,6 +179,51 @@ class ORFFM4IE(InfoExtractor):
         }
 
 
+class ORFFM4IE(ORFRadioIE):
+    IE_NAME = 'orf:fm4'
+    IE_DESC = 'radio FM4'
+    _VALID_URL = r'https?://(?P<station>fm4)\.orf\.at/(?:7tage/?#|player/)(?P<date>[0-9]+)/(?P<show>\w+)'
+
+    _TESTS = [
+        {
+            'url': 'http://fm4.orf.at/player/20170107/CC',
+            'md5': '2b0be47375432a7ef104453432a19212',
+            'info_dict': {
+                'id': '2017-01-07_2100_tl_54_7DaysSat18_31295',
+                'ext': 'mp3',
+                'title': 'Solid Steel Radioshow',
+                'description': 'Die Mixshow von Coldcut und Ninja Tune.',
+                'duration': 3599,
+                'timestamp': 1483819257,
+                'upload_date': '20170107',
+            },
+            'skip': 'Shows from ORF radios are only available for 7 days.'
+        }
+    ]
+
+
+class ORFOE1IE(ORFRadioIE):
+    IE_NAME = 'orf:oe1'
+    IE_DESC = 'Radio Österreich 1'
+    _VALID_URL = r'https?://(?P<station>oe1)\.orf\.at/(?:7tage/?#|player/)(?P<date>[0-9]+)/(?P<show>\w+)'
+
+    _TESTS = [
+       {
+            'url': 'http://oe1.orf.at/player/20170108/456544',
+            'md5': '34d8a6e67ea888293741c86a099b745b',
+            'info_dict': {
+                'id': '2017-01-08_0759_tl_51_7DaysSun6_256141',
+                'ext': 'mp3',
+                'title': 'Morgenjournal',
+                'duration': 609,
+                'timestamp': 1483858796,
+                'upload_date': '20170108',
+            },
+            'skip': 'Shows from ORF radios are only available for 7 days.'
+        }
+    ]
+
+
 class ORFIPTVIE(InfoExtractor):
     IE_NAME = 'orf:iptv'
     IE_DESC = 'iptv.ORF.at'

From 1339ecb2f8cfe96821ba6284cd470e846634f110 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 04:30:29 +0700
Subject: [PATCH 0850/1696] [orf:radio] Cleanup _VALID_URLs (closes #11643)

---
 youtube_dl/extractor/orf.py | 62 +++++++++++++++++--------------------
 1 file changed, 29 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index 10e4a2e1c..cc296eabd 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -182,46 +182,42 @@ class ORFRadioIE(InfoExtractor):
 class ORFFM4IE(ORFRadioIE):
     IE_NAME = 'orf:fm4'
     IE_DESC = 'radio FM4'
-    _VALID_URL = r'https?://(?P<station>fm4)\.orf\.at/(?:7tage/?#|player/)(?P<date>[0-9]+)/(?P<show>\w+)'
+    _VALID_URL = r'https?://(?P<station>fm4)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
 
-    _TESTS = [
-        {
-            'url': 'http://fm4.orf.at/player/20170107/CC',
-            'md5': '2b0be47375432a7ef104453432a19212',
-            'info_dict': {
-                'id': '2017-01-07_2100_tl_54_7DaysSat18_31295',
-                'ext': 'mp3',
-                'title': 'Solid Steel Radioshow',
-                'description': 'Die Mixshow von Coldcut und Ninja Tune.',
-                'duration': 3599,
-                'timestamp': 1483819257,
-                'upload_date': '20170107',
-            },
-            'skip': 'Shows from ORF radios are only available for 7 days.'
-        }
-    ]
+    _TEST = {
+        'url': 'http://fm4.orf.at/player/20170107/CC',
+        'md5': '2b0be47375432a7ef104453432a19212',
+        'info_dict': {
+            'id': '2017-01-07_2100_tl_54_7DaysSat18_31295',
+            'ext': 'mp3',
+            'title': 'Solid Steel Radioshow',
+            'description': 'Die Mixshow von Coldcut und Ninja Tune.',
+            'duration': 3599,
+            'timestamp': 1483819257,
+            'upload_date': '20170107',
+        },
+        'skip': 'Shows from ORF radios are only available for 7 days.'
+    }
 
 
 class ORFOE1IE(ORFRadioIE):
     IE_NAME = 'orf:oe1'
     IE_DESC = 'Radio Österreich 1'
-    _VALID_URL = r'https?://(?P<station>oe1)\.orf\.at/(?:7tage/?#|player/)(?P<date>[0-9]+)/(?P<show>\w+)'
+    _VALID_URL = r'https?://(?P<station>oe1)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
 
-    _TESTS = [
-       {
-            'url': 'http://oe1.orf.at/player/20170108/456544',
-            'md5': '34d8a6e67ea888293741c86a099b745b',
-            'info_dict': {
-                'id': '2017-01-08_0759_tl_51_7DaysSun6_256141',
-                'ext': 'mp3',
-                'title': 'Morgenjournal',
-                'duration': 609,
-                'timestamp': 1483858796,
-                'upload_date': '20170108',
-            },
-            'skip': 'Shows from ORF radios are only available for 7 days.'
-        }
-    ]
+    _TEST = {
+        'url': 'http://oe1.orf.at/player/20170108/456544',
+        'md5': '34d8a6e67ea888293741c86a099b745b',
+        'info_dict': {
+            'id': '2017-01-08_0759_tl_51_7DaysSun6_256141',
+            'ext': 'mp3',
+            'title': 'Morgenjournal',
+            'duration': 609,
+            'timestamp': 1483858796,
+            'upload_date': '20170108',
+        },
+        'skip': 'Shows from ORF radios are only available for 7 days.'
+    }
 
 
 class ORFIPTVIE(InfoExtractor):

From 0de136341a9b6384098e14e4a0a1a5be438abdf2 Mon Sep 17 00:00:00 2001
From: Timendum <timedum@gmail.com>
Date: Tue, 2 May 2017 16:17:15 +0200
Subject: [PATCH 0851/1696] [mediaset] Add extractor

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/mediaset.py   | 118 +++++++++++++++++++++++++++++
 2 files changed, 119 insertions(+)
 create mode 100644 youtube_dl/extractor/mediaset.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index dc4f9f24b..ed603eb29 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -545,6 +545,7 @@ from .mangomolo import (
 )
 from .matchtv import MatchTVIE
 from .mdr import MDRIE
+from .mediaset import MediasetIE
 from .medici import MediciIE
 from .meipai import MeipaiIE
 from .melonvod import MelonVODIE
diff --git a/youtube_dl/extractor/mediaset.py b/youtube_dl/extractor/mediaset.py
new file mode 100644
index 000000000..4b1489524
--- /dev/null
+++ b/youtube_dl/extractor/mediaset.py
@@ -0,0 +1,118 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import (
+    InfoExtractor,
+    ExtractorError
+)
+from ..utils import (
+    determine_ext,
+    parse_duration,
+    unified_strdate
+)
+
+
+class MediasetIE(InfoExtractor):
+    _VALID_URL = r'https?://www\.video\.mediaset\.it/(?:(?:video|on-demand)/(?:.+)_|player/playerIFrame(?:Twitter)?\.shtml\?id=)(?P<id>[0-9]+)(?:.html|&.+)'
+    _TESTS = [{
+        # full episode
+        'url': 'http://www.video.mediaset.it/video/hello_goodbye/full/quarta-puntata_661824.html',
+        'md5': '9b75534d42c44ecef7bf1ffeacb7f85d',
+        'info_dict': {
+            'id': '661824',
+            'ext': 'mp4',
+            'title': 'Quarta puntata',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'md5:7183696d6df570e3412a5ef74b27c5e2',
+            'uploader': 'mediaset'
+        }
+    }, {
+        # on demand
+        'url': 'http://www.video.mediaset.it/video/domenica_live/interviste/il-fenomeno-elettra-lamborghini_716283.html',
+        'md5': '81c57566bf2ee02e995f5342f079ca25',
+        'info_dict': {
+            'id': '716283',
+            'ext': 'mp4',
+            'title': 'Il fenomeno Elettra Lamborghini',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'md5:dabf0e7cf48fc6d0a3417b989028748a',
+            'uploader': 'mediaset'
+        }
+    }, {
+        # clip
+        'url': 'http://www.video.mediaset.it/video/gogglebox/clip/un-grande-classico-della-commedia-sexy_661680.html',
+        'md5': '189ca72fe399db80dbfa595a4abf42d0',
+        'info_dict': {
+            'id': '661680',
+            'ext': 'mp4',
+            'title': 'Un grande classico della commedia sexy',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'Un film che riesce a risvegliare i sensi di Gigi.',
+            'uploader': 'mediaset'
+        }
+    }, {
+        # iframe simple
+        'url': 'http://www.video.mediaset.it/player/playerIFrame.shtml?id=665924&autoplay=true',
+        'md5': '308430901e55e1ad83dddb4be2a4454a',
+        'info_dict': {
+            'id': '665924',
+            'ext': 'mp4',
+            'title': 'Gianna Nannini incontra i fan a Milano',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'La cantante parla del nuovo libro',
+            'uploader': 'mediaset'
+        }
+    }, {
+        # iframe twitter (from http://www.wittytv.it/se-prima-mi-fidavo-zero/)
+        'url': 'https://www.video.mediaset.it/player/playerIFrameTwitter.shtml?id=665104&playrelated=false&autoplay=false&related=true&hidesocial=true',
+        'md5': '6f53a834b3b5eac1ebc2037ccf7194d0',
+        'info_dict': {
+            'id': '665104',
+            'ext': 'mp4',
+            'title': '\"Se prima mi fidavo zero...\"',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'Una piccola anteprima della prossima puntata del Trono Classico',
+            'uploader': 'mediaset'
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        formats = []
+        uploader = None
+        categories = None
+
+        mediainfo = self._download_json(
+            'http://plr.video.mediaset.it/html/metainfo.sjson?id=%s' % video_id,
+            video_id, 'Downloading video info JSON').get('video')
+
+        if 'brand-info' in mediainfo:
+            uploader = mediainfo.get('brand-info').get('publisher')
+            categories = [mediainfo.get('brand-info').get('category')]
+
+        cnd = self._download_json(
+            'http://cdnsel01.mediaset.net/GetCdn.aspx?streamid=%s&format=json' % video_id,
+            video_id, 'Downloading video CND JSON')
+
+        if not cnd.get('videoList'):
+            raise ExtractorError('Video not found')
+
+        for media_url in cnd.get('videoList'):
+            formats.append({
+                'url': media_url,
+                'ext': determine_ext(media_url)
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': mediainfo.get('title'),
+            'formats': formats,
+            'description': mediainfo.get('short-description'),
+            'uploader': uploader,
+            'thumbnail': mediainfo.get('thumbnail'),
+            'duration': parse_duration(mediainfo.get('duration')),
+            'release_date': unified_strdate(mediainfo.get('production-date')),
+            'webpage_url': mediainfo.get('url'),
+            'categories': categories
+        }

From 56f9c77f0e715a26b749d494c99a3f0b076bbbf8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 05:27:51 +0700
Subject: [PATCH 0852/1696] [mediaset] Improve extraction (closes #12708,
 closes #12964)

---
 youtube_dl/extractor/mediaset.py | 128 ++++++++++++++-----------------
 1 file changed, 56 insertions(+), 72 deletions(-)

diff --git a/youtube_dl/extractor/mediaset.py b/youtube_dl/extractor/mediaset.py
index 4b1489524..89e23ca9b 100644
--- a/youtube_dl/extractor/mediaset.py
+++ b/youtube_dl/extractor/mediaset.py
@@ -1,19 +1,25 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .common import (
-    InfoExtractor,
-    ExtractorError
-)
+from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     determine_ext,
     parse_duration,
-    unified_strdate
+    try_get,
+    unified_strdate,
 )
 
 
 class MediasetIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.video\.mediaset\.it/(?:(?:video|on-demand)/(?:.+)_|player/playerIFrame(?:Twitter)?\.shtml\?id=)(?P<id>[0-9]+)(?:.html|&.+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:www\.)?video\.mediaset\.it/
+                        (?:
+                            (?:video|on-demand)/(?:[^/]+/)+[^/]+_|
+                            player/playerIFrame(?:Twitter)?\.shtml\?.*?\bid=
+                        )(?P<id>[0-9]+)
+                    '''
     _TESTS = [{
         # full episode
         'url': 'http://www.video.mediaset.it/video/hello_goodbye/full/quarta-puntata_661824.html',
@@ -22,97 +28,75 @@ class MediasetIE(InfoExtractor):
             'id': '661824',
             'ext': 'mp4',
             'title': 'Quarta puntata',
-            'thumbnail': 're:^https?://.*\.jpg$',
             'description': 'md5:7183696d6df570e3412a5ef74b27c5e2',
-            'uploader': 'mediaset'
-        }
-    }, {
-        # on demand
-        'url': 'http://www.video.mediaset.it/video/domenica_live/interviste/il-fenomeno-elettra-lamborghini_716283.html',
-        'md5': '81c57566bf2ee02e995f5342f079ca25',
-        'info_dict': {
-            'id': '716283',
-            'ext': 'mp4',
-            'title': 'Il fenomeno Elettra Lamborghini',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'description': 'md5:dabf0e7cf48fc6d0a3417b989028748a',
-            'uploader': 'mediaset'
-        }
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 1414,
+            'creator': 'mediaset',
+            'release_date': '20161107',
+            'series': 'Hello Goodbye',
+            'categories': ['reality'],
+        },
+        'expected_warnings': ['is not a supported codec'],
     }, {
         # clip
         'url': 'http://www.video.mediaset.it/video/gogglebox/clip/un-grande-classico-della-commedia-sexy_661680.html',
-        'md5': '189ca72fe399db80dbfa595a4abf42d0',
-        'info_dict': {
-            'id': '661680',
-            'ext': 'mp4',
-            'title': 'Un grande classico della commedia sexy',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'description': 'Un film che riesce a risvegliare i sensi di Gigi.',
-            'uploader': 'mediaset'
-        }
+        'only_matching': True,
     }, {
         # iframe simple
         'url': 'http://www.video.mediaset.it/player/playerIFrame.shtml?id=665924&autoplay=true',
-        'md5': '308430901e55e1ad83dddb4be2a4454a',
-        'info_dict': {
-            'id': '665924',
-            'ext': 'mp4',
-            'title': 'Gianna Nannini incontra i fan a Milano',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'description': 'La cantante parla del nuovo libro',
-            'uploader': 'mediaset'
-        }
+        'only_matching': True,
     }, {
         # iframe twitter (from http://www.wittytv.it/se-prima-mi-fidavo-zero/)
         'url': 'https://www.video.mediaset.it/player/playerIFrameTwitter.shtml?id=665104&playrelated=false&autoplay=false&related=true&hidesocial=true',
-        'md5': '6f53a834b3b5eac1ebc2037ccf7194d0',
-        'info_dict': {
-            'id': '665104',
-            'ext': 'mp4',
-            'title': '\"Se prima mi fidavo zero...\"',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'description': 'Una piccola anteprima della prossima puntata del Trono Classico',
-            'uploader': 'mediaset'
-        }
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
+        video_list = self._download_json(
+            'http://cdnsel01.mediaset.net/GetCdn.aspx',
+            video_id, 'Downloading video CDN JSON', query={
+                'streamid': video_id,
+                'format': 'json',
+            })['videoList']
+
         formats = []
-        uploader = None
-        categories = None
+        for format_url in video_list:
+            if '.ism' in format_url:
+                formats.extend(self._extract_ism_formats(
+                    format_url, video_id, ism_id='mss', fatal=False))
+            else:
+                formats.append({
+                    'url': format_url,
+                    'format_id': determine_ext(format_url),
+                })
+        self._sort_formats(formats)
 
         mediainfo = self._download_json(
-            'http://plr.video.mediaset.it/html/metainfo.sjson?id=%s' % video_id,
-            video_id, 'Downloading video info JSON').get('video')
+            'http://plr.video.mediaset.it/html/metainfo.sjson',
+            video_id, 'Downloading video info JSON', query={
+                'id': video_id,
+            })['video']
 
-        if 'brand-info' in mediainfo:
-            uploader = mediainfo.get('brand-info').get('publisher')
-            categories = [mediainfo.get('brand-info').get('category')]
+        title = mediainfo['title']
 
-        cnd = self._download_json(
-            'http://cdnsel01.mediaset.net/GetCdn.aspx?streamid=%s&format=json' % video_id,
-            video_id, 'Downloading video CND JSON')
-
-        if not cnd.get('videoList'):
-            raise ExtractorError('Video not found')
-
-        for media_url in cnd.get('videoList'):
-            formats.append({
-                'url': media_url,
-                'ext': determine_ext(media_url)
-            })
-        self._sort_formats(formats)
+        creator = try_get(
+            mediainfo, lambda x: x['brand-info']['publisher'], compat_str)
+        category = try_get(
+            mediainfo, lambda x: x['brand-info']['category'], compat_str)
+        categories = [category] if category else None
 
         return {
             'id': video_id,
-            'title': mediainfo.get('title'),
-            'formats': formats,
+            'title': title,
             'description': mediainfo.get('short-description'),
-            'uploader': uploader,
             'thumbnail': mediainfo.get('thumbnail'),
             'duration': parse_duration(mediainfo.get('duration')),
+            'creator': creator,
             'release_date': unified_strdate(mediainfo.get('production-date')),
             'webpage_url': mediainfo.get('url'),
-            'categories': categories
+            'series': mediainfo.get('brand-value'),
+            'categories': categories,
+            'formats': formats,
         }

From 76d5a36391480294b6c26437ccfa34cb43145a7e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 06:11:45 +0700
Subject: [PATCH 0853/1696] [extractor/common] Respect Width and Height
 attributes in ISM manifests

---
 youtube_dl/extractor/common.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b9ad8461a..0bbb1103b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2001,6 +2001,12 @@ class InfoExtractor(object):
             compat_etree_fromstring(ism.encode('utf-8')), urlh.geturl(), ism_id)
 
     def _parse_ism_formats(self, ism_doc, ism_url, ism_id=None):
+        """
+        Parse formats from ISM manifest.
+        References:
+         1. [MS-SSTR]: Smooth Streaming Protocol,
+            https://msdn.microsoft.com/en-us/library/ff469518.aspx
+        """
         if ism_doc.get('IsLive') == 'TRUE' or ism_doc.find('Protection') is not None:
             return []
 
@@ -2022,8 +2028,11 @@ class InfoExtractor(object):
                     self.report_warning('%s is not a supported codec' % fourcc)
                     continue
                 tbr = int(track.attrib['Bitrate']) // 1000
-                width = int_or_none(track.get('MaxWidth'))
-                height = int_or_none(track.get('MaxHeight'))
+                # [1] does not mention Width and Height attributes. However,
+                # they're often present while MaxWidth and MaxHeight are
+                # missing, so should be used as fallbacks
+                width = int_or_none(track.get('MaxWidth') or track.get('Width'))
+                height = int_or_none(track.get('MaxHeight') or track.get('Height'))
                 sampling_rate = int_or_none(track.get('SamplingRate'))
 
                 track_url_pattern = re.sub(r'{[Bb]itrate}', track.attrib['Bitrate'], url_pattern)

From 946826eec75aeb2fe0e9edc8e734b6bfd89ad7c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 06:17:34 +0700
Subject: [PATCH 0854/1696] [extractor/generic] Remove duplicate limelight code

---
 youtube_dl/extractor/generic.py | 23 -----------------------
 1 file changed, 23 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8f9097945..dde65fa96 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2519,29 +2519,6 @@ class GenericIE(InfoExtractor):
             return self.playlist_result(
                 limelight_urls, video_id, video_title, video_description)
 
-        mobj = re.search(r'LimelightPlayer\.doLoad(Media|Channel|ChannelList)\(["\'](?P<id>[a-z0-9]{32})', webpage)
-        if mobj:
-            lm = {
-                'Media': 'media',
-                'Channel': 'channel',
-                'ChannelList': 'channel_list',
-            }
-            return self.url_result(smuggle_url('limelight:%s:%s' % (
-                lm[mobj.group(1)], mobj.group(2)), {'source_url': url}),
-                'Limelight%s' % mobj.group(1), mobj.group(2))
-
-        mobj = re.search(
-            r'''(?sx)
-                <object[^>]+class=(["\'])LimelightEmbeddedPlayerFlash\1[^>]*>.*?
-                    <param[^>]+
-                        name=(["\'])flashVars\2[^>]+
-                        value=(["\'])(?:(?!\3).)*mediaId=(?P<id>[a-z0-9]{32})
-            ''', webpage)
-        if mobj:
-            return self.url_result(smuggle_url(
-                'limelight:media:%s' % mobj.group('id'),
-                {'source_url': url}), 'LimelightMedia', mobj.group('id'))
-
         # Look for Anvato embeds
         anvato_urls = AnvatoIE._extract_urls(self, webpage, video_id)
         if anvato_urls:

From ca04de463d17cc8bf53dafb19eb25e6dff8bd239 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 06:28:40 +0700
Subject: [PATCH 0855/1696] [mediaset] Add support for shortcut

---
 youtube_dl/extractor/mediaset.py | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/mediaset.py b/youtube_dl/extractor/mediaset.py
index 89e23ca9b..3ebc7f2e1 100644
--- a/youtube_dl/extractor/mediaset.py
+++ b/youtube_dl/extractor/mediaset.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
@@ -13,12 +15,15 @@ from ..utils import (
 
 class MediasetIE(InfoExtractor):
     _VALID_URL = r'''(?x)
-                    https?://
-                        (?:www\.)?video\.mediaset\.it/
-                        (?:
-                            (?:video|on-demand)/(?:[^/]+/)+[^/]+_|
-                            player/playerIFrame(?:Twitter)?\.shtml\?.*?\bid=
-                        )(?P<id>[0-9]+)
+                    (?:
+                        mediaset:|
+                        https?://
+                            (?:www\.)?video\.mediaset\.it/
+                            (?:
+                                (?:video|on-demand)/(?:[^/]+/)+[^/]+_|
+                                player/playerIFrame(?:Twitter)?\.shtml\?.*?\bid=
+                            )
+                    )(?P<id>[0-9]+)
                     '''
     _TESTS = [{
         # full episode
@@ -49,6 +54,9 @@ class MediasetIE(InfoExtractor):
         # iframe twitter (from http://www.wittytv.it/se-prima-mi-fidavo-zero/)
         'url': 'https://www.video.mediaset.it/player/playerIFrameTwitter.shtml?id=665104&playrelated=false&autoplay=false&related=true&hidesocial=true',
         'only_matching': True,
+    }, {
+        'url': 'mediaset:661824',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 5d29af3d15eaec8958a4fad00103e18112f0c172 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 06:29:16 +0700
Subject: [PATCH 0856/1696] [extractor/generic] Add support for mediaset embeds

---
 youtube_dl/extractor/generic.py  | 7 +++++++
 youtube_dl/extractor/mediaset.py | 8 ++++++++
 2 files changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index dde65fa96..32f38c401 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -89,6 +89,7 @@ from .limelight import LimelightBaseIE
 from .anvato import AnvatoIE
 from .washingtonpost import WashingtonPostIE
 from .wistia import WistiaIE
+from .mediaset import MediasetIE
 
 
 class GenericIE(InfoExtractor):
@@ -2648,6 +2649,12 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 wapo_urls, video_id, video_title, ie=WashingtonPostIE.ie_key())
 
+        # Look for Mediaset embeds
+        mediaset_urls = MediasetIE._extract_urls(webpage)
+        if mediaset_urls:
+            return self.playlist_from_matches(
+                mediaset_urls, video_id, video_title, ie=MediasetIE.ie_key())
+
         # Looking for http://schema.org/VideoObject
         json_ld = self._search_json_ld(
             webpage, video_id, default={}, expected_type='VideoObject')
diff --git a/youtube_dl/extractor/mediaset.py b/youtube_dl/extractor/mediaset.py
index 3ebc7f2e1..9a0e41663 100644
--- a/youtube_dl/extractor/mediaset.py
+++ b/youtube_dl/extractor/mediaset.py
@@ -59,6 +59,14 @@ class MediasetIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_urls(webpage):
+        return [
+            mobj.group('url')
+            for mobj in re.finditer(
+                r'<iframe\b[^>]+\bsrc=(["\'])(?P<url>https?://(?:www\.)?video\.mediaset\.it/player/playerIFrame(?:Twitter)?\.shtml\?.*?\bid=\d+.*?)\1',
+                webpage)]
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From d105a7edc6e06c4eaf92838217913dde18aaae05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 06:39:47 +0700
Subject: [PATCH 0857/1696] [mediaset] Fix upload date

---
 youtube_dl/extractor/mediaset.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mediaset.py b/youtube_dl/extractor/mediaset.py
index 9a0e41663..9760eafd5 100644
--- a/youtube_dl/extractor/mediaset.py
+++ b/youtube_dl/extractor/mediaset.py
@@ -37,7 +37,7 @@ class MediasetIE(InfoExtractor):
             'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 1414,
             'creator': 'mediaset',
-            'release_date': '20161107',
+            'upload_date': '20161107',
             'series': 'Hello Goodbye',
             'categories': ['reality'],
         },
@@ -110,7 +110,7 @@ class MediasetIE(InfoExtractor):
             'thumbnail': mediainfo.get('thumbnail'),
             'duration': parse_duration(mediainfo.get('duration')),
             'creator': creator,
-            'release_date': unified_strdate(mediainfo.get('production-date')),
+            'upload_date': unified_strdate(mediainfo.get('production-date')),
             'webpage_url': mediainfo.get('url'),
             'series': mediainfo.get('brand-value'),
             'categories': categories,

From 2b8e6a68f8cef6a5e2fd2987446bc19ddc20bbaa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 06:40:19 +0700
Subject: [PATCH 0858/1696] [extractor/generic] Add test for mediaset embed

---
 youtube_dl/extractor/generic.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 32f38c401..c108d4a8a 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1720,6 +1720,19 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': [WashingtonPostIE.ie_key()],
         },
+        {
+            # Mediaset embed
+            'url': 'http://www.tgcom24.mediaset.it/politica/serracchiani-voglio-vivere-in-una-societa-aperta-reazioni-sproporzionate-_3071354-201702a.shtml',
+            'info_dict': {
+                'id': '720642',
+                'ext': 'mp4',
+                'title': 'Serracchiani: "Voglio vivere in una società aperta, con tutela del patto di fiducia"',
+            },
+            'params': {
+                'skip_download': True,
+            },
+            'add_ie': [MediasetIE.ie_key()],
+        },
         # {
         #     # TODO: find another test
         #     # http://schema.org/VideoObject

From 166d12b00cfbc45f60f052c541f567c22a80bb85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 07:33:24 +0700
Subject: [PATCH 0859/1696] [options] PEP 8

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 57f2e2c1e..3021a6f41 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -817,7 +817,7 @@ def parseOpts(overrideArguments=None):
              'The format syntax is the same as --output. Regular expression with '
              'named capture groups may also be used. '
              'The parsed parameters replace existing values. '
-             'Example: --metadata-from-title "%(artist)s - %(title)s" matches a title like ' 
+             'Example: --metadata-from-title "%(artist)s - %(title)s" matches a title like '
              '"Coldplay - Paradise". '
              'Example (regex): --metadata-from-title "(?P<artist>.+?) - (?P<title>.+)"')
     postproc.add_option(

From 09b866e171a6cef8578a2a37f5bd766ae6bfbf50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 07:28:16 +0700
Subject: [PATCH 0860/1696] [ChangeLog] Actualize

---
 ChangeLog | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 96e4ce8a1..0dd213e6d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,8 +1,23 @@
 version <unreleased>
 
+Core
++ [extractor/common] Respect Width and Height attributes in ISM manifests
++ [postprocessor/metadatafromtitle] Add support regular expression syntax for
+  --metadata-from-title (#13065)
+
 Extractor
++ [mediaset] Add support for video.mediaset.it (#12708, #12964)
+* [orf:radio] Fix extraction (#11643, #12926)
+* [aljazeera] Extend URL regular expression (#13053)
+* [imdb] Relax URL regular expression (#13056)
++ [francetv] Add support for mobile.france.tv (#13068)
++ [upskill] Add support for upskillcourses.com (#13043)
+* [thescene] Fix extraction (#13061)
+* [condenast] Improve embed support
 * [liveleak] Fix extraction (#12053)
 + [douyu] Support Douyu shows (#12228)
+* [myspace] Improve URL regular expression (#13040)
+* [adultswim] Use desktop platform in assets URL (#13041)
 
 
 version 2017.05.09

From 6f3c632c24c6f5d1d653273c48e9a612bb04a873 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 07:38:40 +0700
Subject: [PATCH 0861/1696] release 2017.05.14

---
 .github/ISSUE_TEMPLATE.md |  6 +++---
 ChangeLog                 |  2 +-
 README.md                 | 14 ++++++++------
 docs/supportedsites.md    |  4 ++++
 youtube_dl/version.py     |  2 +-
 5 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index f7742ddcf..2955e833d 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.09*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.09**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.14*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.14**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.05.09
+[debug] youtube-dl version 2017.05.14
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 0dd213e6d..91c77515f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.05.14
 
 Core
 + [extractor/common] Respect Width and Height attributes in ISM manifests
diff --git a/README.md b/README.md
index 1ecd2005a..dc0be1f40 100644
--- a/README.md
+++ b/README.md
@@ -400,12 +400,14 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
     --add-metadata                   Write metadata to the video file
     --metadata-from-title FORMAT     Parse additional metadata like song title /
                                      artist from the video title. The format
-                                     syntax is the same as --output, the parsed
-                                     parameters replace existing values.
-                                     Additional templates: %(album)s,
-                                     %(artist)s. Example: --metadata-from-title
-                                     "%(artist)s - %(title)s" matches a title
-                                     like "Coldplay - Paradise"
+                                     syntax is the same as --output. Regular
+                                     expression with named capture groups may
+                                     also be used. The parsed parameters replace
+                                     existing values. Example: --metadata-from-
+                                     title "%(artist)s - %(title)s" matches a
+                                     title like "Coldplay - Paradise". Example
+                                     (regex): --metadata-from-title
+                                     "(?P<artist>.+?) - (?P<title>.+)"
     --xattrs                         Write metadata to the video file's xattrs
                                      (using dublin core and xdg standards)
     --fixup POLICY                   Automatically correct known faults of the
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 2f27686b3..aa6c118f2 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -216,6 +216,7 @@
  - **DiscoveryVR**
  - **Disney**
  - **Dotsub**
+ - **DouyuShow**
  - **DouyuTV**: 斗鱼
  - **DPlay**
  - **DPlayIt**
@@ -433,6 +434,7 @@
  - **MDR**: MDR.DE and KiKA
  - **media.ccc.de**
  - **Medialaan**
+ - **Mediaset**
  - **Medici**
  - **Meipai**: 美拍
  - **MelonVOD**
@@ -861,6 +863,8 @@
  - **uol.com.br**
  - **uplynk**
  - **uplynk:preplay**
+ - **Upskill**
+ - **UpskillCourse**
  - **Urort**: NRK P3 Urørt
  - **URPlay**
  - **USANetwork**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 33a0c2130..9b074fafc 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.05.09'
+__version__ = '2017.05.14'

From c360e641e9f3bd1c5111d03e5d9d90e5b6abdf02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 09:55:40 +0700
Subject: [PATCH 0862/1696] [dailymotion] Extend _VALID_URL (closes #13079)

---
 youtube_dl/extractor/dailymotion.py | 143 ++++++++++++++--------------
 1 file changed, 69 insertions(+), 74 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 441114d19..f8db76c18 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -38,7 +38,7 @@ class DailymotionBaseInfoExtractor(InfoExtractor):
 
 
 class DailymotionIE(DailymotionBaseInfoExtractor):
-    _VALID_URL = r'(?i)(?:https?://)?(?:(www|touch)\.)?dailymotion\.[a-z]{2,3}/(?:(?:embed|swf|#)/)?video/(?P<id>[^/?_]+)'
+    _VALID_URL = r'(?i)https?://(?:(www|touch)\.)?dailymotion\.[a-z]{2,3}/(?:(?:(?:embed|swf|#)/)?video|swf)/(?P<id>[^/?_]+)'
     IE_NAME = 'dailymotion'
 
     _FORMATS = [
@@ -49,87 +49,82 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
         ('stream_h264_hd1080_url', 'hd180'),
     ]
 
-    _TESTS = [
-        {
-            'url': 'http://www.dailymotion.com/video/x5kesuj_office-christmas-party-review-jason-bateman-olivia-munn-t-j-miller_news',
-            'md5': '074b95bdee76b9e3654137aee9c79dfe',
-            'info_dict': {
-                'id': 'x5kesuj',
-                'ext': 'mp4',
-                'title': 'Office Christmas Party Review –  Jason Bateman, Olivia Munn, T.J. Miller',
-                'description': 'Office Christmas Party Review -  Jason Bateman, Olivia Munn, T.J. Miller',
-                'thumbnail': r're:^https?:.*\.(?:jpg|png)$',
-                'duration': 187,
-                'timestamp': 1493651285,
-                'upload_date': '20170501',
-                'uploader': 'Deadline',
-                'uploader_id': 'x1xm8ri',
-                'age_limit': 0,
-                'view_count': int,
-            },
+    _TESTS = [{
+        'url': 'http://www.dailymotion.com/video/x5kesuj_office-christmas-party-review-jason-bateman-olivia-munn-t-j-miller_news',
+        'md5': '074b95bdee76b9e3654137aee9c79dfe',
+        'info_dict': {
+            'id': 'x5kesuj',
+            'ext': 'mp4',
+            'title': 'Office Christmas Party Review –  Jason Bateman, Olivia Munn, T.J. Miller',
+            'description': 'Office Christmas Party Review -  Jason Bateman, Olivia Munn, T.J. Miller',
+            'thumbnail': r're:^https?:.*\.(?:jpg|png)$',
+            'duration': 187,
+            'timestamp': 1493651285,
+            'upload_date': '20170501',
+            'uploader': 'Deadline',
+            'uploader_id': 'x1xm8ri',
+            'age_limit': 0,
+            'view_count': int,
         },
-        {
-            'url': 'https://www.dailymotion.com/video/x2iuewm_steam-machine-models-pricing-listed-on-steam-store-ign-news_videogames',
-            'md5': '2137c41a8e78554bb09225b8eb322406',
-            'info_dict': {
-                'id': 'x2iuewm',
-                'ext': 'mp4',
-                'title': 'Steam Machine Models, Pricing Listed on Steam Store - IGN News',
-                'description': 'Several come bundled with the Steam Controller.',
-                'thumbnail': r're:^https?:.*\.(?:jpg|png)$',
-                'duration': 74,
-                'timestamp': 1425657362,
-                'upload_date': '20150306',
-                'uploader': 'IGN',
-                'uploader_id': 'xijv66',
-                'age_limit': 0,
-                'view_count': int,
-            },
-            'skip': 'video gone',
+    }, {
+        'url': 'https://www.dailymotion.com/video/x2iuewm_steam-machine-models-pricing-listed-on-steam-store-ign-news_videogames',
+        'md5': '2137c41a8e78554bb09225b8eb322406',
+        'info_dict': {
+            'id': 'x2iuewm',
+            'ext': 'mp4',
+            'title': 'Steam Machine Models, Pricing Listed on Steam Store - IGN News',
+            'description': 'Several come bundled with the Steam Controller.',
+            'thumbnail': r're:^https?:.*\.(?:jpg|png)$',
+            'duration': 74,
+            'timestamp': 1425657362,
+            'upload_date': '20150306',
+            'uploader': 'IGN',
+            'uploader_id': 'xijv66',
+            'age_limit': 0,
+            'view_count': int,
         },
+        'skip': 'video gone',
+    }, {
         # Vevo video
-        {
-            'url': 'http://www.dailymotion.com/video/x149uew_katy-perry-roar-official_musi',
-            'info_dict': {
-                'title': 'Roar (Official)',
-                'id': 'USUV71301934',
-                'ext': 'mp4',
-                'uploader': 'Katy Perry',
-                'upload_date': '20130905',
-            },
-            'params': {
-                'skip_download': True,
-            },
-            'skip': 'VEVO is only available in some countries',
+        'url': 'http://www.dailymotion.com/video/x149uew_katy-perry-roar-official_musi',
+        'info_dict': {
+            'title': 'Roar (Official)',
+            'id': 'USUV71301934',
+            'ext': 'mp4',
+            'uploader': 'Katy Perry',
+            'upload_date': '20130905',
         },
+        'params': {
+            'skip_download': True,
+        },
+        'skip': 'VEVO is only available in some countries',
+    }, {
         # age-restricted video
-        {
-            'url': 'http://www.dailymotion.com/video/xyh2zz_leanna-decker-cyber-girl-of-the-year-desires-nude-playboy-plus_redband',
-            'md5': '0d667a7b9cebecc3c89ee93099c4159d',
-            'info_dict': {
-                'id': 'xyh2zz',
-                'ext': 'mp4',
-                'title': 'Leanna Decker - Cyber Girl Of The Year Desires Nude [Playboy Plus]',
-                'uploader': 'HotWaves1012',
-                'age_limit': 18,
-            },
-            'skip': 'video gone',
+        'url': 'http://www.dailymotion.com/video/xyh2zz_leanna-decker-cyber-girl-of-the-year-desires-nude-playboy-plus_redband',
+        'md5': '0d667a7b9cebecc3c89ee93099c4159d',
+        'info_dict': {
+            'id': 'xyh2zz',
+            'ext': 'mp4',
+            'title': 'Leanna Decker - Cyber Girl Of The Year Desires Nude [Playboy Plus]',
+            'uploader': 'HotWaves1012',
+            'age_limit': 18,
         },
+        'skip': 'video gone',
+    }, {
         # geo-restricted, player v5
-        {
-            'url': 'http://www.dailymotion.com/video/xhza0o',
-            'only_matching': True,
-        },
+        'url': 'http://www.dailymotion.com/video/xhza0o',
+        'only_matching': True,
+    }, {
         # with subtitles
-        {
-            'url': 'http://www.dailymotion.com/video/x20su5f_the-power-of-nightmares-1-the-rise-of-the-politics-of-fear-bbc-2004_news',
-            'only_matching': True,
-        },
-        {
-            'url': 'http://www.dailymotion.com/swf/video/x3n92nf',
-            'only_matching': True,
-        }
-    ]
+        'url': 'http://www.dailymotion.com/video/x20su5f_the-power-of-nightmares-1-the-rise-of-the-politics-of-fear-bbc-2004_news',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.dailymotion.com/swf/video/x3n92nf',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.dailymotion.com/swf/x3ss1m_funny-magic-trick-barry-and-stuart_fun',
+        'only_matching': True,
+    }]
 
     @staticmethod
     def _extract_urls(webpage):

From 60f5c9fb191b1ffc11e69dece379a8ad2de22207 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 12:33:33 +0700
Subject: [PATCH 0863/1696] [utils] Recognize more audio codecs (#13081)

---
 youtube_dl/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index aa1900436..4293a77f5 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2360,11 +2360,11 @@ def parse_codecs(codecs_str):
         if codec in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2', 'h263', 'h264', 'mp4v'):
             if not vcodec:
                 vcodec = full_codec
-        elif codec in ('mp4a', 'opus', 'vorbis', 'mp3', 'aac', 'ac-3'):
+        elif codec in ('mp4a', 'opus', 'vorbis', 'mp3', 'aac', 'ac-3', 'ec-3', 'eac3', 'dtsc', 'dtse', 'dtsh', 'dtsl'):
             if not acodec:
                 acodec = full_codec
         else:
-            write_string('WARNING: Unknown codec %s' % full_codec, sys.stderr)
+            write_string('WARNING: Unknown codec %s\n' % full_codec, sys.stderr)
     if not vcodec and not acodec:
         if len(splited_codecs) == 2:
             return {

From 1c45b7a8a9ac50b87b9b1a540e9d9e875ac22a0b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 May 2017 12:47:19 +0700
Subject: [PATCH 0864/1696] [dailymail] Fix sources extraction (closes #13057)

---
 youtube_dl/extractor/dailymail.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dailymail.py b/youtube_dl/extractor/dailymail.py
index 98c835bf1..538565c66 100644
--- a/youtube_dl/extractor/dailymail.py
+++ b/youtube_dl/extractor/dailymail.py
@@ -2,9 +2,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     int_or_none,
     determine_protocol,
+    try_get,
     unescapeHTML,
 )
 
@@ -28,8 +30,14 @@ class DailyMailIE(InfoExtractor):
         video_data = self._parse_json(self._search_regex(
             r"data-opts='({.+?})'", webpage, 'video data'), video_id)
         title = unescapeHTML(video_data['title'])
-        video_sources = self._download_json(video_data.get(
-            'sources', {}).get('url') or 'http://www.dailymail.co.uk/api/player/%s/video-sources.json' % video_id, video_id)
+
+        sources_url = (try_get(
+            video_data,
+            (lambda x: x['plugins']['sources']['url'],
+             lambda x: x['sources']['url']), compat_str) or
+            'http://www.dailymail.co.uk/api/player/%s/video-sources.json' % video_id)
+
+        video_sources = self._download_json(sources_url, video_id)
 
         formats = []
         for rendition in video_sources['renditions']:

From 89fd03079b83e1dfae016e4e28746b00befe8b2d Mon Sep 17 00:00:00 2001
From: mrBliss <dewinant@gmail.com>
Date: Mon, 15 May 2017 16:46:55 +0200
Subject: [PATCH 0865/1696] [vier] Improve extraction

+ Add support for authentication
* Bypass authentication when no credentials provded
* Improve extraction robustness
---
 youtube_dl/extractor/vier.py | 74 +++++++++++++++++++++++++++++++++++-
 1 file changed, 72 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py
index 5ef7635b6..b6efaa284 100644
--- a/youtube_dl/extractor/vier.py
+++ b/youtube_dl/extractor/vier.py
@@ -5,12 +5,17 @@ import re
 import itertools
 
 from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    urlencode_postdata,
+)
 
 
 class VierIE(InfoExtractor):
     IE_NAME = 'vier'
     IE_DESC = 'vier.be and vijf.be'
     _VALID_URL = r'https?://(?:www\.)?(?P<site>vier|vijf)\.be/(?:[^/]+/videos/(?P<display_id>[^/]+)(?:/(?P<id>\d+))?|video/v3/embed/(?P<embed_id>\d+))'
+    _NETRC_MACHINE = 'vier'
     _TESTS = [{
         'url': 'http://www.vier.be/planb/videos/het-wordt-warm-de-moestuin/16129',
         'info_dict': {
@@ -24,19 +29,47 @@ class VierIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         },
+        # 'skip': 'Requires account credentials',
     }, {
         'url': 'http://www.vijf.be/temptationisland/videos/zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas/2561614',
         'info_dict': {
             'id': '2561614',
             'display_id': 'zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas',
             'ext': 'mp4',
-            'title': 'ZO grappig: Temptation Island hosts moeten kiezen tussen onmogelijke dilemma\'s',
+            'title': 'EXTRA: Temptation Island hosts moeten kiezen tussen onmogelijke dilemma\'s',
             'description': 'Het spel is simpel: Annelien Coorevits en Rick Brandsteder krijgen telkens 2 dilemma\'s voorgeschoteld en ze MOETEN een keuze maken.',
         },
         'params': {
             # m3u8 download
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.vier.be/janigaat/videos/jani-gaat-naar-tokio-aflevering-4/2674839',
+        'info_dict': {
+            'id': '2674839',
+            'display_id': 'jani-gaat-naar-tokio-aflevering-4',
+            'ext': 'mp4',
+            'title': 'Jani gaat naar Tokio - Aflevering 4',
+            'description': 'Bekijk hier de volledige vierde aflevering van het 2de seizoen van Jani gaat...',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+        'skip': 'Requires account credentials',
+    }, {
+        'url': 'http://www.vier.be/janigaat/videos/jani-gaat-naar-tokio-aflevering-4/2674839',
+        'info_dict': {
+            'id': '2674839',
+            'display_id': 'jani-gaat-naar-tokio-aflevering-4',
+            'ext': 'mp4',
+            'title': 'jani-gaat-naar-tokio-aflevering-4',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+        'expected_warnings': ['Log in to extract metadata'],
     }, {
         'url': 'http://www.vier.be/planb/videos/mieren-herders-van-de-bladluizen',
         'only_matching': True,
@@ -45,17 +78,54 @@ class VierIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    def _real_initialize(self):
+        self._logged_in = False
+
+    def _login(self, site):
+        username, password = self._get_login_info()
+        if username is None or password is None:
+            return
+
+        login_page = self._download_webpage(
+            'http://www.%s.be/user/login' % site,
+            None, note='Logging in', errnote='Unable to log in',
+            data=urlencode_postdata({
+                'form_id': 'user_login',
+                'name': username,
+                'pass': password,
+            }),
+            headers={'Content-Type': 'application/x-www-form-urlencoded'})
+
+        login_error = self._html_search_regex(
+            r'(?s)<div class="messages error">\s*<div>\s*<h2.+?</h2>(.+?)<',
+            login_page, 'login error', default=None)
+        if login_error:
+            self.report_warning('Unable to log in: %s' % login_error)
+        else:
+            self._logged_in = True
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         embed_id = mobj.group('embed_id')
         display_id = mobj.group('display_id') or embed_id
+        video_id = mobj.group('id') or embed_id
         site = mobj.group('site')
 
+        if not self._logged_in:
+            self._login(site)
+
         webpage = self._download_webpage(url, display_id)
 
+        if r'id="user-login"' in webpage:
+            self.report_warning(
+                'Log in to extract metadata', video_id=display_id)
+            webpage = self._download_webpage(
+                'http://www.%s.be/video/v3/embed/%s' % (site, video_id),
+                display_id)
+
         video_id = self._search_regex(
             [r'data-nid="(\d+)"', r'"nid"\s*:\s*"(\d+)"'],
-            webpage, 'video id')
+            webpage, 'video id', default=video_id)
         application = self._search_regex(
             [r'data-application="([^"]+)"', r'"application"\s*:\s*"([^"]+)"'],
             webpage, 'application', default=site + '_vod')

From 7073015a23f96ce8ca0400286051d53cb9237a9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 May 2017 22:00:53 +0700
Subject: [PATCH 0866/1696] [vier] PEP 8 and cleanup

---
 youtube_dl/extractor/vier.py | 27 ++++++++++-----------------
 1 file changed, 10 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py
index b6efaa284..a98bd5c81 100644
--- a/youtube_dl/extractor/vier.py
+++ b/youtube_dl/extractor/vier.py
@@ -5,10 +5,7 @@ import re
 import itertools
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    urlencode_postdata,
-)
+from ..utils import urlencode_postdata
 
 
 class VierIE(InfoExtractor):
@@ -18,6 +15,7 @@ class VierIE(InfoExtractor):
     _NETRC_MACHINE = 'vier'
     _TESTS = [{
         'url': 'http://www.vier.be/planb/videos/het-wordt-warm-de-moestuin/16129',
+        'md5': 'e4ae2054a6b040ef1e289e20d111b46e',
         'info_dict': {
             'id': '16129',
             'display_id': 'het-wordt-warm-de-moestuin',
@@ -25,22 +23,16 @@ class VierIE(InfoExtractor):
             'title': 'Het wordt warm in De Moestuin',
             'description': 'De vele uren werk eisen hun tol. Wim droomt van assistentie...',
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-        # 'skip': 'Requires account credentials',
     }, {
         'url': 'http://www.vijf.be/temptationisland/videos/zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas/2561614',
         'info_dict': {
             'id': '2561614',
             'display_id': 'zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas',
             'ext': 'mp4',
-            'title': 'EXTRA: Temptation Island hosts moeten kiezen tussen onmogelijke dilemma\'s',
-            'description': 'Het spel is simpel: Annelien Coorevits en Rick Brandsteder krijgen telkens 2 dilemma\'s voorgeschoteld en ze MOETEN een keuze maken.',
+            'title': 'md5:84f45fe48b8c1fa296a7f6d208d080a7',
+            'description': 'md5:0356d4981e58b8cbee19355cbd51a8fe',
         },
         'params': {
-            # m3u8 download
             'skip_download': True,
         },
     }, {
@@ -50,14 +42,15 @@ class VierIE(InfoExtractor):
             'display_id': 'jani-gaat-naar-tokio-aflevering-4',
             'ext': 'mp4',
             'title': 'Jani gaat naar Tokio - Aflevering 4',
-            'description': 'Bekijk hier de volledige vierde aflevering van het 2de seizoen van Jani gaat...',
+            'description': 'md5:2d169e8186ae4247e50c99aaef97f7b2',
         },
         'params': {
-            # m3u8 download
             'skip_download': True,
         },
         'skip': 'Requires account credentials',
     }, {
+        # Requires account credentials but bypassed extraction via v3/embed page
+        # without metadata
         'url': 'http://www.vier.be/janigaat/videos/jani-gaat-naar-tokio-aflevering-4/2674839',
         'info_dict': {
             'id': '2674839',
@@ -66,12 +59,12 @@ class VierIE(InfoExtractor):
             'title': 'jani-gaat-naar-tokio-aflevering-4',
         },
         'params': {
-            # m3u8 download
             'skip_download': True,
         },
         'expected_warnings': ['Log in to extract metadata'],
     }, {
-        'url': 'http://www.vier.be/planb/videos/mieren-herders-van-de-bladluizen',
+        # Without video id in URL
+        'url': 'http://www.vier.be/planb/videos/dit-najaar-plan-b',
         'only_matching': True,
     }, {
         'url': 'http://www.vier.be/video/v3/embed/16129',
@@ -125,7 +118,7 @@ class VierIE(InfoExtractor):
 
         video_id = self._search_regex(
             [r'data-nid="(\d+)"', r'"nid"\s*:\s*"(\d+)"'],
-            webpage, 'video id', default=video_id)
+            webpage, 'video id', default=video_id or display_id)
         application = self._search_regex(
             [r'data-application="([^"]+)"', r'"application"\s*:\s*"([^"]+)"'],
             webpage, 'application', default=site + '_vod')

From 6f766798049418c15e3c1b62c046e507093993dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 16 May 2017 22:11:34 +0700
Subject: [PATCH 0867/1696] [extractor/common] Add support for schemeless URLs
 in _extract_wowza_formats (closes #13088, closes #13092)

---
 youtube_dl/extractor/common.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 0bbb1103b..74b6f1197 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2205,8 +2205,9 @@ class InfoExtractor(object):
 
     def _extract_wowza_formats(self, url, video_id, m3u8_entry_protocol='m3u8_native', skip_protocols=[]):
         url = re.sub(r'/(?:manifest|playlist|jwplayer)\.(?:m3u8|f4m|mpd|smil)', '', url)
-        url_base = self._search_regex(r'(?:https?|rtmp|rtsp)(://[^?]+)', url, 'format url')
-        http_base_url = 'http' + url_base
+        url_base = self._search_regex(
+            r'(?:(?:https?|rtmp|rtsp):)?(//[^?]+)', url, 'format url')
+        http_base_url = self._proto_relative_url(url_base, scheme='http:')
         formats = []
         if 'm3u8' not in skip_protocols:
             formats.extend(self._extract_m3u8_formats(

From f2e2f0c777dc9e541d89b742ac12e7ad37d8cb8e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 May 2017 22:19:33 +0700
Subject: [PATCH 0868/1696] [extractor/common] Fix rtmp and rtsp formats' URLs
 in _extract_wowza_formats

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 74b6f1197..fec39da8b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2207,7 +2207,7 @@ class InfoExtractor(object):
         url = re.sub(r'/(?:manifest|playlist|jwplayer)\.(?:m3u8|f4m|mpd|smil)', '', url)
         url_base = self._search_regex(
             r'(?:(?:https?|rtmp|rtsp):)?(//[^?]+)', url, 'format url')
-        http_base_url = self._proto_relative_url(url_base, scheme='http:')
+        http_base_url = '%s:%s' % ('http', url_base)
         formats = []
         if 'm3u8' not in skip_protocols:
             formats.extend(self._extract_m3u8_formats(
@@ -2241,7 +2241,7 @@ class InfoExtractor(object):
             for protocol in ('rtmp', 'rtsp'):
                 if protocol not in skip_protocols:
                     formats.append({
-                        'url': protocol + url_base,
+                        'url': '%s:%s' % (protocol, url_base),
                         'format_id': protocol,
                         'protocol': protocol,
                     })

From 36b226d48fd0b366dcf99a646550181639301eb2 Mon Sep 17 00:00:00 2001
From: mrBliss <dewinant@gmail.com>
Date: Fri, 24 Mar 2017 10:34:35 +0100
Subject: [PATCH 0869/1696] [vier] Extract more info

Extract the `episode_number` and `upload_date`. Also extract the real
`description`.
---
 youtube_dl/extractor/vier.py | 28 +++++++++++++++++++++++++---
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py
index a98bd5c81..7fb718343 100644
--- a/youtube_dl/extractor/vier.py
+++ b/youtube_dl/extractor/vier.py
@@ -5,7 +5,11 @@ import re
 import itertools
 
 from .common import InfoExtractor
-from ..utils import urlencode_postdata
+from ..utils import (
+    urlencode_postdata,
+    int_or_none,
+    unified_strdate,
+)
 
 
 class VierIE(InfoExtractor):
@@ -22,6 +26,7 @@ class VierIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Het wordt warm in De Moestuin',
             'description': 'De vele uren werk eisen hun tol. Wim droomt van assistentie...',
+            'upload_date': '20121025',
         },
     }, {
         'url': 'http://www.vijf.be/temptationisland/videos/zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas/2561614',
@@ -31,6 +36,7 @@ class VierIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'md5:84f45fe48b8c1fa296a7f6d208d080a7',
             'description': 'md5:0356d4981e58b8cbee19355cbd51a8fe',
+            'upload_date': '20170228',
         },
         'params': {
             'skip_download': True,
@@ -42,7 +48,9 @@ class VierIE(InfoExtractor):
             'display_id': 'jani-gaat-naar-tokio-aflevering-4',
             'ext': 'mp4',
             'title': 'Jani gaat naar Tokio - Aflevering 4',
-            'description': 'md5:2d169e8186ae4247e50c99aaef97f7b2',
+            'description': 'md5:aa8d611541db6ae9e863125704511f88',
+            'upload_date': '20170501',
+            'episode_number': 4,
         },
         'params': {
             'skip_download': True,
@@ -131,14 +139,28 @@ class VierIE(InfoExtractor):
         self._sort_formats(formats)
 
         title = self._og_search_title(webpage, default=display_id)
-        description = self._og_search_description(webpage, default=None)
         thumbnail = self._og_search_thumbnail(webpage, default=None)
+        description = self._html_search_regex(
+            r'''(?x)<div\ class="[^"]*field-type-text-with-summary[^"]*">\s*
+                      (?:<div\ class="[^"]+">\s*)*
+                     <p>\s*(?:<span>)?(.+?)</''',
+            webpage, 'description', default=None)
+        episode_number = int_or_none(self._search_regex(
+            r'(?i)aflevering (\d+)', title, 'episode_number', default=None,
+            fatal=False))
+        upload_date = unified_strdate(self._html_search_regex(
+            r'''(?x)<div\ class="[^"]*field-name-post-date[^"]*">\s*
+                    (?:<div\ class="[^"]+">\s*)*
+                      (\d{2}/\d{2}/\d{4})''',
+            webpage, 'upload_date', default=None))
 
         return {
             'id': video_id,
             'display_id': display_id,
             'title': title,
             'description': description,
+            'episode_number': episode_number,
+            'upload_date': upload_date,
             'thumbnail': thumbnail,
             'formats': formats,
         }

From 650bd9471697390a9cdef2a54d35ec6dd5dc49ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 May 2017 23:38:27 +0700
Subject: [PATCH 0870/1696] [vier] Relax regexes and extract more metadata
 (closes #12539)

---
 youtube_dl/extractor/vier.py | 39 ++++++++++++++++++++++--------------
 1 file changed, 24 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py
index 7fb718343..3e67eb8c2 100644
--- a/youtube_dl/extractor/vier.py
+++ b/youtube_dl/extractor/vier.py
@@ -27,6 +27,8 @@ class VierIE(InfoExtractor):
             'title': 'Het wordt warm in De Moestuin',
             'description': 'De vele uren werk eisen hun tol. Wim droomt van assistentie...',
             'upload_date': '20121025',
+            'series': 'Plan B',
+            'tags': ['De Moestuin', 'Moestuin', 'meisjes', 'Tomaat', 'Wim', 'Droom'],
         },
     }, {
         'url': 'http://www.vijf.be/temptationisland/videos/zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas/2561614',
@@ -37,6 +39,8 @@ class VierIE(InfoExtractor):
             'title': 'md5:84f45fe48b8c1fa296a7f6d208d080a7',
             'description': 'md5:0356d4981e58b8cbee19355cbd51a8fe',
             'upload_date': '20170228',
+            'series': 'Temptation Island',
+            'tags': list,
         },
         'params': {
             'skip_download': True,
@@ -50,7 +54,9 @@ class VierIE(InfoExtractor):
             'title': 'Jani gaat naar Tokio - Aflevering 4',
             'description': 'md5:aa8d611541db6ae9e863125704511f88',
             'upload_date': '20170501',
+            'series': 'Jani gaat',
             'episode_number': 4,
+            'tags': ['Jani Gaat', 'Volledige Aflevering'],
         },
         'params': {
             'skip_download': True,
@@ -135,33 +141,36 @@ class VierIE(InfoExtractor):
             webpage, 'filename')
 
         playlist_url = 'http://vod.streamcloud.be/%s/_definst_/mp4:%s.mp4/playlist.m3u8' % (application, filename)
-        formats = self._extract_wowza_formats(playlist_url, display_id, skip_protocols=['dash'])
+        formats = self._extract_wowza_formats(
+            playlist_url, display_id, skip_protocols=['dash'])
         self._sort_formats(formats)
 
         title = self._og_search_title(webpage, default=display_id)
-        thumbnail = self._og_search_thumbnail(webpage, default=None)
         description = self._html_search_regex(
-            r'''(?x)<div\ class="[^"]*field-type-text-with-summary[^"]*">\s*
-                      (?:<div\ class="[^"]+">\s*)*
-                     <p>\s*(?:<span>)?(.+?)</''',
-            webpage, 'description', default=None)
-        episode_number = int_or_none(self._search_regex(
-            r'(?i)aflevering (\d+)', title, 'episode_number', default=None,
-            fatal=False))
+            r'(?s)<div\b[^>]+\bclass=(["\'])[^>]*?\bfield-type-text-with-summary\b[^>]*?\1[^>]*>.*?<p>(?P<value>.+?)</p>',
+            webpage, 'description', default=None, group='value')
+        thumbnail = self._og_search_thumbnail(webpage, default=None)
         upload_date = unified_strdate(self._html_search_regex(
-            r'''(?x)<div\ class="[^"]*field-name-post-date[^"]*">\s*
-                    (?:<div\ class="[^"]+">\s*)*
-                      (\d{2}/\d{2}/\d{4})''',
-            webpage, 'upload_date', default=None))
+            r'(?s)<div\b[^>]+\bclass=(["\'])[^>]*?\bfield-name-post-date\b[^>]*?\1[^>]*>.*?(?P<value>\d{2}/\d{2}/\d{4})',
+            webpage, 'upload date', default=None, group='value'))
+
+        series = self._search_regex(
+            r'data-program=(["\'])(?P<value>(?:(?!\1).)+)\1', webpage,
+            'series', default=None, group='value')
+        episode_number = int_or_none(self._search_regex(
+            r'(?i)aflevering (\d+)', title, 'episode number', default=None))
+        tags = re.findall(r'<a\b[^>]+\bhref=["\']/tags/[^>]+>([^<]+)<', webpage)
 
         return {
             'id': video_id,
             'display_id': display_id,
             'title': title,
             'description': description,
-            'episode_number': episode_number,
-            'upload_date': upload_date,
             'thumbnail': thumbnail,
+            'upload_date': upload_date,
+            'series': series,
+            'episode_number': episode_number,
+            'tags': tags,
             'formats': formats,
         }
 

From 7769f837018cbfa6cab615b8ae938528f895ce8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 May 2017 22:18:33 +0700
Subject: [PATCH 0871/1696] [jsinterp] Add support for quoted names and
 indexers (closes #13123, closes #13130)

---
 youtube_dl/jsinterp.py | 33 +++++++++++++++++----------------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 24cdec28c..038910049 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -6,6 +6,7 @@ import re
 
 from .utils import (
     ExtractorError,
+    remove_quotes,
 )
 
 _OPERATORS = [
@@ -57,7 +58,6 @@ class JSInterpreter(object):
 
     def interpret_expression(self, expr, local_vars, allow_recursion):
         expr = expr.strip()
-
         if expr == '':  # Empty expression
             return None
 
@@ -121,11 +121,19 @@ class JSInterpreter(object):
             pass
 
         m = re.match(
-            r'(?P<var>%s)\.(?P<member>[^(]+)(?:\(+(?P<args>[^()]*)\))?$' % _NAME_RE,
+            r'(?P<in>%s)\[(?P<idx>.+)\]$' % _NAME_RE, expr)
+        if m:
+            val = local_vars[m.group('in')]
+            idx = self.interpret_expression(
+                m.group('idx'), local_vars, allow_recursion - 1)
+            return val[idx]
+
+        m = re.match(
+            r'(?P<var>%s)(?:\.(?P<member>[^(]+)|\[(?P<member2>[^]]+)\])\s*(?:\(+(?P<args>[^()]*)\))?$' % _NAME_RE,
             expr)
         if m:
             variable = m.group('var')
-            member = m.group('member')
+            member = remove_quotes(m.group('member') or m.group('member2'))
             arg_str = m.group('args')
 
             if variable in local_vars:
@@ -173,14 +181,6 @@ class JSInterpreter(object):
 
             return obj[member](argvals)
 
-        m = re.match(
-            r'(?P<in>%s)\[(?P<idx>.+)\]$' % _NAME_RE, expr)
-        if m:
-            val = local_vars[m.group('in')]
-            idx = self.interpret_expression(
-                m.group('idx'), local_vars, allow_recursion - 1)
-            return val[idx]
-
         for op, opfunc in _OPERATORS:
             m = re.match(r'(?P<x>.+?)%s(?P<y>.+)' % re.escape(op), expr)
             if not m:
@@ -211,21 +211,22 @@ class JSInterpreter(object):
         raise ExtractorError('Unsupported JS expression %r' % expr)
 
     def extract_object(self, objname):
+        _FUNC_NAME_RE = r'''(?:[a-zA-Z$0-9]+|"[a-zA-Z$0-9]+"|'[a-zA-Z$0-9]+')'''
         obj = {}
         obj_m = re.search(
             (r'(?<!this\.)%s\s*=\s*\{' % re.escape(objname)) +
-            r'\s*(?P<fields>([a-zA-Z$0-9]+\s*:\s*function\(.*?\)\s*\{.*?\}(?:,\s*)?)*)' +
-            r'\}\s*;',
+            r'\s*(?P<fields>(%s\s*:\s*function\(.*?\)\s*\{.*?\}(?:,\s*)?)*)' +
+            r'\}\s*;' % _FUNC_NAME_RE,
             self.code)
         fields = obj_m.group('fields')
         # Currently, it only supports function definitions
         fields_m = re.finditer(
-            r'(?P<key>[a-zA-Z$0-9]+)\s*:\s*function'
-            r'\((?P<args>[a-z,]+)\){(?P<code>[^}]+)}',
+            r'(?P<key>%s)\s*:\s*function'
+            r'\((?P<args>[a-z,]+)\){(?P<code>[^}]+)}' % _FUNC_NAME_RE,
             fields)
         for f in fields_m:
             argnames = f.group('args').split(',')
-            obj[f.group('key')] = self.build_function(argnames, f.group('code'))
+            obj[remove_quotes(f.group('key'))] = self.build_function(argnames, f.group('code'))
 
         return obj
 

From 3bc1eea0d841aab62a43ac2358ebdb376d507c01 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 May 2017 22:29:25 +0700
Subject: [PATCH 0872/1696] [ChangeLog] Actualize

---
 ChangeLog | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 91c77515f..06efad4e6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,22 @@
+version <unreleased>
+
+Core
++ [jsinterp] Add support for quoted names and indexers (#13123, #13124, #13125,
+  #13126, #13128, #13129, #13130, #13131, #13132)
++ [extractor/common] Add support for schemeless URLs in _extract_wowza_formats
+  (#13088, #13092)
++ [utils] Recognize more audio codecs (#13081)
+
+Extractors
++ [vier] Extract more metadata (#12539)
+* [vier] Improve extraction (#12801)
+    + Add support for authentication
+    * Bypass authentication when no credentials provided
+    * Improve extraction robustness
+* [dailymail] Fix sources extraction (#13057)
+* [dailymotion] Extend URL regular expression (#13079)
+
+
 version 2017.05.14
 
 Core

From 9d63e57d1f6e828a7c9ff3356763c7666bd0202a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 May 2017 22:30:37 +0700
Subject: [PATCH 0873/1696] release 2017.05.18

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 2955e833d..71a2631b9 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.14*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.14**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.18*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.18**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.05.14
+[debug] youtube-dl version 2017.05.18
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 06efad4e6..443b3aaf6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.05.18
 
 Core
 + [jsinterp] Add support for quoted names and indexers (#13123, #13124, #13125,
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 9b074fafc..7fa23d274 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.05.14'
+__version__ = '2017.05.18'

From 9221d5d7a84fd778d54806b3d72b58392481f2bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 May 2017 22:37:32 +0700
Subject: [PATCH 0874/1696] [ChangeLog] Fix typo

---
 ChangeLog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 443b3aaf6..476387002 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -24,7 +24,7 @@ Core
 + [postprocessor/metadatafromtitle] Add support regular expression syntax for
   --metadata-from-title (#13065)
 
-Extractor
+Extractors
 + [mediaset] Add support for video.mediaset.it (#12708, #12964)
 * [orf:radio] Fix extraction (#11643, #12926)
 * [aljazeera] Extend URL regular expression (#13053)

From 0e2d626ddd2c0c4a8e369026812c7092f5d17de2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 May 2017 22:57:38 +0700
Subject: [PATCH 0875/1696] [jsinterp] Fix typo and cleanup regexes (closes
 #13134)

---
 youtube_dl/jsinterp.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 038910049..7bda59610 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -214,15 +214,18 @@ class JSInterpreter(object):
         _FUNC_NAME_RE = r'''(?:[a-zA-Z$0-9]+|"[a-zA-Z$0-9]+"|'[a-zA-Z$0-9]+')'''
         obj = {}
         obj_m = re.search(
-            (r'(?<!this\.)%s\s*=\s*\{' % re.escape(objname)) +
-            r'\s*(?P<fields>(%s\s*:\s*function\(.*?\)\s*\{.*?\}(?:,\s*)?)*)' +
-            r'\}\s*;' % _FUNC_NAME_RE,
+            r'''(?x)
+                (?<!this\.)%s\s*=\s*{\s*
+                    (?P<fields>(%s\s*:\s*function\s*\(.*?\)\s*{.*?}(?:,\s*)?)*)
+                }\s*;
+            ''' % (re.escape(objname), _FUNC_NAME_RE),
             self.code)
         fields = obj_m.group('fields')
         # Currently, it only supports function definitions
         fields_m = re.finditer(
-            r'(?P<key>%s)\s*:\s*function'
-            r'\((?P<args>[a-z,]+)\){(?P<code>[^}]+)}' % _FUNC_NAME_RE,
+            r'''(?x)
+                (?P<key>%s)\s*:\s*function\s*\((?P<args>[a-z,]+)\){(?P<code>[^}]+)}
+            ''' % _FUNC_NAME_RE,
             fields)
         for f in fields_m:
             argnames = f.group('args').split(',')

From bcaa1dd0609fea4a5ef108c27f53f399c4d24765 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 May 2017 22:58:14 +0700
Subject: [PATCH 0876/1696] [ChangeLog] Actualize

---
 ChangeLog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 476387002..606e60313 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Core
+* [jsinterp] Fix typo and cleanup regular expressions (#13134)
+
+
 version 2017.05.18
 
 Core

From bfdf6fcc6677531a296d51ba97e110f0a4556fbf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 May 2017 23:00:03 +0700
Subject: [PATCH 0877/1696] release 2017.05.18.1

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 71a2631b9..b714fcfa8 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.18*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.18**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.18.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.18.1**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.05.18
+[debug] youtube-dl version 2017.05.18.1
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 606e60313..1637876b3 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.05.18.1
 
 Core
 * [jsinterp] Fix typo and cleanup regular expressions (#13134)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 7fa23d274..5e963e71c 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.05.18'
+__version__ = '2017.05.18.1'

From 00cb0faca8a2875355651611bd90c4e8a856a2bf Mon Sep 17 00:00:00 2001
From: Logan B <einstein95@users.noreply.github.com>
Date: Sat, 20 May 2017 02:54:57 +1200
Subject: [PATCH 0878/1696] [mitele] Update app key regex

---
 youtube_dl/extractor/mitele.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index 28b743cca..964dc542c 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -136,11 +136,9 @@ class MiTeleIE(InfoExtractor):
             video_id, 'Downloading gigya script')
 
         # Get a appKey/uuid for getting the session key
-        appKey_var = self._search_regex(
-            r'value\s*\(\s*["\']appGridApplicationKey["\']\s*,\s*([0-9a-f]+)',
-            gigya_sc, 'appKey variable')
         appKey = self._search_regex(
-            r'var\s+%s\s*=\s*["\']([0-9a-f]+)' % appKey_var, gigya_sc, 'appKey')
+            r'constant\s*\(\s*["\']_appGridApplicationKey["\']\s*,\s*["\']([0-9a-f]+)',
+            gigya_sc, 'appKey')
 
         session_json = self._download_json(
             'https://appgrid-api.cloud.accedo.tv/session',

From 5f6fbcea08bc6473cf24c8f2ffee38bc01c80de0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 19 May 2017 22:29:54 +0700
Subject: [PATCH 0879/1696] [hitbox] Add support for smashcast.tv (closes
 #13154)

---
 youtube_dl/extractor/hitbox.py | 52 ++++++++++++++++++++--------------
 1 file changed, 30 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/hitbox.py b/youtube_dl/extractor/hitbox.py
index e21ebb8fb..1d905dc81 100644
--- a/youtube_dl/extractor/hitbox.py
+++ b/youtube_dl/extractor/hitbox.py
@@ -16,8 +16,8 @@ from ..utils import (
 
 class HitboxIE(InfoExtractor):
     IE_NAME = 'hitbox'
-    _VALID_URL = r'https?://(?:www\.)?hitbox\.tv/video/(?P<id>[0-9]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?(?:hitbox|smashcast)\.tv/(?:[^/]+/)*videos?/(?P<id>[0-9]+)'
+    _TESTS = [{
         'url': 'http://www.hitbox.tv/video/203213',
         'info_dict': {
             'id': '203213',
@@ -38,13 +38,15 @@ class HitboxIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         },
-    }
+    }, {
+        'url': 'https://www.smashcast.tv/hitboxlive/videos/203213',
+        'only_matching': True,
+    }]
 
     def _extract_metadata(self, url, video_id):
         thumb_base = 'https://edge.sf.hitbox.tv'
         metadata = self._download_json(
-            '%s/%s' % (url, video_id), video_id,
-            'Downloading metadata JSON')
+            '%s/%s' % (url, video_id), video_id, 'Downloading metadata JSON')
 
         date = 'media_live_since'
         media_type = 'livestream'
@@ -63,14 +65,15 @@ class HitboxIE(InfoExtractor):
         views = int_or_none(video_meta.get('media_views'))
         timestamp = parse_iso8601(video_meta.get(date), ' ')
         categories = [video_meta.get('category_name')]
-        thumbs = [
-            {'url': thumb_base + video_meta.get('media_thumbnail'),
-             'width': 320,
-             'height': 180},
-            {'url': thumb_base + video_meta.get('media_thumbnail_large'),
-             'width': 768,
-             'height': 432},
-        ]
+        thumbs = [{
+            'url': thumb_base + video_meta.get('media_thumbnail'),
+            'width': 320,
+            'height': 180
+        }, {
+            'url': thumb_base + video_meta.get('media_thumbnail_large'),
+            'width': 768,
+            'height': 432
+        }]
 
         return {
             'id': video_id,
@@ -90,7 +93,7 @@ class HitboxIE(InfoExtractor):
         video_id = self._match_id(url)
 
         player_config = self._download_json(
-            'https://www.hitbox.tv/api/player/config/video/%s' % video_id,
+            'https://www.smashcast.tv/api/player/config/video/%s' % video_id,
             video_id, 'Downloading video JSON')
 
         formats = []
@@ -121,8 +124,7 @@ class HitboxIE(InfoExtractor):
         self._sort_formats(formats)
 
         metadata = self._extract_metadata(
-            'https://www.hitbox.tv/api/media/video',
-            video_id)
+            'https://www.smashcast.tv/api/media/video', video_id)
         metadata['formats'] = formats
 
         return metadata
@@ -130,8 +132,8 @@ class HitboxIE(InfoExtractor):
 
 class HitboxLiveIE(HitboxIE):
     IE_NAME = 'hitbox:live'
-    _VALID_URL = r'https?://(?:www\.)?hitbox\.tv/(?!video)(?P<id>.+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?(?:hitbox|smashcast)\.tv/(?P<id>[^/?#&]+)'
+    _TESTS = [{
         'url': 'http://www.hitbox.tv/dimak',
         'info_dict': {
             'id': 'dimak',
@@ -146,13 +148,20 @@ class HitboxLiveIE(HitboxIE):
             # live
             'skip_download': True,
         },
-    }
+    }, {
+        'url': 'https://www.smashcast.tv/dimak',
+        'only_matching': True,
+    }]
+
+    @classmethod
+    def suitable(cls, url):
+        return False if HitboxIE.suitable(url) else super(HitboxLiveIE, cls).suitable(url)
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         player_config = self._download_json(
-            'https://www.hitbox.tv/api/player/config/live/%s' % video_id,
+            'https://www.smashcast.tv/api/player/config/live/%s' % video_id,
             video_id)
 
         formats = []
@@ -197,8 +206,7 @@ class HitboxLiveIE(HitboxIE):
         self._sort_formats(formats)
 
         metadata = self._extract_metadata(
-            'https://www.hitbox.tv/api/media/live',
-            video_id)
+            'https://www.smashcast.tv/api/media/live', video_id)
         metadata['formats'] = formats
         metadata['is_live'] = True
         metadata['title'] = self._live_title(metadata.get('title'))

From 8fffac6927b4144bfb3e473b58fa52313042ecbb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 19 May 2017 23:11:02 +0700
Subject: [PATCH 0880/1696] [njpwworld] Fix extraction (closes #13162)

---
 youtube_dl/extractor/njpwworld.py | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/njpwworld.py b/youtube_dl/extractor/njpwworld.py
index f5e3f6815..9b5ad5a9f 100644
--- a/youtube_dl/extractor/njpwworld.py
+++ b/youtube_dl/extractor/njpwworld.py
@@ -6,6 +6,7 @@ import re
 from .common import InfoExtractor
 from ..compat import compat_urlparse
 from ..utils import (
+    extract_attributes,
     get_element_by_class,
     urlencode_postdata,
 )
@@ -56,17 +57,24 @@ class NJPWWorldIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         formats = []
-        for player_url, kind in re.findall(r'<a[^>]+href="(/player[^"]+)".+?<img[^>]+src="[^"]+qf_btn_([^".]+)', webpage):
-            player_url = compat_urlparse.urljoin(url, player_url)
-
+        for mobj in re.finditer(r'<a[^>]+\bhref=(["\'])/player.+?[^>]*>', webpage):
+            player = extract_attributes(mobj.group(0))
+            player_path = player.get('href')
+            if not player_path:
+                continue
+            kind = self._search_regex(
+                r'(low|high)$', player.get('class') or '', 'kind',
+                default='low')
+            player_url = compat_urlparse.urljoin(url, player_path)
             player_page = self._download_webpage(
                 player_url, video_id, note='Downloading player page')
-
             entries = self._parse_html5_media_entries(
                 player_url, player_page, video_id, m3u8_id='hls-%s' % kind,
-                m3u8_entry_protocol='m3u8_native',
-                preference=2 if 'hq' in kind else 1)
-            formats.extend(entries[0]['formats'])
+                m3u8_entry_protocol='m3u8_native')
+            kind_formats = entries[0]['formats']
+            for f in kind_formats:
+                f['quality'] = 2 if kind == 'high' else 1
+            formats.extend(kind_formats)
 
         self._sort_formats(formats)
 

From 77d682da9dc2e62a1505cccb77408b8fa638a7f1 Mon Sep 17 00:00:00 2001
From: vobe <vobe@jackal.fr>
Date: Fri, 19 May 2017 20:18:03 +0200
Subject: [PATCH 0881/1696] [toypics] Fix extraction

---
 youtube_dl/extractor/toypics.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/toypics.py b/youtube_dl/extractor/toypics.py
index 938e05076..774fa565c 100644
--- a/youtube_dl/extractor/toypics.py
+++ b/youtube_dl/extractor/toypics.py
@@ -24,15 +24,16 @@ class ToypicsIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         page = self._download_webpage(url, video_id)
-        video_url = self._html_search_regex(
-            r'src:\s+"(http://static[0-9]+\.toypics\.net/flvideo/[^"]+)"', page, 'video URL')
-        title = self._html_search_regex(
-            r'<title>Toypics - ([^<]+)</title>', page, 'title')
+        formats = self._parse_html5_media_entries(url, page, video_id)[0]['formats']
+        title = self._html_search_regex([
+            r'<h1[^>]+class=["\']view-video-title[^>]+>([^<]+)</h',
+            r'<title>([^<]+) - Toypics</title>',
+        ], page, 'title')
         username = self._html_search_regex(
-            r'toypics.net/([^/"]+)" class="user-name">', page, 'username')
+            r'More videos from <strong>([^<]+)</strong>', page, 'username')
         return {
             'id': video_id,
-            'url': video_url,
+            'formats': formats,
             'title': title,
             'uploader': username,
             'age_limit': 18,
@@ -71,7 +72,7 @@ class ToypicsUserIE(InfoExtractor):
                 note='Downloading page %d/%d' % (n, page_count))
             urls.extend(
                 re.findall(
-                    r'<p class="video-entry-title">\s+<a href="(https?://videos.toypics.net/view/[^"]+)">',
+                    r'<div[^>]+class=["\']preview[^>]+>\s*<a[^>]+href="(https?://videos.toypics.net/view/[^"]+)"',
                     lpage))
 
         return {

From 2ca29f1aafda58fd7fd6db55d8b99eb89e07f2df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 20 May 2017 01:28:42 +0700
Subject: [PATCH 0882/1696] [toypics] Improve and modernize

---
 youtube_dl/extractor/toypics.py | 34 ++++++++++++++++++---------------
 1 file changed, 19 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/toypics.py b/youtube_dl/extractor/toypics.py
index 774fa565c..f705a06c9 100644
--- a/youtube_dl/extractor/toypics.py
+++ b/youtube_dl/extractor/toypics.py
@@ -6,43 +6,48 @@ import re
 
 
 class ToypicsIE(InfoExtractor):
-    IE_DESC = 'Toypics user profile'
-    _VALID_URL = r'https?://videos\.toypics\.net/view/(?P<id>[0-9]+)/.*'
+    IE_DESC = 'Toypics video'
+    _VALID_URL = r'https?://videos\.toypics\.net/view/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://videos.toypics.net/view/514/chancebulged,-2-1/',
         'md5': '16e806ad6d6f58079d210fe30985e08b',
         'info_dict': {
             'id': '514',
             'ext': 'mp4',
-            'title': 'Chance-Bulge\'d, 2',
+            'title': "Chance-Bulge'd, 2",
             'age_limit': 18,
             'uploader': 'kidsune',
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        page = self._download_webpage(url, video_id)
-        formats = self._parse_html5_media_entries(url, page, video_id)[0]['formats']
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        formats = self._parse_html5_media_entries(
+            url, webpage, video_id)[0]['formats']
         title = self._html_search_regex([
             r'<h1[^>]+class=["\']view-video-title[^>]+>([^<]+)</h',
             r'<title>([^<]+) - Toypics</title>',
-        ], page, 'title')
-        username = self._html_search_regex(
-            r'More videos from <strong>([^<]+)</strong>', page, 'username')
+        ], webpage, 'title')
+
+        uploader = self._html_search_regex(
+            r'More videos from <strong>([^<]+)</strong>', webpage, 'uploader',
+            fatal=False)
+
         return {
             'id': video_id,
             'formats': formats,
             'title': title,
-            'uploader': username,
+            'uploader': uploader,
             'age_limit': 18,
         }
 
 
 class ToypicsUserIE(InfoExtractor):
     IE_DESC = 'Toypics user profile'
-    _VALID_URL = r'https?://videos\.toypics\.net/(?P<username>[^/?]+)(?:$|[?#])'
+    _VALID_URL = r'https?://videos\.toypics\.net/(?!view)(?P<id>[^/?#&]+)'
     _TEST = {
         'url': 'http://videos.toypics.net/Mikey',
         'info_dict': {
@@ -52,8 +57,7 @@ class ToypicsUserIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        username = mobj.group('username')
+        username = self._match_id(url)
 
         profile_page = self._download_webpage(
             url, username, note='Retrieving profile page')
@@ -72,7 +76,7 @@ class ToypicsUserIE(InfoExtractor):
                 note='Downloading page %d/%d' % (n, page_count))
             urls.extend(
                 re.findall(
-                    r'<div[^>]+class=["\']preview[^>]+>\s*<a[^>]+href="(https?://videos.toypics.net/view/[^"]+)"',
+                    r'<div[^>]+class=["\']preview[^>]+>\s*<a[^>]+href="(https?://videos\.toypics\.net/view/[^"]+)"',
                     lpage))
 
         return {

From daa4e9ff90e9c017238dae48e9862ebe02bc3ff2 Mon Sep 17 00:00:00 2001
From: Nii-90 <Nii-90@users.noreply.github.com>
Date: Sat, 20 May 2017 09:50:46 -0400
Subject: [PATCH 0883/1696] [adobepass] Add support for Brighthouse MSO

---
 youtube_dl/extractor/adobepass.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
index 7da96c65c..d57ad85c2 100644
--- a/youtube_dl/extractor/adobepass.py
+++ b/youtube_dl/extractor/adobepass.py
@@ -36,6 +36,11 @@ MSO_INFO = {
         'username_field': 'Ecom_User_ID',
         'password_field': 'Ecom_Password',
     },
+    'Brighthouse': {
+        'name': 'Bright House Networks | Spectrum',
+        'username_field': 'j_username',
+        'password_field': 'j_password',
+    },
     'Charter_Direct': {
         'name': 'Charter Spectrum',
         'username_field': 'IDToken1',

From 35a2d221a35caf8aad533b5f8ef6d7b12acd4e41 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 20 May 2017 23:06:30 +0700
Subject: [PATCH 0884/1696] [toggle] Relax _VALID_URL (closes #13172)

---
 youtube_dl/extractor/toggle.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/toggle.py b/youtube_dl/extractor/toggle.py
index c54b876d3..348d6ecdf 100644
--- a/youtube_dl/extractor/toggle.py
+++ b/youtube_dl/extractor/toggle.py
@@ -17,7 +17,7 @@ from ..utils import (
 
 class ToggleIE(InfoExtractor):
     IE_NAME = 'toggle'
-    _VALID_URL = r'https?://video\.toggle\.sg/(?:en|zh)/(?:series|clips|movies)/(?:[^/]+/)+(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://video\.toggle\.sg/(?:en|zh)/(?:[^/]+/){2,}(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://video.toggle.sg/en/series/lion-moms-tif/trailers/lion-moms-premier/343115',
         'info_dict': {
@@ -73,6 +73,12 @@ class ToggleIE(InfoExtractor):
     }, {
         'url': 'http://video.toggle.sg/en/movies/seven-days/321936',
         'only_matching': True,
+    }, {
+        'url': 'https://video.toggle.sg/en/tv-show/news/may-2017-cna-singapore-tonight/fri-19-may-2017/512456',
+        'only_matching': True,
+    }, {
+        'url': 'http://video.toggle.sg/en/channels/eleven-plus/401585',
+        'only_matching': True,
     }]
 
     _FORMAT_PREFERENCES = {

From fdc20f87a62ff973b97c9ea2b0e68b394960b8fa Mon Sep 17 00:00:00 2001
From: Protuhj <ihatespam234@gmail.com>
Date: Sat, 20 May 2017 14:11:37 -0400
Subject: [PATCH 0885/1696] [youtube] Fix DASH manifest signature decryption
 (closes #8944)

---
 youtube_dl/extractor/youtube.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 44a39282f..ae832cac5 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1435,6 +1435,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         else:
             age_gate = False
             video_info = None
+            sts = ''
             # Try looking directly into the video webpage
             ytplayer_config = self._get_ytplayer_config(video_id, video_webpage)
             if ytplayer_config:
@@ -1451,6 +1452,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         args['ypc_vid'], YoutubeIE.ie_key(), video_id=args['ypc_vid'])
                 if args.get('livestream') == '1' or args.get('live_playback') == 1:
                     is_live = True
+                sts = ytplayer_config.get('sts', '')
             if not video_info or self._downloader.params.get('youtube_include_dash_manifest', True):
                 # We also try looking in get_video_info since it may contain different dashmpd
                 # URL that points to a DASH manifest with possibly different itag set (some itags
@@ -1461,8 +1463,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 self.report_video_info_webpage_download(video_id)
                 for el_type in ['&el=info', '&el=embedded', '&el=detailpage', '&el=vevo', '']:
                     video_info_url = (
-                        '%s://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
-                        % (proto, video_id, el_type))
+                        '%s://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en&sts=%s'
+                        % (proto, video_id, el_type, sts))
                     video_info_webpage = self._download_webpage(
                         video_info_url,
                         video_id, note=False,

From dc4e4f90a285dfebd060da55973ad3d1bc88ab7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 May 2017 01:18:56 +0700
Subject: [PATCH 0886/1696] [youtube] Modernize

---
 youtube_dl/extractor/youtube.py | 27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index ae832cac5..d66693c0c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1435,7 +1435,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         else:
             age_gate = False
             video_info = None
-            sts = ''
+            sts = None
             # Try looking directly into the video webpage
             ytplayer_config = self._get_ytplayer_config(video_id, video_webpage)
             if ytplayer_config:
@@ -1452,7 +1452,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         args['ypc_vid'], YoutubeIE.ie_key(), video_id=args['ypc_vid'])
                 if args.get('livestream') == '1' or args.get('live_playback') == 1:
                     is_live = True
-                sts = ytplayer_config.get('sts', '')
+                sts = ytplayer_config.get('sts')
             if not video_info or self._downloader.params.get('youtube_include_dash_manifest', True):
                 # We also try looking in get_video_info since it may contain different dashmpd
                 # URL that points to a DASH manifest with possibly different itag set (some itags
@@ -1461,14 +1461,25 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 # The general idea is to take a union of itags of both DASH manifests (for example
                 # video with such 'manifest behavior' see https://github.com/rg3/youtube-dl/issues/6093)
                 self.report_video_info_webpage_download(video_id)
-                for el_type in ['&el=info', '&el=embedded', '&el=detailpage', '&el=vevo', '']:
-                    video_info_url = (
-                        '%s://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en&sts=%s'
-                        % (proto, video_id, el_type, sts))
+                for el in ('info', 'embedded', 'detailpage', 'vevo', ''):
+                    query = {
+                        'video_id': video_id,
+                        'ps': 'default',
+                        'eurl': '',
+                        'gl': 'US',
+                        'hl': 'en',
+                    }
+                    if el:
+                        query['el'] = el
+                    if sts:
+                        query['sts'] = sts
                     video_info_webpage = self._download_webpage(
-                        video_info_url,
+                        '%s://www.youtube.com/get_video_info' % proto,
                         video_id, note=False,
-                        errnote='unable to download video info webpage')
+                        errnote='unable to download video info webpage',
+                        fatal=False, query=query)
+                    if not video_info_webpage:
+                        continue
                     get_video_info = compat_parse_qs(video_info_webpage)
                     if get_video_info.get('use_cipher_signature') != ['True']:
                         add_dash_mpd(get_video_info)

From afdb387cd8f9f54ee666cf1f64416d877f4eba19 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ond=C5=99ej=20Caletka?= <ondrej@caletka.cz>
Date: Sun, 21 May 2017 10:41:52 +0200
Subject: [PATCH 0887/1696] [streamcz] Add support for subtitles

---
 youtube_dl/extractor/streamcz.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/streamcz.py b/youtube_dl/extractor/streamcz.py
index 9e533103c..58e0b4c80 100644
--- a/youtube_dl/extractor/streamcz.py
+++ b/youtube_dl/extractor/streamcz.py
@@ -26,7 +26,7 @@ class StreamCZIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.stream.cz/peklonataliri/765767-ecka-pro-deti',
-        'md5': '6d3ca61a8d0633c9c542b92fcb936b0c',
+        'md5': '934bb6a6d220d99c010783c9719960d5',
         'info_dict': {
             'id': '765767',
             'ext': 'mp4',
@@ -37,7 +37,7 @@ class StreamCZIE(InfoExtractor):
         },
     }, {
         'url': 'http://www.stream.cz/blanik/10002447-tri-roky-pro-mazanka',
-        'md5': 'e54a254fb8b871968fd8403255f28589',
+        'md5': '849a88c1e1ca47d41403c2ba5e59e261',
         'info_dict': {
             'id': '10002447',
             'ext': 'mp4',
@@ -85,6 +85,14 @@ class StreamCZIE(InfoExtractor):
         else:
             title = data['name']
 
+        subtitles = {}
+        srt_url = data.get('subtitles_srt')
+        if srt_url:
+            subtitles['cs'] = [{
+                'ext': 'srt',
+                'url': srt_url,
+            }]
+
         return {
             'id': video_id,
             'title': title,
@@ -93,4 +101,5 @@ class StreamCZIE(InfoExtractor):
             'description': data.get('web_site_text'),
             'duration': int_or_none(data.get('duration')),
             'view_count': int_or_none(data.get('views')),
+            'subtitles': subtitles,
         }

From a609e61a90ad8f3b472e2e221f9f75c8408b65f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 22 May 2017 23:40:07 +0700
Subject: [PATCH 0888/1696] [downloader/external] Pass -loglevel to ffmpeg
 downloader (closes #13183)

---
 youtube_dl/downloader/external.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index e78169a0d..db018fa89 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -212,6 +212,11 @@ class FFmpegFD(ExternalFD):
 
         args = [ffpp.executable, '-y']
 
+        for log_level in ('quiet', 'verbose'):
+            if self.params.get(log_level, False):
+                args += ['-loglevel', log_level]
+                break
+
         seekable = info_dict.get('_seekable')
         if seekable is not None:
             # setting -seekable prevents ffmpeg from guessing if the server

From 0b75e42dfb36e973e7448de4b031bde24bd3e614 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 23 May 2017 00:00:49 +0700
Subject: [PATCH 0889/1696] Credit @zurfyx for atresplayer improvements
 (#12548)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 1bdb74285..7fd47d5fb 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -212,3 +212,4 @@ Xiao Di Guan
 Thomas Winant
 Daniel Twardowski
 Jeremie Jarosh
+Gerard Rovira

From 06c1b3ce070a36bc769b599cb36bc71dd59bed86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 23 May 2017 00:07:31 +0700
Subject: [PATCH 0890/1696] Credit @mphe for streamango (#12643)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 7fd47d5fb..487a2bdab 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -213,3 +213,4 @@ Thomas Winant
 Daniel Twardowski
 Jeremie Jarosh
 Gerard Rovira
+Marvin Ewald

From 01f3c8e29003b3b82f33849975e36962ba34372c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 23 May 2017 00:08:14 +0700
Subject: [PATCH 0891/1696] Credit @fredbourni for noovo (#12792)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 487a2bdab..b5e8c074b 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -214,3 +214,4 @@ Daniel Twardowski
 Jeremie Jarosh
 Gerard Rovira
 Marvin Ewald
+Frédéric Bournival

From 9dc5ab041f62d9e968f9c08f26fc98b92d819a4e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 23 May 2017 00:15:44 +0700
Subject: [PATCH 0892/1696] [ChangeLog] Actualize

---
 ChangeLog | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 1637876b3..373fa12a6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,19 @@
+version <unreleased>
+
+Core
++ [downloader/external] Pass -loglevel to ffmpeg downloader (#13183)
++ [adobepass] Add support for Bright House Networks (#13149)
+
+Extractors
++ [streamcz] Add support for subtitles (#13174)
+* [youtube] Fix DASH manifest signature decryption (#8944, #13156)
+* [toggle] Relax URL regular expression (#13172)
+* [toypics] Fix extraction (#13077)
+* [njpwworld] Fix extraction (#13162, #13169)
++ [hitbox] Add support for smashcast.tv (#13154)
+* [mitele] Update app key regular expression (#13158)
+
+
 version 2017.05.18.1
 
 Core

From 2570e851677f1930c32b953424fa6fe8d8769856 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 23 May 2017 00:17:48 +0700
Subject: [PATCH 0893/1696] release 2017.05.23

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 +-
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index b714fcfa8..730a07ed1 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.18.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.18.1**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.23*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.23**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.05.18.1
+[debug] youtube-dl version 2017.05.23
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 373fa12a6..75a95f0b0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.05.23
 
 Core
 + [downloader/external] Pass -loglevel to ffmpeg downloader (#13183)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index aa6c118f2..5545cff23 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -803,7 +803,7 @@
  - **ToonGoggles**
  - **Tosh**: Tosh.0
  - **tou.tv**
- - **Toypics**: Toypics user profile
+ - **Toypics**: Toypics video
  - **ToypicsUser**: Toypics user profile
  - **TrailerAddict** (Currently broken)
  - **Trilulilu**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 5e963e71c..c006eba76 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.05.18.1'
+__version__ = '2017.05.23'

From de53511201589f12b252680cfb2c35ac73fa8c9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 23 May 2017 00:41:53 +0700
Subject: [PATCH 0894/1696] Credit @timendum for rai (#11790) and mediaset
 (#12964)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index b5e8c074b..9a8a72265 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -215,3 +215,4 @@ Jeremie Jarosh
 Gerard Rovira
 Marvin Ewald
 Frédéric Bournival
+Timendum

From 0551f1b07b41bf64b4b8c1150abec2903e37a81f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 23 May 2017 21:47:40 +0700
Subject: [PATCH 0895/1696] Credit @gritstub for vevo fix (#12879)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 9a8a72265..f80c34a08 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -216,3 +216,4 @@ Gerard Rovira
 Marvin Ewald
 Frédéric Bournival
 Timendum
+gritstub

From 884d09f3304992837be140e1e22ff44a87cbf6ff Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 24 May 2017 09:44:41 +0100
Subject: [PATCH 0896/1696] [cbsnews] fix extraction for 60 Minutes videos

---
 youtube_dl/extractor/cbsnews.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 17bb9af4f..651ae2665 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -65,7 +65,7 @@ class CBSNewsIE(CBSIE):
 
         item = video_info['item'] if 'item' in video_info else video_info
         guid = item['mpxRefId']
-        return self._extract_video_info(guid)
+        return self._extract_video_info(guid, 'cbsnews')
 
 
 class CBSNewsLiveVideoIE(InfoExtractor):

From e834f04400ba419b842743224510ca8abaf933c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 May 2017 22:58:16 +0700
Subject: [PATCH 0897/1696] [vimeo] Fix formats' sorting (closes #13189)

---
 youtube_dl/extractor/vimeo.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 61cc469bf..a483c8409 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -151,10 +151,16 @@ class VimeoBaseInfoExtractor(InfoExtractor):
                     else:
                         mpd_manifest_urls = [(format_id, manifest_url)]
                     for f_id, m_url in mpd_manifest_urls:
-                        formats.extend(self._extract_mpd_formats(
+                        mpd_formats = self._extract_mpd_formats(
                             m_url.replace('/master.json', '/master.mpd'), video_id, f_id,
                             'Downloading %s MPD information' % cdn_name,
-                            fatal=False))
+                            fatal=False)
+                        for f in mpd_formats:
+                            if f.get('vcodec') == 'none':
+                                f['preference'] = -50
+                            elif f.get('acodec') == 'none':
+                                f['preference'] = -40
+                        formats.extend(mpd_formats)
 
         subtitles = {}
         text_tracks = config['request'].get('text_tracks')

From 5192ee17e79ec103abf1e5a06faa0b0019ab00f2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 25 May 2017 22:04:12 +0800
Subject: [PATCH 0898/1696] [postprocessor/ffmpeg] Fix metadata filename
 handling on Python 2

Fixes #13182
---
 ChangeLog                          | 6 ++++++
 youtube_dl/postprocessor/ffmpeg.py | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 75a95f0b0..9e4f53093 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Core
+* [postprocessor/ffmpeg] Fix metadata filename handling on Python 2 (#13182)
+
+
 version 2017.05.23
 
 Core
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index c91ec8588..f021ea8fd 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -444,7 +444,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
 
         chapters = info.get('chapters', [])
         if chapters:
-            metadata_filename = encodeFilename(replace_extension(filename, 'meta'))
+            metadata_filename = replace_extension(filename, 'meta')
             with io.open(metadata_filename, 'wt', encoding='utf-8') as f:
                 def ffmpeg_escape(text):
                     return re.sub(r'(=|;|#|\\|\n)', r'\\\1', text)

From b7f8749304aca8e8f338f40a1b2877e4ac732c73 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 May 2017 22:28:26 +0700
Subject: [PATCH 0899/1696] [udemy] Fix extraction for outputs' format entries
 without URL (closes #13192)

---
 youtube_dl/extractor/udemy.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index dae1aa3c6..160be1b1b 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -52,6 +52,10 @@ class UdemyIE(InfoExtractor):
         # new URL schema
         'url': 'https://www.udemy.com/electric-bass-right-from-the-start/learn/v4/t/lecture/4580906',
         'only_matching': True,
+    }, {
+        # no url in outputs format entry
+        'url': 'https://www.udemy.com/learn-web-development-complete-step-by-step-guide-to-success/learn/v4/t/lecture/4125812',
+        'only_matching': True,
     }]
 
     def _extract_course_info(self, webpage, video_id):
@@ -219,7 +223,7 @@ class UdemyIE(InfoExtractor):
 
         def extract_output_format(src, f_id):
             return {
-                'url': src['url'],
+                'url': src.get('url'),
                 'format_id': '%sp' % (src.get('height') or f_id),
                 'width': int_or_none(src.get('width')),
                 'height': int_or_none(src.get('height')),

From 59ed87cbd9ea08c889514a05b646141004f432a1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 26 May 2017 14:15:46 +0800
Subject: [PATCH 0900/1696] [youku] Fix extraction (closes #13191)

---
 ChangeLog                     |   3 +
 youtube_dl/extractor/youku.py | 214 +++++++---------------------------
 2 files changed, 45 insertions(+), 172 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 9e4f53093..6a05657ab 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -3,6 +3,9 @@ version <unreleased>
 Core
 * [postprocessor/ffmpeg] Fix metadata filename handling on Python 2 (#13182)
 
+Extractors
+* [youku] Fix extraction (#13191)
+
 
 version 2017.05.23
 
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 73ebe5759..37be440fa 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -1,7 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import base64
 import itertools
 import random
 import re
@@ -9,15 +8,9 @@ import string
 import time
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_ord,
-    compat_str,
-    compat_urllib_parse_urlencode,
-)
 from ..utils import (
     ExtractorError,
     get_element_by_attribute,
-    try_get,
 )
 
 
@@ -35,9 +28,9 @@ class YoukuIE(InfoExtractor):
         # MD5 is unstable
         'url': 'http://v.youku.com/v_show/id_XMTc1ODE5Njcy.html',
         'info_dict': {
-            'id': 'XMTc1ODE5Njcy_part1',
+            'id': 'XMTc1ODE5Njcy',
             'title': '★Smile﹗♡ Git Fresh -Booty Music舞蹈.',
-            'ext': 'flv'
+            'ext': 'mp4',
         }
     }, {
         'url': 'http://player.youku.com/player.php/sid/XNDgyMDQ2NTQw/v.swf',
@@ -46,25 +39,24 @@ class YoukuIE(InfoExtractor):
         'url': 'http://v.youku.com/v_show/id_XODgxNjg1Mzk2_ev_1.html',
         'info_dict': {
             'id': 'XODgxNjg1Mzk2',
+            'ext': 'mp4',
             'title': '武媚娘传奇 85',
         },
-        'playlist_count': 11,
-        'skip': 'Available in China only',
     }, {
         'url': 'http://v.youku.com/v_show/id_XMTI1OTczNDM5Mg==.html',
         'info_dict': {
             'id': 'XMTI1OTczNDM5Mg',
+            'ext': 'mp4',
             'title': '花千骨 04',
         },
-        'playlist_count': 13,
     }, {
         'url': 'http://v.youku.com/v_show/id_XNjA1NzA2Njgw.html',
         'note': 'Video protected with password',
         'info_dict': {
             'id': 'XNjA1NzA2Njgw',
+            'ext': 'mp4',
             'title': '邢義田复旦讲座之想象中的胡人—从“左衽孔子”说起',
         },
-        'playlist_count': 19,
         'params': {
             'videopassword': '100600',
         },
@@ -73,130 +65,16 @@ class YoukuIE(InfoExtractor):
         'url': 'http://v.youku.com/v_show/id_XOTUxMzg4NDMy.html',
         'info_dict': {
             'id': 'XOTUxMzg4NDMy',
+            'ext': 'mp4',
             'title': '我的世界☆明月庄主☆车震猎杀☆杀人艺术Minecraft',
         },
-        'playlist_count': 6,
     }]
 
-    def construct_video_urls(self, data):
-        # get sid, token
-        def yk_t(s1, s2):
-            ls = list(range(256))
-            t = 0
-            for i in range(256):
-                t = (t + ls[i] + compat_ord(s1[i % len(s1)])) % 256
-                ls[i], ls[t] = ls[t], ls[i]
-            s = bytearray()
-            x, y = 0, 0
-            for i in range(len(s2)):
-                y = (y + 1) % 256
-                x = (x + ls[y]) % 256
-                ls[x], ls[y] = ls[y], ls[x]
-                s.append(compat_ord(s2[i]) ^ ls[(ls[x] + ls[y]) % 256])
-            return bytes(s)
-
-        sid, token = yk_t(
-            b'becaf9be', base64.b64decode(data['security']['encrypt_string'].encode('ascii'))
-        ).decode('ascii').split('_')
-
-        # get oip
-        oip = data['security']['ip']
-
-        fileid_dict = {}
-        for stream in data['stream']:
-            if stream.get('channel_type') == 'tail':
-                continue
-            format = stream.get('stream_type')
-            fileid = try_get(
-                stream, lambda x: x['segs'][0]['fileid'],
-                compat_str) or stream['stream_fileid']
-            fileid_dict[format] = fileid
-
-        def get_fileid(format, n):
-            number = hex(int(str(n), 10))[2:].upper()
-            if len(number) == 1:
-                number = '0' + number
-            streamfileids = fileid_dict[format]
-            fileid = streamfileids[0:8] + number + streamfileids[10:]
-            return fileid
-
-        # get ep
-        def generate_ep(format, n):
-            fileid = get_fileid(format, n)
-            ep_t = yk_t(
-                b'bf7e5f01',
-                ('%s_%s_%s' % (sid, fileid, token)).encode('ascii')
-            )
-            ep = base64.b64encode(ep_t).decode('ascii')
-            return ep
-
-        # generate video_urls
-        video_urls_dict = {}
-        for stream in data['stream']:
-            if stream.get('channel_type') == 'tail':
-                continue
-            format = stream.get('stream_type')
-            video_urls = []
-            for dt in stream['segs']:
-                n = str(stream['segs'].index(dt))
-                param = {
-                    'K': dt['key'],
-                    'hd': self.get_hd(format),
-                    'myp': 0,
-                    'ypp': 0,
-                    'ctype': 12,
-                    'ev': 1,
-                    'token': token,
-                    'oip': oip,
-                    'ep': generate_ep(format, n)
-                }
-                video_url = \
-                    'http://k.youku.com/player/getFlvPath/' + \
-                    'sid/' + sid + \
-                    '_00' + \
-                    '/st/' + self.parse_ext_l(format) + \
-                    '/fileid/' + get_fileid(format, n) + '?' + \
-                    compat_urllib_parse_urlencode(param)
-                video_urls.append(video_url)
-            video_urls_dict[format] = video_urls
-
-        return video_urls_dict
-
     @staticmethod
     def get_ysuid():
         return '%d%s' % (int(time.time()), ''.join([
             random.choice(string.ascii_letters) for i in range(3)]))
 
-    def get_hd(self, fm):
-        hd_id_dict = {
-            '3gp': '0',
-            '3gphd': '1',
-            'flv': '0',
-            'flvhd': '0',
-            'mp4': '1',
-            'mp4hd': '1',
-            'mp4hd2': '1',
-            'mp4hd3': '1',
-            'hd2': '2',
-            'hd3': '3',
-        }
-        return hd_id_dict[fm]
-
-    def parse_ext_l(self, fm):
-        ext_dict = {
-            '3gp': 'flv',
-            '3gphd': 'mp4',
-            'flv': 'flv',
-            'flvhd': 'flv',
-            'mp4': 'mp4',
-            'mp4hd': 'mp4',
-            'mp4hd2': 'flv',
-            'mp4hd3': 'flv',
-            'hd2': 'flv',
-            'hd3': 'flv',
-        }
-        return ext_dict[fm]
-
     def get_format_name(self, fm):
         _dict = {
             '3gp': 'h6',
@@ -210,32 +88,40 @@ class YoukuIE(InfoExtractor):
             'hd2': 'h2',
             'hd3': 'h1',
         }
-        return _dict[fm]
+        return _dict.get(fm)
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         self._set_cookie('youku.com', '__ysuid', self.get_ysuid())
+        self._set_cookie('youku.com', 'xreferrer', 'http://www.youku.com')
 
-        def retrieve_data(req_url, note):
-            headers = {
-                'Referer': req_url,
-            }
-            headers.update(self.geo_verification_headers())
-            self._set_cookie('youku.com', 'xreferrer', 'http://www.youku.com')
-
-            raw_data = self._download_json(req_url, video_id, note=note, headers=headers)
-
-            return raw_data['data']
-
-        video_password = self._downloader.params.get('videopassword')
+        _, urlh = self._download_webpage_handle(
+            'https://log.mmstat.com/eg.js', video_id, 'Retrieving cna info')
+        # The etag header is '"foobar"'; let's remove the double quotes
+        cna = urlh.headers['etag'][1:-1]
 
         # request basic data
-        basic_data_url = 'http://play.youku.com/play/get.json?vid=%s&ct=12' % video_id
-        if video_password:
-            basic_data_url += '&pwd=%s' % video_password
+        basic_data_params = {
+            'vid': video_id,
+            'ccode': '0401',
+            'client_ip': '192.168.1.1',
+            'utid': cna,
+            'client_ts': time.time() / 1000,
+        }
 
-        data = retrieve_data(basic_data_url, 'Downloading JSON metadata')
+        video_password = self._downloader.params.get('videopassword')
+        if video_password:
+            basic_data_params['password'] = video_password
+
+        headers = {
+            'Referer': url,
+        }
+        headers.update(self.geo_verification_headers())
+        data = self._download_json(
+            'https://ups.youku.com/ups/get.json', video_id,
+            'Downloading JSON metadata',
+            query=basic_data_params, headers=headers)['data']
 
         error = data.get('error')
         if error:
@@ -255,37 +141,21 @@ class YoukuIE(InfoExtractor):
         # get video title
         title = data['video']['title']
 
-        # generate video_urls_dict
-        video_urls_dict = self.construct_video_urls(data)
-
-        # construct info
-        entries = [{
-            'id': '%s_part%d' % (video_id, i + 1),
-            'title': title,
-            'formats': [],
-            # some formats are not available for all parts, we have to detect
-            # which one has all
-        } for i in range(max(len(v.get('segs')) for v in data['stream']))]
-        for stream in data['stream']:
-            if stream.get('channel_type') == 'tail':
-                continue
-            fm = stream.get('stream_type')
-            video_urls = video_urls_dict[fm]
-            for video_url, seg, entry in zip(video_urls, stream['segs'], entries):
-                entry['formats'].append({
-                    'url': video_url,
-                    'format_id': self.get_format_name(fm),
-                    'ext': self.parse_ext_l(fm),
-                    'filesize': int(seg['size']),
-                    'width': stream.get('width'),
-                    'height': stream.get('height'),
-                })
+        formats = [{
+            'url': stream['m3u8_url'],
+            'format_id': self.get_format_name(stream.get('stream_type')),
+            'ext': 'mp4',
+            'protocol': 'm3u8_native',
+            'filesize': int(stream.get('size')),
+            'width': stream.get('width'),
+            'height': stream.get('height'),
+        } for stream in data['stream'] if stream.get('channel_type') != 'tail']
+        self._sort_formats(formats)
 
         return {
-            '_type': 'multi_video',
             'id': video_id,
             'title': title,
-            'entries': entries,
+            'formats': formats,
         }
 
 

From 5552c9eb0fece567f7dda13810939fca32d7d65a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 26 May 2017 21:58:18 +0800
Subject: [PATCH 0901/1696] [utils] Recognize more patterns in strip_jsonp()

Used in Youku Show pages
---
 ChangeLog           | 1 +
 test/test_utils.py  | 8 ++++++++
 youtube_dl/utils.py | 7 ++++++-
 3 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 6a05657ab..d6e980c5a 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Core
++ [utils] strip_jsonp() can recognize more patterns
 * [postprocessor/ffmpeg] Fix metadata filename handling on Python 2 (#13182)
 
 Extractors
diff --git a/test/test_utils.py b/test/test_utils.py
index f31559e71..d7e05817c 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -678,6 +678,14 @@ class TestUtil(unittest.TestCase):
         d = json.loads(stripped)
         self.assertEqual(d, {'status': 'success'})
 
+        stripped = strip_jsonp('window.cb && window.cb({"status": "success"});')
+        d = json.loads(stripped)
+        self.assertEqual(d, {'status': 'success'})
+
+        stripped = strip_jsonp('window.cb && cb({"status": "success"});')
+        d = json.loads(stripped)
+        self.assertEqual(d, {'status': 'success'})
+
     def test_uppercase_escape(self):
         self.assertEqual(uppercase_escape('aä'), 'aä')
         self.assertEqual(uppercase_escape('\\U0001d550'), '𝕐')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 4293a77f5..6c84bfe0f 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2211,7 +2211,12 @@ def parse_age_limit(s):
 
 def strip_jsonp(code):
     return re.sub(
-        r'(?s)^[a-zA-Z0-9_.$]+\s*\(\s*(.*)\);?\s*?(?://[^\n]*)*$', r'\1', code)
+        r'''(?sx)^
+            (?:window\.)?(?P<func_name>[a-zA-Z0-9_.$]+)
+            (?:\s*&&\s*(?P=func_name))?
+            \s*\(\s*(?P<callback_data>.*)\);?
+            \s*?(?://[^\n]*)*$''',
+        r'\g<callback_data>', code)
 
 
 def js_to_json(code):

From d3d4ba7f241bb3c8d6478a36917ff6c81e97a85d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 26 May 2017 21:59:16 +0800
Subject: [PATCH 0902/1696] [youku:show] Fix extraction

---
 ChangeLog                     |  1 +
 youtube_dl/extractor/youku.py | 67 +++++++++++++++++++++--------------
 2 files changed, 41 insertions(+), 27 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index d6e980c5a..2ac471f8e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -5,6 +5,7 @@ Core
 * [postprocessor/ffmpeg] Fix metadata filename handling on Python 2 (#13182)
 
 Extractors
+* [youku:show] Fix extraction
 * [youku] Fix extraction (#13191)
 
 
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 37be440fa..e081cfc19 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -10,7 +10,10 @@ import time
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
-    get_element_by_attribute,
+    get_element_by_class,
+    js_to_json,
+    strip_jsonp,
+    urljoin,
 )
 
 
@@ -160,49 +163,59 @@ class YoukuIE(InfoExtractor):
 
 
 class YoukuShowIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?youku\.com/show_page/id_(?P<id>[0-9a-z]+)\.html'
+    _VALID_URL = r'https?://list\.youku\.com/show/id_(?P<id>[0-9a-z]+)\.html'
     IE_NAME = 'youku:show'
 
     _TEST = {
-        'url': 'http://www.youku.com/show_page/id_zc7c670be07ff11e48b3f.html',
+        'url': 'http://list.youku.com/show/id_zc7c670be07ff11e48b3f.html',
         'info_dict': {
             'id': 'zc7c670be07ff11e48b3f',
             'title': '花千骨 未删减版',
-            'description': 'md5:578d4f2145ae3f9128d9d4d863312910',
+            'description': 'md5:a1ae6f5618571bbeb5c9821f9c81b558',
         },
         'playlist_count': 50,
     }
 
     _PAGE_SIZE = 40
 
-    def _find_videos_in_page(self, webpage):
-        videos = re.findall(
-            r'<li><a[^>]+href="(?P<url>https?://v\.youku\.com/[^"]+)"[^>]+title="(?P<title>[^"]+)"', webpage)
-        return [
-            self.url_result(video_url, YoukuIE.ie_key(), title)
-            for video_url, title in videos]
-
     def _real_extract(self, url):
         show_id = self._match_id(url)
         webpage = self._download_webpage(url, show_id)
 
-        entries = self._find_videos_in_page(webpage)
-
-        playlist_title = self._html_search_regex(
-            r'<span[^>]+class="name">([^<]+)</span>', webpage, 'playlist title', fatal=False)
-        detail_div = get_element_by_attribute('class', 'detail', webpage) or ''
-        playlist_description = self._html_search_regex(
-            r'<span[^>]+style="display:none"[^>]*>([^<]+)</span>',
-            detail_div, 'playlist description', fatal=False)
-
-        for idx in itertools.count(1):
-            episodes_page = self._download_webpage(
-                'http://www.youku.com/show_episode/id_%s.html' % show_id,
-                show_id, query={'divid': 'reload_%d' % (idx * self._PAGE_SIZE + 1)},
-                note='Downloading episodes page %d' % idx)
-            new_entries = self._find_videos_in_page(episodes_page)
+        entries = []
+        page_config = self._parse_json(self._search_regex(
+            r'var\s+PageConfig\s*=\s*({.+});', webpage, 'page config'),
+            show_id, transform_source=js_to_json)
+        for idx in itertools.count(0):
+            if idx == 0:
+                playlist_data_url = 'http://list.youku.com/show/module'
+                query = {'id': page_config['showid'], 'tab': 'point'}
+            else:
+                playlist_data_url = 'http://list.youku.com/show/point'
+                query = {
+                    'id': page_config['showid'],
+                    'stage': 'reload_%d' % (self._PAGE_SIZE * idx + 1),
+                }
+            query['callback'] = 'cb'
+            playlist_data = self._download_json(
+                playlist_data_url, show_id, query=query,
+                note='Downloading playlist data page %d' % (idx + 1),
+                transform_source=lambda s: js_to_json(strip_jsonp(s)))['html']
+            video_urls = re.findall(
+                r'<div[^>]+class="p-thumb"[^<]+<a[^>]+href="([^"]+)"',
+                playlist_data)
+            new_entries = [
+                self.url_result(urljoin(url, video_url), YoukuIE.ie_key())
+                for video_url in video_urls]
             entries.extend(new_entries)
             if len(new_entries) < self._PAGE_SIZE:
                 break
 
-        return self.playlist_result(entries, show_id, playlist_title, playlist_description)
+        desc = self._html_search_meta('description', webpage, fatal=False)
+        playlist_title = desc.split(',')[0] if desc else None
+        detail_li = get_element_by_class('p-intro', webpage)
+        playlist_description = get_element_by_class(
+            'intro-more', detail_li) if detail_li else None
+
+        return self.playlist_result(
+            entries, show_id, playlist_title, playlist_description)

From c130f0a37b5c31fa559373ec319f79830c591c22 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 26 May 2017 23:04:42 +0800
Subject: [PATCH 0903/1696] [tudou] Merge into youku extractor (fixes #12214)

Also, there are no tudou playlists anymore. All playlist URLs points to youku
playlists.
---
 ChangeLog                          |   1 +
 youtube_dl/extractor/extractors.py |   5 --
 youtube_dl/extractor/tudou.py      | 132 -----------------------------
 youtube_dl/extractor/youku.py      |  16 +++-
 4 files changed, 15 insertions(+), 139 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 2ac471f8e..936ba512c 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -5,6 +5,7 @@ Core
 * [postprocessor/ffmpeg] Fix metadata filename handling on Python 2 (#13182)
 
 Extractors
+* [tudou] Merge into youku extractor (#12214)
 * [youku:show] Fix extraction
 * [youku] Fix extraction (#13191)
 
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index ed603eb29..558c910d4 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1019,11 +1019,6 @@ from .trilulilu import TriluliluIE
 from .trutv import TruTVIE
 from .tube8 import Tube8IE
 from .tubitv import TubiTvIE
-from .tudou import (
-    TudouIE,
-    TudouPlaylistIE,
-    TudouAlbumIE,
-)
 from .tumblr import TumblrIE
 from .tunein import (
     TuneInClipIE,
diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index 2aae55e7e..7421378a8 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -3,138 +3,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_str
-from ..utils import (
-    ExtractorError,
-    int_or_none,
-    InAdvancePagedList,
-    float_or_none,
-    unescapeHTML,
-)
-
-
-class TudouIE(InfoExtractor):
-    IE_NAME = 'tudou'
-    _VALID_URL = r'https?://(?:www\.)?tudou\.com/(?:(?:programs|wlplay)/view|(?:listplay|albumplay)/[\w-]{11})/(?P<id>[\w-]{11})'
-    _TESTS = [{
-        'url': 'http://www.tudou.com/listplay/zzdE77v6Mmo/2xN2duXMxmw.html',
-        'md5': '140a49ed444bd22f93330985d8475fcb',
-        'info_dict': {
-            'id': '159448201',
-            'ext': 'f4v',
-            'title': '卡马乔国足开大脚长传冲吊集锦',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'timestamp': 1372113489000,
-            'description': '卡马乔卡家军，开大脚先进战术不完全集锦！',
-            'duration': 289.04,
-            'view_count': int,
-            'filesize': int,
-        }
-    }, {
-        'url': 'http://www.tudou.com/programs/view/ajX3gyhL0pc/',
-        'info_dict': {
-            'id': '117049447',
-            'ext': 'f4v',
-            'title': 'La Sylphide-Bolshoi-Ekaterina Krysanova & Vyacheslav Lopatin 2012',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'timestamp': 1349207518000,
-            'description': 'md5:294612423894260f2dcd5c6c04fe248b',
-            'duration': 5478.33,
-            'view_count': int,
-            'filesize': int,
-        }
-    }]
-
-    _PLAYER_URL = 'http://js.tudouui.com/bin/lingtong/PortalPlayer_177.swf'
-
-    # Translated from tudou/tools/TVCHelper.as in PortalPlayer_193.swf
-    # 0001, 0002 and 4001 are not included as they indicate temporary issues
-    TVC_ERRORS = {
-        '0003': 'The video is deleted or does not exist',
-        '1001': 'This video is unavailable due to licensing issues',
-        '1002': 'This video is unavailable as it\'s under review',
-        '1003': 'This video is unavailable as it\'s under review',
-        '3001': 'Password required',
-        '5001': 'This video is available in Mainland China only due to licensing issues',
-        '7001': 'This video is unavailable',
-        '8001': 'This video is unavailable due to licensing issues',
-    }
-
-    def _url_for_id(self, video_id, quality=None):
-        info_url = 'http://v2.tudou.com/f?id=' + compat_str(video_id)
-        if quality:
-            info_url += '&hd' + quality
-        xml_data = self._download_xml(info_url, video_id, 'Opening the info XML page')
-        error = xml_data.attrib.get('error')
-        if error is not None:
-            raise ExtractorError('Tudou said: %s' % error, expected=True)
-        final_url = xml_data.text
-        return final_url
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        item_data = self._download_json(
-            'http://www.tudou.com/tvp/getItemInfo.action?ic=%s' % video_id, video_id)
-
-        youku_vcode = item_data.get('vcode')
-        if youku_vcode:
-            return self.url_result('youku:' + youku_vcode, ie='Youku')
-
-        if not item_data.get('itemSegs'):
-            tvc_code = item_data.get('tvcCode')
-            if tvc_code:
-                err_msg = self.TVC_ERRORS.get(tvc_code)
-                if err_msg:
-                    raise ExtractorError('Tudou said: %s' % err_msg, expected=True)
-                raise ExtractorError('Unexpected error %s returned from Tudou' % tvc_code)
-            raise ExtractorError('Unxpected error returned from Tudou')
-
-        title = unescapeHTML(item_data['kw'])
-        description = item_data.get('desc')
-        thumbnail_url = item_data.get('pic')
-        view_count = int_or_none(item_data.get('playTimes'))
-        timestamp = int_or_none(item_data.get('pt'))
-
-        segments = self._parse_json(item_data['itemSegs'], video_id)
-        # It looks like the keys are the arguments that have to be passed as
-        # the hd field in the request url, we pick the higher
-        # Also, filter non-number qualities (see issue #3643).
-        quality = sorted(filter(lambda k: k.isdigit(), segments.keys()),
-                         key=lambda k: int(k))[-1]
-        parts = segments[quality]
-        len_parts = len(parts)
-        if len_parts > 1:
-            self.to_screen('%s: found %s parts' % (video_id, len_parts))
-
-        def part_func(partnum):
-            part = parts[partnum]
-            part_id = part['k']
-            final_url = self._url_for_id(part_id, quality)
-            ext = (final_url.split('?')[0]).split('.')[-1]
-            return [{
-                'id': '%s' % part_id,
-                'url': final_url,
-                'ext': ext,
-                'title': title,
-                'thumbnail': thumbnail_url,
-                'description': description,
-                'view_count': view_count,
-                'timestamp': timestamp,
-                'duration': float_or_none(part.get('seconds'), 1000),
-                'filesize': int_or_none(part.get('size')),
-                'http_headers': {
-                    'Referer': self._PLAYER_URL,
-                },
-            }]
-
-        entries = InAdvancePagedList(part_func, len_parts, 1)
-
-        return {
-            '_type': 'multi_video',
-            'entries': entries,
-            'id': video_id,
-            'title': title,
-        }
 
 
 class TudouPlaylistIE(InfoExtractor):
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index e081cfc19..ab97466d2 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -22,7 +22,9 @@ class YoukuIE(InfoExtractor):
     IE_DESC = '优酷'
     _VALID_URL = r'''(?x)
         (?:
-            http://(?:v|player)\.youku\.com/(?:v_show/id_|player\.php/sid/)|
+            https?://(
+                (?:v|player)\.youku\.com/(?:v_show/id_|player\.php/sid/)|
+                video\.tudou\.com/v/)|
             youku:)
         (?P<id>[A-Za-z0-9]+)(?:\.html|/v\.swf|)
     '''
@@ -71,6 +73,16 @@ class YoukuIE(InfoExtractor):
             'ext': 'mp4',
             'title': '我的世界☆明月庄主☆车震猎杀☆杀人艺术Minecraft',
         },
+    }, {
+        'url': 'http://video.tudou.com/v/XMjIyNzAzMTQ4NA==.html?f=46177805',
+        'info_dict': {
+            'id': 'XMjIyNzAzMTQ4NA',
+            'ext': 'mp4',
+            'title': '卡马乔国足开大脚长传冲吊集锦',
+        },
+    }, {
+        'url': 'http://video.tudou.com/v/XMjE4ODI3OTg2MA==.html',
+        'only_matching': True,
     }]
 
     @staticmethod
@@ -107,7 +119,7 @@ class YoukuIE(InfoExtractor):
         # request basic data
         basic_data_params = {
             'vid': video_id,
-            'ccode': '0401',
+            'ccode': '0402' if 'tudou.com' in url else '0401',
             'client_ip': '192.168.1.1',
             'utid': cna,
             'client_ts': time.time() / 1000,

From 97067db2ae32b18ef573af99b91d36026b4b1d05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 26 May 2017 22:12:24 +0700
Subject: [PATCH 0904/1696] [bbc] Add support for authentication

---
 youtube_dl/extractor/bbc.py | 40 +++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index dd65b8d86..7cd4446b4 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -6,14 +6,18 @@ import itertools
 
 from .common import InfoExtractor
 from ..utils import (
+    clean_html,
     dict_get,
     ExtractorError,
     float_or_none,
+    get_element_by_class,
     int_or_none,
     parse_duration,
     parse_iso8601,
     try_get,
     unescapeHTML,
+    urlencode_postdata,
+    urljoin,
 )
 from ..compat import (
     compat_etree_fromstring,
@@ -38,6 +42,9 @@ class BBCCoUkIE(InfoExtractor):
                         (?P<id>%s)(?!/(?:episodes|broadcasts|clips))
                     ''' % _ID_REGEX
 
+    _LOGIN_URL = 'https://account.bbc.com/signin'
+    _NETRC_MACHINE = 'bbc'
+
     _MEDIASELECTOR_URLS = [
         # Provides HQ HLS streams with even better quality that pc mediaset but fails
         # with geolocation in some cases when it's even not geo restricted at all (e.g.
@@ -227,6 +234,39 @@ class BBCCoUkIE(InfoExtractor):
 
     _USP_RE = r'/([^/]+?)\.ism(?:\.hlsv2\.ism)?/[^/]+\.m3u8'
 
+    def _login(self):
+        username, password = self._get_login_info()
+        if username is None:
+            return
+
+        login_page = self._download_webpage(
+            self._LOGIN_URL, None, 'Downloading signin page')
+
+        login_form = self._hidden_inputs(login_page)
+
+        login_form.update({
+            'username': username,
+            'password': password,
+        })
+
+        post_url = urljoin(self._LOGIN_URL, self._search_regex(
+            r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
+            'post url', default=self._LOGIN_URL, group='url'))
+
+        response, urlh = self._download_webpage_handle(
+            post_url, None, 'Logging in', data=urlencode_postdata(login_form),
+            headers={'Referer': self._LOGIN_URL})
+
+        if self._LOGIN_URL in urlh.geturl():
+            error = clean_html(get_element_by_class('form-message', response))
+            if error:
+                raise ExtractorError(
+                    'Unable to login: %s' % error, expected=True)
+            raise ExtractorError('Unable to log in')
+
+    def _real_initialize(self):
+        self._login()
+
     class MediaSelectionError(Exception):
         def __init__(self, id):
             self.id = id

From fd545fc6d1a5b489830e80c421f7e09264e0c80f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 26 May 2017 12:43:14 +0800
Subject: [PATCH 0905/1696] Revert "[youtube] Don't use the DASH manifest from
 'get_video_info' if 'use_cipher_signature' is True (#5118)"

This reverts commit 87dc451108b278eb5e5600a0159bdbd337272392.
---
 youtube_dl/extractor/youtube.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index d66693c0c..d6c098706 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1481,8 +1481,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     if not video_info_webpage:
                         continue
                     get_video_info = compat_parse_qs(video_info_webpage)
-                    if get_video_info.get('use_cipher_signature') != ['True']:
-                        add_dash_mpd(get_video_info)
+                    add_dash_mpd(get_video_info)
                     if not video_info:
                         video_info = get_video_info
                     if 'token' in get_video_info:

From 98a061894176aa791e645dda0285c04da41241f1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 26 May 2017 23:21:32 +0800
Subject: [PATCH 0906/1696] [ChangeLog] Update after the fix for #11381

---
 ChangeLog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChangeLog b/ChangeLog
index 936ba512c..a622f704d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -5,6 +5,7 @@ Core
 * [postprocessor/ffmpeg] Fix metadata filename handling on Python 2 (#13182)
 
 Extractors
++ [youtube] DASH MPDs with cipher signatures are recognized now (#11381)
 * [tudou] Merge into youku extractor (#12214)
 * [youku:show] Fix extraction
 * [youku] Fix extraction (#13191)

From cc304ce588ca95ddeca67d376e6656eb2bb2b988 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 26 May 2017 22:27:56 +0700
Subject: [PATCH 0907/1696] [ChangeLog] Actualize

---
 ChangeLog | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index a622f704d..c38f414f4 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -6,9 +6,13 @@ Core
 
 Extractors
 + [youtube] DASH MPDs with cipher signatures are recognized now (#11381)
++ [bbc] Add support for authentication
 * [tudou] Merge into youku extractor (#12214)
 * [youku:show] Fix extraction
 * [youku] Fix extraction (#13191)
+* [udemy] Fix extraction for outputs' format entries without URL (#13192)
+* [vimeo] Fix formats' sorting (#13189)
+* [cbsnews] Fix extraction for 60 Minutes videos (#12861)
 
 
 version 2017.05.23

From 28dbde9cc3e9921737b7677409fef073ea207363 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 26 May 2017 22:29:16 +0700
Subject: [PATCH 0908/1696] release 2017.05.26

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 3 ---
 youtube_dl/version.py     | 2 +-
 4 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 730a07ed1..5ca69ffa1 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.23*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.23**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.26*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.26**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.05.23
+[debug] youtube-dl version 2017.05.26
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index c38f414f4..ff6bccf61 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.05.26
 
 Core
 + [utils] strip_jsonp() can recognize more patterns
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 5545cff23..9791d585b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -810,9 +810,6 @@
  - **TruTV**
  - **Tube8**
  - **TubiTv**
- - **tudou**
- - **tudou:album**
- - **tudou:playlist**
  - **Tumblr**
  - **tunein:clip**
  - **tunein:program**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index c006eba76..16d91dfad 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.05.23'
+__version__ = '2017.05.26'

From 7f4c3a7439a66e522a4cd5d7ef3f0fb19bf155af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 26 May 2017 22:42:27 +0700
Subject: [PATCH 0909/1696] [cbsnews] Fix extraction (closes #13205)

---
 youtube_dl/extractor/cbsnews.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 651ae2665..5b61c3f52 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -61,11 +61,17 @@ class CBSNewsIE(CBSIE):
 
         video_info = self._parse_json(self._html_search_regex(
             r'(?:<ul class="media-list items" id="media-related-items"><li data-video-info|<div id="cbsNewsVideoPlayer" data-video-player-options)=\'({.+?})\'',
-            webpage, 'video JSON info'), video_id)
+            webpage, 'video JSON info', default='{}'), video_id, fatal=False)
 
-        item = video_info['item'] if 'item' in video_info else video_info
-        guid = item['mpxRefId']
-        return self._extract_video_info(guid, 'cbsnews')
+        if video_info:
+            item = video_info['item'] if 'item' in video_info else video_info
+        else:
+            state = self._parse_json(self._search_regex(
+                r'data-cbsvideoui-options=(["\'])(?P<json>{.+?})\1', webpage,
+                'playlist JSON info', group='json'), video_id)['state']
+            item = state['playlist'][state['pid']]
+
+        return self._extract_video_info(item['mpxRefId'], 'cbsnews')
 
 
 class CBSNewsLiveVideoIE(InfoExtractor):

From d16c0121b91f8f2b1b7131f3dcad06902d39613c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 27 May 2017 00:06:45 +0800
Subject: [PATCH 0910/1696] [youku] Extract more metadata (closes #10433)

---
 ChangeLog                     |  6 +++++
 youtube_dl/extractor/youku.py | 46 ++++++++++++++++++++++++++++++++++-
 2 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index ff6bccf61..65a3ca468 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
++ [youku] Extract more metadata (#10433)
+
+
 version 2017.05.26
 
 Core
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index ab97466d2..dcce15d77 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -12,6 +12,7 @@ from ..utils import (
     ExtractorError,
     get_element_by_class,
     js_to_json,
+    str_or_none,
     strip_jsonp,
     urljoin,
 )
@@ -36,6 +37,12 @@ class YoukuIE(InfoExtractor):
             'id': 'XMTc1ODE5Njcy',
             'title': '★Smile﹗♡ Git Fresh -Booty Music舞蹈.',
             'ext': 'mp4',
+            'duration': 74.73,
+            'thumbnail': r're:^https?://.*',
+            'uploader': '。躲猫猫、',
+            'uploader_id': '36017967',
+            'uploader_url': 'http://i.youku.com/u/UMTQ0MDcxODY4',
+            'tags': list,
         }
     }, {
         'url': 'http://player.youku.com/player.php/sid/XNDgyMDQ2NTQw/v.swf',
@@ -46,6 +53,12 @@ class YoukuIE(InfoExtractor):
             'id': 'XODgxNjg1Mzk2',
             'ext': 'mp4',
             'title': '武媚娘传奇 85',
+            'duration': 1999.61,
+            'thumbnail': r're:^https?://.*',
+            'uploader': '疯狂豆花',
+            'uploader_id': '62583473',
+            'uploader_url': 'http://i.youku.com/u/UMjUwMzMzODky',
+            'tags': list,
         },
     }, {
         'url': 'http://v.youku.com/v_show/id_XMTI1OTczNDM5Mg==.html',
@@ -53,6 +66,12 @@ class YoukuIE(InfoExtractor):
             'id': 'XMTI1OTczNDM5Mg',
             'ext': 'mp4',
             'title': '花千骨 04',
+            'duration': 2363,
+            'thumbnail': r're:^https?://.*',
+            'uploader': '放剧场-花千骨',
+            'uploader_id': '772849359',
+            'uploader_url': 'http://i.youku.com/u/UMzA5MTM5NzQzNg==',
+            'tags': list,
         },
     }, {
         'url': 'http://v.youku.com/v_show/id_XNjA1NzA2Njgw.html',
@@ -61,6 +80,12 @@ class YoukuIE(InfoExtractor):
             'id': 'XNjA1NzA2Njgw',
             'ext': 'mp4',
             'title': '邢義田复旦讲座之想象中的胡人—从“左衽孔子”说起',
+            'duration': 7264.5,
+            'thumbnail': r're:^https?://.*',
+            'uploader': 'FoxJin1006',
+            'uploader_id': '322014285',
+            'uploader_url': 'http://i.youku.com/u/UMTI4ODA1NzE0MA==',
+            'tags': list,
         },
         'params': {
             'videopassword': '100600',
@@ -72,6 +97,12 @@ class YoukuIE(InfoExtractor):
             'id': 'XOTUxMzg4NDMy',
             'ext': 'mp4',
             'title': '我的世界☆明月庄主☆车震猎杀☆杀人艺术Minecraft',
+            'duration': 702.08,
+            'thumbnail': r're:^https?://.*',
+            'uploader': '明月庄主moon',
+            'uploader_id': '38465621',
+            'uploader_url': 'http://i.youku.com/u/UMTUzODYyNDg0',
+            'tags': list,
         },
     }, {
         'url': 'http://video.tudou.com/v/XMjIyNzAzMTQ4NA==.html?f=46177805',
@@ -79,6 +110,12 @@ class YoukuIE(InfoExtractor):
             'id': 'XMjIyNzAzMTQ4NA',
             'ext': 'mp4',
             'title': '卡马乔国足开大脚长传冲吊集锦',
+            'duration': 289,
+            'thumbnail': r're:^https?://.*',
+            'uploader': '阿卜杜拉之星',
+            'uploader_id': '2382249',
+            'uploader_url': 'http://i.youku.com/u/UOTUyODk5Ng==',
+            'tags': list,
         },
     }, {
         'url': 'http://video.tudou.com/v/XMjE4ODI3OTg2MA==.html',
@@ -154,7 +191,8 @@ class YoukuIE(InfoExtractor):
                 raise ExtractorError(msg)
 
         # get video title
-        title = data['video']['title']
+        video_data = data['video']
+        title = video_data['title']
 
         formats = [{
             'url': stream['m3u8_url'],
@@ -171,6 +209,12 @@ class YoukuIE(InfoExtractor):
             'id': video_id,
             'title': title,
             'formats': formats,
+            'duration': video_data.get('seconds'),
+            'thumbnail': video_data.get('logo'),
+            'uploader': video_data.get('username'),
+            'uploader_id': str_or_none(video_data.get('userid')),
+            'uploader_url': data.get('uploader', {}).get('homepage'),
+            'tags': video_data.get('tags'),
         }
 
 

From 20e2c9de0414f18dff26020045445cf91cad6b70 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 26 May 2017 20:00:44 +0100
Subject: [PATCH 0911/1696] [adn] fix formats extraction

---
 youtube_dl/extractor/adn.py | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/adn.py b/youtube_dl/extractor/adn.py
index 66caf6a81..39f80b23f 100644
--- a/youtube_dl/extractor/adn.py
+++ b/youtube_dl/extractor/adn.py
@@ -15,6 +15,7 @@ from ..utils import (
     intlist_to_bytes,
     srt_subtitles_timecode,
     strip_or_none,
+    urljoin,
 )
 
 
@@ -31,25 +32,28 @@ class ADNIE(InfoExtractor):
             'description': 'md5:2f7b5aa76edbc1a7a92cedcda8a528d5',
         }
     }
+    _BASE_URL = 'http://animedigitalnetwork.fr'
 
     def _get_subtitles(self, sub_path, video_id):
         if not sub_path:
             return None
 
         enc_subtitles = self._download_webpage(
-            'http://animedigitalnetwork.fr/' + sub_path,
-            video_id, fatal=False)
+            urljoin(self._BASE_URL, sub_path),
+            video_id, fatal=False, headers={
+                'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:53.0) Gecko/20100101 Firefox/53.0',
+            })
         if not enc_subtitles:
             return None
 
         # http://animedigitalnetwork.fr/components/com_vodvideo/videojs/adn-vjs.min.js
         dec_subtitles = intlist_to_bytes(aes_cbc_decrypt(
             bytes_to_intlist(base64.b64decode(enc_subtitles[24:])),
-            bytes_to_intlist(b'\nd\xaf\xd2J\xd0\xfc\xe1\xfc\xdf\xb61\xe8\xe1\xf0\xcc'),
+            bytes_to_intlist(b'\x1b\xe0\x29\x61\x38\x94\x24\x00\x12\xbd\xc5\x80\xac\xce\xbe\xb0'),
             bytes_to_intlist(base64.b64decode(enc_subtitles[:24]))
         ))
         subtitles_json = self._parse_json(
-            dec_subtitles[:-compat_ord(dec_subtitles[-1])],
+            dec_subtitles[:-compat_ord(dec_subtitles[-1])].decode(),
             None, fatal=False)
         if not subtitles_json:
             return None
@@ -103,9 +107,16 @@ class ADNIE(InfoExtractor):
         metas = options.get('metas') or {}
         title = metas.get('title') or video_info['title']
         links = player_config.get('links') or {}
+        if not links:
+            links_url = player_config['linksurl']
+            links_data = self._download_json(urljoin(
+                self._BASE_URL, links_url), video_id)
+            links = links_data.get('links') or {}
 
         formats = []
         for format_id, qualities in links.items():
+            if not isinstance(qualities, dict):
+                continue
             for load_balancer_url in qualities.values():
                 load_balancer_data = self._download_json(
                     load_balancer_url, video_id, fatal=False) or {}

From b407d8533d3956a7c27ad42cbde9a877c36df72c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 27 May 2017 23:05:02 +0800
Subject: [PATCH 0912/1696] [utils] Drop an compatibility wrapper for Python <
 2.6

addinfourl.getcode is added since Python 2.6a1. As youtube-dl now
requires 2.6+, this is no longer necessary.

See https://github.com/python/cpython/commit/9b0d46db115c10767b240a0a64286214b50fe6ad
---
 youtube_dl/utils.py | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 6c84bfe0f..d79410e58 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -932,14 +932,6 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
         except zlib.error:
             return zlib.decompress(data)
 
-    @staticmethod
-    def addinfourl_wrapper(stream, headers, url, code):
-        if hasattr(compat_urllib_request.addinfourl, 'getcode'):
-            return compat_urllib_request.addinfourl(stream, headers, url, code)
-        ret = compat_urllib_request.addinfourl(stream, headers, url)
-        ret.code = code
-        return ret
-
     def http_request(self, req):
         # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
         # always respected by websites, some tend to give out URLs with non percent-encoded
@@ -991,13 +983,13 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
                     break
                 else:
                     raise original_ioerror
-            resp = self.addinfourl_wrapper(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
+            resp = compat_urllib_request.addinfourl(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
             resp.msg = old_resp.msg
             del resp.headers['Content-encoding']
         # deflate
         if resp.headers.get('Content-encoding', '') == 'deflate':
             gz = io.BytesIO(self.deflate(resp.read()))
-            resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
+            resp = compat_urllib_request.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code)
             resp.msg = old_resp.msg
             del resp.headers['Content-encoding']
         # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see

From 03327bc9a62bba1e6181e22e0ada82fb2f087fb0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 May 2017 22:36:58 +0700
Subject: [PATCH 0913/1696] [cbsinteractive] Relax _VALID_URL (closes #13213)

---
 youtube_dl/extractor/cbsinteractive.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cbsinteractive.py b/youtube_dl/extractor/cbsinteractive.py
index 86f7eee20..681d63e29 100644
--- a/youtube_dl/extractor/cbsinteractive.py
+++ b/youtube_dl/extractor/cbsinteractive.py
@@ -8,7 +8,7 @@ from ..utils import int_or_none
 
 
 class CBSInteractiveIE(CBSIE):
-    _VALID_URL = r'https?://(?:www\.)?(?P<site>cnet|zdnet)\.com/(?:videos|video/share)/(?P<id>[^/?]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<site>cnet|zdnet)\.com/(?:videos|video(?:/share)?)/(?P<id>[^/?]+)'
     _TESTS = [{
         'url': 'http://www.cnet.com/videos/hands-on-with-microsofts-windows-8-1-update/',
         'info_dict': {
@@ -60,6 +60,9 @@ class CBSInteractiveIE(CBSIE):
             # m3u8 download
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.zdnet.com/video/huawei-matebook-x-video/',
+        'only_matching': True,
     }]
 
     MPX_ACCOUNTS = {

From 1e0d65f0bded292440d55ca37449038c8b5d9344 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Sun, 14 May 2017 10:04:42 +0200
Subject: [PATCH 0914/1696] [beam:vod] Add extractor

---
 youtube_dl/extractor/beampro.py    | 121 ++++++++++++++++++++++++++---
 youtube_dl/extractor/extractors.py |   5 +-
 2 files changed, 116 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/beampro.py b/youtube_dl/extractor/beampro.py
index f3a9e3278..79c5fe5cd 100644
--- a/youtube_dl/extractor/beampro.py
+++ b/youtube_dl/extractor/beampro.py
@@ -6,16 +6,30 @@ from ..utils import (
     ExtractorError,
     clean_html,
     compat_str,
+    float_or_none,
     int_or_none,
     parse_iso8601,
     try_get,
+    urljoin,
 )
 
 
-class BeamProLiveIE(InfoExtractor):
+class BeamProBaseIE(InfoExtractor):
+    _RATINGS = {'family': 0, 'teen': 13, '18+': 18}
+
+    def _extract_channel_info(self, chan):
+        user_id = chan.get('userId') or try_get(chan, lambda x: x['user']['id'])
+        return {
+            'uploader': chan.get('token') or try_get(
+                chan, lambda x: x['user']['username'], compat_str),
+            'uploader_id': compat_str(user_id) if user_id else None,
+            'age_limit': self._RATINGS.get(chan.get('audience')),
+        }
+
+
+class BeamProLiveIE(BeamProBaseIE):
     IE_NAME = 'Beam:live'
     _VALID_URL = r'https?://(?:\w+\.)?beam\.pro/(?P<id>[^/?#&]+)'
-    _RATINGS = {'family': 0, 'teen': 13, '18+': 18}
     _TEST = {
         'url': 'http://www.beam.pro/niterhayven',
         'info_dict': {
@@ -38,6 +52,10 @@ class BeamProLiveIE(InfoExtractor):
         },
     }
 
+    @classmethod
+    def suitable(cls, url):
+        return False if BeamProVodIE.suitable(url) else super(BeamProLiveIE, cls).suitable(url)
+
     def _real_extract(self, url):
         channel_name = self._match_id(url)
 
@@ -55,19 +73,104 @@ class BeamProLiveIE(InfoExtractor):
             channel_name, ext='mp4', m3u8_id='hls', fatal=False)
         self._sort_formats(formats)
 
-        user_id = chan.get('userId') or try_get(chan, lambda x: x['user']['id'])
-
-        return {
+        info = {
             'id': compat_str(chan.get('id') or channel_name),
             'title': self._live_title(chan.get('name') or channel_name),
             'description': clean_html(chan.get('description')),
             'thumbnail': try_get(chan, lambda x: x['thumbnail']['url'], compat_str),
             'timestamp': parse_iso8601(chan.get('updatedAt')),
-            'uploader': chan.get('token') or try_get(
-                chan, lambda x: x['user']['username'], compat_str),
-            'uploader_id': compat_str(user_id) if user_id else None,
-            'age_limit': self._RATINGS.get(chan.get('audience')),
             'is_live': True,
             'view_count': int_or_none(chan.get('viewersTotal')),
             'formats': formats,
         }
+        info.update(self._extract_channel_info(chan))
+
+        return info
+
+
+class BeamProVodIE(BeamProBaseIE):
+    IE_NAME = 'Beam:vod'
+    _VALID_URL = r'https?://(?:\w+\.)?beam\.pro/[^/?#&]+.*[?&]vod=(?P<id>\d+)'
+    _TEST = {
+        'url': 'https://beam.pro/willow8714?vod=2259830',
+        'md5': 'b2431e6e8347dc92ebafb565d368b76b',
+        'info_dict': {
+            'id': '2259830',
+            'ext': 'mp4',
+            'title': 'willow8714\'s Channel',
+            'duration': 6828.15,
+            'thumbnail': r're:https://.*source\.png$',
+            'timestamp': 1494046474,
+            'upload_date': '20170506',
+            'uploader': 'willow8714',
+            'uploader_id': '6085379',
+            'age_limit': 13,
+            'view_count': int,
+        },
+    }
+
+    def _extract_format(self, vod, vod_type):
+        if not vod.get('baseUrl'):
+            return []
+
+        if vod_type == 'hls':
+            filename, protocol = 'manifest.m3u8', 'm3u8'
+        elif vod_type == 'raw':
+            filename, protocol = 'source.mp4', 'https'
+        else:
+            return []
+
+        data = vod.get('data') or {}
+
+        format_id = [vod_type]
+        if 'Height' in data:
+            format_id.append('%sp' % data['Height'])
+
+        return [{
+            'url': urljoin(vod['baseUrl'], filename),
+            'format_id': '-'.join(format_id),
+            'ext': 'mp4',
+            'protocol': protocol,
+            'width': int_or_none(data.get('Width')),
+            'height': int_or_none(data.get('Height')),
+            'fps': int_or_none(data.get('Fps')),
+            'tbr': int_or_none(data.get('Bitrate'), 1000),
+        }]
+
+    def _real_extract(self, url):
+        vod_id = self._match_id(url)
+
+        vod_info = self._download_json(
+            'https://beam.pro/api/v1/recordings/%s' % vod_id, vod_id)
+
+        state = vod_info.get('state')
+        if state != 'AVAILABLE':
+            raise ExtractorError(
+                'VOD %s is not available (state: %s)' % (vod_id, state), expected=True)
+
+        formats = []
+        thumbnail_url = None
+
+        for vod in vod_info['vods']:
+            vod_type = vod.get('format')
+            if vod_type in ('hls', 'raw'):
+                formats.extend(self._extract_format(vod, vod_type))
+            elif vod_type == 'thumbnail':
+                thumbnail_url = urljoin(vod.get('baseUrl'), 'source.png')
+
+        self._sort_formats(formats)
+
+        info = {
+            'id': vod_id,
+            'title': vod_info.get('name') or vod_id,
+            'duration': float_or_none(vod_info.get('duration')),
+            'thumbnail': thumbnail_url,
+            'timestamp': parse_iso8601(vod_info.get('createdAt')),
+            'view_count': int_or_none(vod_info.get('viewsTotal')),
+            'formats': formats,
+        }
+
+        chan = vod_info.get('channel') or {}
+        info.update(self._extract_channel_info(chan))
+
+        return info
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 558c910d4..aecd08ac1 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -98,7 +98,10 @@ from .bbc import (
     BBCCoUkPlaylistIE,
     BBCIE,
 )
-from .beampro import BeamProLiveIE
+from .beampro import (
+    BeamProLiveIE,
+    BeamProVodIE,
+)
 from .beeg import BeegIE
 from .behindkink import BehindKinkIE
 from .bellmedia import BellMediaIE

From 6bceb36b9939b2fcc8417cc734fa9e376d67e4f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 May 2017 05:39:28 +0700
Subject: [PATCH 0915/1696] [beam] Improve and add support for mixer.com
 (closes #13032)

---
 youtube_dl/extractor/beampro.py | 52 ++++++++++++++++++++-------------
 1 file changed, 32 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/beampro.py b/youtube_dl/extractor/beampro.py
index 79c5fe5cd..2eaec1ab4 100644
--- a/youtube_dl/extractor/beampro.py
+++ b/youtube_dl/extractor/beampro.py
@@ -15,6 +15,7 @@ from ..utils import (
 
 
 class BeamProBaseIE(InfoExtractor):
+    _API_BASE = 'https://mixer.com/api/v1'
     _RATINGS = {'family': 0, 'teen': 13, '18+': 18}
 
     def _extract_channel_info(self, chan):
@@ -28,10 +29,10 @@ class BeamProBaseIE(InfoExtractor):
 
 
 class BeamProLiveIE(BeamProBaseIE):
-    IE_NAME = 'Beam:live'
-    _VALID_URL = r'https?://(?:\w+\.)?beam\.pro/(?P<id>[^/?#&]+)'
+    IE_NAME = 'Mixer:live'
+    _VALID_URL = r'https?://(?:\w+\.)?(?:beam\.pro|mixer\.com)/(?P<id>[^/?#&]+)'
     _TEST = {
-        'url': 'http://www.beam.pro/niterhayven',
+        'url': 'http://mixer.com/niterhayven',
         'info_dict': {
             'id': '261562',
             'ext': 'mp4',
@@ -52,6 +53,8 @@ class BeamProLiveIE(BeamProBaseIE):
         },
     }
 
+    _MANIFEST_URL_TEMPLATE = '%s/channels/%%s/manifest.%%s' % BeamProBaseIE._API_BASE
+
     @classmethod
     def suitable(cls, url):
         return False if BeamProVodIE.suitable(url) else super(BeamProLiveIE, cls).suitable(url)
@@ -60,7 +63,7 @@ class BeamProLiveIE(BeamProBaseIE):
         channel_name = self._match_id(url)
 
         chan = self._download_json(
-            'https://beam.pro/api/v1/channels/%s' % channel_name, channel_name)
+            '%s/channels/%s' % (self._API_BASE, channel_name), channel_name)
 
         if chan.get('online') is False:
             raise ExtractorError(
@@ -68,16 +71,22 @@ class BeamProLiveIE(BeamProBaseIE):
 
         channel_id = chan['id']
 
+        def manifest_url(kind):
+            return self._MANIFEST_URL_TEMPLATE % (channel_id, kind)
+
         formats = self._extract_m3u8_formats(
-            'https://beam.pro/api/v1/channels/%s/manifest.m3u8' % channel_id,
-            channel_name, ext='mp4', m3u8_id='hls', fatal=False)
+            manifest_url('m3u8'), channel_name, ext='mp4', m3u8_id='hls',
+            fatal=False)
+        formats.extend(self._extract_smil_formats(
+            manifest_url('smil'), channel_name, fatal=False))
         self._sort_formats(formats)
 
         info = {
             'id': compat_str(chan.get('id') or channel_name),
             'title': self._live_title(chan.get('name') or channel_name),
             'description': clean_html(chan.get('description')),
-            'thumbnail': try_get(chan, lambda x: x['thumbnail']['url'], compat_str),
+            'thumbnail': try_get(
+                chan, lambda x: x['thumbnail']['url'], compat_str),
             'timestamp': parse_iso8601(chan.get('updatedAt')),
             'is_live': True,
             'view_count': int_or_none(chan.get('viewersTotal')),
@@ -89,10 +98,10 @@ class BeamProLiveIE(BeamProBaseIE):
 
 
 class BeamProVodIE(BeamProBaseIE):
-    IE_NAME = 'Beam:vod'
-    _VALID_URL = r'https?://(?:\w+\.)?beam\.pro/[^/?#&]+.*[?&]vod=(?P<id>\d+)'
+    IE_NAME = 'Mixer:vod'
+    _VALID_URL = r'https?://(?:\w+\.)?(?:beam\.pro|mixer\.com)/[^/?#&]+\?.*?\bvod=(?P<id>\d+)'
     _TEST = {
-        'url': 'https://beam.pro/willow8714?vod=2259830',
+        'url': 'https://mixer.com/willow8714?vod=2259830',
         'md5': 'b2431e6e8347dc92ebafb565d368b76b',
         'info_dict': {
             'id': '2259830',
@@ -107,23 +116,27 @@ class BeamProVodIE(BeamProBaseIE):
             'age_limit': 13,
             'view_count': int,
         },
+        'params': {
+            'skip_download': True,
+        },
     }
 
-    def _extract_format(self, vod, vod_type):
+    @staticmethod
+    def _extract_format(vod, vod_type):
         if not vod.get('baseUrl'):
             return []
 
         if vod_type == 'hls':
-            filename, protocol = 'manifest.m3u8', 'm3u8'
+            filename, protocol = 'manifest.m3u8', 'm3u8_native'
         elif vod_type == 'raw':
             filename, protocol = 'source.mp4', 'https'
         else:
-            return []
+            assert False
 
-        data = vod.get('data') or {}
+        data = vod.get('data') if isinstance(vod.get('data'), dict) else {}
 
         format_id = [vod_type]
-        if 'Height' in data:
+        if isinstance(data.get('Height'), compat_str):
             format_id.append('%sp' % data['Height'])
 
         return [{
@@ -141,12 +154,13 @@ class BeamProVodIE(BeamProBaseIE):
         vod_id = self._match_id(url)
 
         vod_info = self._download_json(
-            'https://beam.pro/api/v1/recordings/%s' % vod_id, vod_id)
+            '%s/recordings/%s' % (self._API_BASE, vod_id), vod_id)
 
         state = vod_info.get('state')
         if state != 'AVAILABLE':
             raise ExtractorError(
-                'VOD %s is not available (state: %s)' % (vod_id, state), expected=True)
+                'VOD %s is not available (state: %s)' % (vod_id, state),
+                expected=True)
 
         formats = []
         thumbnail_url = None
@@ -169,8 +183,6 @@ class BeamProVodIE(BeamProBaseIE):
             'view_count': int_or_none(vod_info.get('viewsTotal')),
             'formats': formats,
         }
-
-        chan = vod_info.get('channel') or {}
-        info.update(self._extract_channel_info(chan))
+        info.update(self._extract_channel_info(vod_info.get('channel') or {}))
 
         return info

From 765522345fcdf726df9b520314fb8802c2fa8459 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Kundr=C3=A1t?= <jkt@flaska.net>
Date: Fri, 12 May 2017 17:16:50 +0200
Subject: [PATCH 0916/1696] [dvtv] Parse adaptive formats as well

The old code hit an error when it attempted to parse the string
"adaptive" for video height. Actually parsing the returned playlists is
a good idea because it adds more output formats, including some
audio-only-ones.
---
 youtube_dl/extractor/dvtv.py | 31 ++++++++++++++++++++++++-------
 1 file changed, 24 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/dvtv.py b/youtube_dl/extractor/dvtv.py
index 974c69dbc..b1ae37d78 100644
--- a/youtube_dl/extractor/dvtv.py
+++ b/youtube_dl/extractor/dvtv.py
@@ -71,6 +71,14 @@ class DVTVIE(InfoExtractor):
     }, {
         'url': 'http://video.aktualne.cz/v-cechach-poprve-zazni-zelenkova-zrestaurovana-mse/r~45b4b00483ec11e4883b002590604f2e/',
         'only_matching': True,
+    }, {
+        'url': 'https://video.aktualne.cz/dvtv/zeman-si-jen-leci-mindraky-sobotku-nenavidi-a-babis-se-mu-te/r~960cdb3a365a11e7a83b0025900fea04/',
+        'md5': 'f8efe9656017da948369aa099788c8ea',
+        'info_dict': {
+            'id': '3c496fec365911e7a6500025900fea04',
+            'ext': 'm3u8',
+            'title': 'Zeman si jen léčí mindráky, Sobotku nenávidí a Babiš se mu teď hodí, tvrdí Kmenta',
+        }
     }]
 
     def _parse_video_metadata(self, js, video_id):
@@ -79,13 +87,22 @@ class DVTVIE(InfoExtractor):
         formats = []
         for video in metadata['sources']:
             ext = video['type'][6:]
-            formats.append({
-                'url': video['file'],
-                'ext': ext,
-                'format_id': '%s-%s' % (ext, video['label']),
-                'height': int(video['label'].rstrip('p')),
-                'fps': 25,
-            })
+            if video['label'] != 'adaptive':
+                formats.append({
+                    'url': video['file'],
+                    'ext': ext,
+                    'format_id': '%s-%s' % (ext, video['label']),
+                    'height': int(video['label'].rstrip('p')),
+                    'fps': 25,
+                })
+            elif video['type'] == 'application/dash+xml':
+                formats.extend(self._extract_mpd_formats(video['file'],
+                                                         video_id,
+                                                         fatal=False))
+            elif video['type'] == 'application/vnd.apple.mpegurl':
+                formats.extend(self._extract_m3u8_formats(video['file'],
+                                                          video_id,
+                                                          fatal=False))
 
         self._sort_formats(formats)
 

From 4d9fc40100eee316da261c9e60eed9dab5c8ef71 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 May 2017 06:19:21 +0700
Subject: [PATCH 0917/1696] [dvtv] Improve and fix playlists support (closes
 #13063)

---
 youtube_dl/extractor/dvtv.py | 108 ++++++++++++++++++++++-------------
 1 file changed, 67 insertions(+), 41 deletions(-)

diff --git a/youtube_dl/extractor/dvtv.py b/youtube_dl/extractor/dvtv.py
index b1ae37d78..e85c58bd5 100644
--- a/youtube_dl/extractor/dvtv.py
+++ b/youtube_dl/extractor/dvtv.py
@@ -5,9 +5,12 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
-    js_to_json,
-    unescapeHTML,
+    determine_ext,
     ExtractorError,
+    int_or_none,
+    js_to_json,
+    mimetype2ext,
+    unescapeHTML,
 )
 
 
@@ -24,14 +27,7 @@ class DVTVIE(InfoExtractor):
             'id': 'dc0768de855511e49e4b0025900fea04',
             'ext': 'mp4',
             'title': 'Vondra o Českém století: Při pohledu na Havla mi bylo trapně',
-        }
-    }, {
-        'url': 'http://video.aktualne.cz/dvtv/stropnicky-policie-vrbetice-preventivne-nekontrolovala/r~82ed4322849211e4a10c0025900fea04/',
-        'md5': '6388f1941b48537dbd28791f712af8bf',
-        'info_dict': {
-            'id': '72c02230849211e49f60002590604f2e',
-            'ext': 'mp4',
-            'title': 'Stropnický: Policie Vrbětice preventivně nekontrolovala',
+            'duration': 1484,
         }
     }, {
         'url': 'http://video.aktualne.cz/dvtv/dvtv-16-12-2014-utok-talibanu-boj-o-kliniku-uprchlici/r~973eb3bc854e11e498be002590604f2e/',
@@ -44,72 +40,100 @@ class DVTVIE(InfoExtractor):
             'info_dict': {
                 'id': 'b0b40906854d11e4bdad0025900fea04',
                 'ext': 'mp4',
-                'title': 'Drtinová Veselovský TV 16. 12. 2014: Témata dne'
+                'title': 'Drtinová Veselovský TV 16. 12. 2014: Témata dne',
+                'description': 'md5:0916925dea8e30fe84222582280b47a0',
+                'timestamp': 1418760010,
+                'upload_date': '20141216',
             }
         }, {
             'md5': '5f7652a08b05009c1292317b449ffea2',
             'info_dict': {
                 'id': '420ad9ec854a11e4bdad0025900fea04',
                 'ext': 'mp4',
-                'title': 'Školní masakr možná změní boj s Talibanem, říká novinářka'
+                'title': 'Školní masakr možná změní boj s Talibanem, říká novinářka',
+                'description': 'md5:ff2f9f6de73c73d7cef4f756c1c1af42',
+                'timestamp': 1418760010,
+                'upload_date': '20141216',
             }
         }, {
             'md5': '498eb9dfa97169f409126c617e2a3d64',
             'info_dict': {
                 'id': '95d35580846a11e4b6d20025900fea04',
                 'ext': 'mp4',
-                'title': 'Boj o kliniku: Veřejný zájem, nebo právo na majetek?'
+                'title': 'Boj o kliniku: Veřejný zájem, nebo právo na majetek?',
+                'description': 'md5:889fe610a70fee5511dc3326a089188e',
+                'timestamp': 1418760010,
+                'upload_date': '20141216',
             }
         }, {
             'md5': 'b8dc6b744844032dab6ba3781a7274b9',
             'info_dict': {
                 'id': '6fe14d66853511e4833a0025900fea04',
                 'ext': 'mp4',
-                'title': 'Pánek: Odmítání syrských uprchlíků je ostudou české vlády'
+                'title': 'Pánek: Odmítání syrských uprchlíků je ostudou české vlády',
+                'description': 'md5:544f86de6d20c4815bea11bf2ac3004f',
+                'timestamp': 1418760010,
+                'upload_date': '20141216',
             }
         }],
-    }, {
-        'url': 'http://video.aktualne.cz/v-cechach-poprve-zazni-zelenkova-zrestaurovana-mse/r~45b4b00483ec11e4883b002590604f2e/',
-        'only_matching': True,
     }, {
         'url': 'https://video.aktualne.cz/dvtv/zeman-si-jen-leci-mindraky-sobotku-nenavidi-a-babis-se-mu-te/r~960cdb3a365a11e7a83b0025900fea04/',
         'md5': 'f8efe9656017da948369aa099788c8ea',
         'info_dict': {
             'id': '3c496fec365911e7a6500025900fea04',
-            'ext': 'm3u8',
+            'ext': 'mp4',
             'title': 'Zeman si jen léčí mindráky, Sobotku nenávidí a Babiš se mu teď hodí, tvrdí Kmenta',
-        }
+            'duration': 1103,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://video.aktualne.cz/v-cechach-poprve-zazni-zelenkova-zrestaurovana-mse/r~45b4b00483ec11e4883b002590604f2e/',
+        'only_matching': True,
     }]
 
     def _parse_video_metadata(self, js, video_id):
-        metadata = self._parse_json(js, video_id, transform_source=js_to_json)
+        data = self._parse_json(js, video_id, transform_source=js_to_json)
+
+        title = unescapeHTML(data['title'])
 
         formats = []
-        for video in metadata['sources']:
-            ext = video['type'][6:]
-            if video['label'] != 'adaptive':
+        for video in data['sources']:
+            video_url = video.get('file')
+            if not video_url:
+                continue
+            video_type = video.get('type')
+            ext = determine_ext(video_url, mimetype2ext(video_type))
+            if video_type == 'application/vnd.apple.mpegurl' or ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    video_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls', fatal=False))
+            elif video_type == 'application/dash+xml' or ext == 'mpd':
+                formats.extend(self._extract_mpd_formats(
+                    video_url, video_id, mpd_id='dash', fatal=False))
+            else:
+                label = video.get('label')
+                height = self._search_regex(
+                    r'^(\d+)[pP]', label or '', 'height', default=None)
+                format_id = ['http']
+                for f in (ext, label):
+                    if f:
+                        format_id.append(f)
                 formats.append({
-                    'url': video['file'],
-                    'ext': ext,
-                    'format_id': '%s-%s' % (ext, video['label']),
-                    'height': int(video['label'].rstrip('p')),
-                    'fps': 25,
+                    'url': video_url,
+                    'format_id': '-'.join(format_id),
+                    'height': int_or_none(height),
                 })
-            elif video['type'] == 'application/dash+xml':
-                formats.extend(self._extract_mpd_formats(video['file'],
-                                                         video_id,
-                                                         fatal=False))
-            elif video['type'] == 'application/vnd.apple.mpegurl':
-                formats.extend(self._extract_m3u8_formats(video['file'],
-                                                          video_id,
-                                                          fatal=False))
-
         self._sort_formats(formats)
 
         return {
-            'id': metadata['mediaid'],
-            'title': unescapeHTML(metadata['title']),
-            'thumbnail': self._proto_relative_url(metadata['image'], 'http:'),
+            'id': data.get('mediaid') or video_id,
+            'title': title,
+            'description': data.get('description'),
+            'thumbnail': data.get('image'),
+            'duration': int_or_none(data.get('duration')),
+            'timestamp': int_or_none(data.get('pubtime')),
             'formats': formats
         }
 
@@ -120,7 +144,7 @@ class DVTVIE(InfoExtractor):
 
         # single video
         item = self._search_regex(
-            r"(?s)embedData[0-9a-f]{32}\['asset'\]\s*=\s*(\{.+?\});",
+            r'(?s)embedData[0-9a-f]{32}\[["\']asset["\']\]\s*=\s*(\{.+?\});',
             webpage, 'video', default=None, fatal=False)
 
         if item:
@@ -130,6 +154,8 @@ class DVTVIE(InfoExtractor):
         items = re.findall(
             r"(?s)BBX\.context\.assets\['[0-9a-f]{32}'\]\.push\(({.+?})\);",
             webpage)
+        if not items:
+            items = re.findall(r'(?s)var\s+asset\s*=\s*({.+?});\n', webpage)
 
         if items:
             return {

From 4050be78e5c2e5b77a747c20db0a12e11a218db5 Mon Sep 17 00:00:00 2001
From: midas02 <midas02@users.noreply.github.com>
Date: Tue, 18 Apr 2017 05:44:33 +0200
Subject: [PATCH 0918/1696] [medialaan] Fix videos with missing videoUrl

A rough trick to get around the two different json styles medialaan seems to be using.
Fix for these example videos:
https://vtmkzoom.be/video?aid=45724
https://vtmkzoom.be/video?aid=45425
---
 youtube_dl/extractor/medialaan.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/medialaan.py b/youtube_dl/extractor/medialaan.py
index 6e067474b..fc8f2550b 100644
--- a/youtube_dl/extractor/medialaan.py
+++ b/youtube_dl/extractor/medialaan.py
@@ -17,7 +17,7 @@ from ..utils import (
 class MedialaanIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                     https?://
-                        (?:www\.)?
+                        (?:www\.|nieuws\.)?
                         (?:
                             (?P<site_id>vtm|q2|vtmkzoom)\.be/
                             (?:
@@ -85,6 +85,22 @@ class MedialaanIE(InfoExtractor):
         # clip
         'url': 'http://vtmkzoom.be/k3-dansstudio/een-nieuw-seizoen-van-k3-dansstudio',
         'only_matching': True,
+    }, {
+        # http/s redirect
+        'url': 'https://vtmkzoom.be/video?aid=45724',
+        'info_dict': {
+            'id': '257136373657000',
+            'ext': 'mp4',
+            'title': 'K3 Dansstudio Ushuaia afl.6',
+        },
+        'params': {
+                'skip_download': True,
+        },
+        'skip': 'Requires account credentials',
+    }, {
+        # nieuws.vtm.be
+        'url': 'https://nieuws.vtm.be/stadion/stadion/genk-nog-moeilijk-programma',
+        'only_matching': True,
     }]
 
     def _real_initialize(self):
@@ -146,6 +162,8 @@ class MedialaanIE(InfoExtractor):
                 video_id, transform_source=lambda s: '[%s]' % s, fatal=False)
             if player:
                 video = player[-1]
+                if video['videoUrl'] in ('http', 'https'):
+                    return self.url_result(video['url'], MedialaanIE.ie_key())
                 info = {
                     'id': video_id,
                     'url': video['videoUrl'],

From 703751add4ec7b7b8a66d892ea893e62763faca0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 May 2017 06:25:50 +0700
Subject: [PATCH 0919/1696] [medialaan] PEP 8 (closes #12774)

---
 youtube_dl/extractor/medialaan.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/medialaan.py b/youtube_dl/extractor/medialaan.py
index fc8f2550b..4c32fbc2c 100644
--- a/youtube_dl/extractor/medialaan.py
+++ b/youtube_dl/extractor/medialaan.py
@@ -94,7 +94,7 @@ class MedialaanIE(InfoExtractor):
             'title': 'K3 Dansstudio Ushuaia afl.6',
         },
         'params': {
-                'skip_download': True,
+            'skip_download': True,
         },
         'skip': 'Requires account credentials',
     }, {

From 5a6d1da442579fc06e458406dba0f8b60102dc1c Mon Sep 17 00:00:00 2001
From: motophil <phil@rrr.de>
Date: Sun, 19 Mar 2017 12:54:02 +0100
Subject: [PATCH 0920/1696] [gaskrank] Fix extraction

---
 youtube_dl/extractor/gaskrank.py | 38 ++++++++------------------------
 1 file changed, 9 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/gaskrank.py b/youtube_dl/extractor/gaskrank.py
index 36ba7d8cf..baf513665 100644
--- a/youtube_dl/extractor/gaskrank.py
+++ b/youtube_dl/extractor/gaskrank.py
@@ -6,7 +6,6 @@ from .common import InfoExtractor
 from ..utils import (
     float_or_none,
     int_or_none,
-    js_to_json,
     unified_strdate,
 )
 
@@ -49,19 +48,12 @@ class GaskrankIE(InfoExtractor):
 
     def _real_extract(self, url):
         """extract information from gaskrank.tv"""
-        def fix_json(code):
-            """Removes trailing comma in json: {{},} --> {{}}"""
-            return re.sub(r',\s*}', r'}', js_to_json(code))
 
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
         categories = [re.match(self._VALID_URL, url).group('categories')]
-        title = self._search_regex(
-            r'movieName\s*:\s*\'([^\']*)\'',
-            webpage, 'title')
-        thumbnail = self._search_regex(
-            r'poster\s*:\s*\'([^\']*)\'',
-            webpage, 'thumbnail', default=None)
+        title = self._og_search_title(webpage, default=None) or self._html_search_meta(
+            'title', webpage, fatal=True)
 
         mobj = re.search(
             r'Video von:\s*(?P<uploader_id>[^|]*?)\s*\|\s*vom:\s*(?P<upload_date>[0-9][0-9]\.[0-9][0-9]\.[0-9][0-9][0-9][0-9])',
@@ -89,29 +81,14 @@ class GaskrankIE(InfoExtractor):
         if average_rating:
             average_rating = float_or_none(average_rating.replace(',', '.'))
 
-        playlist = self._parse_json(
-            self._search_regex(
-                r'playlist\s*:\s*\[([^\]]*)\]',
-                webpage, 'playlist', default='{}'),
-            display_id, transform_source=fix_json, fatal=False)
-
         video_id = self._search_regex(
             r'https?://movies\.gaskrank\.tv/([^-]*?)(-[^\.]*)?\.mp4',
-            playlist.get('0').get('src'), 'video id')
+            webpage, 'video id')
 
-        formats = []
-        for key in playlist:
-            formats.append({
-                'url': playlist[key]['src'],
-                'format_id': key,
-                'quality': playlist[key].get('quality')})
-        self._sort_formats(formats, field_preference=['format_id'])
-
-        return {
+        entry = self._parse_html5_media_entries(url, webpage, video_id)[0]
+        entry.update({
             'id': video_id,
             'title': title,
-            'formats': formats,
-            'thumbnail': thumbnail,
             'categories': categories,
             'display_id': display_id,
             'uploader_id': uploader_id,
@@ -120,4 +97,7 @@ class GaskrankIE(InfoExtractor):
             'tags': tags,
             'view_count': view_count,
             'average_rating': average_rating,
-        }
+        })
+        self._sort_formats(entry['formats'])
+
+        return entry

From 7e688d2f6ade3e39423161c7837e48ca2b8243a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 May 2017 06:46:52 +0700
Subject: [PATCH 0921/1696] [gaskrank] Improve (closes #12493)

---
 youtube_dl/extractor/gaskrank.py | 74 ++++++++++++++++----------------
 1 file changed, 36 insertions(+), 38 deletions(-)

diff --git a/youtube_dl/extractor/gaskrank.py b/youtube_dl/extractor/gaskrank.py
index baf513665..1726a6704 100644
--- a/youtube_dl/extractor/gaskrank.py
+++ b/youtube_dl/extractor/gaskrank.py
@@ -11,50 +11,48 @@ from ..utils import (
 
 
 class GaskrankIE(InfoExtractor):
-    """InfoExtractor for gaskrank.tv"""
-    _VALID_URL = r'https?://(?:www\.)?gaskrank\.tv/tv/(?P<categories>[^/]+)/(?P<id>[^/]+)\.html?'
-    _TESTS = [
-        {
-            'url': 'http://www.gaskrank.tv/tv/motorrad-fun/strike-einparken-durch-anfaenger-crash-mit-groesserem-flurschaden.htm',
-            'md5': '1ae88dbac97887d85ebd1157a95fc4f9',
-            'info_dict': {
-                'id': '201601/26955',
-                'ext': 'mp4',
-                'title': 'Strike! Einparken können nur Männer - Flurschaden hält sich in Grenzen *lol*',
-                'thumbnail': r're:^https?://.*\.jpg$',
-                'categories': ['motorrad-fun'],
-                'display_id': 'strike-einparken-durch-anfaenger-crash-mit-groesserem-flurschaden',
-                'uploader_id': 'Bikefun',
-                'upload_date': '20170110',
-                'uploader_url': None,
-            }
-        },
-        {
-            'url': 'http://www.gaskrank.tv/tv/racing/isle-of-man-tt-2011-michael-du-15920.htm',
-            'md5': 'c33ee32c711bc6c8224bfcbe62b23095',
-            'info_dict': {
-                'id': '201106/15920',
-                'ext': 'mp4',
-                'title': 'Isle of Man - Michael Dunlop vs Guy Martin - schwindelig kucken',
-                'thumbnail': r're:^https?://.*\.jpg$',
-                'categories': ['racing'],
-                'display_id': 'isle-of-man-tt-2011-michael-du-15920',
-                'uploader_id': 'IOM',
-                'upload_date': '20160506',
-                'uploader_url': 'www.iomtt.com',
-            }
+    _VALID_URL = r'https?://(?:www\.)?gaskrank\.tv/tv/(?P<categories>[^/]+)/(?P<id>[^/]+)\.htm'
+    _TESTS = [{
+        'url': 'http://www.gaskrank.tv/tv/motorrad-fun/strike-einparken-durch-anfaenger-crash-mit-groesserem-flurschaden.htm',
+        'md5': '1ae88dbac97887d85ebd1157a95fc4f9',
+        'info_dict': {
+            'id': '201601/26955',
+            'ext': 'mp4',
+            'title': 'Strike! Einparken können nur Männer - Flurschaden hält sich in Grenzen *lol*',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'categories': ['motorrad-fun'],
+            'display_id': 'strike-einparken-durch-anfaenger-crash-mit-groesserem-flurschaden',
+            'uploader_id': 'Bikefun',
+            'upload_date': '20170110',
+            'uploader_url': None,
         }
-    ]
+    }, {
+        'url': 'http://www.gaskrank.tv/tv/racing/isle-of-man-tt-2011-michael-du-15920.htm',
+        'md5': 'c33ee32c711bc6c8224bfcbe62b23095',
+        'info_dict': {
+            'id': '201106/15920',
+            'ext': 'mp4',
+            'title': 'Isle of Man - Michael Dunlop vs Guy Martin - schwindelig kucken',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'categories': ['racing'],
+            'display_id': 'isle-of-man-tt-2011-michael-du-15920',
+            'uploader_id': 'IOM',
+            'upload_date': '20170523',
+            'uploader_url': 'www.iomtt.com',
+        }
+    }]
 
     def _real_extract(self, url):
-        """extract information from gaskrank.tv"""
-
         display_id = self._match_id(url)
+
         webpage = self._download_webpage(url, display_id)
-        categories = [re.match(self._VALID_URL, url).group('categories')]
-        title = self._og_search_title(webpage, default=None) or self._html_search_meta(
+
+        title = self._og_search_title(
+            webpage, default=None) or self._html_search_meta(
             'title', webpage, fatal=True)
 
+        categories = [re.match(self._VALID_URL, url).group('categories')]
+
         mobj = re.search(
             r'Video von:\s*(?P<uploader_id>[^|]*?)\s*\|\s*vom:\s*(?P<upload_date>[0-9][0-9]\.[0-9][0-9]\.[0-9][0-9][0-9][0-9])',
             webpage)
@@ -83,7 +81,7 @@ class GaskrankIE(InfoExtractor):
 
         video_id = self._search_regex(
             r'https?://movies\.gaskrank\.tv/([^-]*?)(-[^\.]*)?\.mp4',
-            webpage, 'video id')
+            webpage, 'video id', default=display_id)
 
         entry = self._parse_html5_media_entries(url, webpage, video_id)[0]
         entry.update({

From bc22df29c49523e41abb84eb09232784b2e07d34 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Wed, 26 Apr 2017 21:21:17 +0200
Subject: [PATCH 0922/1696] [abcnews] Add support for embed URLs

---
 youtube_dl/extractor/abcnews.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/abcnews.py b/youtube_dl/extractor/abcnews.py
index 4f56c4c11..e18be2497 100644
--- a/youtube_dl/extractor/abcnews.py
+++ b/youtube_dl/extractor/abcnews.py
@@ -12,7 +12,11 @@ from ..compat import compat_urlparse
 
 class AbcNewsVideoIE(AMPIE):
     IE_NAME = 'abcnews:video'
-    _VALID_URL = r'https?://abcnews\.go\.com/[^/]+/video/(?P<display_id>[0-9a-z-]+)-(?P<id>\d+)'
+    _VALID_URL = r'''(?x)
+        https?://abcnews\.go\.com/
+            (?:[^/]+/video/(?P<display_id>[0-9a-z-]+)-|
+               video/embed\?id=)
+       (?P<id>\d+)'''
 
     _TESTS = [{
         'url': 'http://abcnews.go.com/ThisWeek/video/week-exclusive-irans-foreign-minister-zarif-20411932',
@@ -29,6 +33,17 @@ class AbcNewsVideoIE(AMPIE):
             # m3u8 download
             'skip_download': True,
         },
+    }, {
+        'url': 'http://abcnews.go.com/video/embed?id=46979033',
+        'md5': 'd19d160cfa8372fc197f406d4821dd9f',
+        'info_dict': {
+            'id': '46979033',
+            'ext': 'flv',
+            'title': 'Nikki Haley reacts to French election results',
+            'description': 'md5:df1b5ad1b8ad76c4d4e01d3c67d6b38e',
+            'duration': 274,
+            'thumbnail': r're:^https?://.*\.jpg$',
+        },
     }, {
         'url': 'http://abcnews.go.com/2020/video/2020-husband-stands-teacher-jail-student-affairs-26119478',
         'only_matching': True,

From 164fcbfeb75787cfeb1f0a84188ba4c11ac4a009 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 May 2017 07:05:50 +0700
Subject: [PATCH 0923/1696] [abcnews] Improve and remove duplicate test (closes
 #12851)

---
 youtube_dl/extractor/abcnews.py | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/abcnews.py b/youtube_dl/extractor/abcnews.py
index e18be2497..74d54560c 100644
--- a/youtube_dl/extractor/abcnews.py
+++ b/youtube_dl/extractor/abcnews.py
@@ -13,10 +13,14 @@ from ..compat import compat_urlparse
 class AbcNewsVideoIE(AMPIE):
     IE_NAME = 'abcnews:video'
     _VALID_URL = r'''(?x)
-        https?://abcnews\.go\.com/
-            (?:[^/]+/video/(?P<display_id>[0-9a-z-]+)-|
-               video/embed\?id=)
-       (?P<id>\d+)'''
+                    https?://
+                        abcnews\.go\.com/
+                        (?:
+                            [^/]+/video/(?P<display_id>[0-9a-z-]+)-|
+                            video/embed\?.*?\bid=
+                        )
+                        (?P<id>\d+)
+                    '''
 
     _TESTS = [{
         'url': 'http://abcnews.go.com/ThisWeek/video/week-exclusive-irans-foreign-minister-zarif-20411932',
@@ -35,15 +39,7 @@ class AbcNewsVideoIE(AMPIE):
         },
     }, {
         'url': 'http://abcnews.go.com/video/embed?id=46979033',
-        'md5': 'd19d160cfa8372fc197f406d4821dd9f',
-        'info_dict': {
-            'id': '46979033',
-            'ext': 'flv',
-            'title': 'Nikki Haley reacts to French election results',
-            'description': 'md5:df1b5ad1b8ad76c4d4e01d3c67d6b38e',
-            'duration': 274,
-            'thumbnail': r're:^https?://.*\.jpg$',
-        },
+        'only_matching': True,
     }, {
         'url': 'http://abcnews.go.com/2020/video/2020-husband-stands-teacher-jail-student-affairs-26119478',
         'only_matching': True,

From 855520427458f68076143c4079b038f4d15e5a13 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 May 2017 07:50:15 +0700
Subject: [PATCH 0924/1696] [xhamster] Extract categories (closes #11728)

---
 youtube_dl/extractor/xhamster.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 7b6703714..7cf0629e7 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -4,6 +4,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    clean_html,
     dict_get,
     ExtractorError,
     int_or_none,
@@ -25,6 +26,7 @@ class XHamsterIE(InfoExtractor):
             'uploader': 'Ruseful2011',
             'duration': 893,
             'age_limit': 18,
+            'categories': ['Fake Hub', 'Amateur', 'MILFs', 'POV', 'Boss', 'Office', 'Oral', 'Reality', 'Sexy'],
         },
     }, {
         'url': 'http://xhamster.com/movies/2221348/britney_spears_sexy_booty.html?hd',
@@ -36,6 +38,7 @@ class XHamsterIE(InfoExtractor):
             'uploader': 'jojo747400',
             'duration': 200,
             'age_limit': 18,
+            'categories': ['Britney Spears', 'Celebrities', 'HD Videos', 'Sexy', 'Sexy Booty'],
         },
         'params': {
             'skip_download': True,
@@ -51,6 +54,7 @@ class XHamsterIE(InfoExtractor):
             'uploader': 'parejafree',
             'duration': 72,
             'age_limit': 18,
+            'categories': ['Amateur', 'Blowjobs'],
         },
         'params': {
             'skip_download': True,
@@ -152,6 +156,12 @@ class XHamsterIE(InfoExtractor):
 
         self._sort_formats(formats)
 
+        categories_html = self._search_regex(
+            r'(?s)<table.+?(<span>Categories:.+?)</table>', webpage,
+            'categories', default=None)
+        categories = [clean_html(category) for category in re.findall(
+            r'<a[^>]+>(.+?)</a>', categories_html)] if categories_html else None
+
         return {
             'id': video_id,
             'title': title,
@@ -165,6 +175,7 @@ class XHamsterIE(InfoExtractor):
             'dislike_count': int_or_none(dislike_count),
             'comment_count': int_or_none(comment_count),
             'age_limit': age_limit,
+            'categories': categories,
             'formats': formats,
         }
 

From 78e210dea554144c8c1890c617808b0acdab9bfc Mon Sep 17 00:00:00 2001
From: fiocfun <fiocfun@gmail.com>
Date: Sat, 27 May 2017 13:28:08 +0300
Subject: [PATCH 0925/1696] [xhamster] Fix author and like/dislike count
 extraction

---
 youtube_dl/extractor/xhamster.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 7cf0629e7..ae0c167c4 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -108,7 +108,7 @@ class XHamsterIE(InfoExtractor):
             webpage, 'upload date', fatal=False))
 
         uploader = self._html_search_regex(
-            r'<span[^>]+itemprop=["\']author[^>]+><a[^>]+href=["\'].+?xhamster\.com/user/[^>]+>(?P<uploader>.+?)</a>',
+            r'<span[^>]+?itemprop=["\']author[^>]+?><a[^>]+?href=["\'][^>]+?><span[^>]+?itemprop=["\']name[^>]+?>(.+?)</span',
             webpage, 'uploader', default='anonymous')
 
         thumbnail = self._search_regex(
@@ -124,7 +124,7 @@ class XHamsterIE(InfoExtractor):
             r'content=["\']User(?:View|Play)s:(\d+)',
             webpage, 'view count', fatal=False))
 
-        mobj = re.search(r"hint='(?P<likecount>\d+) Likes / (?P<dislikecount>\d+) Dislikes'", webpage)
+        mobj = re.search(r'hint=[\'"](?P<likecount>\d+) Likes / (?P<dislikecount>\d+) Dislikes[\'"]', webpage)
         (like_count, dislike_count) = (mobj.group('likecount'), mobj.group('dislikecount')) if mobj else (None, None)
 
         mobj = re.search(r'</label>Comments \((?P<commentcount>\d+)\)</div>', webpage)

From a846173d9304c958bd7bedb89fe767490c885ae1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 May 2017 07:55:56 +0700
Subject: [PATCH 0926/1696] [xhamster] Simplify (closes #13216)

---
 youtube_dl/extractor/xhamster.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index ae0c167c4..6987b2e3a 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -108,7 +108,7 @@ class XHamsterIE(InfoExtractor):
             webpage, 'upload date', fatal=False))
 
         uploader = self._html_search_regex(
-            r'<span[^>]+?itemprop=["\']author[^>]+?><a[^>]+?href=["\'][^>]+?><span[^>]+?itemprop=["\']name[^>]+?>(.+?)</span',
+            r'<span[^>]+itemprop=["\']author[^>]+><a[^>]+><span[^>]+>([^<]+)',
             webpage, 'uploader', default='anonymous')
 
         thumbnail = self._search_regex(
@@ -124,7 +124,7 @@ class XHamsterIE(InfoExtractor):
             r'content=["\']User(?:View|Play)s:(\d+)',
             webpage, 'view count', fatal=False))
 
-        mobj = re.search(r'hint=[\'"](?P<likecount>\d+) Likes / (?P<dislikecount>\d+) Dislikes[\'"]', webpage)
+        mobj = re.search(r'hint=[\'"](?P<likecount>\d+) Likes / (?P<dislikecount>\d+) Dislikes', webpage)
         (like_count, dislike_count) = (mobj.group('likecount'), mobj.group('dislikecount')) if mobj else (None, None)
 
         mobj = re.search(r'</label>Comments \((?P<commentcount>\d+)\)</div>', webpage)

From a49eccdfa7e1ba4d834a6f65d15ef88a37bd78dd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 28 May 2017 00:47:41 +0800
Subject: [PATCH 0927/1696] [youtube] Parse player_url if format URLs are
 encrypted or DASH MPDs are requested

Fixes #13211
---
 ChangeLog                       |  2 ++
 youtube_dl/extractor/youtube.py | 11 ++++++-----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 65a3ca468..c8dd70fe5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,8 @@
 version <unreleased>
 
 Extractors
+* [youtube] Fix DASH MPD extraction for videos with non-encrypted format URLs
+  (#13211)
 + [youku] Extract more metadata (#10433)
 
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index d6c098706..40ac1a019 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1715,12 +1715,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 format_id = url_data['itag'][0]
                 url = url_data['url'][0]
 
-                if 'sig' in url_data:
-                    url += '&signature=' + url_data['sig'][0]
-                elif 's' in url_data:
-                    encrypted_sig = url_data['s'][0]
+                if 's' in url_data or self._downloader.params.get('youtube_include_dash_manifest', True):
                     ASSETS_RE = r'"assets":.+?"js":\s*("[^"]+")'
-
                     jsplayer_url_json = self._search_regex(
                         ASSETS_RE,
                         embed_webpage if age_gate else video_webpage,
@@ -1741,6 +1737,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                             video_webpage, 'age gate player URL')
                         player_url = json.loads(player_url_json)
 
+                if 'sig' in url_data:
+                    url += '&signature=' + url_data['sig'][0]
+                elif 's' in url_data:
+                    encrypted_sig = url_data['s'][0]
+
                     if self._downloader.params.get('verbose'):
                         if player_url is None:
                             player_version = 'unknown'

From 99bea8d298ae68a14bd86675db1331bef09bf193 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 May 2017 00:33:24 +0700
Subject: [PATCH 0928/1696] [ChangeLog] Actualize

---
 ChangeLog | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index c8dd70fe5..dfe505e95 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -3,7 +3,18 @@ version <unreleased>
 Extractors
 * [youtube] Fix DASH MPD extraction for videos with non-encrypted format URLs
   (#13211)
+* [xhamster] Fix uploader and like/dislike count extraction (#13216))
++ [xhamster] Extract categories (#11728)
++ [abcnews] Add support for embed URLs (#12851)
+* [gaskrank] Fix extraction (#12493)
+* [medialaan] Fix videos with missing videoUrl (#12774)
+* [dvtv] Fix playlist support
++ [dvtv] Add support for DASH and HLS formats (#3063)
++ [beam:vod] Add support for beam.pro/mixer.com VODs (#13032))
+* [cbsinteractive] Relax URL regular expression (#13213)
+* [adn] Fix formats extraction
 + [youku] Extract more metadata (#10433)
+* [cbsnews] Fix extraction (#13205)
 
 
 version 2017.05.26

From 941ea38ef535ec2923c224a88cdedeae3eb42a68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 May 2017 00:42:18 +0700
Subject: [PATCH 0929/1696] release 2017.05.29

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 3 ++-
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 5ca69ffa1..24a0d72c5 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.26*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.26**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.29*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.29**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.05.26
+[debug] youtube-dl version 2017.05.29
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index dfe505e95..7d78c87e6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.05.29
 
 Extractors
 * [youtube] Fix DASH MPD extraction for videos with non-encrypted format URLs
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 9791d585b..8ddc800b4 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -93,7 +93,6 @@
  - **bbc.co.uk:article**: BBC articles
  - **bbc.co.uk:iplayer:playlist**
  - **bbc.co.uk:playlist**
- - **Beam:live**
  - **Beatport**
  - **Beeg**
  - **BehindKink**
@@ -453,6 +452,8 @@
  - **mixcloud:playlist**
  - **mixcloud:stream**
  - **mixcloud:user**
+ - **Mixer:live**
+ - **Mixer:vod**
  - **MLB**
  - **Mnet**
  - **MoeVideo**: LetitBit video services: moevideo.net, playreplay.net and videochart.net
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 16d91dfad..d4ee43122 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.05.26'
+__version__ = '2017.05.29'

From d10d0e3cf8c38fba27d873fd3cb60572327fe5dc Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 29 May 2017 14:58:07 +0800
Subject: [PATCH 0930/1696] [README.md] Add an example for how to use .netrc on
 Windows

That's a Python bug: http://bugs.python.org/issue28334
Most likely it will be fixed in Python 3.7: https://github.com/python/cpython/pull/123
---
 README.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index dc0be1f40..284ce8ca3 100644
--- a/README.md
+++ b/README.md
@@ -474,7 +474,10 @@ machine twitch login my_twitch_account_name password my_twitch_password
 ```
 To activate authentication with the `.netrc` file you should pass `--netrc` to youtube-dl or place it in the [configuration file](#configuration).
 
-On Windows you may also need to setup the `%HOME%` environment variable manually.
+On Windows you may also need to setup the `%HOME%` environment variable manually. For example:
+```
+set HOME=%USERPROFILE%
+```
 
 # OUTPUT TEMPLATE
 

From 6b7ce85cdcba2449455fb79e84ce2014b18b5779 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 May 2017 23:50:48 +0700
Subject: [PATCH 0931/1696] [README.md] Mention http_dash_segments protocol

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 284ce8ca3..fa901a46d 100644
--- a/README.md
+++ b/README.md
@@ -652,7 +652,7 @@ Also filtering work for comparisons `=` (equals), `!=` (not equals), `^=` (begin
  - `acodec`: Name of the audio codec in use
  - `vcodec`: Name of the video codec in use
  - `container`: Name of the container format
- - `protocol`: The protocol that will be used for the actual download, lower-case (`http`, `https`, `rtsp`, `rtmp`, `rtmpe`, `mms`, `f4m`, `ism`, `m3u8`, or `m3u8_native`)
+ - `protocol`: The protocol that will be used for the actual download, lower-case (`http`, `https`, `rtsp`, `rtmp`, `rtmpe`, `mms`, `f4m`, `ism`, `http_dash_segments`, `m3u8`, or `m3u8_native`)
  - `format_id`: A short description of the format
 
 Note that none of the aforementioned meta fields are guaranteed to be present since this solely depends on the metadata obtained by particular extractor, i.e. the metadata offered by the video hoster.

From c56ad5c97580f4883bef375427caa62c851dc7a8 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 31 May 2017 15:43:54 +0100
Subject: [PATCH 0932/1696] [packtpub] Fix authentication(closes #13240)

---
 youtube_dl/extractor/packtpub.py | 23 ++++++++---------------
 1 file changed, 8 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/packtpub.py b/youtube_dl/extractor/packtpub.py
index bb668c999..8ed3c6347 100644
--- a/youtube_dl/extractor/packtpub.py
+++ b/youtube_dl/extractor/packtpub.py
@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 
+import json
 import re
 
 from .common import InfoExtractor
@@ -14,7 +15,6 @@ from ..utils import (
     strip_or_none,
     unified_timestamp,
     urljoin,
-    urlencode_postdata,
 )
 
 
@@ -45,22 +45,15 @@ class PacktPubIE(PacktPubBaseIE):
         (username, password) = self._get_login_info()
         if username is None:
             return
-        webpage = self._download_webpage(self._PACKT_BASE, None)
-        login_form = self._form_hidden_inputs(
-            'packt-user-login-form', webpage)
-        login_form.update({
-            'email': username,
-            'password': password,
-        })
-        self._download_webpage(
-            self._PACKT_BASE, None, 'Logging in as %s' % username,
-            data=urlencode_postdata(login_form))
         try:
             self._TOKEN = self._download_json(
-                '%s/users/tokens/sessions' % self._MAPT_REST, None,
-                'Downloading Authorization Token')['data']['token']
+                self._MAPT_REST + '/users/tokens', None,
+                'Downloading Authorization Token', data=json.dumps({
+                    'email': username,
+                    'password': password,
+                }).encode())['data']['access']
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code in (401, 404):
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code in (400, 401, 404):
                 message = self._parse_json(e.cause.read().decode(), None)['message']
                 raise ExtractorError(message, expected=True)
             raise
@@ -83,7 +76,7 @@ class PacktPubIE(PacktPubBaseIE):
 
         headers = {}
         if self._TOKEN:
-            headers['Authorization'] = self._TOKEN
+            headers['Authorization'] = 'Bearer ' + self._TOKEN
         video = self._download_json(
             '%s/users/me/products/%s/chapters/%s/sections/%s'
             % (self._MAPT_REST, course_id, chapter_id, video_id), video_id,

From 87b5184a0d7e32f595fca19b1ec227164ccd0acd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 31 May 2017 23:56:32 +0700
Subject: [PATCH 0933/1696] [drbonanza] Fix extraction (closes #13231)

---
 youtube_dl/extractor/drbonanza.py | 146 +++++++-----------------------
 1 file changed, 35 insertions(+), 111 deletions(-)

diff --git a/youtube_dl/extractor/drbonanza.py b/youtube_dl/extractor/drbonanza.py
index 79ec212c8..164e97c36 100644
--- a/youtube_dl/extractor/drbonanza.py
+++ b/youtube_dl/extractor/drbonanza.py
@@ -1,135 +1,59 @@
 from __future__ import unicode_literals
 
-import json
 import re
 
 from .common import InfoExtractor
 from ..utils import (
-    int_or_none,
-    parse_iso8601,
+    js_to_json,
+    parse_duration,
+    unescapeHTML,
 )
 
 
 class DRBonanzaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?dr\.dk/bonanza/(?:[^/]+/)+(?:[^/])+?(?:assetId=(?P<id>\d+))?(?:[#&]|$)'
-
-    _TESTS = [{
-        'url': 'http://www.dr.dk/bonanza/serie/portraetter/Talkshowet.htm?assetId=65517',
+    _VALID_URL = r'https?://(?:www\.)?dr\.dk/bonanza/[^/]+/\d+/[^/]+/(?P<id>\d+)/(?P<display_id>[^/?#&]+)'
+    _TEST = {
+        'url': 'http://www.dr.dk/bonanza/serie/154/matador/40312/matador---0824-komme-fremmede-',
         'info_dict': {
-            'id': '65517',
+            'id': '40312',
+            'display_id': 'matador---0824-komme-fremmede-',
             'ext': 'mp4',
-            'title': 'Talkshowet - Leonard Cohen',
-            'description': 'md5:8f34194fb30cd8c8a30ad8b27b70c0ca',
+            'title': 'MATADOR - 08:24. "Komme fremmede".',
+            'description': 'md5:77b4c1ac4d4c1b9d610ab4395212ff84',
             'thumbnail': r're:^https?://.*\.(?:gif|jpg)$',
-            'timestamp': 1295537932,
-            'upload_date': '20110120',
-            'duration': 3664,
+            'duration': 4613,
         },
-        'params': {
-            'skip_download': True,  # requires rtmp
-        },
-    }, {
-        'url': 'http://www.dr.dk/bonanza/radio/serie/sport/fodbold.htm?assetId=59410',
-        'md5': '6dfe039417e76795fb783c52da3de11d',
-        'info_dict': {
-            'id': '59410',
-            'ext': 'mp3',
-            'title': 'EM fodbold 1992 Danmark - Tyskland finale Transmission',
-            'description': 'md5:501e5a195749480552e214fbbed16c4e',
-            'thumbnail': r're:^https?://.*\.(?:gif|jpg)$',
-            'timestamp': 1223274900,
-            'upload_date': '20081006',
-            'duration': 7369,
-        },
-    }]
+    }
 
     def _real_extract(self, url):
-        url_id = self._match_id(url)
-        webpage = self._download_webpage(url, url_id)
+        mobj = re.match(self._VALID_URL, url)
+        video_id, display_id = mobj.group('id', 'display_id')
 
-        if url_id:
-            info = json.loads(self._html_search_regex(r'({.*?%s.*})' % url_id, webpage, 'json'))
-        else:
-            # Just fetch the first video on that page
-            info = json.loads(self._html_search_regex(r'bonanzaFunctions.newPlaylist\(({.*})\)', webpage, 'json'))
+        webpage = self._download_webpage(url, display_id)
 
-        asset_id = str(info['AssetId'])
-        title = info['Title'].rstrip(' \'\"-,.:;!?')
-        duration = int_or_none(info.get('Duration'), scale=1000)
-        # First published online. "FirstPublished" contains the date for original airing.
-        timestamp = parse_iso8601(
-            re.sub(r'\.\d+$', '', info['Created']))
+        info = self._parse_html5_media_entries(
+            url, webpage, display_id, m3u8_id='hls',
+            m3u8_entry_protocol='m3u8_native')[0]
+        self._sort_formats(info['formats'])
 
-        def parse_filename_info(url):
-            match = re.search(r'/\d+_(?P<width>\d+)x(?P<height>\d+)x(?P<bitrate>\d+)K\.(?P<ext>\w+)$', url)
-            if match:
-                return {
-                    'width': int(match.group('width')),
-                    'height': int(match.group('height')),
-                    'vbr': int(match.group('bitrate')),
-                    'ext': match.group('ext')
-                }
-            match = re.search(r'/\d+_(?P<bitrate>\d+)K\.(?P<ext>\w+)$', url)
-            if match:
-                return {
-                    'vbr': int(match.group('bitrate')),
-                    'ext': match.group(2)
-                }
-            return {}
+        asset = self._parse_json(
+            self._search_regex(
+                r'(?s)currentAsset\s*=\s*({.+?})\s*</script', webpage, 'asset'),
+            display_id, transform_source=js_to_json)
 
-        video_types = ['VideoHigh', 'VideoMid', 'VideoLow']
-        preferencemap = {
-            'VideoHigh': -1,
-            'VideoMid': -2,
-            'VideoLow': -3,
-            'Audio': -4,
-        }
+        title = unescapeHTML(asset['AssetTitle']).strip()
 
-        formats = []
-        for file in info['Files']:
-            if info['Type'] == 'Video':
-                if file['Type'] in video_types:
-                    format = parse_filename_info(file['Location'])
-                    format.update({
-                        'url': file['Location'],
-                        'format_id': file['Type'].replace('Video', ''),
-                        'preference': preferencemap.get(file['Type'], -10),
-                    })
-                    if format['url'].startswith('rtmp'):
-                        rtmp_url = format['url']
-                        format['rtmp_live'] = True  # --resume does not work
-                        if '/bonanza/' in rtmp_url:
-                            format['play_path'] = rtmp_url.split('/bonanza/')[1]
-                    formats.append(format)
-                elif file['Type'] == 'Thumb':
-                    thumbnail = file['Location']
-            elif info['Type'] == 'Audio':
-                if file['Type'] == 'Audio':
-                    format = parse_filename_info(file['Location'])
-                    format.update({
-                        'url': file['Location'],
-                        'format_id': file['Type'],
-                        'vcodec': 'none',
-                    })
-                    formats.append(format)
-                elif file['Type'] == 'Thumb':
-                    thumbnail = file['Location']
+        def extract(field):
+            return self._search_regex(
+                r'<div[^>]+>\s*<p>%s:<p>\s*</div>\s*<div[^>]+>\s*<p>([^<]+)</p>' % field,
+                webpage, field, default=None)
 
-        description = '%s\n%s\n%s\n' % (
-            info['Description'], info['Actors'], info['Colophon'])
-
-        self._sort_formats(formats)
-
-        display_id = re.sub(r'[^\w\d-]', '', re.sub(r' ', '-', title.lower())) + '-' + asset_id
-        display_id = re.sub(r'-+', '-', display_id)
-
-        return {
-            'id': asset_id,
+        info.update({
+            'id': asset.get('AssetId') or video_id,
             'display_id': display_id,
             'title': title,
-            'formats': formats,
-            'description': description,
-            'thumbnail': thumbnail,
-            'timestamp': timestamp,
-            'duration': duration,
-        }
+            'description': extract('Programinfo'),
+            'duration': parse_duration(extract('Tid')),
+            'thumbnail': asset.get('AssetImageUrl'),
+        })
+        return info

From 4489d418163def3bc94fc1b9be999ae946621530 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Jun 2017 00:15:15 +0700
Subject: [PATCH 0934/1696] [francetv] Relax _VALID_URL

---
 youtube_dl/extractor/francetv.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 546d5caa0..2bcbb3e39 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -112,7 +112,7 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
 
 
 class FranceTVIE(FranceTVBaseInfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www\.)?france\.tv|mobile\.france\.tv)/(?:[^/]+/)+(?P<id>[^/]+)\.html'
+    _VALID_URL = r'https?://(?:(?:www\.)?france\.tv|mobile\.france\.tv)/(?:[^/]+/)*(?P<id>[^/]+)\.html'
 
     _TESTS = [{
         'url': 'https://www.france.tv/france-2/13h15-le-dimanche/140921-les-mysteres-de-jesus.html',
@@ -157,6 +157,9 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
     }, {
         'url': 'https://mobile.france.tv/france-5/c-dans-l-air/137347-emission-du-vendredi-12-mai-2017.html',
         'only_matching': True,
+    }, {
+        'url': 'https://www.france.tv/142749-rouge-sang.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From f7a747ce59036dca4451037bff4b75cf358e4ad5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Jun 2017 22:19:52 +0700
Subject: [PATCH 0935/1696] [1tv] Lower preference for http formats (closes
 #13246)

---
 youtube_dl/extractor/firsttv.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/firsttv.py b/youtube_dl/extractor/firsttv.py
index 081c71842..4803a22c8 100644
--- a/youtube_dl/extractor/firsttv.py
+++ b/youtube_dl/extractor/firsttv.py
@@ -102,6 +102,8 @@ class FirstTVIE(InfoExtractor):
                     'format_id': f.get('name'),
                     'tbr': tbr,
                     'source_preference': quality(f.get('name')),
+                    # quality metadata of http formats may be incorrect
+                    'preference': -1,
                 })
             # m3u8 URL format is reverse engineered from [1] (search for
             # master.m3u8). dashEdges (that is currently balancer-vod.1tv.ru)

From 39d4c1be4d601a3297157315652637ca52000965 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Jun 2017 23:29:45 +0700
Subject: [PATCH 0936/1696] [youtube] Improve chapters extraction (closes
 #13247)

---
 test/test_youtube_chapters.py   | 7 +++++++
 youtube_dl/extractor/youtube.py | 6 ++++++
 2 files changed, 13 insertions(+)

diff --git a/test/test_youtube_chapters.py b/test/test_youtube_chapters.py
index cb12f8384..324ca8525 100644
--- a/test/test_youtube_chapters.py
+++ b/test/test_youtube_chapters.py
@@ -254,6 +254,13 @@ class TestYoutubeChapters(unittest.TestCase):
                 'title': '3 - Из серпов луны...[Iz serpov luny]',
             }]
         ),
+        (
+            # https://www.youtube.com/watch?v=xZW70zEasOk
+            # time point more than duration
+            '''● LCS Spring finals: Saturday and Sunday from <a href="#" onclick="yt.www.watch.player.seekTo(13*60+30);return false;">13:30</a> outside the venue! <br />● PAX East: Fri, Sat & Sun - more info in tomorrows video on the main channel!''',
+            283,
+            []
+        ),
     ]
 
     def test_youtube_chapters(self):
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 40ac1a019..bf4f4e139 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1353,10 +1353,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             start_time = parse_duration(time_point)
             if start_time is None:
                 continue
+            if start_time > duration:
+                break
             end_time = (duration if next_num == len(chapter_lines)
                         else parse_duration(chapter_lines[next_num][1]))
             if end_time is None:
                 continue
+            if end_time > duration:
+                end_time = duration
+            if start_time > end_time:
+                break
             chapter_title = re.sub(
                 r'<a[^>]+>[^<]+</a>', '', chapter_line).strip(' \t-')
             chapter_title = re.sub(r'\s+', ' ', chapter_title)

From f129c3f349b41ef8f454d67af761e1c15ad30a92 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jun 2017 01:03:51 +0700
Subject: [PATCH 0937/1696] [safari] Fix typo (closes #13252)

---
 youtube_dl/extractor/safari.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index c3aec1edd..794e4ee6f 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -67,7 +67,7 @@ class SafariBaseIE(InfoExtractor):
                 'Login failed; make sure your credentials are correct and try again.',
                 expected=True)
 
-        SafariBaseIE.LOGGED_IN = True
+        self.LOGGED_IN = True
 
         self.to_screen('Login successful')
 

From 4bede0d8f5b6fc8d8e46ee240f808935e03eafa2 Mon Sep 17 00:00:00 2001
From: CeruleanSky <CeruleanSky@users.noreply.github.com>
Date: Sat, 3 Jun 2017 08:14:23 -0400
Subject: [PATCH 0938/1696] [YoutubeDL] Don't emit ANSI escape codes on Windows

---
 youtube_dl/YoutubeDL.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4c33d494a..7efa0c948 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -498,24 +498,25 @@ class YoutubeDL(object):
     def to_console_title(self, message):
         if not self.params.get('consoletitle', False):
             return
-        if compat_os_name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
-            # c_wchar_p() might not be necessary if `message` is
-            # already of type unicode()
-            ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
+        if compat_os_name == 'nt':
+            if ctypes.windll.kernel32.GetConsoleWindow():
+                # c_wchar_p() might not be necessary if `message` is
+                # already of type unicode()
+                ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
         elif 'TERM' in os.environ:
             self._write_string('\033]0;%s\007' % message, self._screen_file)
 
     def save_console_title(self):
         if not self.params.get('consoletitle', False):
             return
-        if 'TERM' in os.environ:
+        if compat_os_name != 'nt' and 'TERM' in os.environ:
             # Save the title on stack
             self._write_string('\033[22;0t', self._screen_file)
 
     def restore_console_title(self):
         if not self.params.get('consoletitle', False):
             return
-        if 'TERM' in os.environ:
+        if compat_os_name != 'nt' and 'TERM' in os.environ:
             # Restore the title from stack
             self._write_string('\033[23;0t', self._screen_file)
 

From cd750b731c41cd9e84c7b16f7c22b6a34da18452 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 3 Jun 2017 22:07:47 +0700
Subject: [PATCH 0939/1696] [godtv] Remove extractor (closes #13175)

---
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/godtv.py      | 66 ------------------------------
 2 files changed, 67 deletions(-)
 delete mode 100644 youtube_dl/extractor/godtv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index aecd08ac1..31e790155 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -392,7 +392,6 @@ from .globo import (
 from .go import GoIE
 from .go90 import Go90IE
 from .godtube import GodTubeIE
-from .godtv import GodTVIE
 from .golem import GolemIE
 from .googledrive import GoogleDriveIE
 from .googleplus import GooglePlusIE
diff --git a/youtube_dl/extractor/godtv.py b/youtube_dl/extractor/godtv.py
deleted file mode 100644
index c5d3b4e6a..000000000
--- a/youtube_dl/extractor/godtv.py
+++ /dev/null
@@ -1,66 +0,0 @@
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from .ooyala import OoyalaIE
-from ..utils import js_to_json
-
-
-class GodTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?god\.tv(?:/[^/]+)*/(?P<id>[^/?#&]+)'
-    _TESTS = [{
-        'url': 'http://god.tv/jesus-image/video/jesus-conference-2016/randy-needham',
-        'info_dict': {
-            'id': 'lpd3g2MzE6D1g8zFAKz8AGpxWcpu6o_3',
-            'ext': 'mp4',
-            'title': 'Randy Needham',
-            'duration': 3615.08,
-        },
-        'params': {
-            'skip_download': True,
-        }
-    }, {
-        'url': 'http://god.tv/playlist/bible-study',
-        'info_dict': {
-            'id': 'bible-study',
-        },
-        'playlist_mincount': 37,
-    }, {
-        'url': 'http://god.tv/node/15097',
-        'only_matching': True,
-    }, {
-        'url': 'http://god.tv/live/africa',
-        'only_matching': True,
-    }, {
-        'url': 'http://god.tv/liveevents',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, display_id)
-
-        settings = self._parse_json(
-            self._search_regex(
-                r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);',
-                webpage, 'settings', default='{}'),
-            display_id, transform_source=js_to_json, fatal=False)
-
-        ooyala_id = None
-
-        if settings:
-            playlist = settings.get('playlist')
-            if playlist and isinstance(playlist, list):
-                entries = [
-                    OoyalaIE._build_url_result(video['content_id'])
-                    for video in playlist if video.get('content_id')]
-                if entries:
-                    return self.playlist_result(entries, display_id)
-            ooyala_id = settings.get('ooyala', {}).get('content_id')
-
-        if not ooyala_id:
-            ooyala_id = self._search_regex(
-                r'["\']content_id["\']\s*:\s*(["\'])(?P<id>[\w-]+)\1',
-                webpage, 'ooyala id', group='id')
-
-        return OoyalaIE._build_url_result(ooyala_id)

From cd138d8bd44ad9845453f7a7fa945f9740425809 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Jun 2017 15:54:19 +0700
Subject: [PATCH 0940/1696] [pornhub:playlist] Fix extraction (closes #13281)

---
 youtube_dl/extractor/pornhub.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 1dcc8df00..d90c6ce1a 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -252,11 +252,14 @@ class PornHubPlaylistBaseIE(InfoExtractor):
 
         playlist = self._parse_json(
             self._search_regex(
-                r'playlistObject\s*=\s*({.+?});', webpage, 'playlist'),
-            playlist_id)
+                r'(?:playlistObject|PLAYLIST_VIEW)\s*=\s*({.+?});', webpage,
+                'playlist', default='{}'),
+            playlist_id, fatal=False)
+        title = playlist.get('title') or self._search_regex(
+            r'>Videos\s+in\s+(.+?)\s+[Pp]laylist<', webpage, 'title', fatal=False)
 
         return self.playlist_result(
-            entries, playlist_id, playlist.get('title'), playlist.get('description'))
+            entries, playlist_id, title, playlist.get('description'))
 
 
 class PornHubPlaylistIE(PornHubPlaylistBaseIE):

From 9edcdac90cc871e6d6e497dc0fb099e40b99ff02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Jun 2017 20:39:55 +0700
Subject: [PATCH 0941/1696] [pornhub:uservideos] Add missing raise

---
 youtube_dl/extractor/pornhub.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index d90c6ce1a..e032817f2 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -299,6 +299,7 @@ class PornHubUserVideosIE(PornHubPlaylistBaseIE):
             except ExtractorError as e:
                 if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
                     break
+                raise
             page_entries = self._extract_entries(webpage)
             if not page_entries:
                 break

From 62bafabc099de4ededf9e136e3c11e878749d066 Mon Sep 17 00:00:00 2001
From: Adam Voss <vossad01@gmail.com>
Date: Sun, 4 Jun 2017 08:47:05 -0500
Subject: [PATCH 0942/1696] [bandcamp:weekly] Add extractor

---
 youtube_dl/extractor/bandcamp.py   | 67 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  2 +-
 2 files changed, 68 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index 489d0ba53..54fa8634c 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -17,6 +17,7 @@ from ..utils import (
     parse_filesize,
     unescapeHTML,
     update_url_query,
+    unified_strdate,
 )
 
 
@@ -222,6 +223,10 @@ class BandcampAlbumIE(InfoExtractor):
         'playlist_count': 2,
     }]
 
+    @classmethod
+    def suitable(cls, url):
+        return False if BandcampWeeklyIE.suitable(url) else super(BandcampAlbumIE, cls).suitable(url)
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         uploader_id = mobj.group('subdomain')
@@ -250,3 +255,65 @@ class BandcampAlbumIE(InfoExtractor):
             'title': title,
             'entries': entries,
         }
+
+
+class BandcampWeeklyIE(InfoExtractor):
+    IE_NAME = 'Bandcamp:bandcamp_weekly'
+    _VALID_URL = r'https?://(?:www\.)?bandcamp\.com/?\?(?:.*&)?show=(?P<id>\d+)(?:$|[&#])'
+    _TESTS = [{
+        'url': 'https://bandcamp.com/?show=224',
+        'md5': 'b00df799c733cf7e0c567ed187dea0fd',
+        'info_dict': {
+            'id': '224',
+            'ext': 'opus',
+            'title': 'BC Weekly April 4th 2017: Magic Moments',
+            'description': 'Stones Throw\'s Vex Ruffin, plus up and coming singer Salami Rose Joe Louis, in conversation about their fantastic DIY albums.',
+        }
+    }, {
+        'url': 'https://bandcamp.com/?blah/blah@&show=228',
+        'only_matching': True
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        blob = self._parse_json(
+            self._search_regex(
+                r'data-blob=(["\'])(?P<blob>{.+?})\1', webpage,
+                'blob', group='blob'),
+            video_id, transform_source=unescapeHTML)
+
+        show = blob['bcw_show']
+
+        # This is desired because any invalid show id redirects to `bandcamp.com`
+        # which happens to expose the latest Bandcamp Weekly episode.
+        video_id = compat_str(show['show_id'])
+
+        def to_format_dictionaries(audio_stream):
+            dictionaries = [{'format_id': kvp[0], 'url': kvp[1]} for kvp in audio_stream.items()]
+            known_extensions = ['mp3', 'opus']
+
+            for dictionary in dictionaries:
+                for ext in known_extensions:
+                    if ext in dictionary['format_id']:
+                        dictionary['ext'] = ext
+                        break
+
+            return dictionaries
+
+        formats = to_format_dictionaries(show['audio_stream'])
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': show['audio_title'] + ': ' + show['subtitle'],
+            'description': show.get('desc'),
+            'duration': float_or_none(show.get('audio_duration')),
+            'webpage_url': 'https://bandcamp.com/?show=' + video_id,
+            'is_live': False,
+            'release_date': unified_strdate(show.get('published_date')),
+            'series': 'Bandcamp Weekly',
+            'episode_id': compat_str(video_id),
+            'formats': formats
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 31e790155..e1907314d 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -90,7 +90,7 @@ from .azmedien import (
 )
 from .baidu import BaiduVideoIE
 from .bambuser import BambuserIE, BambuserChannelIE
-from .bandcamp import BandcampIE, BandcampAlbumIE
+from .bandcamp import BandcampIE, BandcampAlbumIE, BandcampWeeklyIE
 from .bbc import (
     BBCCoUkIE,
     BBCCoUkArticleIE,

From 6d923aab35f186ca5c4c7fce6b7c43145aa192db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Jun 2017 23:21:30 +0700
Subject: [PATCH 0943/1696] [bandcamp:weekly] Improve and extract more metadata
 (closes #12758)

---
 youtube_dl/extractor/bandcamp.py | 78 ++++++++++++++++++++++----------
 1 file changed, 54 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index 54fa8634c..9ddb9af17 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -14,6 +14,7 @@ from ..utils import (
     ExtractorError,
     float_or_none,
     int_or_none,
+    KNOWN_EXTENSIONS,
     parse_filesize,
     unescapeHTML,
     update_url_query,
@@ -22,7 +23,7 @@ from ..utils import (
 
 
 class BandcampIE(InfoExtractor):
-    _VALID_URL = r'https?://.*?\.bandcamp\.com/track/(?P<title>.*)'
+    _VALID_URL = r'https?://.*?\.bandcamp\.com/track/(?P<title>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://youtube-dl.bandcamp.com/track/youtube-dl-test-song',
         'md5': 'c557841d5e50261777a6585648adf439',
@@ -156,7 +157,7 @@ class BandcampIE(InfoExtractor):
 
 class BandcampAlbumIE(InfoExtractor):
     IE_NAME = 'Bandcamp:album'
-    _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?:/album/(?P<album_id>[^?#]+)|/?(?:$|[?#]))'
+    _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?:/album/(?P<album_id>[^/?#&]+))?'
 
     _TESTS = [{
         'url': 'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1',
@@ -225,7 +226,9 @@ class BandcampAlbumIE(InfoExtractor):
 
     @classmethod
     def suitable(cls, url):
-        return False if BandcampWeeklyIE.suitable(url) else super(BandcampAlbumIE, cls).suitable(url)
+        return (False
+                if BandcampWeeklyIE.suitable(url) or BandcampIE.suitable(url)
+                else super(BandcampAlbumIE, cls).suitable(url))
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -258,16 +261,22 @@ class BandcampAlbumIE(InfoExtractor):
 
 
 class BandcampWeeklyIE(InfoExtractor):
-    IE_NAME = 'Bandcamp:bandcamp_weekly'
-    _VALID_URL = r'https?://(?:www\.)?bandcamp\.com/?\?(?:.*&)?show=(?P<id>\d+)(?:$|[&#])'
+    IE_NAME = 'Bandcamp:weekly'
+    _VALID_URL = r'https?://(?:www\.)?bandcamp\.com/?\?(?:.*?&)?show=(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://bandcamp.com/?show=224',
         'md5': 'b00df799c733cf7e0c567ed187dea0fd',
         'info_dict': {
             'id': '224',
             'ext': 'opus',
-            'title': 'BC Weekly April 4th 2017: Magic Moments',
-            'description': 'Stones Throw\'s Vex Ruffin, plus up and coming singer Salami Rose Joe Louis, in conversation about their fantastic DIY albums.',
+            'title': 'BC Weekly April 4th 2017 - Magic Moments',
+            'description': 'md5:5d48150916e8e02d030623a48512c874',
+            'duration': 5829.77,
+            'release_date': '20170404',
+            'series': 'Bandcamp Weekly',
+            'episode': 'Magic Moments',
+            'episode_number': 208,
+            'episode_id': '224',
         }
     }, {
         'url': 'https://bandcamp.com/?blah/blah@&show=228',
@@ -288,32 +297,53 @@ class BandcampWeeklyIE(InfoExtractor):
 
         # This is desired because any invalid show id redirects to `bandcamp.com`
         # which happens to expose the latest Bandcamp Weekly episode.
-        video_id = compat_str(show['show_id'])
+        show_id = int_or_none(show.get('show_id')) or int_or_none(video_id)
 
-        def to_format_dictionaries(audio_stream):
-            dictionaries = [{'format_id': kvp[0], 'url': kvp[1]} for kvp in audio_stream.items()]
-            known_extensions = ['mp3', 'opus']
-
-            for dictionary in dictionaries:
-                for ext in known_extensions:
-                    if ext in dictionary['format_id']:
-                        dictionary['ext'] = ext
-                        break
-
-            return dictionaries
-
-        formats = to_format_dictionaries(show['audio_stream'])
+        formats = []
+        for format_id, format_url in show['audio_stream'].items():
+            if not isinstance(format_url, compat_str):
+                continue
+            for known_ext in KNOWN_EXTENSIONS:
+                if known_ext in format_id:
+                    ext = known_ext
+                    break
+            else:
+                ext = None
+            formats.append({
+                'format_id': format_id,
+                'url': format_url,
+                'ext': ext,
+                'vcodec': 'none',
+            })
         self._sort_formats(formats)
 
+        title = show.get('audio_title') or 'Bandcamp Weekly'
+        subtitle = show.get('subtitle')
+        if subtitle:
+            title += ' - %s' % subtitle
+
+        episode_number = None
+        seq = blob.get('bcw_seq')
+
+        if seq and isinstance(seq, list):
+            try:
+                episode_number = next(
+                    int_or_none(e.get('episode_number'))
+                    for e in seq
+                    if isinstance(e, dict) and int_or_none(e.get('id')) == show_id)
+            except StopIteration:
+                pass
+
         return {
             'id': video_id,
-            'title': show['audio_title'] + ': ' + show['subtitle'],
-            'description': show.get('desc'),
+            'title': title,
+            'description': show.get('desc') or show.get('short_desc'),
             'duration': float_or_none(show.get('audio_duration')),
-            'webpage_url': 'https://bandcamp.com/?show=' + video_id,
             'is_live': False,
             'release_date': unified_strdate(show.get('published_date')),
             'series': 'Bandcamp Weekly',
+            'episode': show.get('subtitle'),
+            'episode_number': episode_number,
             'episode_id': compat_str(video_id),
             'formats': formats
         }

From 43e6579558290db833171272412a2d4f007e853d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Jun 2017 23:22:19 +0700
Subject: [PATCH 0944/1696] Credit @adamvoss for bandcamp:weekly (#12758)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index f80c34a08..f5a08e974 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -217,3 +217,4 @@ Marvin Ewald
 Frédéric Bournival
 Timendum
 gritstub
+Adam Voss

From e1b90cc3db81c96abdf9a532aa4816b571ad5e88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Jun 2017 00:35:41 +0700
Subject: [PATCH 0945/1696] Credit @mikf for beam:vod (#13032)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index f5a08e974..bd61240c0 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -218,3 +218,4 @@ Frédéric Bournival
 Timendum
 gritstub
 Adam Voss
+Mike Fährmann

From f8ba3fda4dedfe1e480b6b929bc374766c722234 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Jun 2017 00:38:44 +0700
Subject: [PATCH 0946/1696] Credit @jktjkt for dvtv formats (#13063)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index bd61240c0..e2bdebe5e 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -219,3 +219,4 @@ Timendum
 gritstub
 Adam Voss
 Mike Fährmann
+Jan Kundrát

From 130880ba48fe545c9464ac5854cbc88915f84b52 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Jun 2017 00:43:38 +0700
Subject: [PATCH 0947/1696] [ChangeLog] Actualize

---
 ChangeLog | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 7d78c87e6..639693413 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,20 @@
+version <unreleased>
+
+Core
+* [YoutubeDL] Don't emit ANSI escape codes on Windows (#13270)
+
+Extractors
++ [bandcamp:weekly] Add support for bandcamp weekly (#12758)
+* [pornhub:playlist] Fix extraction (#13281)
+- [godtv] Remove extractor (#13175)
+* [safari] Fix typo (#13252)
+* [youtube] Improve chapters extraction (#13247)
+* [1tv] Lower preference for HTTP formats (#13246)
+* [francetv] Relax URL regular expression
+* [drbonanza] Fix extraction (#13231)
+* [packtpub] Fix authentication (#13240)
+
+
 version 2017.05.29
 
 Extractors

From 537191826f0c778ed3e2eaef4168a3aa358fb7e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Jun 2017 00:48:07 +0700
Subject: [PATCH 0948/1696] release 2017.06.05

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 +-
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 24a0d72c5..ce1e46085 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.05.29*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.05.29**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.06.05*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.06.05**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.05.29
+[debug] youtube-dl version 2017.06.05
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 639693413..c605c5bd6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.06.05
 
 Core
 * [YoutubeDL] Don't emit ANSI escape codes on Windows (#13270)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 8ddc800b4..fe022d158 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -87,6 +87,7 @@
  - **bambuser:channel**
  - **Bandcamp**
  - **Bandcamp:album**
+ - **Bandcamp:weekly**
  - **bangumi.bilibili.com**: BiliBili番剧
  - **bbc**: BBC
  - **bbc.co.uk**: BBC iPlayer
@@ -310,7 +311,6 @@
  - **Go**
  - **Go90**
  - **GodTube**
- - **GodTV**
  - **Golem**
  - **GoogleDrive**
  - **Goshgay**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d4ee43122..d3aa39977 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.05.29'
+__version__ = '2017.06.05'

From e1db730d86302f1f96f20d44510e8fb060366bff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 6 Jun 2017 00:13:57 +0700
Subject: [PATCH 0949/1696] [tvplayer] Fix extraction (closes #13291)

---
 youtube_dl/extractor/tvplayer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tvplayer.py b/youtube_dl/extractor/tvplayer.py
index ebde6053f..8f8686a65 100644
--- a/youtube_dl/extractor/tvplayer.py
+++ b/youtube_dl/extractor/tvplayer.py
@@ -48,7 +48,7 @@ class TVPlayerIE(InfoExtractor):
             'https://tvplayer.com/watch/context', display_id,
             'Downloading JSON context', query={
                 'resource': resource_id,
-                'nonce': token,
+                'gen': token,
             })
 
         validate = context['validate']

From 0a3924e74622175dc9798e01e60d6bd3b3b36018 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 6 Jun 2017 21:21:44 +0700
Subject: [PATCH 0950/1696] [rtlnl] Improve _VALID_URL (closes #13295)

---
 youtube_dl/extractor/rtlnl.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index 54076de28..3e22998c6 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -15,7 +15,7 @@ class RtlNlIE(InfoExtractor):
         https?://(?:www\.)?
         (?:
             rtlxl\.nl/[^\#]*\#!/[^/]+/|
-            rtl\.nl/system/videoplayer/(?:[^/]+/)+(?:video_)?embed\.html\b.+?\buuid=
+            rtl\.nl/(?:system/videoplayer/(?:[^/]+/)+(?:video_)?embed\.html\b.+?\buuid=|video/)
         )
         (?P<id>[0-9a-f-]+)'''
 
@@ -70,6 +70,9 @@ class RtlNlIE(InfoExtractor):
     }, {
         'url': 'http://rtlxl.nl/?_ga=1.204735956.572365465.1466978370#!/rtl-nieuws-132237/3c487912-023b-49ac-903e-2c5d79f8410f',
         'only_matching': True,
+    }, {
+        'url': 'https://www.rtl.nl/video/c603c9c2-601d-4b5e-8175-64f1e942dc7d/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From eb703e538035b0ebdc076be0ba982aac97d9ef08 Mon Sep 17 00:00:00 2001
From: Luca Steeb <contact@luca-steeb.com>
Date: Mon, 5 Jun 2017 20:17:43 +0200
Subject: [PATCH 0951/1696] [streamango] Make title optional

---
 youtube_dl/extractor/streamango.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/streamango.py b/youtube_dl/extractor/streamango.py
index aa4fad162..cf03d5b09 100644
--- a/youtube_dl/extractor/streamango.py
+++ b/youtube_dl/extractor/streamango.py
@@ -21,6 +21,13 @@ class StreamangoIE(InfoExtractor):
             'ext': 'mp4',
             'title': '20170315_150006.mp4',
         }
+    }, {
+        'url': 'https://streamango.com/embed/foqebrpftarclpob/asdf_asd_2_mp4',
+        'info_dict': {
+            'id': 'foqebrpftarclpob',
+            'ext': 'mp4',
+            'title': 'foqebrpftarclpob',
+        }
     }, {
         'url': 'https://streamango.com/embed/clapasobsptpkdfe/20170315_150006_mp4',
         'only_matching': True,
@@ -31,7 +38,7 @@ class StreamangoIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        title = self._og_search_title(webpage)
+        title = self._og_search_title(webpage, default=video_id)
 
         formats = []
         for format_ in re.findall(r'({[^}]*\bsrc\s*:\s*[^}]*})', webpage):

From 1508da30c28f2047a6d9d1dcebd529fa6ea56f15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 7 Jun 2017 21:53:13 +0700
Subject: [PATCH 0952/1696] [streamango] Skip download for test (closes #13292)

---
 youtube_dl/extractor/streamango.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/streamango.py b/youtube_dl/extractor/streamango.py
index cf03d5b09..a9e34c027 100644
--- a/youtube_dl/extractor/streamango.py
+++ b/youtube_dl/extractor/streamango.py
@@ -22,12 +22,16 @@ class StreamangoIE(InfoExtractor):
             'title': '20170315_150006.mp4',
         }
     }, {
+        # no og:title
         'url': 'https://streamango.com/embed/foqebrpftarclpob/asdf_asd_2_mp4',
         'info_dict': {
             'id': 'foqebrpftarclpob',
             'ext': 'mp4',
             'title': 'foqebrpftarclpob',
-        }
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'https://streamango.com/embed/clapasobsptpkdfe/20170315_150006_mp4',
         'only_matching': True,

From a355b57f5893997567fe635f730a7e88d57cbca1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Jun 2017 21:52:19 +0700
Subject: [PATCH 0953/1696] [README.md] Clarify output template references
 (closes #13316)

---
 youtube_dl/options.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 3021a6f41..79e9fd12c 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -310,7 +310,7 @@ def parseOpts(overrideArguments=None):
         metavar='FILTER', dest='match_filter', default=None,
         help=(
             'Generic video filter. '
-            'Specify any key (see help for -o for a list of available keys) to '
+            'Specify any key (see the "OUTPUT TEMPLATE" for a list of available keys) to '
             'match if the key is present, '
             '!key to check if the key is not present, '
             'key > NUMBER (like "comment_count > 12", also works with '
@@ -618,7 +618,7 @@ def parseOpts(overrideArguments=None):
     verbosity.add_option(
         '-j', '--dump-json',
         action='store_true', dest='dumpjson', default=False,
-        help='Simulate, quiet but print JSON information. See --output for a description of available keys.')
+        help='Simulate, quiet but print JSON information. See the "OUTPUT TEMPLATE" for a description of available keys.')
     verbosity.add_option(
         '-J', '--dump-single-json',
         action='store_true', dest='dump_single_json', default=False,

From 76e6378358a618a20051e9f9fd38e43af3169683 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Jun 2017 22:02:42 +0700
Subject: [PATCH 0954/1696] [README.md] Improve man page formatting

---
 README.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index fa901a46d..2189552a9 100644
--- a/README.md
+++ b/README.md
@@ -535,13 +535,14 @@ The basic usage is not to set any template arguments when downloading a single f
  - `playlist_id` (string): Playlist identifier
  - `playlist_title` (string): Playlist title
 
-
 Available for the video that belongs to some logical chapter or section:
+
  - `chapter` (string): Name or title of the chapter the video belongs to
  - `chapter_number` (numeric): Number of the chapter the video belongs to
  - `chapter_id` (string): Id of the chapter the video belongs to
 
 Available for the video that is an episode of some series or programme:
+
  - `series` (string): Title of the series or programme the video episode belongs to
  - `season` (string): Title of the season the video episode belongs to
  - `season_number` (numeric): Number of the season the video episode belongs to
@@ -551,6 +552,7 @@ Available for the video that is an episode of some series or programme:
  - `episode_id` (string): Id of the video episode
 
 Available for the media that is a track or a part of a music album:
+
  - `track` (string): Title of the track
  - `track_number` (numeric): Number of the track within an album or a disc
  - `track_id` (string): Id of the track

From c9969434183c562eb9935aa20f147f234aa61e53 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Jun 2017 22:53:14 +0700
Subject: [PATCH 0955/1696] [YoutubeDL] Sanitize more fields (#13313)

---
 youtube_dl/YoutubeDL.py | 52 +++++++++++++++++++++++++++++------------
 1 file changed, 37 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 7efa0c948..c05103bb6 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -58,6 +58,7 @@ from .utils import (
     format_bytes,
     formatSeconds,
     GeoRestrictedError,
+    int_or_none,
     ISO3166Utils,
     locked_file,
     make_HTTPS_handler,
@@ -302,6 +303,17 @@ class YoutubeDL(object):
                         postprocessor.
     """
 
+    _NUMERIC_FIELDS = set((
+        'width', 'height', 'tbr', 'abr', 'asr', 'vbr', 'fps', 'filesize', 'filesize_approx',
+        'timestamp', 'upload_year', 'upload_month', 'upload_day',
+        'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
+        'average_rating', 'comment_count', 'age_limit',
+        'start_time', 'end_time',
+        'chapter_number', 'season_number', 'episode_number',
+        'track_number', 'disc_number', 'release_year',
+        'playlist_index',
+    ))
+
     params = None
     _ies = []
     _pps = []
@@ -639,22 +651,11 @@ class YoutubeDL(object):
                     r'%%(\1)0%dd' % field_size_compat_map[mobj.group('field')],
                     outtmpl)
 
-            NUMERIC_FIELDS = set((
-                'width', 'height', 'tbr', 'abr', 'asr', 'vbr', 'fps', 'filesize', 'filesize_approx',
-                'timestamp', 'upload_year', 'upload_month', 'upload_day',
-                'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
-                'average_rating', 'comment_count', 'age_limit',
-                'start_time', 'end_time',
-                'chapter_number', 'season_number', 'episode_number',
-                'track_number', 'disc_number', 'release_year',
-                'playlist_index',
-            ))
-
             # Missing numeric fields used together with integer presentation types
             # in format specification will break the argument substitution since
             # string 'NA' is returned for missing fields. We will patch output
             # template for missing fields to meet string presentation type.
-            for numeric_field in NUMERIC_FIELDS:
+            for numeric_field in self._NUMERIC_FIELDS:
                 if numeric_field not in template_dict:
                     # As of [1] format syntax is:
                     #  %[mapping_key][conversion_flags][minimum_width][.precision][length_modifier]type
@@ -1345,9 +1346,28 @@ class YoutubeDL(object):
         if 'title' not in info_dict:
             raise ExtractorError('Missing "title" field in extractor result')
 
-        if not isinstance(info_dict['id'], compat_str):
-            self.report_warning('"id" field is not a string - forcing string conversion')
-            info_dict['id'] = compat_str(info_dict['id'])
+        def report_force_conversion(field, field_not, conversion):
+            self.report_warning(
+                '"%s" field is not %s - forcing %s conversion, there is an error in extractor'
+                % (field, field_not, conversion))
+
+        def sanitize_string_field(info, string_field):
+            field = info.get(string_field)
+            if field is None or isinstance(field, compat_str):
+                return
+            report_force_conversion(string_field, 'a string', 'string')
+            info[string_field] = compat_str(field)
+
+        def sanitize_numeric_fields(info):
+            for numeric_field in self._NUMERIC_FIELDS:
+                field = info.get(numeric_field)
+                if field is None or isinstance(field, compat_numeric_types):
+                    continue
+                report_force_conversion(numeric_field, 'numeric', 'int')
+                info[numeric_field] = int_or_none(field)
+
+        sanitize_string_field(info_dict, 'id')
+        sanitize_numeric_fields(info_dict)
 
         if 'playlist' not in info_dict:
             # It isn't part of a playlist
@@ -1435,6 +1455,8 @@ class YoutubeDL(object):
             if 'url' not in format:
                 raise ExtractorError('Missing "url" key in result (index %d)' % i)
 
+            sanitize_string_field(format, 'format_id')
+            sanitize_numeric_fields(format)
             format['url'] = sanitize_url(format['url'])
 
             if format.get('format_id') is None:

From 931adf8cc18e1cc2199d0d4ba5d852f8119ebdb7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Jun 2017 22:54:30 +0700
Subject: [PATCH 0956/1696] [liveleak] Ensure height is int (closes #13313)

---
 youtube_dl/extractor/liveleak.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index c54519636..b2247a84d 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -115,8 +115,9 @@ class LiveLeakIE(InfoExtractor):
 
         for a_format in info_dict['formats']:
             if not a_format.get('height'):
-                a_format['height'] = self._search_regex(
-                    r'([0-9]+)p\.mp4', a_format['url'], 'height label', default=None)
+                a_format['height'] = int_or_none(self._search_regex(
+                    r'([0-9]+)p\.mp4', a_format['url'], 'height label',
+                    default=None))
 
         self._sort_formats(info_dict['formats'])
 

From 4244a13a1d7420d8e8a2a51bc4e7d2fa17e9f844 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Jun 2017 23:20:48 +0700
Subject: [PATCH 0957/1696] [safari] Improve authentication detection (closes
 #13319)

---
 youtube_dl/extractor/safari.py | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index 794e4ee6f..909a6ba97 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -16,7 +16,6 @@ from ..utils import (
 
 class SafariBaseIE(InfoExtractor):
     _LOGIN_URL = 'https://www.safaribooksonline.com/accounts/login/'
-    _SUCCESSFUL_LOGIN_REGEX = r'<a href="/accounts/logout/"[^>]*>Sign Out</a>'
     _NETRC_MACHINE = 'safari'
 
     _API_BASE = 'https://www.safaribooksonline.com/api/v1'
@@ -28,10 +27,6 @@ class SafariBaseIE(InfoExtractor):
         self._login()
 
     def _login(self):
-        # We only need to log in once for courses or individual videos
-        if self.LOGGED_IN:
-            return
-
         (username, password) = self._get_login_info()
         if username is None:
             return
@@ -39,11 +34,17 @@ class SafariBaseIE(InfoExtractor):
         headers = std_headers.copy()
         if 'Referer' not in headers:
             headers['Referer'] = self._LOGIN_URL
-        login_page_request = sanitized_Request(self._LOGIN_URL, headers=headers)
 
         login_page = self._download_webpage(
-            login_page_request, None,
-            'Downloading login form')
+            self._LOGIN_URL, None, 'Downloading login form', headers=headers)
+
+        def is_logged(webpage):
+            return any(re.search(p, webpage) for p in (
+                r'href=["\']/accounts/logout/', r'>Sign Out<'))
+
+        if is_logged(login_page):
+            self.LOGGED_IN = True
+            return
 
         csrf = self._html_search_regex(
             r"name='csrfmiddlewaretoken'\s+value='([^']+)'",
@@ -62,15 +63,13 @@ class SafariBaseIE(InfoExtractor):
         login_page = self._download_webpage(
             request, None, 'Logging in as %s' % username)
 
-        if re.search(self._SUCCESSFUL_LOGIN_REGEX, login_page) is None:
+        if not is_logged(login_page):
             raise ExtractorError(
                 'Login failed; make sure your credentials are correct and try again.',
                 expected=True)
 
         self.LOGGED_IN = True
 
-        self.to_screen('Login successful')
-
 
 class SafariIE(SafariBaseIE):
     IE_NAME = 'safari'

From 1693bebe4d3e605001b1df8b6e1f238c2e3ff897 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 9 Jun 2017 00:16:42 +0700
Subject: [PATCH 0958/1696] [sohu] Fix numeric fields

---
 youtube_dl/extractor/sohu.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 7da12cef8..a62ed84f1 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -8,7 +8,11 @@ from ..compat import (
     compat_str,
     compat_urllib_parse_urlencode,
 )
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    try_get,
+)
 
 
 class SohuIE(InfoExtractor):
@@ -169,10 +173,11 @@ class SohuIE(InfoExtractor):
                 formats.append({
                     'url': video_url,
                     'format_id': format_id,
-                    'filesize': data['clipsBytes'][i],
-                    'width': data['width'],
-                    'height': data['height'],
-                    'fps': data['fps'],
+                    'filesize': int_or_none(
+                        try_get(data, lambda x: x['clipsBytes'][i])),
+                    'width': int_or_none(data.get('width')),
+                    'height': int_or_none(data.get('height')),
+                    'fps': int_or_none(data.get('fps')),
                 })
             self._sort_formats(formats)
 

From 15aeeb11882c9597614ce6e249968bfdf7830e8a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 9 Jun 2017 00:20:07 +0700
Subject: [PATCH 0959/1696] [flickr] Ensure format id is string

---
 youtube_dl/extractor/flickr.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/flickr.py b/youtube_dl/extractor/flickr.py
index a8e1bf42a..9f166efd4 100644
--- a/youtube_dl/extractor/flickr.py
+++ b/youtube_dl/extractor/flickr.py
@@ -1,7 +1,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlencode
+from ..compat import (
+    compat_str,
+    compat_urllib_parse_urlencode,
+)
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -81,7 +84,7 @@ class FlickrIE(InfoExtractor):
 
             formats = []
             for stream in streams['stream']:
-                stream_type = str(stream.get('type'))
+                stream_type = compat_str(stream.get('type'))
                 formats.append({
                     'format_id': stream_type,
                     'url': stream['_content'],

From cc69a3de1b0980d34e21719fd4c264575d620bb4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 9 Jun 2017 00:22:14 +0700
Subject: [PATCH 0960/1696] [foxgay] Ensure height is int

---
 youtube_dl/extractor/foxgay.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/foxgay.py b/youtube_dl/extractor/foxgay.py
index e887ae488..512a10645 100644
--- a/youtube_dl/extractor/foxgay.py
+++ b/youtube_dl/extractor/foxgay.py
@@ -5,6 +5,7 @@ import itertools
 from .common import InfoExtractor
 from ..utils import (
     get_element_by_id,
+    int_or_none,
     remove_end,
 )
 
@@ -46,7 +47,7 @@ class FoxgayIE(InfoExtractor):
 
         formats = [{
             'url': source,
-            'height': resolution,
+            'height': int_or_none(resolution),
         } for source, resolution in zip(
             video_data['sources'], video_data.get('resolutions', itertools.repeat(None)))]
 

From 87f5646937f52643a32f80210adf2647e3bd5d25 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 9 Jun 2017 00:24:23 +0700
Subject: [PATCH 0961/1696] [gfycat] Ensure filesize is int

---
 youtube_dl/extractor/gfycat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gfycat.py b/youtube_dl/extractor/gfycat.py
index 884700c52..45ccc11c1 100644
--- a/youtube_dl/extractor/gfycat.py
+++ b/youtube_dl/extractor/gfycat.py
@@ -82,7 +82,7 @@ class GfycatIE(InfoExtractor):
             video_url = gfy.get('%sUrl' % format_id)
             if not video_url:
                 continue
-            filesize = gfy.get('%sSize' % format_id)
+            filesize = int_or_none(gfy.get('%sSize' % format_id))
             formats.append({
                 'url': video_url,
                 'format_id': format_id,

From e4d2e76d8ed6d125f671987b855c6d3eef1b38f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 9 Jun 2017 00:27:11 +0700
Subject: [PATCH 0962/1696] [golem] Ensure format id is string

---
 youtube_dl/extractor/golem.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/golem.py b/youtube_dl/extractor/golem.py
index 2bfb99040..47a068e74 100644
--- a/youtube_dl/extractor/golem.py
+++ b/youtube_dl/extractor/golem.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_str,
     compat_urlparse,
 )
 from ..utils import (
@@ -46,7 +47,7 @@ class GolemIE(InfoExtractor):
                 continue
 
             formats.append({
-                'format_id': e.tag,
+                'format_id': compat_str(e.tag),
                 'url': compat_urlparse.urljoin(self._PREFIX, url),
                 'height': self._int(e.get('height'), 'height'),
                 'width': self._int(e.get('width'), 'width'),

From 34646967ba8a09c4868f08f1903425b6d721c33d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 9 Jun 2017 00:29:20 +0700
Subject: [PATCH 0963/1696] [jove] Ensure comment count is int

---
 youtube_dl/extractor/jove.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/jove.py b/youtube_dl/extractor/jove.py
index f9a034b78..27e0e37f6 100644
--- a/youtube_dl/extractor/jove.py
+++ b/youtube_dl/extractor/jove.py
@@ -65,9 +65,9 @@ class JoveIE(InfoExtractor):
             webpage, 'description', fatal=False)
         publish_date = unified_strdate(self._html_search_meta(
             'citation_publication_date', webpage, 'publish date', fatal=False))
-        comment_count = self._html_search_regex(
+        comment_count = int(self._html_search_regex(
             r'<meta name="num_comments" content="(\d+) Comments?"',
-            webpage, 'comment count', fatal=False)
+            webpage, 'comment count', fatal=False))
 
         return {
             'id': video_id,

From f5521ea209d1b20ec0bc6eeeb16e99101d05c854 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 9 Jun 2017 00:30:23 +0700
Subject: [PATCH 0964/1696] [sexu] Ensure height is int

---
 youtube_dl/extractor/sexu.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/sexu.py b/youtube_dl/extractor/sexu.py
index 5e22ea730..3df51520b 100644
--- a/youtube_dl/extractor/sexu.py
+++ b/youtube_dl/extractor/sexu.py
@@ -32,8 +32,9 @@ class SexuIE(InfoExtractor):
         formats = [{
             'url': source['file'].replace('\\', ''),
             'format_id': source.get('label'),
-            'height': self._search_regex(
-                r'^(\d+)[pP]', source.get('label', ''), 'height', default=None),
+            'height': int(self._search_regex(
+                r'^(\d+)[pP]', source.get('label', ''), 'height',
+                default=None)),
         } for source in sources if source.get('file')]
         self._sort_formats(formats)
 

From 751583042299138ef4deef5862e6fe2ada078417 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 9 Jun 2017 00:31:47 +0700
Subject: [PATCH 0965/1696] [turbo] Ensure format id is string

---
 youtube_dl/extractor/turbo.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/turbo.py b/youtube_dl/extractor/turbo.py
index 25aa9c58e..be3eaa5c2 100644
--- a/youtube_dl/extractor/turbo.py
+++ b/youtube_dl/extractor/turbo.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -49,7 +50,7 @@ class TurboIE(InfoExtractor):
         for child in item:
             m = re.search(r'url_video_(?P<quality>.+)', child.tag)
             if m:
-                quality = m.group('quality')
+                quality = compat_str(m.group('quality'))
                 formats.append({
                     'format_id': quality,
                     'url': child.text,

From 1afd0b0da70b6ed709c610aff98786d71511a629 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 9 Jun 2017 00:40:03 +0700
Subject: [PATCH 0966/1696] [extractor/common] Return unicode string from
 _match_id

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index fec39da8b..f027447c8 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -376,7 +376,7 @@ class InfoExtractor(object):
             cls._VALID_URL_RE = re.compile(cls._VALID_URL)
         m = cls._VALID_URL_RE.match(url)
         assert m
-        return m.group('id')
+        return compat_str(m.group('id'))
 
     @classmethod
     def working(cls):

From 4e3637034cf2b3630e6f9c6d34aa0177a8c83950 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Jun 2017 23:55:05 +0700
Subject: [PATCH 0967/1696] [extractor/generic] Ensure format id is unicode
 string

---
 youtube_dl/extractor/generic.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c108d4a8a..8ef1a2980 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -10,6 +10,7 @@ from .common import InfoExtractor
 from .youtube import YoutubeIE
 from ..compat import (
     compat_etree_fromstring,
+    compat_str,
     compat_urllib_parse_unquote,
     compat_urlparse,
     compat_xml_parse_error,
@@ -1907,14 +1908,14 @@ class GenericIE(InfoExtractor):
         content_type = head_response.headers.get('Content-Type', '').lower()
         m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>[^;\s]+)', content_type)
         if m:
-            format_id = m.group('format_id')
+            format_id = compat_str(m.group('format_id'))
             if format_id.endswith('mpegurl'):
                 formats = self._extract_m3u8_formats(url, video_id, 'mp4')
             elif format_id == 'f4m':
                 formats = self._extract_f4m_formats(url, video_id)
             else:
                 formats = [{
-                    'format_id': m.group('format_id'),
+                    'format_id': format_id,
                     'url': url,
                     'vcodec': 'none' if m.group('type') == 'audio' else None
                 }]

From 7409af9eb322b0e76a4ae2d713020c45d178fabe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Jun 2017 08:56:53 +0700
Subject: [PATCH 0968/1696] [msn] Fix formats extraction

---
 youtube_dl/extractor/msn.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/msn.py b/youtube_dl/extractor/msn.py
index 1473bcf48..650731fdc 100644
--- a/youtube_dl/extractor/msn.py
+++ b/youtube_dl/extractor/msn.py
@@ -68,10 +68,6 @@ class MSNIE(InfoExtractor):
             format_url = file_.get('url')
             if not format_url:
                 continue
-            ext = determine_ext(format_url)
-            if ext == 'ism':
-                formats.extend(self._extract_ism_formats(
-                    format_url + '/Manifest', display_id, 'mss', fatal=False))
             if 'm3u8' in format_url:
                 # m3u8_native should not be used here until
                 # https://github.com/rg3/youtube-dl/issues/9913 is fixed
@@ -79,6 +75,9 @@ class MSNIE(InfoExtractor):
                     format_url, display_id, 'mp4',
                     m3u8_id='hls', fatal=False)
                 formats.extend(m3u8_formats)
+            elif determine_ext(format_url) == 'ism':
+                formats.extend(self._extract_ism_formats(
+                    format_url + '/Manifest', display_id, 'mss', fatal=False))
             else:
                 formats.append({
                     'url': format_url,

From 6e999fbc120dfdbd8ec5ca056105420fd7cbffe8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Jun 2017 19:44:44 +0700
Subject: [PATCH 0969/1696] [newgrounds] Improve formats and uploader
 extraction (closes #13346)

---
 youtube_dl/extractor/newgrounds.py | 48 ++++++++++++++++++++++++++----
 1 file changed, 43 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/newgrounds.py b/youtube_dl/extractor/newgrounds.py
index 9bea610c8..ae4fe95ef 100644
--- a/youtube_dl/extractor/newgrounds.py
+++ b/youtube_dl/extractor/newgrounds.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import int_or_none
 
 
 class NewgroundsIE(InfoExtractor):
@@ -23,24 +24,61 @@ class NewgroundsIE(InfoExtractor):
             'title': 'Dancin',
             'uploader': 'Squirrelman82',
         },
+    }, {
+        # source format unavailable, additional mp4 formats
+        'url': 'http://www.newgrounds.com/portal/view/689400',
+        'info_dict': {
+            'id': '689400',
+            'ext': 'mp4',
+            'title': 'ZTV News Episode 8',
+            'uploader': 'BennettTheSage',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
         media_id = self._match_id(url)
+
         webpage = self._download_webpage(url, media_id)
 
         title = self._html_search_regex(
             r'<title>([^>]+)</title>', webpage, 'title')
 
-        uploader = self._html_search_regex(
-            r'Author\s*<a[^>]+>([^<]+)', webpage, 'uploader', fatal=False)
+        video_url = self._parse_json(self._search_regex(
+            r'"url"\s*:\s*("[^"]+"),', webpage, ''), media_id)
 
-        music_url = self._parse_json(self._search_regex(
-            r'"url":("[^"]+"),', webpage, ''), media_id)
+        formats = [{
+            'url': video_url,
+            'format_id': 'source',
+            'quality': 1,
+        }]
+
+        max_resolution = int_or_none(self._search_regex(
+            r'max_resolution["\']\s*:\s*(\d+)', webpage, 'max resolution',
+            default=None))
+        if max_resolution:
+            url_base = video_url.rpartition('.')[0]
+            for resolution in (360, 720, 1080):
+                if resolution > max_resolution:
+                    break
+                formats.append({
+                    'url': '%s.%dp.mp4' % (url_base, resolution),
+                    'format_id': '%dp' % resolution,
+                    'height': resolution,
+                })
+
+        self._check_formats(formats, media_id)
+        self._sort_formats(formats)
+
+        uploader = self._html_search_regex(
+            r'(?:Author|Writer)\s*<a[^>]+>([^<]+)', webpage, 'uploader',
+            fatal=False)
 
         return {
             'id': media_id,
             'title': title,
-            'url': music_url,
             'uploader': uploader,
+            'formats': formats,
         }

From 70e7967202efa0acb6202ceedf87049ba72b94e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Jun 2017 20:50:33 +0700
Subject: [PATCH 0970/1696] [newgrounds:playlist] Add extractor (closes #10611)

---
 youtube_dl/extractor/extractors.py |  5 ++-
 youtube_dl/extractor/newgrounds.py | 56 +++++++++++++++++++++++++++++-
 2 files changed, 59 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index e1907314d..7e45232dd 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -636,7 +636,10 @@ from .neteasemusic import (
     NetEaseMusicProgramIE,
     NetEaseMusicDjRadioIE,
 )
-from .newgrounds import NewgroundsIE
+from .newgrounds import (
+    NewgroundsIE,
+    NewgroundsPlaylistIE,
+)
 from .newstube import NewstubeIE
 from .nextmedia import (
     NextMediaIE,
diff --git a/youtube_dl/extractor/newgrounds.py b/youtube_dl/extractor/newgrounds.py
index ae4fe95ef..dc183caec 100644
--- a/youtube_dl/extractor/newgrounds.py
+++ b/youtube_dl/extractor/newgrounds.py
@@ -1,7 +1,12 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    extract_attributes,
+    int_or_none,
+)
 
 
 class NewgroundsIE(InfoExtractor):
@@ -82,3 +87,52 @@ class NewgroundsIE(InfoExtractor):
             'uploader': uploader,
             'formats': formats,
         }
+
+
+class NewgroundsPlaylistIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?newgrounds\.com/(?:collection|[^/]+/search/[^/]+)/(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'https://www.newgrounds.com/collection/cats',
+        'info_dict': {
+            'id': 'cats',
+            'title': 'Cats',
+        },
+        'playlist_mincount': 46,
+    }, {
+        'url': 'http://www.newgrounds.com/portal/search/author/ZONE-SAMA',
+        'info_dict': {
+            'id': 'ZONE-SAMA',
+            'title': 'Portal Search: ZONE-SAMA',
+        },
+        'playlist_mincount': 47,
+    }, {
+        'url': 'http://www.newgrounds.com/audio/search/title/cats',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        title = self._search_regex(
+            r'<title>([^>]+)</title>', webpage, 'title', default=None)
+
+        # cut left menu
+        webpage = self._search_regex(
+            r'(?s)<div[^>]+\bclass=["\']column wide(.+)',
+            webpage, 'wide column', default=webpage)
+
+        entries = []
+        for a, path, media_id in re.findall(
+                r'(<a[^>]+\bhref=["\']/?((?:portal/view|audio/listen)/(\d+))[^>]+>)',
+                webpage):
+            a_class = extract_attributes(a).get('class')
+            if a_class not in ('item-portalsubmission', 'item-audiosubmission'):
+                continue
+            entries.append(
+                self.url_result(
+                    'https://www.newgrounds.com/%s' % path,
+                    ie=NewgroundsIE.ie_key(), video_id=media_id))
+
+        return self.playlist_result(entries, playlist_id, title)

From 2ae2ffda5eae9c64d40d2fec839ba5deb07717f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Jun 2017 21:27:22 +0700
Subject: [PATCH 0971/1696] [utils] Improve unified_timestamp

---
 test/test_utils.py  | 1 +
 youtube_dl/utils.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index d7e05817c..41b094d89 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -340,6 +340,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(unified_timestamp('May 16, 2016 11:15 PM'), 1463440500)
         self.assertEqual(unified_timestamp('Feb 7, 2016 at 6:35 pm'), 1454870100)
         self.assertEqual(unified_timestamp('2017-03-30T17:52:41Q'), 1490896361)
+        self.assertEqual(unified_timestamp('Sep 11, 2013 | 5:49 AM'), 1378878540)
 
     def test_determine_ext(self):
         self.assertEqual(determine_ext('http://example.com/foo/bar.mp4/?download'), 'mp4')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d79410e58..16bf49408 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1179,7 +1179,7 @@ def unified_timestamp(date_str, day_first=True):
     if date_str is None:
         return None
 
-    date_str = date_str.replace(',', ' ')
+    date_str = re.sub(r'[,|]', '', date_str)
 
     pm_delta = 12 if re.search(r'(?i)PM', date_str) else 0
     timezone, date_str = extract_timezone(date_str)

From 28a4d6cce89d159f9fe701e6cf716c7e3ffa4415 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Jun 2017 21:27:32 +0700
Subject: [PATCH 0972/1696] [newgrounds] Extract more metadata (closes #13232)

---
 youtube_dl/extractor/newgrounds.py | 38 ++++++++++++++++++++++++++----
 1 file changed, 33 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/newgrounds.py b/youtube_dl/extractor/newgrounds.py
index dc183caec..0e26f8399 100644
--- a/youtube_dl/extractor/newgrounds.py
+++ b/youtube_dl/extractor/newgrounds.py
@@ -6,6 +6,9 @@ from .common import InfoExtractor
 from ..utils import (
     extract_attributes,
     int_or_none,
+    parse_duration,
+    parse_filesize,
+    unified_timestamp,
 )
 
 
@@ -19,7 +22,10 @@ class NewgroundsIE(InfoExtractor):
             'ext': 'mp3',
             'title': 'B7 - BusMode',
             'uploader': 'Burn7',
-        }
+            'timestamp': 1378878540,
+            'upload_date': '20130911',
+            'duration': 143,
+        },
     }, {
         'url': 'https://www.newgrounds.com/portal/view/673111',
         'md5': '3394735822aab2478c31b1004fe5e5bc',
@@ -28,6 +34,8 @@ class NewgroundsIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Dancin',
             'uploader': 'Squirrelman82',
+            'timestamp': 1460256780,
+            'upload_date': '20160410',
         },
     }, {
         # source format unavailable, additional mp4 formats
@@ -37,6 +45,8 @@ class NewgroundsIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'ZTV News Episode 8',
             'uploader': 'BennettTheSage',
+            'timestamp': 1487965140,
+            'upload_date': '20170224',
         },
         'params': {
             'skip_download': True,
@@ -51,11 +61,11 @@ class NewgroundsIE(InfoExtractor):
         title = self._html_search_regex(
             r'<title>([^>]+)</title>', webpage, 'title')
 
-        video_url = self._parse_json(self._search_regex(
+        media_url = self._parse_json(self._search_regex(
             r'"url"\s*:\s*("[^"]+"),', webpage, ''), media_id)
 
         formats = [{
-            'url': video_url,
+            'url': media_url,
             'format_id': 'source',
             'quality': 1,
         }]
@@ -64,7 +74,7 @@ class NewgroundsIE(InfoExtractor):
             r'max_resolution["\']\s*:\s*(\d+)', webpage, 'max resolution',
             default=None))
         if max_resolution:
-            url_base = video_url.rpartition('.')[0]
+            url_base = media_url.rpartition('.')[0]
             for resolution in (360, 720, 1080):
                 if resolution > max_resolution:
                     break
@@ -77,14 +87,32 @@ class NewgroundsIE(InfoExtractor):
         self._check_formats(formats, media_id)
         self._sort_formats(formats)
 
-        uploader = self._html_search_regex(
+        uploader = self._search_regex(
             r'(?:Author|Writer)\s*<a[^>]+>([^<]+)', webpage, 'uploader',
             fatal=False)
 
+        timestamp = unified_timestamp(self._search_regex(
+            r'<dt>Uploaded</dt>\s*<dd>([^<]+)', webpage, 'timestamp',
+            default=None))
+        duration = parse_duration(self._search_regex(
+            r'<dd>Song\s*</dd><dd>.+?</dd><dd>([^<]+)', webpage, 'duration',
+            default=None))
+
+        filesize_approx = parse_filesize(self._html_search_regex(
+            r'<dd>Song\s*</dd><dd>(.+?)</dd>', webpage, 'filesize',
+            default=None))
+        if len(formats) == 1:
+            formats[0]['filesize_approx'] = filesize_approx
+
+        if '<dd>Song' in webpage:
+            formats[0]['vcodec'] = 'none'
+
         return {
             'id': media_id,
             'title': title,
             'uploader': uploader,
+            'timestamp': timestamp,
+            'duration': duration,
             'formats': formats,
         }
 

From 624bd0104c3c0f367ecb129c438da57398668801 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Jun 2017 21:36:19 +0700
Subject: [PATCH 0973/1696] [rutv] Add support for testplayer.vgtrk.com (closes
 #13347)

---
 youtube_dl/extractor/rutv.py | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/rutv.py b/youtube_dl/extractor/rutv.py
index a5e672c0a..d2713c19a 100644
--- a/youtube_dl/extractor/rutv.py
+++ b/youtube_dl/extractor/rutv.py
@@ -13,11 +13,15 @@ from ..utils import (
 class RUTVIE(InfoExtractor):
     IE_DESC = 'RUTV.RU'
     _VALID_URL = r'''(?x)
-        https?://player\.(?:rutv\.ru|vgtrk\.com)/
-            (?P<path>flash\d+v/container\.swf\?id=
-            |iframe/(?P<type>swf|video|live)/id/
-            |index/iframe/cast_id/)
-            (?P<id>\d+)'''
+                    https?://
+                        (?:test)?player\.(?:rutv\.ru|vgtrk\.com)/
+                        (?P<path>
+                            flash\d+v/container\.swf\?id=|
+                            iframe/(?P<type>swf|video|live)/id/|
+                            index/iframe/cast_id/
+                        )
+                        (?P<id>\d+)
+                    '''
 
     _TESTS = [
         {
@@ -99,17 +103,21 @@ class RUTVIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            'url': 'https://testplayer.vgtrk.com/iframe/live/id/19201/showZoomBtn/false/isPlay/true/',
+            'only_matching': True,
+        },
     ]
 
     @classmethod
     def _extract_url(cls, webpage):
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>https?://player\.(?:rutv\.ru|vgtrk\.com)/(?:iframe/(?:swf|video|live)/id|index/iframe/cast_id)/.+?)\1', webpage)
+            r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:test)?player\.(?:rutv\.ru|vgtrk\.com)/(?:iframe/(?:swf|video|live)/id|index/iframe/cast_id)/.+?)\1', webpage)
         if mobj:
             return mobj.group('url')
 
         mobj = re.search(
-            r'<meta[^>]+?property=(["\'])og:video\1[^>]+?content=(["\'])(?P<url>https?://player\.(?:rutv\.ru|vgtrk\.com)/flash\d+v/container\.swf\?id=.+?\2)',
+            r'<meta[^>]+?property=(["\'])og:video\1[^>]+?content=(["\'])(?P<url>https?://(?:test)?player\.(?:rutv\.ru|vgtrk\.com)/flash\d+v/container\.swf\?id=.+?\2)',
             webpage)
         if mobj:
             return mobj.group('url')

From 16bc958287b25566dbe22d22274d4c992278d2f7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Jun 2017 00:14:04 +0700
Subject: [PATCH 0974/1696] [xfileshare] Modernize and pass referrer

---
 youtube_dl/extractor/xfileshare.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index 13f8be6cb..bfec17e3c 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -130,12 +130,12 @@ class XFileShareIE(InfoExtractor):
             if countdown:
                 self._sleep(countdown, video_id)
 
-            post = urlencode_postdata(fields)
-
-            req = sanitized_Request(url, post)
-            req.add_header('Content-type', 'application/x-www-form-urlencoded')
-
-            webpage = self._download_webpage(req, video_id, 'Downloading video page')
+            webpage = self._download_webpage(
+                url, video_id, 'Downloading video page',
+                data=urlencode_postdata(fields), headers={
+                    'Referer': url,
+                    'Content-type': 'application/x-www-form-urlencoded',
+                })
 
         title = (self._search_regex(
             (r'style="z-index: [0-9]+;">([^<]+)</span>',

From 534863e057b155cd71035f05600444c2be62ca95 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Jun 2017 00:16:47 +0700
Subject: [PATCH 0975/1696] [xfileshare] Add support for rapidvideo (closes
 #13348)

---
 youtube_dl/extractor/xfileshare.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index bfec17e3c..32e16bc73 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -30,6 +30,7 @@ class XFileShareIE(InfoExtractor):
         (r'vidabc\.com', 'Vid ABC'),
         (r'vidbom\.com', 'VidBom'),
         (r'vidlo\.us', 'vidlo'),
+        (r'rapidvideo\.(?:cool|org)', 'RapidVideo.TV'),
     )
 
     IE_DESC = 'XFileShare based sites: %s' % ', '.join(list(zip(*_SITES))[1])
@@ -109,6 +110,9 @@ class XFileShareIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.rapidvideo.cool/b667kprndr8w',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 72b409559c984bb116015ada55acd38120b24cc0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Jun 2017 01:50:32 +0700
Subject: [PATCH 0976/1696] [compat] Introduce compat_HTMLParseError

---
 youtube_dl/compat.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 39527117f..bbc499eda 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2322,6 +2322,19 @@ try:
 except ImportError:  # Python 2
     from HTMLParser import HTMLParser as compat_HTMLParser
 
+try:  # Python 2
+    from HTMLParser import HTMLParseError as compat_HTMLParseError
+except ImportError:  # Python <3.4
+    try:
+        from html.parser import HTMLParseError as compat_HTMLParseError
+    except ImportError:  # Python >3.4
+
+        # HTMLParseError has been deprecated in Python 3.3 and removed in
+        # Python 3.5. Introducing dummy exception for Python >3.5 for compatible
+        # and uniform cross-version exceptiong handling
+        class compat_HTMLParseError(Exception):
+            pass
+
 try:
     from subprocess import DEVNULL
     compat_subprocess_get_DEVNULL = lambda: DEVNULL

From b4a3d461e4a00dfc60047b667aa3136c8b03eda8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Jun 2017 01:52:24 +0700
Subject: [PATCH 0977/1696] [utils] Handle HTMLParseError in extract_attributes
 (closes #13349)

---
 test/test_utils.py  | 2 ++
 youtube_dl/utils.py | 9 +++++++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 41b094d89..2b93b3604 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -916,6 +916,8 @@ class TestUtil(unittest.TestCase):
             supports_outside_bmp = False
         if supports_outside_bmp:
             self.assertEqual(extract_attributes('<e x="Smile &#128512;!">'), {'x': 'Smile \U0001f600!'})
+        # Malformed HTML should not break attributes extraction on older Python
+        self.assertEqual(extract_attributes('<mal"formed/>'), {})
 
     def test_clean_html(self):
         self.assertEqual(clean_html('a:\nb'), 'a: b')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 16bf49408..1973bd483 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -36,6 +36,7 @@ import xml.etree.ElementTree
 import zlib
 
 from .compat import (
+    compat_HTMLParseError,
     compat_HTMLParser,
     compat_basestring,
     compat_chr,
@@ -409,8 +410,12 @@ def extract_attributes(html_element):
     but the cases in the unit test will work for all of 2.6, 2.7, 3.2-3.5.
     """
     parser = HTMLAttributeParser()
-    parser.feed(html_element)
-    parser.close()
+    try:
+        parser.feed(html_element)
+        parser.close()
+    # Older Python may throw HTMLParseError in case of malformed HTML
+    except compat_HTMLParseError:
+        pass
     return parser.attrs
 
 

From bf87c36c93aee81a1bf26c8213262724955d52a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Jun 2017 02:01:12 +0700
Subject: [PATCH 0978/1696] [xfileshare] PEP 8

---
 youtube_dl/extractor/xfileshare.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index 32e16bc73..a32789ef4 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -10,7 +10,6 @@ from ..utils import (
     ExtractorError,
     int_or_none,
     NO_DEFAULT,
-    sanitized_Request,
     urlencode_postdata,
 )
 

From 9932ac5c58080e52b3111f5f8dec15c65c4079ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Jun 2017 02:01:15 +0700
Subject: [PATCH 0979/1696] [ChangeLog] Actualize

---
 ChangeLog | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index c605c5bd6..19f160c81 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,36 @@
+version <unreleased>
+
+Core
+* [utils] Handle compat_HTMLParseError in extract_attributes (#13349)
++ [compat] Introduce compat_HTMLParseError
+* [utils] Improve unified_timestamp
+* [extractor/generic] Ensure format id is unicode string
+* [extractor/common] Return unicode string from _match_id
++ [YoutubeDL] Sanitize more fields (#13313)
+
+Extractors
++ [xfileshare] Add support for rapidvideo.tv (#13348)
+* [xfileshare] Modernize and pass Referer
++ [rutv] Add support for testplayer.vgtrk.com (#13347)
++ [newgrounds] Extract more metadata (#13232)
++ [newgrounds:playlist] Add support for playlists (#10611)
+* [newgrounds] Improve formats and uploader extraction (#13346)
+* [msn] Fix formats extraction
+* [turbo] Ensure format id is string
+* [sexu] Ensure height is int
+* [jove] Ensure comment count is int
+* [golem] Ensure format id is string
+* [gfycat] Ensure filesize is int
+* [foxgay] Ensure height is int
+* [flickr] Ensure format id is string
+* [sohu] Fix numeric fields
+* [safari] Improve authentication detection (#13319)
+* [liveleak] Ensure height is int (#13313)
+* [streamango] Make title optional (#13292)
+* [rtlnl] Improve URL regular expression (#13295)
+* [tvplayer] Fix extraction (#13291)
+
+
 version 2017.06.05
 
 Core

From cb1e6d898532e86b0752117337b7519d22c63274 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Jun 2017 02:23:17 +0700
Subject: [PATCH 0980/1696] release 2017.06.12

---
 .github/ISSUE_TEMPLATE.md |  6 +++---
 ChangeLog                 |  2 +-
 README.md                 | 28 ++++++++++++++--------------
 docs/supportedsites.md    |  3 ++-
 youtube_dl/version.py     |  2 +-
 5 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index ce1e46085..b8806e7b3 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.06.05*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.06.05**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.06.12*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.06.12**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.06.05
+[debug] youtube-dl version 2017.06.12
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 19f160c81..e3b7f7d60 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.06.12
 
 Core
 * [utils] Handle compat_HTMLParseError in extract_attributes (#13349)
diff --git a/README.md b/README.md
index 2189552a9..fe2bebc2a 100644
--- a/README.md
+++ b/README.md
@@ -145,18 +145,18 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
     --max-views COUNT                Do not download any videos with more than
                                      COUNT views
     --match-filter FILTER            Generic video filter. Specify any key (see
-                                     help for -o for a list of available keys)
-                                     to match if the key is present, !key to
-                                     check if the key is not present, key >
-                                     NUMBER (like "comment_count > 12", also
-                                     works with >=, <, <=, !=, =) to compare
-                                     against a number, key = 'LITERAL' (like
-                                     "uploader = 'Mike Smith'", also works with
-                                     !=) to match against a string literal and &
-                                     to require multiple matches. Values which
-                                     are not known are excluded unless you put a
-                                     question mark (?) after the operator. For
-                                     example, to only match videos that have
+                                     the "OUTPUT TEMPLATE" for a list of
+                                     available keys) to match if the key is
+                                     present, !key to check if the key is not
+                                     present, key > NUMBER (like "comment_count
+                                     > 12", also works with >=, <, <=, !=, =) to
+                                     compare against a number, key = 'LITERAL'
+                                     (like "uploader = 'Mike Smith'", also works
+                                     with !=) to match against a string literal
+                                     and & to require multiple matches. Values
+                                     which are not known are excluded unless you
+                                     put a question mark (?) after the operator.
+                                     For example, to only match videos that have
                                      been liked more than 100 times and disliked
                                      less than 50 times (or the dislike
                                      functionality is not available at the given
@@ -277,8 +277,8 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
     --get-filename                   Simulate, quiet but print output filename
     --get-format                     Simulate, quiet but print output format
     -j, --dump-json                  Simulate, quiet but print JSON information.
-                                     See --output for a description of available
-                                     keys.
+                                     See the "OUTPUT TEMPLATE" for a description
+                                     of available keys.
     -J, --dump-single-json           Simulate, quiet but print JSON information
                                      for each command-line argument. If the URL
                                      refers to a playlist, dump the whole
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index fe022d158..7189f31d9 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -512,6 +512,7 @@
  - **netease:song**: 网易云音乐
  - **Netzkino**
  - **Newgrounds**
+ - **NewgroundsPlaylist**
  - **Newstube**
  - **NextMedia**: 蘋果日報
  - **NextMediaActionNews**: 蘋果日報 - 動新聞
@@ -974,7 +975,7 @@
  - **WSJArticle**
  - **XBef**
  - **XboxClips**
- - **XFileShare**: XFileShare based sites: DaClips, FileHoot, GorillaVid, MovPod, PowerWatch, Rapidvideo.ws, TheVideoBee, Vidto, Streamin.To, XVIDSTAGE, Vid ABC, VidBom, vidlo
+ - **XFileShare**: XFileShare based sites: DaClips, FileHoot, GorillaVid, MovPod, PowerWatch, Rapidvideo.ws, TheVideoBee, Vidto, Streamin.To, XVIDSTAGE, Vid ABC, VidBom, vidlo, RapidVideo.TV
  - **XHamster**
  - **XHamsterEmbed**
  - **xiami:album**: 虾米音乐 - 专辑
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d3aa39977..a7386c3a8 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.06.05'
+__version__ = '2017.06.12'

From b081f53b08e791d9e688bfa1a17caf3bb3630693 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Jun 2017 02:35:57 +0700
Subject: [PATCH 0981/1696] [compat] Add compat_HTMLParseError to __all__

---
 youtube_dl/compat.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index bbc499eda..7ef327451 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2895,6 +2895,7 @@ else:
 
 
 __all__ = [
+    'compat_HTMLParseError',
     'compat_HTMLParser',
     'compat_HTTPError',
     'compat_basestring',

From 97fa1f8dc4df553624b76a3da2b00edc47744a16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 13 Jun 2017 23:15:06 +0700
Subject: [PATCH 0982/1696] [corus] Add support for history.ca (closes #13359)

---
 youtube_dl/extractor/corus.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/corus.py b/youtube_dl/extractor/corus.py
index 7b2f5008b..9cdd34636 100644
--- a/youtube_dl/extractor/corus.py
+++ b/youtube_dl/extractor/corus.py
@@ -8,7 +8,16 @@ from ..utils import int_or_none
 
 
 class CorusIE(ThePlatformFeedIE):
-    _VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:globaltv|etcanada)\.com|(?:hgtv|foodnetwork|slice)\.ca)/(?:video/|(?:[^/]+/)+(?:videos/[a-z0-9-]+-|video\.html\?.*?\bv=))(?P<id>\d+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:www\.)?
+                        (?P<domain>
+                            (?:globaltv|etcanada)\.com|
+                            (?:hgtv|foodnetwork|slice|history)\.ca
+                        )
+                        /(?:video/|(?:[^/]+/)+(?:videos/[a-z0-9-]+-|video\.html\?.*?\bv=))
+                        (?P<id>\d+)
+                    '''
     _TESTS = [{
         'url': 'http://www.hgtv.ca/shows/bryan-inc/videos/movie-night-popcorn-with-bryan-870923331648/',
         'md5': '05dcbca777bf1e58c2acbb57168ad3a6',
@@ -27,6 +36,9 @@ class CorusIE(ThePlatformFeedIE):
     }, {
         'url': 'http://etcanada.com/video/873675331955/meet-the-survivor-game-changers-castaways-part-2/',
         'only_matching': True,
+    }, {
+        'url': 'http://www.history.ca/the-world-without-canada/video/full-episodes/natural-resources/video.html?v=955054659646#video',
+        'only_matching': True,
     }]
 
     _TP_FEEDS = {
@@ -50,6 +62,10 @@ class CorusIE(ThePlatformFeedIE):
             'feed_id': '5tUJLgV2YNJ5',
             'account_id': 2414427935,
         },
+        'history': {
+            'feed_id': 'tQFx_TyyEq4J',
+            'account_id': 2369613659,
+        },
     }
 
     def _real_extract(self, url):

From b5dc33daa96fd59ed5e0dc38cb5bb5157433a72b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 13 Jun 2017 23:27:27 +0700
Subject: [PATCH 0983/1696] [corus] Add support for showcase.ca

---
 youtube_dl/extractor/corus.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/corus.py b/youtube_dl/extractor/corus.py
index 9cdd34636..807a29eea 100644
--- a/youtube_dl/extractor/corus.py
+++ b/youtube_dl/extractor/corus.py
@@ -13,7 +13,7 @@ class CorusIE(ThePlatformFeedIE):
                         (?:www\.)?
                         (?P<domain>
                             (?:globaltv|etcanada)\.com|
-                            (?:hgtv|foodnetwork|slice|history)\.ca
+                            (?:hgtv|foodnetwork|slice|history|showcase)\.ca
                         )
                         /(?:video/|(?:[^/]+/)+(?:videos/[a-z0-9-]+-|video\.html\?.*?\bv=))
                         (?P<id>\d+)
@@ -39,6 +39,9 @@ class CorusIE(ThePlatformFeedIE):
     }, {
         'url': 'http://www.history.ca/the-world-without-canada/video/full-episodes/natural-resources/video.html?v=955054659646#video',
         'only_matching': True,
+    }, {
+        'url': 'http://www.showcase.ca/eyewitness/video/eyewitness++106/video.html?v=955070531919&p=1&s=da#video',
+        'only_matching': True,
     }]
 
     _TP_FEEDS = {
@@ -66,6 +69,10 @@ class CorusIE(ThePlatformFeedIE):
             'feed_id': 'tQFx_TyyEq4J',
             'account_id': 2369613659,
         },
+        'showcase': {
+            'feed_id': '9H6qyshBZU3E',
+            'account_id': 2414426607,
+        },
     }
 
     def _real_extract(self, url):

From 7dd5415cd0e824b00e6abf9a18d55701d52babec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Jun 2017 21:33:40 +0700
Subject: [PATCH 0984/1696] [npo] Improve _VALID_URL (closes #13376)

---
 youtube_dl/extractor/npo.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 79296f0ef..5f8b6def1 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -35,7 +35,7 @@ class NPOIE(NPOBaseIE):
                         https?://
                             (?:www\.)?
                             (?:
-                                npo\.nl/(?!live|radio)(?:[^/]+/){2}|
+                                npo\.nl/(?!(?:live|radio)/)(?:[^/]+/){2}|
                                 ntr\.nl/(?:[^/]+/){2,}|
                                 omroepwnl\.nl/video/fragment/[^/]+__|
                                 zapp\.nl/[^/]+/[^/]+/
@@ -150,6 +150,9 @@ class NPOIE(NPOBaseIE):
         # live stream
         'url': 'npo:LI_NL1_4188102',
         'only_matching': True,
+    }, {
+        'url': 'http://www.npo.nl/radio-gaga/13-06-2017/BNN_101383373',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 0a268c6e11e9fb55f41b474497997bff61d97cd2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Jun 2017 22:02:15 +0700
Subject: [PATCH 0985/1696] [extractor/common] Improve jwplayer formats
 extraction (closes #13379)

---
 youtube_dl/extractor/common.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index f027447c8..941385ae2 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2328,6 +2328,8 @@ class InfoExtractor(object):
         urls = []
         formats = []
         for source in jwplayer_sources_data:
+            if not isinstance(source, dict):
+                continue
             source_url = self._proto_relative_url(source.get('file'))
             if not source_url:
                 continue

From 5215f45327a1ca84c73b98df05c1977520cfb0a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marcin=20Cie=C5=9Blak?= <saper@saper.info>
Date: Wed, 14 Jun 2017 23:13:04 +0200
Subject: [PATCH 0986/1696] [disney] Add support for disneychannel.de

---
 youtube_dl/extractor/disney.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/disney.py b/youtube_dl/extractor/disney.py
index 939d1338c..968c4c7fd 100644
--- a/youtube_dl/extractor/disney.py
+++ b/youtube_dl/extractor/disney.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 class DisneyIE(InfoExtractor):
     _VALID_URL = r'''(?x)
-        https?://(?P<domain>(?:[^/]+\.)?(?:disney\.[a-z]{2,3}(?:\.[a-z]{2})?|disney(?:(?:me|latino)\.com|turkiye\.com\.tr)|(?:starwars|marvelkids)\.com))/(?:(?:embed/|(?:[^/]+/)+[\w-]+-)(?P<id>[a-z0-9]{24})|(?:[^/]+/)?(?P<display_id>[^/?#]+))'''
+        https?://(?P<domain>(?:[^/]+\.)?(?:disney\.[a-z]{2,3}(?:\.[a-z]{2})?|disney(?:(?:me|latino)\.com|turkiye\.com\.tr|channel\.de)|(?:starwars|marvelkids)\.com))/(?:(?:embed/|(?:[^/]+/)+[\w-]+-)(?P<id>[a-z0-9]{24})|(?:[^/]+/)?(?P<display_id>[^/?#]+))'''
     _TESTS = [{
         # Disney.EmbedVideo
         'url': 'http://video.disney.com/watch/moana-trailer-545ed1857afee5a0ec239977',
@@ -68,6 +68,9 @@ class DisneyIE(InfoExtractor):
     }, {
         'url': 'http://disneyjunior.en.disneyme.com/dj/watch-my-friends-tigger-and-pooh-promo',
         'only_matching': True,
+    }, {
+        'url': 'http://disneychannel.de/sehen/soy-luna-folge-118-5518518987ba27f3cc729268',
+        'only_matching': True,
     }, {
         'url': 'http://disneyjunior.disney.com/galactech-the-galactech-grab-galactech-an-admiral-rescue',
         'only_matching': True,

From 0bf4b71b753534ac4a93d9893dcd7333b5f229b3 Mon Sep 17 00:00:00 2001
From: gfabiano <gfabiano40@gmail.com>
Date: Wed, 14 Jun 2017 23:16:50 +0200
Subject: [PATCH 0987/1696] [4tube] Fix extraction (closes #13381)

---
 youtube_dl/extractor/fourtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index 9776c8422..e3fd08bcf 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -85,11 +85,11 @@ class FourTubeIE(InfoExtractor):
             media_id = params[0]
             sources = ['%s' % p for p in params[2]]
 
-        token_url = 'http://tkn.4tube.com/{0}/desktop/{1}'.format(
+        token_url = 'https://tkn.kodicdn.com/{0}/desktop/{1}'.format(
             media_id, '+'.join(sources))
         headers = {
             b'Content-Type': b'application/x-www-form-urlencoded',
-            b'Origin': b'http://www.4tube.com',
+            b'Origin': b'https://www.4tube.com',
         }
         token_req = sanitized_Request(token_url, b'{}', headers)
         tokens = self._download_json(token_req, video_id)

From ca270371718fda18b3331dd645a200a365b096c9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 15 Jun 2017 11:19:03 +0800
Subject: [PATCH 0988/1696] [bilibili] Fix extraction of videos with double
 quotes in titles

Closes #13387
---
 ChangeLog                        |  6 ++++++
 youtube_dl/extractor/bilibili.py | 18 +++++++++++++++++-
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index e3b7f7d60..9b45f08e3 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [bilibili] Fix extraction of videos with double quotes in titles (#13387)
+
+
 version 2017.06.12
 
 Core
diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 1e3f25515..1e57310d6 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -54,6 +54,22 @@ class BiliBiliIE(InfoExtractor):
             'description': '如果你是神明，并且能够让妄想成为现实。那你会进行怎么样的妄想？是淫靡的世界？独裁社会？毁灭性的制裁？还是……2015年，涩谷。从6年前发生的大灾害“涩谷地震”之后复兴了的这个街区里新设立的私立高中...',
         },
         'skip': 'Geo-restricted to China',
+    }, {
+        # Title with double quotes
+        'url': 'http://www.bilibili.com/video/av8903802/',
+        'info_dict': {
+            'id': '8903802',
+            'ext': 'mp4',
+            'title': '阿滴英文｜英文歌分享#6 "Closer',
+            'description': '滴妹今天唱Closer給你聽! 有史以来，被推最多次也是最久的歌曲，其实歌词跟我原本想像差蛮多的，不过还是好听！ 微博@阿滴英文',
+            'uploader': '阿滴英文',
+            'uploader_id': '65880958',
+            'timestamp': 1488382620,
+            'upload_date': '20170301',
+        },
+        'params': {
+            'skip_download': True,  # Test metadata only
+        },
     }]
 
     _APP_KEY = '84956560bc028eb7'
@@ -135,7 +151,7 @@ class BiliBiliIE(InfoExtractor):
                 'formats': formats,
             })
 
-        title = self._html_search_regex('<h1[^>]+title="([^"]+)">', webpage, 'title')
+        title = self._html_search_regex('<h1[^>]*>([^<]+)</h1>', webpage, 'title')
         description = self._html_search_meta('description', webpage)
         timestamp = unified_timestamp(self._html_search_regex(
             r'<time[^>]+datetime="([^"]+)"', webpage, 'upload time', default=None))

From 6a9cb29509a3b1fa2c1ffce82acb2fa7a2699c29 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 15 Jun 2017 13:04:36 +0800
Subject: [PATCH 0989/1696] [extractor/common] Fix json dumping with
 --geo-bypass

The line "[debug] Using fake IP %s (%s) as X-Forwarded-For." was printed
to stdout even with -j/-J, which breaks the resultant JSON.
---
 ChangeLog                      | 3 +++
 youtube_dl/extractor/common.py | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 9b45f08e3..5cd0b3393 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,8 @@
 version <unreleased>
 
+Core
+* [extractor/common] Fix json dumping with --geo-bypass
+
 Extractors
 * [bilibili] Fix extraction of videos with double quotes in titles (#13387)
 
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 941385ae2..6e415ea41 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -420,7 +420,7 @@ class InfoExtractor(object):
             if country_code:
                 self._x_forwarded_for_ip = GeoUtils.random_ipv4(country_code)
                 if self._downloader.params.get('verbose', False):
-                    self._downloader.to_stdout(
+                    self._downloader.to_screen(
                         '[debug] Using fake IP %s (%s) as X-Forwarded-For.'
                         % (self._x_forwarded_for_ip, country_code.upper()))
 

From 0ea6efbb7a1434b83c871577845383a472324cb5 Mon Sep 17 00:00:00 2001
From: gfabiano <gfabiano40@gmail.com>
Date: Wed, 14 Jun 2017 23:38:21 +0200
Subject: [PATCH 0990/1696] [xfileshare] Add support for fastvideo.me

---
 youtube_dl/extractor/xfileshare.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index a32789ef4..74aac85a3 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -30,6 +30,7 @@ class XFileShareIE(InfoExtractor):
         (r'vidbom\.com', 'VidBom'),
         (r'vidlo\.us', 'vidlo'),
         (r'rapidvideo\.(?:cool|org)', 'RapidVideo.TV'),
+        (r'fastvideo\.me', 'FastVideo.me'),
     )
 
     IE_DESC = 'XFileShare based sites: %s' % ', '.join(list(zip(*_SITES))[1])
@@ -112,6 +113,9 @@ class XFileShareIE(InfoExtractor):
     }, {
         'url': 'http://www.rapidvideo.cool/b667kprndr8w',
         'only_matching': True,
+    }, {
+        'url': 'http://www.fastvideo.me/k8604r8nk8sn/FAST_FURIOUS_8_-_Trailer_italiano_ufficiale.mp4.html',
+        'only_matching': True
     }]
 
     def _real_extract(self, url):

From 96a2daa1ee5de9ddff5d0180fccb8376725de60c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 15 Jun 2017 23:40:39 +0700
Subject: [PATCH 0991/1696] [extractor/common] Improve jwplayer subtitles
 extraction

---
 youtube_dl/extractor/common.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 6e415ea41..9751ab021 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2299,6 +2299,8 @@ class InfoExtractor(object):
             tracks = video_data.get('tracks')
             if tracks and isinstance(tracks, list):
                 for track in tracks:
+                    if not isinstance(track, dict):
+                        continue
                     if track.get('kind') != 'captions':
                         continue
                     track_url = urljoin(base_url, track.get('file'))

From b230fefc3c436b4d3ff278a11552b8a964b7056e Mon Sep 17 00:00:00 2001
From: Jakub Adam Wieczorek <ja.wieczorek@student.uw.edu.pl>
Date: Thu, 15 Jun 2017 23:57:56 +0200
Subject: [PATCH 0992/1696] [polskieradio] Fix extraction

---
 youtube_dl/extractor/polskieradio.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/polskieradio.py b/youtube_dl/extractor/polskieradio.py
index 2ac1fcb0b..978d6f813 100644
--- a/youtube_dl/extractor/polskieradio.py
+++ b/youtube_dl/extractor/polskieradio.py
@@ -65,7 +65,7 @@ class PolskieRadioIE(InfoExtractor):
         webpage = self._download_webpage(url, playlist_id)
 
         content = self._search_regex(
-            r'(?s)<div[^>]+class="audio atarticle"[^>]*>(.+?)<script>',
+            r'(?s)<div[^>]+class="\s*this-article\s*"[^>]*>(.+?)<div[^>]+class="tags"[^>]*>',
             webpage, 'content')
 
         timestamp = unified_timestamp(self._html_search_regex(

From 4f90d2aeacccab6dc461e92236480701c4a0900b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 17 Jun 2017 17:09:24 +0800
Subject: [PATCH 0993/1696] [Makefile] Excluding __pycache__ correctly (#13400)

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 023556391..84ccce2b3 100644
--- a/Makefile
+++ b/Makefile
@@ -101,7 +101,7 @@ youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-
 		--exclude '*.pyc' \
 		--exclude '*.pyo' \
 		--exclude '*~' \
-		--exclude '__pycache' \
+		--exclude '__pycache__' \
 		--exclude '.git' \
 		--exclude 'testdata' \
 		--exclude 'docs/_build' \

From 473e87064b7b60ea68147401a2e6487b715f25c8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 17 Jun 2017 17:37:25 +0800
Subject: [PATCH 0994/1696] [devscripts/prepare_manpage] Fix deprecated escape
 sequence on py36

---
 devscripts/prepare_manpage.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/devscripts/prepare_manpage.py b/devscripts/prepare_manpage.py
index f9fe63f1f..76bf873e1 100644
--- a/devscripts/prepare_manpage.py
+++ b/devscripts/prepare_manpage.py
@@ -8,7 +8,7 @@ import re
 ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 README_FILE = os.path.join(ROOT_DIR, 'README.md')
 
-PREFIX = '''%YOUTUBE-DL(1)
+PREFIX = r'''%YOUTUBE-DL(1)
 
 # NAME
 

From be80986ed9175af0a0fb216edfdfaeeb9769d1dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Jun 2017 19:01:27 +0700
Subject: [PATCH 0995/1696] [postprocessor/metadatafromtitle] Fix missing
 optional meta fields (closes #13408)

---
 youtube_dl/postprocessor/metadatafromtitle.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/postprocessor/metadatafromtitle.py b/youtube_dl/postprocessor/metadatafromtitle.py
index c73f02447..f5c14d974 100644
--- a/youtube_dl/postprocessor/metadatafromtitle.py
+++ b/youtube_dl/postprocessor/metadatafromtitle.py
@@ -35,11 +35,14 @@ class MetadataFromTitlePP(PostProcessor):
         title = info['title']
         match = re.match(self._titleregex, title)
         if match is None:
-            self._downloader.to_screen('[fromtitle] Could not interpret title of video as "%s"' % self._titleformat)
+            self._downloader.to_screen(
+                '[fromtitle] Could not interpret title of video as "%s"'
+                % self._titleformat)
             return [], info
         for attribute, value in match.groupdict().items():
-            value = match.group(attribute)
             info[attribute] = value
-            self._downloader.to_screen('[fromtitle] parsed ' + attribute + ': ' + value)
+            self._downloader.to_screen(
+                '[fromtitle] parsed %s: %s'
+                % (attribute, value if value is not None else 'NA'))
 
         return [], info

From b08e235f09b360262b97ef361a2321839a0534cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Jun 2017 23:14:24 +0700
Subject: [PATCH 0996/1696] [compat] Fix compat_shlex_quote on Windows (closes
 #5889, closes #10254)

---
 youtube_dl/compat.py | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 7ef327451..9e4e13bcf 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2617,14 +2617,22 @@ except ImportError:  # Python 2
                 parsed_result[name] = [value]
         return parsed_result
 
-try:
-    from shlex import quote as compat_shlex_quote
-except ImportError:  # Python < 3.3
+
+compat_os_name = os._name if os.name == 'java' else os.name
+
+
+if compat_os_name == 'nt':
     def compat_shlex_quote(s):
-        if re.match(r'^[-_\w./]+$', s):
-            return s
-        else:
-            return "'" + s.replace("'", "'\"'\"'") + "'"
+        return s if re.match(r'^[-_\w./]+$', s) else '"%s"' % s.replace('"', '\\"')
+else:
+    try:
+        from shlex import quote as compat_shlex_quote
+    except ImportError:  # Python < 3.3
+        def compat_shlex_quote(s):
+            if re.match(r'^[-_\w./]+$', s):
+                return s
+            else:
+                return "'" + s.replace("'", "'\"'\"'") + "'"
 
 
 try:
@@ -2649,9 +2657,6 @@ def compat_ord(c):
         return ord(c)
 
 
-compat_os_name = os._name if os.name == 'java' else os.name
-
-
 if sys.version_info >= (3, 0):
     compat_getenv = os.getenv
     compat_expanduser = os.path.expanduser

From 8b6ac49ecc873f787f647f1e623af989f85de644 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Jun 2017 23:15:57 +0700
Subject: [PATCH 0997/1696] [postprocessor/execafterdownload] Encode command
 line (closes #13407)

---
 youtube_dl/postprocessor/execafterdownload.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py
index 90630c2d7..64dabe790 100644
--- a/youtube_dl/postprocessor/execafterdownload.py
+++ b/youtube_dl/postprocessor/execafterdownload.py
@@ -4,7 +4,10 @@ import subprocess
 
 from .common import PostProcessor
 from ..compat import compat_shlex_quote
-from ..utils import PostProcessingError
+from ..utils import (
+    encodeArgument,
+    PostProcessingError,
+)
 
 
 class ExecAfterDownloadPP(PostProcessor):
@@ -20,7 +23,7 @@ class ExecAfterDownloadPP(PostProcessor):
         cmd = cmd.replace('{}', compat_shlex_quote(information['filepath']))
 
         self._downloader.to_screen('[exec] Executing command: %s' % cmd)
-        retCode = subprocess.call(cmd, shell=True)
+        retCode = subprocess.call(encodeArgument(cmd), shell=True)
         if retCode != 0:
             raise PostProcessingError(
                 'Command returned error code %d' % retCode)

From aefce8e6dca8a0ae784468a5e1a4ba01f1471380 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Jun 2017 23:48:58 +0700
Subject: [PATCH 0998/1696] [utils] Use compat_shlex_quote in shell_quote

---
 youtube_dl/utils.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 1973bd483..39860e9d1 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -22,7 +22,6 @@ import locale
 import math
 import operator
 import os
-import pipes
 import platform
 import random
 import re
@@ -1535,7 +1534,7 @@ def shell_quote(args):
         if isinstance(a, bytes):
             # We may get a filename encoded with 'encodeFilename'
             a = a.decode(encoding)
-        quoted_args.append(pipes.quote(a))
+        quoted_args.append(compat_shlex_quote(a))
     return ' '.join(quoted_args)
 
 

From 1433734c35dbfe16193ddcb6e59f55b58e76ba74 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Jun 2017 23:50:21 +0700
Subject: [PATCH 0999/1696] [downloader/common] Use utils.shell_quote for debug
 command line

---
 youtube_dl/downloader/common.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 5d6621147..77242dacc 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -8,10 +8,11 @@ import random
 
 from ..compat import compat_os_name
 from ..utils import (
+    decodeArgument,
     encodeFilename,
     error_to_compat_str,
-    decodeArgument,
     format_bytes,
+    shell_quote,
     timeconvert,
 )
 
@@ -381,10 +382,5 @@ class FileDownloader(object):
         if exe is None:
             exe = os.path.basename(str_args[0])
 
-        try:
-            import pipes
-            shell_quote = lambda args: ' '.join(map(pipes.quote, str_args))
-        except ImportError:
-            shell_quote = repr
         self.to_screen('[debug] %s command line: %s' % (
             exe, shell_quote(str_args)))

From a6f29820c6b4911a3c0a38ede6cff0bb3efdab9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Jun 2017 00:15:43 +0700
Subject: [PATCH 1000/1696] [ChangeLog] Actualize

---
 ChangeLog | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 5cd0b3393..b738d2063 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,10 +1,25 @@
 version <unreleased>
 
 Core
+* [downloader/common] Use utils.shell_quote for debug command line
+* [utils] Use compat_shlex_quote in shell_quote
+* [postprocessor/execafterdownload] Encode command line (#13407)
+* [compat] Fix compat_shlex_quote on Windows (#5889, #10254)
+* [postprocessor/metadatafromtitle] Fix missing optional meta fields processing
+   in --metadata-from-title (#13408)
 * [extractor/common] Fix json dumping with --geo-bypass
++ [extractor/common] Improve jwplayer subtitles extraction
++ [extractor/common] Improve jwplayer formats extraction (#13379)
 
 Extractors
+* [polskieradio] Fix extraction (#13392)
++ [xfileshare] Add support for fastvideo.me (#13385)
 * [bilibili] Fix extraction of videos with double quotes in titles (#13387)
+* [4tube] Fix extraction (#13381, #13382)
++ [disney] Add support for disneychannel.de (#13383)
+* [npo] Improve URL regular expression (#13376)
++ [corus] Add support for showcase.ca
++ [corus] Add support for history.ca (#13359)
 
 
 version 2017.06.12

From eaa006d1bd6a5fd1d3c482a04963068061fd2682 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Jun 2017 00:16:49 +0700
Subject: [PATCH 1001/1696] release 2017.06.18

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 +-
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index b8806e7b3..002c1274a 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.06.12*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.06.12**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.06.18*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.06.18**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.06.12
+[debug] youtube-dl version 2017.06.18
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index b738d2063..29d21e4e4 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.06.18
 
 Core
 * [downloader/common] Use utils.shell_quote for debug command line
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 7189f31d9..0f21be0a2 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -975,7 +975,7 @@
  - **WSJArticle**
  - **XBef**
  - **XboxClips**
- - **XFileShare**: XFileShare based sites: DaClips, FileHoot, GorillaVid, MovPod, PowerWatch, Rapidvideo.ws, TheVideoBee, Vidto, Streamin.To, XVIDSTAGE, Vid ABC, VidBom, vidlo, RapidVideo.TV
+ - **XFileShare**: XFileShare based sites: DaClips, FileHoot, GorillaVid, MovPod, PowerWatch, Rapidvideo.ws, TheVideoBee, Vidto, Streamin.To, XVIDSTAGE, Vid ABC, VidBom, vidlo, RapidVideo.TV, FastVideo.me
  - **XHamster**
  - **XHamsterEmbed**
  - **xiami:album**: 虾米音乐 - 专辑
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a7386c3a8..8782a6a1e 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.06.12'
+__version__ = '2017.06.18'

From 560050669b1e98bc76c797256c6e21719541b2ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Jun 2017 20:18:51 +0700
Subject: [PATCH 1002/1696] [asiancrush] Add extractor (closes #13420)

---
 youtube_dl/extractor/asiancrush.py | 93 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  4 ++
 2 files changed, 97 insertions(+)
 create mode 100644 youtube_dl/extractor/asiancrush.py

diff --git a/youtube_dl/extractor/asiancrush.py b/youtube_dl/extractor/asiancrush.py
new file mode 100644
index 000000000..594c88c9c
--- /dev/null
+++ b/youtube_dl/extractor/asiancrush.py
@@ -0,0 +1,93 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from .kaltura import KalturaIE
+from ..utils import (
+    extract_attributes,
+    remove_end,
+    urlencode_postdata,
+)
+
+
+class AsianCrushIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?asiancrush\.com/video/(?:[^/]+/)?0+(?P<id>\d+)v\b'
+    _TESTS = [{
+        'url': 'https://www.asiancrush.com/video/012869v/women-who-flirt/',
+        'md5': 'c3b740e48d0ba002a42c0b72857beae6',
+        'info_dict': {
+            'id': '1_y4tmjm5r',
+            'ext': 'mp4',
+            'title': 'Women Who Flirt',
+            'description': 'md5:3db14e9186197857e7063522cb89a805',
+            'timestamp': 1496936429,
+            'upload_date': '20170608',
+            'uploader_id': 'craig@crifkin.com',
+        },
+    }, {
+        'url': 'https://www.asiancrush.com/video/she-was-pretty/011886v-pretty-episode-3/',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        data = self._download_json(
+            'https://www.asiancrush.com/wp-admin/admin-ajax.php', video_id,
+            data=urlencode_postdata({
+                'postid': video_id,
+                'action': 'get_channel_kaltura_vars',
+            }))
+
+        entry_id = data['entry_id']
+
+        return self.url_result(
+            'kaltura:%s:%s' % (data['partner_id'], entry_id),
+            ie=KalturaIE.ie_key(), video_id=entry_id,
+            video_title=data.get('vid_label'))
+
+
+class AsianCrushPlaylistIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?asiancrush\.com/series/0+(?P<id>\d+)s\b'
+    _TEST = {
+        'url': 'https://www.asiancrush.com/series/012481s/scholar-walks-night/',
+        'info_dict': {
+            'id': '12481',
+            'title': 'Scholar Who Walks the Night',
+            'description': 'md5:7addd7c5132a09fd4741152d96cce886',
+        },
+        'playlist_count': 20,
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        entries = []
+
+        for mobj in re.finditer(
+                r'<a[^>]+href=(["\'])(?P<url>%s.*?)\1[^>]*>' % AsianCrushIE._VALID_URL,
+                webpage):
+            attrs = extract_attributes(mobj.group(0))
+            if attrs.get('class') == 'clearfix':
+                entries.append(self.url_result(
+                    mobj.group('url'), ie=AsianCrushIE.ie_key()))
+
+        title = remove_end(
+            self._html_search_regex(
+                r'(?s)<h1\b[^>]\bid=["\']movieTitle[^>]+>(.+?)</h1>', webpage,
+                'title', default=None) or self._og_search_title(
+                webpage, default=None) or self._html_search_meta(
+                'twitter:title', webpage, 'title',
+                default=None) or self._search_regex(
+                r'<title>([^<]+)</title>', webpage, 'title', fatal=False),
+            ' | AsianCrush')
+
+        description = self._og_search_description(
+            webpage, default=None) or self._html_search_meta(
+            'twitter:description', webpage, 'description', fatal=False)
+
+        return self.playlist_result(entries, playlist_id, title, description)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 7e45232dd..edbb4bdde 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -71,6 +71,10 @@ from .arte import (
     TheOperaPlatformIE,
     ArteTVPlaylistIE,
 )
+from .asiancrush import (
+    AsianCrushIE,
+    AsianCrushPlaylistIE,
+)
 from .atresplayer import AtresPlayerIE
 from .atttechchannel import ATTTechChannelIE
 from .atvat import ATVAtIE

From cc2ffe5afe692bccc9214110bd253b87f4361856 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 19 Jun 2017 16:20:36 +0800
Subject: [PATCH 1003/1696] [pandora.tv] Fix upload_date extraction (closes
 #12846)

---
 ChangeLog                         |  6 ++++++
 youtube_dl/extractor/pandoratv.py | 25 ++++++++++++++++++++++---
 2 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 29d21e4e4..7f077fd24 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [pandora.tv] Fix upload_date extraction (#12846)
+
+
 version 2017.06.18
 
 Core
diff --git a/youtube_dl/extractor/pandoratv.py b/youtube_dl/extractor/pandoratv.py
index 89c95fffb..fc7bd3411 100644
--- a/youtube_dl/extractor/pandoratv.py
+++ b/youtube_dl/extractor/pandoratv.py
@@ -19,7 +19,7 @@ class PandoraTVIE(InfoExtractor):
     IE_NAME = 'pandora.tv'
     IE_DESC = '판도라TV'
     _VALID_URL = r'https?://(?:.+?\.)?channel\.pandora\.tv/channel/video\.ptv\?'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://jp.channel.pandora.tv/channel/video.ptv?c1=&prgid=53294230&ch_userid=mikakim&ref=main&lot=cate_01_2',
         'info_dict': {
             'id': '53294230',
@@ -34,7 +34,26 @@ class PandoraTVIE(InfoExtractor):
             'view_count': int,
             'like_count': int,
         }
-    }
+    }, {
+        'url': 'http://channel.pandora.tv/channel/video.ptv?ch_userid=gogoucc&prgid=54721744',
+        'info_dict': {
+            'id': '54721744',
+            'ext': 'flv',
+            'title': '[HD] JAPAN COUNTDOWN 170423',
+            'description': '[HD] JAPAN COUNTDOWN 170423',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 1704.9,
+            'upload_date': '20170423',
+            'uploader': 'GOGO_UCC',
+            'uploader_id': 'gogoucc',
+            'view_count': int,
+            'like_count': int,
+        },
+        'params': {
+            # Test metadata only
+            'skip_download': True,
+        },
+    }]
 
     def _real_extract(self, url):
         qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
@@ -86,7 +105,7 @@ class PandoraTVIE(InfoExtractor):
             'description': info.get('body'),
             'thumbnail': info.get('thumbnail') or info.get('poster'),
             'duration': float_or_none(info.get('runtime'), 1000) or parse_duration(info.get('time')),
-            'upload_date': info['fid'][:8] if isinstance(info.get('fid'), compat_str) else None,
+            'upload_date': info['fid'].split('/')[-1][:8] if isinstance(info.get('fid'), compat_str) else None,
             'uploader': info.get('nickname'),
             'uploader_id': info.get('upload_userid'),
             'view_count': str_to_int(info.get('hit')),

From 2d2132ac6e75932c9c7e6c143f20156482043729 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 19 Jun 2017 22:53:39 +0700
Subject: [PATCH 1004/1696] [adobepass] Fix extraction on older python 2.6

---
 youtube_dl/extractor/adobepass.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
index d57ad85c2..3dfc632e7 100644
--- a/youtube_dl/extractor/adobepass.py
+++ b/youtube_dl/extractor/adobepass.py
@@ -6,7 +6,10 @@ import time
 import xml.etree.ElementTree as etree
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
+from ..compat import (
+    compat_kwargs,
+    compat_urlparse,
+)
 from ..utils import (
     unescapeHTML,
     urlencode_postdata,
@@ -1317,7 +1320,8 @@ class AdobePassIE(InfoExtractor):
         headers = kwargs.get('headers', {})
         headers.update(self.geo_verification_headers())
         kwargs['headers'] = headers
-        return super(AdobePassIE, self)._download_webpage_handle(*args, **kwargs)
+        return super(AdobePassIE, self)._download_webpage_handle(
+            *args, **compat_kwargs(kwargs))
 
     @staticmethod
     def _get_mvpd_resource(provider_id, title, guid, rating):

From fee00b3884922f6cb44926349df788d0fc589811 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 19 Jun 2017 22:57:37 +0700
Subject: [PATCH 1005/1696] [viu] Fix extraction on older python 2.6

---
 youtube_dl/extractor/viu.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viu.py b/youtube_dl/extractor/viu.py
index db6a65d2e..5cf93591c 100644
--- a/youtube_dl/extractor/viu.py
+++ b/youtube_dl/extractor/viu.py
@@ -4,7 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_str
+from ..compat import (
+    compat_kwargs,
+    compat_str,
+)
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -36,7 +39,8 @@ class ViuBaseIE(InfoExtractor):
         headers.update(kwargs.get('headers', {}))
         kwargs['headers'] = headers
         response = self._download_json(
-            'https://www.viu.com/api/' + path, *args, **kwargs)['response']
+            'https://www.viu.com/api/' + path, *args,
+            **compat_kwargs(kwargs))['response']
         if response.get('status') != 'success':
             raise ExtractorError('%s said: %s' % (
                 self.IE_NAME, response['message']), expected=True)

From a1de83e5f01cc220fa45caee80b9159cc555609d Mon Sep 17 00:00:00 2001
From: Orn <orngudjonsson@gmail.com>
Date: Thu, 15 Jun 2017 22:29:27 +0000
Subject: [PATCH 1006/1696] [ruv] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/ruv.py        | 31 ++++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+)
 create mode 100644 youtube_dl/extractor/ruv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index edbb4bdde..e97691daa 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -875,6 +875,7 @@ from .rutube import (
 )
 from .rutv import RUTVIE
 from .ruutu import RuutuIE
+from .ruv import RuvIE
 from .sandia import SandiaIE
 from .safari import (
     SafariIE,
diff --git a/youtube_dl/extractor/ruv.py b/youtube_dl/extractor/ruv.py
new file mode 100644
index 000000000..518fb5027
--- /dev/null
+++ b/youtube_dl/extractor/ruv.py
@@ -0,0 +1,31 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class RuvIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?ruv\.is/sarpurinn/ruv/\w+/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://ruv.is/sarpurinn/ruv/frettir/20170614',
+        'md5': 'a07ea1ebaba64082d90323b1c96f264b',
+        'info_dict': {
+            'id': '20170614',
+            'ext': 'mp4',
+            'title': 'Fréttir',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._og_search_title(webpage)
+        video_url = self._html_search_regex(r'video\.src\s*=\s*["\'](.+?)["\']', webpage, 'video URL')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': video_url,
+            'ext': 'mp4'
+        }

From 85cbcede5b9ad2b059e799bb9047264c2a38b7fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 19 Jun 2017 23:45:09 +0700
Subject: [PATCH 1007/1696] [ruv] Improve, extract all formats and metadata
 (closes #13396)

---
 youtube_dl/extractor/ruv.py | 96 ++++++++++++++++++++++++++++++++-----
 1 file changed, 83 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/ruv.py b/youtube_dl/extractor/ruv.py
index 518fb5027..8f3cc4095 100644
--- a/youtube_dl/extractor/ruv.py
+++ b/youtube_dl/extractor/ruv.py
@@ -2,30 +2,100 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    unified_timestamp,
+)
 
 
 class RuvIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ruv\.is/sarpurinn/ruv/\w+/(?P<id>[0-9]+)'
-    _TEST = {
-        'url': 'http://ruv.is/sarpurinn/ruv/frettir/20170614',
-        'md5': 'a07ea1ebaba64082d90323b1c96f264b',
+    _VALID_URL = r'https?://(?:www\.)?ruv\.is/(?:sarpurinn/[^/]+|node)/(?P<id>[^/]+(?:/\d+)?)'
+    _TESTS = [{
+        # m3u8
+        'url': 'http://ruv.is/sarpurinn/ruv-aukaras/fh-valur/20170516',
+        'md5': '66347652f4e13e71936817102acc1724',
         'info_dict': {
-            'id': '20170614',
+            'id': '1144499',
+            'display_id': 'fh-valur/20170516',
             'ext': 'mp4',
-            'title': 'Fréttir',
-        }
-    }
+            'title': 'FH - Valur',
+            'description': 'Bein útsending frá 3. leik FH og Vals í úrslitum Olísdeildar karla í handbolta.',
+            'timestamp': 1494963600,
+            'upload_date': '20170516',
+        },
+    }, {
+        # mp3
+        'url': 'http://ruv.is/sarpurinn/ras-2/morgunutvarpid/20170619',
+        'md5': '395ea250c8a13e5fdb39d4670ef85378',
+        'info_dict': {
+            'id': '1153630',
+            'display_id': 'morgunutvarpid/20170619',
+            'ext': 'mp3',
+            'title': 'Morgunútvarpið',
+            'description': 'md5:a4cf1202c0a1645ca096b06525915418',
+            'timestamp': 1497855000,
+            'upload_date': '20170619',
+        },
+    }, {
+        'url': 'http://ruv.is/sarpurinn/ruv/frettir/20170614',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.ruv.is/node/1151854',
+        'only_matching': True,
+    }, {
+        'url': 'http://ruv.is/sarpurinn/klippa/secret-soltice-hefst-a-morgun',
+        'only_matching': True,
+    }, {
+        'url': 'http://ruv.is/sarpurinn/ras-1/morgunvaktin/20170619',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
 
         title = self._og_search_title(webpage)
-        video_url = self._html_search_regex(r'video\.src\s*=\s*["\'](.+?)["\']', webpage, 'video URL')
+
+        FIELD_RE = r'video\.%s\s*=\s*(["\'])(?P<url>(?:(?!\1).)+)\1'
+
+        media_url = self._html_search_regex(
+            FIELD_RE % 'src', webpage, 'video URL', group='url')
+
+        video_id = self._search_regex(
+            r'<link\b[^>]+\bhref=["\']https?://www\.ruv\.is/node/(\d+)',
+            webpage, 'video id', default=display_id)
+
+        ext = determine_ext(media_url)
+
+        if ext == 'm3u8':
+            formats = self._extract_m3u8_formats(
+                media_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                m3u8_id='hls')
+        elif ext == 'mp3':
+            formats = [{
+                'format_id': 'mp3',
+                'url': media_url,
+                'vcodec': 'none',
+            }]
+        else:
+            formats = [{
+                'url': media_url,
+            }]
+
+        description = self._og_search_description(webpage, default=None)
+        thumbnail = self._og_search_thumbnail(
+            webpage, default=None) or self._search_regex(
+            FIELD_RE % 'poster', webpage, 'thumbnail', fatal=False)
+        timestamp = unified_timestamp(self._html_search_meta(
+            'article:published_time', webpage, 'timestamp', fatal=False))
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': title,
-            'url': video_url,
-            'ext': 'mp4'
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'formats': formats,
         }

From 1641ca402d03b36d28bab94eb898997cadf69993 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 20 Jun 2017 01:27:59 +0700
Subject: [PATCH 1008/1696] [vimeo] Add fallback mp4 extension for original
 format

---
 youtube_dl/extractor/vimeo.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index a483c8409..c3f71b45e 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -615,7 +615,10 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 if download_url and not source_file.get('is_cold') and not source_file.get('is_defrosting'):
                     source_name = source_file.get('public_name', 'Original')
                     if self._is_valid_url(download_url, video_id, '%s video' % source_name):
-                        ext = source_file.get('extension', determine_ext(download_url)).lower()
+                        ext = (try_get(
+                            source_file, lambda x: x['extension'],
+                            compat_str) or determine_ext(
+                            download_url, None) or 'mp4').lower()
                         formats.append({
                             'url': download_url,
                             'ext': ext,

From 6ce79d7ac067ef5c286661923b8df6dc7dc7b0da Mon Sep 17 00:00:00 2001
From: Giuseppe Fabiano <gfabiano40@gmail.com>
Date: Mon, 19 Jun 2017 23:07:00 +0200
Subject: [PATCH 1009/1696] [abcotvs] Fix test md5

---
 youtube_dl/extractor/abcotvs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/abcotvs.py b/youtube_dl/extractor/abcotvs.py
index 76e98132b..03b92a39c 100644
--- a/youtube_dl/extractor/abcotvs.py
+++ b/youtube_dl/extractor/abcotvs.py
@@ -22,7 +22,7 @@ class ABCOTVSIE(InfoExtractor):
                 'display_id': 'east-bay-museum-celebrates-vintage-synthesizers',
                 'ext': 'mp4',
                 'title': 'East Bay museum celebrates vintage synthesizers',
-                'description': 'md5:a4f10fb2f2a02565c1749d4adbab4b10',
+                'description': 'md5:24ed2bd527096ec2a5c67b9d5a9005f3',
                 'thumbnail': r're:^https?://.*\.jpg$',
                 'timestamp': 1421123075,
                 'upload_date': '20150113',

From 048b55804da21c2b2978cd2f710b2c7b438e24ba Mon Sep 17 00:00:00 2001
From: Giuseppe Fabiano <gfabiano40@gmail.com>
Date: Mon, 19 Jun 2017 23:30:45 +0200
Subject: [PATCH 1010/1696] [watchindianporn] Fix extraction (closes #13411)

---
 youtube_dl/extractor/watchindianporn.py | 50 +++++++------------------
 1 file changed, 14 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/watchindianporn.py b/youtube_dl/extractor/watchindianporn.py
index ed099beea..fadc539ee 100644
--- a/youtube_dl/extractor/watchindianporn.py
+++ b/youtube_dl/extractor/watchindianporn.py
@@ -4,11 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    unified_strdate,
-    parse_duration,
-    int_or_none,
-)
+from ..utils import parse_duration
 
 
 class WatchIndianPornIE(InfoExtractor):
@@ -23,11 +19,8 @@ class WatchIndianPornIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Hot milf from kerala shows off her gorgeous large breasts on camera',
             'thumbnail': r're:^https?://.*\.jpg$',
-            'uploader': 'LoveJay',
-            'upload_date': '20160428',
             'duration': 226,
             'view_count': int,
-            'comment_count': int,
             'categories': list,
             'age_limit': 18,
         }
@@ -40,51 +33,36 @@ class WatchIndianPornIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
-        video_url = self._html_search_regex(
-            r"url: escape\('([^']+)'\)", webpage, 'url')
+        info_dict = self._parse_html5_media_entries(url, webpage, video_id)[0]
 
-        title = self._html_search_regex(
-            r'<h2 class="he2"><span>(.*?)</span>',
-            webpage, 'title')
-        thumbnail = self._html_search_regex(
-            r'<span id="container"><img\s+src="([^"]+)"',
-            webpage, 'thumbnail', fatal=False)
-
-        uploader = self._html_search_regex(
-            r'class="aupa">\s*(.*?)</a>',
-            webpage, 'uploader')
-        upload_date = unified_strdate(self._html_search_regex(
-            r'Added: <strong>(.+?)</strong>', webpage, 'upload date', fatal=False))
+        title = self._html_search_regex((
+            r'<title>(.+?)\s*-\s*Indian\s+Porn</title>',
+            r'<h4>(.+?)</h4>'
+        ), webpage, 'title')
 
         duration = parse_duration(self._search_regex(
-            r'<td>Time:\s*</td>\s*<td align="right"><span>\s*(.+?)\s*</span>',
+            r'Time:\s*<strong>\s*(.+?)\s*</strong>',
             webpage, 'duration', fatal=False))
 
-        view_count = int_or_none(self._search_regex(
-            r'<td>Views:\s*</td>\s*<td align="right"><span>\s*(\d+)\s*</span>',
+        view_count = int(self._search_regex(
+            r'(?s)Time:\s*<strong>.*?</strong>.*?<strong>\s*(\d+)\s*</strong>',
             webpage, 'view count', fatal=False))
-        comment_count = int_or_none(self._search_regex(
-            r'<td>Comments:\s*</td>\s*<td align="right"><span>\s*(\d+)\s*</span>',
-            webpage, 'comment count', fatal=False))
 
         categories = re.findall(
-            r'<a href="[^"]+/search/video/desi"><span>([^<]+)</span></a>',
+            r'<a[^>]+class=[\'"]categories[\'"][^>]*>\s*([^<]+)\s*</a>',
             webpage)
 
-        return {
+        info_dict.update({
             'id': video_id,
             'display_id': display_id,
-            'url': video_url,
             'http_headers': {
                 'Referer': url,
             },
             'title': title,
-            'thumbnail': thumbnail,
-            'uploader': uploader,
-            'upload_date': upload_date,
             'duration': duration,
             'view_count': view_count,
-            'comment_count': comment_count,
             'categories': categories,
             'age_limit': 18,
-        }
+        })
+
+        return info_dict

From 9be9ec5980c1c53642eb112ccced3246ac8a391e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 20 Jun 2017 22:58:33 +0700
Subject: [PATCH 1011/1696] [googledrive] Fix formats' sorting (closes #13443)

---
 youtube_dl/extractor/googledrive.py | 37 +++++++++++++++++++----------
 1 file changed, 25 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
index fec36cbbb..9705cfadd 100644
--- a/youtube_dl/extractor/googledrive.py
+++ b/youtube_dl/extractor/googledrive.py
@@ -69,19 +69,32 @@ class GoogleDriveIE(InfoExtractor):
             r'"fmt_stream_map"\s*,\s*"([^"]+)', webpage, 'fmt stream map').split(',')
         fmt_list = self._search_regex(r'"fmt_list"\s*,\s*"([^"]+)', webpage, 'fmt_list').split(',')
 
+        resolutions = {}
+        for fmt in fmt_list:
+            mobj = re.search(
+                r'^(?P<format_id>\d+)/(?P<width>\d+)[xX](?P<height>\d+)', fmt)
+            if mobj:
+                resolutions[mobj.group('format_id')] = (
+                    int(mobj.group('width')), int(mobj.group('height')))
+
         formats = []
-        for fmt, fmt_stream in zip(fmt_list, fmt_stream_map):
-            fmt_id, fmt_url = fmt_stream.split('|')
-            resolution = fmt.split('/')[1]
-            width, height = resolution.split('x')
-            formats.append({
-                'url': lowercase_escape(fmt_url),
-                'format_id': fmt_id,
-                'resolution': resolution,
-                'width': int_or_none(width),
-                'height': int_or_none(height),
-                'ext': self._FORMATS_EXT[fmt_id],
-            })
+        for fmt_stream in fmt_stream_map:
+            fmt_stream_split = fmt_stream.split('|')
+            if len(fmt_stream_split) < 2:
+                continue
+            format_id, format_url = fmt_stream_split[:2]
+            f = {
+                'url': lowercase_escape(format_url),
+                'format_id': format_id,
+                'ext': self._FORMATS_EXT[format_id],
+            }
+            resolution = resolutions.get(format_id)
+            if resolution:
+                f.update({
+                    'width': resolution[0],
+                    'height': resolution[0],
+                })
+            formats.append(f)
         self._sort_formats(formats)
 
         return {

From 97b6e3011370a851b942bca144afb7cb08a57f5d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 22 Jun 2017 00:20:45 +0700
Subject: [PATCH 1012/1696] [youporn] Fix title extraction (closes #13456)

---
 youtube_dl/extractor/youporn.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 34ab878a4..7bc2cefc7 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -68,9 +68,12 @@ class YouPornIE(InfoExtractor):
         webpage = self._download_webpage(request, display_id)
 
         title = self._search_regex(
-            [r'(?:video_titles|videoTitle)\s*[:=]\s*(["\'])(?P<title>.+?)\1',
-             r'<h1[^>]+class=["\']heading\d?["\'][^>]*>([^<])<'],
-            webpage, 'title', group='title')
+            [r'(?:video_titles|videoTitle|title)\s*[:=]\s*(["\'])(?P<title>(?:(?!\1).)+)\1',
+             r'<h1[^>]+class=["\']heading\d?["\'][^>]*>(?P<title>[^<]+)<'],
+            webpage, 'title', group='title',
+            default=None) or self._og_search_title(
+            webpage, default=None) or self._html_search_meta(
+            'title', webpage, fatal=True)
 
         links = []
 

From d4893e764bbda206b6194884e67b0acaf231d0d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 22 Jun 2017 00:40:15 +0700
Subject: [PATCH 1013/1696] [youporn] Improve formats extraction

---
 youtube_dl/extractor/youporn.py | 32 ++++++++++++++++++++++++--------
 1 file changed, 24 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 7bc2cefc7..64e3c3c94 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     int_or_none,
     sanitized_Request,
@@ -68,7 +69,7 @@ class YouPornIE(InfoExtractor):
         webpage = self._download_webpage(request, display_id)
 
         title = self._search_regex(
-            [r'(?:video_titles|videoTitle|title)\s*[:=]\s*(["\'])(?P<title>(?:(?!\1).)+)\1',
+            [r'(?:video_titles|videoTitle)\s*[:=]\s*(["\'])(?P<title>(?:(?!\1).)+)\1',
              r'<h1[^>]+class=["\']heading\d?["\'][^>]*>(?P<title>[^<]+)<'],
             webpage, 'title', group='title',
             default=None) or self._og_search_title(
@@ -77,22 +78,37 @@ class YouPornIE(InfoExtractor):
 
         links = []
 
+        # Main source
+        definitions = self._parse_json(
+            self._search_regex(
+                r'mediaDefinition\s*=\s*(\[.+?\]);', webpage,
+                'media definitions', default='[]'),
+            video_id, fatal=False)
+        if definitions:
+            for definition in definitions:
+                if not isinstance(definition, dict):
+                    continue
+                video_url = definition.get('videoUrl')
+                if isinstance(video_url, compat_str) and video_url:
+                    links.append(video_url)
+
+        # Fallback #1, this also contains extra low quality 180p format
+        for _, link in re.findall(r'<a[^>]+href=(["\'])(http.+?)\1[^>]+title=["\']Download [Vv]ideo', webpage):
+            links.append(link)
+
+        # Fallback #2 (unavailable as at 22.06.2017)
         sources = self._search_regex(
             r'(?s)sources\s*:\s*({.+?})', webpage, 'sources', default=None)
         if sources:
             for _, link in re.findall(r'[^:]+\s*:\s*(["\'])(http.+?)\1', sources):
                 links.append(link)
 
-        # Fallback #1
+        # Fallback #3 (unavailable as at 22.06.2017)
         for _, link in re.findall(
-                r'(?:videoUrl|videoSrc|videoIpadUrl|html5PlayerSrc)\s*[:=]\s*(["\'])(http.+?)\1', webpage):
+                r'(?:videoSrc|videoIpadUrl|html5PlayerSrc)\s*[:=]\s*(["\'])(http.+?)\1', webpage):
             links.append(link)
 
-        # Fallback #2, this also contains extra low quality 180p format
-        for _, link in re.findall(r'<a[^>]+href=(["\'])(http.+?)\1[^>]+title=["\']Download [Vv]ideo', webpage):
-            links.append(link)
-
-        # Fallback #3, encrypted links
+        # Fallback #4, encrypted links (unavailable as at 22.06.2017)
         for _, encrypted_link in re.findall(
                 r'encryptedQuality\d{3,4}URL\s*=\s*(["\'])([\da-zA-Z+/=]+)\1', webpage):
             links.append(aes_decrypt_text(encrypted_link, title, 32).decode('utf-8'))

From 18166bb8e8db6bbeb1f279e236b9808e7d197dd8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 22 Jun 2017 00:47:02 +0700
Subject: [PATCH 1014/1696] [youporn] Fix upload date extraction

---
 youtube_dl/extractor/youporn.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 64e3c3c94..547adefeb 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -27,7 +27,7 @@ class YouPornIE(InfoExtractor):
             'description': 'Love & Sex Answers: http://bit.ly/DanAndJenn -- Is It Unhealthy To Masturbate Daily?',
             'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': 'Ask Dan And Jennifer',
-            'upload_date': '20101221',
+            'upload_date': '20101217',
             'average_rating': int,
             'view_count': int,
             'comment_count': int,
@@ -46,7 +46,7 @@ class YouPornIE(InfoExtractor):
             'description': 'http://sweetlivegirls.com Big Tits Awesome Brunette On amazing webcam show.mp4',
             'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': 'Unknown',
-            'upload_date': '20111125',
+            'upload_date': '20110418',
             'average_rating': int,
             'view_count': int,
             'comment_count': int,
@@ -143,7 +143,8 @@ class YouPornIE(InfoExtractor):
             r'(?s)<div[^>]+class=["\']submitByLink["\'][^>]*>(.+?)</div>',
             webpage, 'uploader', fatal=False)
         upload_date = unified_strdate(self._html_search_regex(
-            r'(?s)<div[^>]+class=["\']videoInfo(?:Date|Time)["\'][^>]*>(.+?)</div>',
+            [r'Date\s+[Aa]dded:\s*<span>([^<]+)',
+             r'(?s)<div[^>]+class=["\']videoInfo(?:Date|Time)["\'][^>]*>(.+?)</div>'],
             webpage, 'upload date', fatal=False))
 
         age_limit = self._rta_search(webpage)

From 0f4a5a73e70172c0accbd2e936d08988d065b3b1 Mon Sep 17 00:00:00 2001
From: Parmjit Virk <pvirk@mts.net>
Date: Thu, 22 Jun 2017 12:08:36 -0500
Subject: [PATCH 1015/1696] [drtuber] Fix formats extraction (fixes 12058)

---
 youtube_dl/extractor/drtuber.py | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/drtuber.py b/youtube_dl/extractor/drtuber.py
index 1eca82b3b..c5d56a9ad 100644
--- a/youtube_dl/extractor/drtuber.py
+++ b/youtube_dl/extractor/drtuber.py
@@ -44,8 +44,23 @@ class DrTuberIE(InfoExtractor):
         webpage = self._download_webpage(
             'http://www.drtuber.com/video/%s' % video_id, display_id)
 
-        video_url = self._html_search_regex(
-            r'<source src="([^"]+)"', webpage, 'video URL')
+        video_data = self._download_json(
+            'http://www.drtuber.com/player_config_json/', video_id, query={
+                'vid': video_id,
+                'embed': 0,
+                'aid': 0,
+                'domain_id': 0,
+            })
+
+        formats = []
+        for format_id, video_url in video_data['files'].items():
+            if video_url:
+                formats.append({
+                    'format_id': format_id,
+                    'quality': 2 if format_id == 'hq' else 1,
+                    'url': video_url
+                })
+        self._sort_formats(formats)
 
         title = self._html_search_regex(
             (r'class="title_watch"[^>]*><(?:p|h\d+)[^>]*>([^<]+)<',
@@ -75,7 +90,7 @@ class DrTuberIE(InfoExtractor):
         return {
             'id': video_id,
             'display_id': display_id,
-            'url': video_url,
+            'formats': formats,
             'title': title,
             'thumbnail': thumbnail,
             'like_count': like_count,

From fa3ea7223ac4d547c848e2df44504158ee0099f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Jun 2017 00:42:42 +0700
Subject: [PATCH 1016/1696] [hgtv.com:show] Relax video config regex and update
 test (closes #13279, closes #13461)

---
 youtube_dl/extractor/hgtv.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/hgtv.py b/youtube_dl/extractor/hgtv.py
index e854300c7..4d4b06824 100644
--- a/youtube_dl/extractor/hgtv.py
+++ b/youtube_dl/extractor/hgtv.py
@@ -7,14 +7,19 @@ from .common import InfoExtractor
 class HGTVComShowIE(InfoExtractor):
     IE_NAME = 'hgtv.com:show'
     _VALID_URL = r'https?://(?:www\.)?hgtv\.com/shows/[^/]+/(?P<id>[^/?#&]+)'
-    _TEST = {
-        'url': 'http://www.hgtv.com/shows/flip-or-flop/flip-or-flop-full-episodes-videos',
+    _TESTS = [{
+        # data-module="video"
+        'url': 'http://www.hgtv.com/shows/flip-or-flop/flip-or-flop-full-episodes-season-4-videos',
         'info_dict': {
-            'id': 'flip-or-flop-full-episodes-videos',
+            'id': 'flip-or-flop-full-episodes-season-4-videos',
             'title': 'Flip or Flop Full Episodes',
         },
         'playlist_mincount': 15,
-    }
+    }, {
+        # data-deferred-module="video"
+        'url': 'http://www.hgtv.com/shows/good-bones/episodes/an-old-victorian-house-gets-a-new-facelift',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
@@ -23,7 +28,7 @@ class HGTVComShowIE(InfoExtractor):
 
         config = self._parse_json(
             self._search_regex(
-                r'(?s)data-module=["\']video["\'][^>]*>.*?<script[^>]+type=["\']text/x-config["\'][^>]*>(.+?)</script',
+                r'(?s)data-(?:deferred)?-module=["\']video["\'][^>]*>.*?<script[^>]+type=["\']text/x-config["\'][^>]*>(.+?)</script',
                 webpage, 'video config'),
             display_id)['channels'][0]
 

From ddbb4c5c3ea7a64e09834d9628e7e34e03675289 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Jun 2017 02:00:19 +0700
Subject: [PATCH 1017/1696] [youtube] Adapt to new automatic captions rendition
 (closes #13467)

---
 youtube_dl/extractor/youtube.py | 62 ++++++++++++++++++++++-----------
 1 file changed, 41 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index bf4f4e139..77cd271ef 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1269,37 +1269,57 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     sub_lang_list[sub_lang] = sub_formats
                 return sub_lang_list
 
+            def make_captions(sub_url, sub_langs):
+                parsed_sub_url = compat_urllib_parse_urlparse(sub_url)
+                caption_qs = compat_parse_qs(parsed_sub_url.query)
+                captions = {}
+                for sub_lang in sub_langs:
+                    sub_formats = []
+                    for ext in self._SUBTITLE_FORMATS:
+                        caption_qs.update({
+                            'tlang': [sub_lang],
+                            'fmt': [ext],
+                        })
+                        sub_url = compat_urlparse.urlunparse(parsed_sub_url._replace(
+                            query=compat_urllib_parse_urlencode(caption_qs, True)))
+                        sub_formats.append({
+                            'url': sub_url,
+                            'ext': ext,
+                        })
+                    captions[sub_lang] = sub_formats
+                return captions
+
+            # New captions format as of 22.06.2017
+            player_response = args.get('player_response')
+            if player_response and isinstance(player_response, compat_str):
+                player_response = self._parse_json(
+                    player_response, video_id, fatal=False)
+                if player_response:
+                    renderer = player_response['captions']['playerCaptionsTracklistRenderer']
+                    base_url = renderer['captionTracks'][0]['baseUrl']
+                    sub_lang_list = []
+                    for lang in renderer['translationLanguages']:
+                        lang_code = lang.get('languageCode')
+                        if lang_code:
+                            sub_lang_list.append(lang_code)
+                    return make_captions(base_url, sub_lang_list)
+
             # Some videos don't provide ttsurl but rather caption_tracks and
             # caption_translation_languages (e.g. 20LmZk1hakA)
+            # Does not used anymore as of 22.06.2017
             caption_tracks = args['caption_tracks']
             caption_translation_languages = args['caption_translation_languages']
             caption_url = compat_parse_qs(caption_tracks.split(',')[0])['u'][0]
-            parsed_caption_url = compat_urllib_parse_urlparse(caption_url)
-            caption_qs = compat_parse_qs(parsed_caption_url.query)
-
-            sub_lang_list = {}
+            sub_lang_list = []
             for lang in caption_translation_languages.split(','):
                 lang_qs = compat_parse_qs(compat_urllib_parse_unquote_plus(lang))
                 sub_lang = lang_qs.get('lc', [None])[0]
-                if not sub_lang:
-                    continue
-                sub_formats = []
-                for ext in self._SUBTITLE_FORMATS:
-                    caption_qs.update({
-                        'tlang': [sub_lang],
-                        'fmt': [ext],
-                    })
-                    sub_url = compat_urlparse.urlunparse(parsed_caption_url._replace(
-                        query=compat_urllib_parse_urlencode(caption_qs, True)))
-                    sub_formats.append({
-                        'url': sub_url,
-                        'ext': ext,
-                    })
-                sub_lang_list[sub_lang] = sub_formats
-            return sub_lang_list
+                if sub_lang:
+                    sub_lang_list.append(sub_lang)
+            return make_captions(caption_url, sub_lang_list)
         # An extractor error can be raise by the download process if there are
         # no automatic captions but there are subtitles
-        except (KeyError, ExtractorError):
+        except (KeyError, IndexError, ExtractorError):
             self._downloader.report_warning(err_msg)
             return {}
 

From 38dad4737fd18ff380d38ce85ad200d1d0931b8e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Jun 2017 02:10:54 +0700
Subject: [PATCH 1018/1696] [ChangeLog] Actualize

---
 ChangeLog | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 7f077fd24..f12e74f1d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,22 @@
 version <unreleased>
 
+Core
+* [adobepass] Fix extraction on older python 2.6
+
 Extractors
+* [youtube] Adapt to new automatic captions rendition (#13467)
+* [hgtv.com:show] Relax video config regular expression (#13279, #13461)
+* [drtuber] Fix formats extraction (#12058)
+* [youporn] Fix upload date extraction
+* [youporn] Improve formats extraction
+* [youporn] Fix title extraction (#13456)
+* [googledrive] Fix formats sorting (#13443)
+* [watchindianporn] Fix extraction (#13411, #13415)
++ [vimeo] Add fallback mp4 extension for original format
++ [ruv] Add support for ruv.is (#13396)
+* [viu] Fix extraction on older python 2.6
 * [pandora.tv] Fix upload_date extraction (#12846)
++ [asiancrush] Add support for asiancrush.com (#13420)
 
 
 version 2017.06.18

From 170719414da3b516af464c4b0fb8a6a94c398e14 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Jun 2017 02:13:21 +0700
Subject: [PATCH 1019/1696] release 2017.06.23

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 3 +++
 youtube_dl/version.py     | 2 +-
 4 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 002c1274a..4e7ceafd8 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.06.18*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.06.18**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.06.23*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.06.23**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.06.18
+[debug] youtube-dl version 2017.06.23
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index f12e74f1d..746250db9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.06.23
 
 Core
 * [adobepass] Fix extraction on older python 2.6
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 0f21be0a2..e827ec0cf 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -67,6 +67,8 @@
  - **arte.tv:info**
  - **arte.tv:magazine**
  - **arte.tv:playlist**
+ - **AsianCrush**
+ - **AsianCrushPlaylist**
  - **AtresPlayer**
  - **ATTTechChannel**
  - **ATVAt**
@@ -686,6 +688,7 @@
  - **rutube:person**: Rutube person videos
  - **RUTV**: RUTV.RU
  - **Ruutu**
+ - **Ruv**
  - **safari**: safaribooksonline.com online video
  - **safari:api**
  - **safari:course**: safaribooksonline.com online courses
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 8782a6a1e..dfb69ab8d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.06.18'
+__version__ = '2017.06.23'

From ac7409eec5be3e120928e66af0d5fd7f38a06c5f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Jun 2017 02:54:12 +0700
Subject: [PATCH 1020/1696] [hgtv.com:show] Fix typo

---
 youtube_dl/extractor/hgtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/hgtv.py b/youtube_dl/extractor/hgtv.py
index 4d4b06824..a4f332565 100644
--- a/youtube_dl/extractor/hgtv.py
+++ b/youtube_dl/extractor/hgtv.py
@@ -28,7 +28,7 @@ class HGTVComShowIE(InfoExtractor):
 
         config = self._parse_json(
             self._search_regex(
-                r'(?s)data-(?:deferred)?-module=["\']video["\'][^>]*>.*?<script[^>]+type=["\']text/x-config["\'][^>]*>(.+?)</script',
+                r'(?s)data-(?:deferred-)?module=["\']video["\'][^>]*>.*?<script[^>]+type=["\']text/x-config["\'][^>]*>(.+?)</script',
                 webpage, 'video config'),
             display_id)['channels'][0]
 

From 4cb18ab1b96f1302f482f73989e263933e71d2e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Jun 2017 20:50:48 +0700
Subject: [PATCH 1021/1696] [ooyala] Skip empty format URLs (closes #13471,
 closes #13476)

---
 youtube_dl/extractor/ooyala.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index 84be2b1e3..ef8adccc1 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -41,7 +41,7 @@ class OoyalaBaseIE(InfoExtractor):
             for stream in cur_auth_data['streams']:
                 s_url = base64.b64decode(
                     stream['url']['data'].encode('ascii')).decode('utf-8')
-                if s_url in urls:
+                if not s_url or s_url in urls:
                     continue
                 urls.append(s_url)
                 ext = determine_ext(s_url, None)

From 4f4dd8d797538476a5a97f90e7074df2105d7573 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Jun 2017 20:56:21 +0700
Subject: [PATCH 1022/1696] [ooyala] Make more robust

---
 youtube_dl/extractor/ooyala.py | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index ef8adccc1..a25f326b5 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -3,12 +3,14 @@ import re
 import base64
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
-    int_or_none,
-    float_or_none,
-    ExtractorError,
-    unsmuggle_url,
     determine_ext,
+    ExtractorError,
+    float_or_none,
+    int_or_none,
+    try_get,
+    unsmuggle_url,
 )
 from ..compat import compat_urllib_parse_urlencode
 
@@ -39,13 +41,15 @@ class OoyalaBaseIE(InfoExtractor):
         formats = []
         if cur_auth_data['authorized']:
             for stream in cur_auth_data['streams']:
-                s_url = base64.b64decode(
-                    stream['url']['data'].encode('ascii')).decode('utf-8')
+                url_data = try_get(stream, lambda x: x['url']['data'], compat_str)
+                if not url_data:
+                    continue
+                s_url = base64.b64decode(url_data.encode('ascii')).decode('utf-8')
                 if not s_url or s_url in urls:
                     continue
                 urls.append(s_url)
                 ext = determine_ext(s_url, None)
-                delivery_type = stream['delivery_type']
+                delivery_type = stream.get('delivery_type')
                 if delivery_type == 'hls' or ext == 'm3u8':
                     formats.extend(self._extract_m3u8_formats(
                         re.sub(r'/ip(?:ad|hone)/', '/all/', s_url), embed_code, 'mp4', 'm3u8_native',
@@ -65,7 +69,7 @@ class OoyalaBaseIE(InfoExtractor):
                 else:
                     formats.append({
                         'url': s_url,
-                        'ext': ext or stream.get('delivery_type'),
+                        'ext': ext or delivery_type,
                         'vcodec': stream.get('video_codec'),
                         'format_id': delivery_type,
                         'width': int_or_none(stream.get('width')),

From b5f523ed62f6c84fe0c58274f1751e66c58282d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Jun 2017 20:56:48 +0700
Subject: [PATCH 1023/1696] [ooyala] Add test for missing stream['url']['data']

---
 youtube_dl/extractor/ooyala.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index a25f326b5..52580baed 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -140,6 +140,11 @@ class OoyalaIE(OoyalaBaseIE):
                 'title': 'Divide Tool Path.mp4',
                 'duration': 204.405,
             }
+        },
+        {
+            # empty stream['url']['data']
+            'url': 'http://player.ooyala.com/player.js?embedCode=w2bnZtYjE6axZ_dw1Cd0hQtXd_ige2Is',
+            'only_matching': True,
         }
     ]
 

From 73af5cc817ff19d21cb432c5a4e9e37dd35a353d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Jun 2017 21:18:33 +0700
Subject: [PATCH 1024/1696] [YoutubeDL] Skip malformed formats for better
 extraction robustness

---
 youtube_dl/YoutubeDL.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index c05103bb6..b3a6d4d3b 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1448,17 +1448,25 @@ class YoutubeDL(object):
         if not formats:
             raise ExtractorError('No video formats found!')
 
+        def is_wellformed(f):
+            url = f.get('url')
+            valid_url = url and isinstance(url, compat_str)
+            if not valid_url:
+                self.report_warning(
+                    '"url" field is missing or empty - skipping format, '
+                    'there is an error in extractor')
+            return valid_url
+
+        # Filter out malformed formats for better extraction robustness
+        formats = list(filter(is_wellformed, formats))
+
         formats_dict = {}
 
         # We check that all the formats have the format and format_id fields
         for i, format in enumerate(formats):
-            if 'url' not in format:
-                raise ExtractorError('Missing "url" key in result (index %d)' % i)
-
             sanitize_string_field(format, 'format_id')
             sanitize_numeric_fields(format)
             format['url'] = sanitize_url(format['url'])
-
             if format.get('format_id') is None:
                 format['format_id'] = compat_str(i)
             else:

From bd65f181532ab4e535b408d3ccf99723534eb326 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Jun 2017 18:33:31 +0700
Subject: [PATCH 1025/1696] [onetpl] Add support for videos embedded via
 pulsembed (closes #13482)

---
 youtube_dl/extractor/onet.py | 30 ++++++++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/onet.py b/youtube_dl/extractor/onet.py
index 94f57990b..58da1bc27 100644
--- a/youtube_dl/extractor/onet.py
+++ b/youtube_dl/extractor/onet.py
@@ -11,6 +11,7 @@ from ..utils import (
     get_element_by_class,
     int_or_none,
     js_to_json,
+    NO_DEFAULT,
     parse_iso8601,
     remove_start,
     strip_or_none,
@@ -198,6 +199,19 @@ class OnetPlIE(InfoExtractor):
             'upload_date': '20170214',
             'timestamp': 1487078046,
         },
+    }, {
+        # embedded via pulsembed
+        'url': 'http://film.onet.pl/pensjonat-nad-rozlewiskiem-relacja-z-planu-serialu/y428n0',
+        'info_dict': {
+            'id': '501235.965429946',
+            'ext': 'mp4',
+            'title': '"Pensjonat nad rozlewiskiem": relacja z planu serialu',
+            'upload_date': '20170622',
+            'timestamp': 1498159955,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'http://film.onet.pl/zwiastuny/ghost-in-the-shell-drugi-zwiastun-pl/5q6yl3',
         'only_matching': True,
@@ -212,13 +226,25 @@ class OnetPlIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    def _search_mvp_id(self, webpage, default=NO_DEFAULT):
+        return self._search_regex(
+            r'data-(?:params-)?mvp=["\'](\d+\.\d+)', webpage, 'mvp id',
+            default=default)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 
-        mvp_id = self._search_regex(
-            r'data-params-mvp=["\'](\d+\.\d+)', webpage, 'mvp id')
+        mvp_id = self._search_mvp_id(webpage, default=None)
+
+        if not mvp_id:
+            pulsembed_url = self._search_regex(
+                r'data-src=(["\'])(?P<url>(?:https?:)?//pulsembed\.eu/.+?)\1',
+                webpage, 'pulsembed url', group='url')
+            webpage = self._download_webpage(
+                pulsembed_url, video_id, 'Downloading pulsembed webpage')
+            mvp_id = self._search_mvp_id(webpage)
 
         return self.url_result(
             'onetmvp:%s' % mvp_id, OnetMVPIE.ie_key(), video_id=mvp_id)

From 27449ad894f7d49e189686a15399ea66cfb63667 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Jun 2017 01:09:12 +0700
Subject: [PATCH 1026/1696] [redbulltv] Add support for lives and segments
 (closes #13486))

---
 youtube_dl/extractor/redbulltv.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/redbulltv.py b/youtube_dl/extractor/redbulltv.py
index afab62426..c5918afee 100644
--- a/youtube_dl/extractor/redbulltv.py
+++ b/youtube_dl/extractor/redbulltv.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class RedBullTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?redbull\.tv/(?:video|film)/(?P<id>AP-\w+)'
+    _VALID_URL = r'https?://(?:www\.)?redbull\.tv/(?:video|film|live)/(?:AP-\w+/segment/)?(?P<id>AP-\w+)'
     _TESTS = [{
         # film
         'url': 'https://www.redbull.tv/video/AP-1Q756YYX51W11/abc-of-wrc',
@@ -42,6 +42,22 @@ class RedBullTVIE(InfoExtractor):
             'season_number': 2,
             'episode_number': 4,
         },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        # segment
+        'url': 'https://www.redbull.tv/live/AP-1R5DX49XS1W11/segment/AP-1QSAQJ6V52111/semi-finals',
+        'info_dict': {
+            'id': 'AP-1QSAQJ6V52111',
+            'ext': 'mp4',
+            'title': 'Semi Finals - Vans Park Series Pro Tour',
+            'description': 'md5:306a2783cdafa9e65e39aa62f514fd97',
+            'duration': 11791.991,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'https://www.redbull.tv/film/AP-1MSKKF5T92111/in-motion',
         'only_matching': True,

From 23aec3d623146d06535a4f5388693c1e9a2bbfde Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Jun 2017 01:10:31 +0700
Subject: [PATCH 1027/1696] [redbulltv] Restore hls format prefix

---
 youtube_dl/extractor/redbulltv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/redbulltv.py b/youtube_dl/extractor/redbulltv.py
index c5918afee..5d6cc3610 100644
--- a/youtube_dl/extractor/redbulltv.py
+++ b/youtube_dl/extractor/redbulltv.py
@@ -98,7 +98,8 @@ class RedBullTVIE(InfoExtractor):
         title = info['title'].strip()
 
         formats = self._extract_m3u8_formats(
-            video['url'], video_id, 'mp4', 'm3u8_native')
+            video['url'], video_id, 'mp4', entry_protocol='m3u8_native',
+            m3u8_id='hls')
         self._sort_formats(formats)
 
         subtitles = {}

From 449c66577640a0c3f0b383204a1e7284429a61c3 Mon Sep 17 00:00:00 2001
From: james <jrabblac@gmail.com>
Date: Sat, 17 Jun 2017 17:15:41 +0200
Subject: [PATCH 1028/1696] [raiplay:live] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/rai.py        | 19 ++++++++++++++++++-
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index e97691daa..a263c88b3 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -824,6 +824,7 @@ from .radiobremen import RadioBremenIE
 from .radiofrance import RadioFranceIE
 from .rai import (
     RaiPlayIE,
+    RaiPlayLiveIE,
     RaiIE,
 )
 from .rbmaradio import RBMARadioIE
diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 81eb9db85..ed15a5f10 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -208,10 +208,27 @@ class RaiPlayIE(RaiBaseIE):
         }
 
         info.update(relinker_info)
-
         return info
 
 
+class RaiPlayLiveIE(RaiBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?raiplay\.it/dirette/(?P<id>\w*)'
+    _TEST = {
+        'url': 'http://www.raiplay.it/dirette/rai3',
+        'only_matching': True,
+    }
+
+    def _real_extract(self, url):
+        channel = self._match_id(url)
+
+        webpage = self._download_webpage(url, channel)
+        re_id = r'<div([^>]*)data-uniquename=(["\'])[\w-]*(?P<id>%s)(\2)([^>]*?)>' % RaiBaseIE._UUID_RE
+        video_id = self._html_search_regex(re_id, webpage, 'livestream-id', group='id')
+
+        return self.url_result('http://www.raiplay.it/dirette/ContentItem-%s.html' % video_id,
+                               RaiPlayIE.ie_key(), video_id)
+
+
 class RaiIE(RaiBaseIE):
     _VALID_URL = r'https?://[^/]+\.(?:rai\.(?:it|tv)|rainews\.it)/dl/.+?-(?P<id>%s)(?:-.+?)?\.html' % RaiBaseIE._UUID_RE
     _TESTS = [{

From 9c48b5a193de754403f4a1ced78f6cf6b3893676 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Jun 2017 01:48:02 +0700
Subject: [PATCH 1029/1696] [raiplay:live] Improve and add test (closes #13414)

---
 youtube_dl/extractor/rai.py | 44 +++++++++++++++++++++++++++----------
 1 file changed, 32 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index ed15a5f10..e11bf8f9a 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -191,11 +191,12 @@ class RaiPlayIE(RaiBaseIE):
 
         info = {
             'id': video_id,
-            'title': title,
+            'title': self._live_title(title) if relinker_info.get(
+                'is_live') else title,
             'alt_title': media.get('subtitle'),
             'description': media.get('description'),
-            'uploader': media.get('channel'),
-            'creator': media.get('editor'),
+            'uploader': strip_or_none(media.get('channel')),
+            'creator': strip_or_none(media.get('editor')),
             'duration': parse_duration(video.get('duration')),
             'timestamp': timestamp,
             'thumbnails': thumbnails,
@@ -212,21 +213,40 @@ class RaiPlayIE(RaiBaseIE):
 
 
 class RaiPlayLiveIE(RaiBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?raiplay\.it/dirette/(?P<id>\w*)'
+    _VALID_URL = r'https?://(?:www\.)?raiplay\.it/dirette/(?P<id>[^/?#&]+)'
     _TEST = {
-        'url': 'http://www.raiplay.it/dirette/rai3',
-        'only_matching': True,
+        'url': 'http://www.raiplay.it/dirette/rainews24',
+        'info_dict': {
+            'id': 'd784ad40-e0ae-4a69-aa76-37519d238a9c',
+            'display_id': 'rainews24',
+            'ext': 'mp4',
+            'title': 're:^Diretta di Rai News 24 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'description': 'md5:6eca31500550f9376819f174e5644754',
+            'uploader': 'Rai News 24',
+            'creator': 'Rai News 24',
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):
-        channel = self._match_id(url)
+        display_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, channel)
-        re_id = r'<div([^>]*)data-uniquename=(["\'])[\w-]*(?P<id>%s)(\2)([^>]*?)>' % RaiBaseIE._UUID_RE
-        video_id = self._html_search_regex(re_id, webpage, 'livestream-id', group='id')
+        webpage = self._download_webpage(url, display_id)
 
-        return self.url_result('http://www.raiplay.it/dirette/ContentItem-%s.html' % video_id,
-                               RaiPlayIE.ie_key(), video_id)
+        video_id = self._search_regex(
+            r'data-uniquename=["\']ContentItem-(%s)' % RaiBaseIE._UUID_RE,
+            webpage, 'content id')
+
+        return {
+            '_type': 'url_transparent',
+            'ie_key': RaiPlayIE.ie_key(),
+            'url': 'http://www.raiplay.it/dirette/ContentItem-%s.html' % video_id,
+            'id': video_id,
+            'display_id': display_id,
+        }
 
 
 class RaiIE(RaiBaseIE):

From 5744cf6c03e6f89914d44be7c5d77fca3b121bef Mon Sep 17 00:00:00 2001
From: Argn0 <Argn0@users.noreply.github.com>
Date: Sat, 24 Jun 2017 20:59:15 +0200
Subject: [PATCH 1030/1696] [ign] Add another video id pattern (closes #13328)

---
 youtube_dl/extractor/ign.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py
index c45c68c1d..c1367cf51 100644
--- a/youtube_dl/extractor/ign.py
+++ b/youtube_dl/extractor/ign.py
@@ -89,6 +89,11 @@ class IGNIE(InfoExtractor):
             'url': 'http://me.ign.com/ar/angry-birds-2/106533/video/lrd-ldyy-lwl-lfylm-angry-birds',
             'only_matching': True,
         },
+        {
+            # videoId pattern
+            'url': 'http://www.ign.com/articles/2017/06/08/new-ducktales-short-donalds-birthday-doesnt-go-as-planned',
+            'only_matching': True,
+        },
     ]
 
     def _find_video_id(self, webpage):
@@ -98,6 +103,8 @@ class IGNIE(InfoExtractor):
             r'data-video-id="(.+?)"',
             r'<object id="vid_(.+?)"',
             r'<meta name="og:image" content=".*/(.+?)-(.+?)/.+.jpg"',
+            r'videoId&quot;\s*:\s*&quot;(.+?)&quot;',
+            r'videoId["\']\s*:\s*["\']([^"\']+?)["\']',
         ]
         return self._search_regex(res_id, webpage, 'video id', default=None)
 

From fd9ee4de8c3aa640d51c5ff42f5d8241970e894d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Jun 2017 02:14:10 +0700
Subject: [PATCH 1031/1696] [wsj] Add support for barrons.com (closes #13470)

---
 youtube_dl/extractor/wsj.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/wsj.py b/youtube_dl/extractor/wsj.py
index 45cfca7c5..9b5487710 100644
--- a/youtube_dl/extractor/wsj.py
+++ b/youtube_dl/extractor/wsj.py
@@ -13,7 +13,7 @@ class WSJIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                         (?:
                             https?://video-api\.wsj\.com/api-video/player/iframe\.html\?.*?\bguid=|
-                            https?://(?:www\.)?wsj\.com/video/[^/]+/|
+                            https?://(?:www\.)?(?:wsj|barrons)\.com/video/[^/]+/|
                             wsj:
                         )
                         (?P<id>[a-fA-F0-9-]{36})
@@ -35,6 +35,9 @@ class WSJIE(InfoExtractor):
     }, {
         'url': 'http://www.wsj.com/video/can-alphabet-build-a-smarter-city/359DDAA8-9AC1-489C-82E6-0429C1E430E0.html',
         'only_matching': True,
+    }, {
+        'url': 'http://www.barrons.com/video/capitalism-deserves-more-respect-from-millennials/F301217E-6F46-43AE-B8D2-B7180D642EE9.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 0c7a631b613fe8ec443e6b17ea8f7a17fb8abb5b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Jun 2017 04:54:56 +0700
Subject: [PATCH 1032/1696] [adobepass] Add support for ATTOTT MSO (DIRECTV
 NOW) (closes #13472)

---
 youtube_dl/extractor/adobepass.py | 43 ++++++++++++++++++++++++++-----
 1 file changed, 37 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
index 3dfc632e7..b83b51efb 100644
--- a/youtube_dl/extractor/adobepass.py
+++ b/youtube_dl/extractor/adobepass.py
@@ -15,6 +15,7 @@ from ..utils import (
     urlencode_postdata,
     unified_timestamp,
     ExtractorError,
+    NO_DEFAULT,
 )
 
 
@@ -24,6 +25,11 @@ MSO_INFO = {
         'username_field': 'username',
         'password_field': 'password',
     },
+    'ATTOTT': {
+        'name': 'DIRECTV NOW',
+        'username_field': 'email',
+        'password_field': 'loginpassword',
+    },
     'Rogers': {
         'name': 'Rogers',
         'username_field': 'UserName',
@@ -1316,6 +1322,8 @@ class AdobePassIE(InfoExtractor):
     _USER_AGENT = 'Mozilla/5.0 (X11; Linux i686; rv:47.0) Gecko/20100101 Firefox/47.0'
     _MVPD_CACHE = 'ap-mvpd'
 
+    _DOWNLOADING_LOGIN_PAGE = 'Downloading Provider Login Page'
+
     def _download_webpage_handle(self, *args, **kwargs):
         headers = kwargs.get('headers', {})
         headers.update(self.geo_verification_headers())
@@ -1365,6 +1373,21 @@ class AdobePassIE(InfoExtractor):
                 'Use --ap-mso to specify Adobe Pass Multiple-system operator Identifier '
                 'and --ap-username and --ap-password or --netrc to provide account credentials.', expected=True)
 
+        def extract_redirect_url(html, url=None, fatal=False):
+            # TODO: eliminate code duplication with generic extractor and move
+            # redirection code into _download_webpage_handle
+            REDIRECT_REGEX = r'[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)'
+            redirect_url = self._search_regex(
+                r'(?i)<meta\s+(?=(?:[a-z-]+="[^"]+"\s+)*http-equiv="refresh")'
+                r'(?:[a-z-]+="[^"]+"\s+)*?content="%s' % REDIRECT_REGEX,
+                html, 'meta refresh redirect',
+                default=NO_DEFAULT if fatal else None, fatal=fatal)
+            if not redirect_url:
+                return None
+            if url:
+                redirect_url = compat_urlparse.urljoin(url, unescapeHTML(redirect_url))
+            return redirect_url
+
         mvpd_headers = {
             'ap_42': 'anonymous',
             'ap_11': 'Linux i686',
@@ -1414,16 +1437,15 @@ class AdobePassIE(InfoExtractor):
                         if '<form name="signin"' in provider_redirect_page:
                             provider_login_page_res = provider_redirect_page_res
                         elif 'http-equiv="refresh"' in provider_redirect_page:
-                            oauth_redirect_url = self._html_search_regex(
-                                r'content="0;\s*url=([^\'"]+)',
-                                provider_redirect_page, 'meta refresh redirect')
+                            oauth_redirect_url = extract_redirect_url(
+                                provider_redirect_page, fatal=True)
                             provider_login_page_res = self._download_webpage_handle(
                                 oauth_redirect_url, video_id,
-                                'Downloading Provider Login Page')
+                                self._DOWNLOADING_LOGIN_PAGE)
                         else:
                             provider_login_page_res = post_form(
                                 provider_redirect_page_res,
-                                'Downloading Provider Login Page')
+                                self._DOWNLOADING_LOGIN_PAGE)
 
                         mvpd_confirm_page_res = post_form(
                             provider_login_page_res, 'Logging in', {
@@ -1470,8 +1492,17 @@ class AdobePassIE(InfoExtractor):
                             'Content-Type': 'application/x-www-form-urlencoded'
                         })
                 else:
+                    # Some providers (e.g. DIRECTV NOW) have another meta refresh
+                    # based redirect that should be followed.
+                    provider_redirect_page, urlh = provider_redirect_page_res
+                    provider_refresh_redirect_url = extract_redirect_url(
+                        provider_redirect_page, url=urlh.geturl())
+                    if provider_refresh_redirect_url:
+                        provider_redirect_page_res = self._download_webpage_handle(
+                            provider_refresh_redirect_url, video_id,
+                            'Downloading Provider Redirect Page (meta refresh)')
                     provider_login_page_res = post_form(
-                        provider_redirect_page_res, 'Downloading Provider Login Page')
+                        provider_redirect_page_res, self._DOWNLOADING_LOGIN_PAGE)
                     mvpd_confirm_page_res = post_form(provider_login_page_res, 'Logging in', {
                         mso_info.get('username_field', 'username'): username,
                         mso_info.get('password_field', 'password'): password,

From a5aea53fc88a87889a8ce4189e28c28386bb6bb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Jun 2017 05:13:12 +0700
Subject: [PATCH 1033/1696] [ChangeLog] Actualize

---
 ChangeLog | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 746250db9..d3343c760 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,20 @@
+version <unreleased>
+
+Core
++ [adobepass] Add support for DIRECTV NOW (mso ATTOTT) (#13472)
+* [YoutubeDL] Skip malformed formats for better extraction robustness
+
+Extractors
++ [wsj] Add support for barrons.com (#13470)
++ [ign] Add another video id pattern (#13328)
++ [raiplay:live] Add support for live streams (#13414)
++ [redbulltv] Add support for live videos and segments (#13486)
++ [onetpl] Add support for videos embedded via pulsembed (#13482)
+* [ooyala] Make more robust
+* [ooyala] Skip empty format URLs (#13471, #13476)
+* [hgtv.com:show] Fix typo
+
+
 version 2017.06.23
 
 Core

From a7ce8f16c4ee4b8f567d943ada8ade8a50f99860 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Jun 2017 05:16:06 +0700
Subject: [PATCH 1034/1696] release 2017.06.25

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 1 +
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 4e7ceafd8..82bbbdaa6 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.06.23*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.06.23**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.06.25*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.06.25**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.06.23
+[debug] youtube-dl version 2017.06.25
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index d3343c760..a0072ffe3 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.06.25
 
 Core
 + [adobepass] Add support for DIRECTV NOW (mso ATTOTT) (#13472)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index e827ec0cf..010ff762c 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -644,6 +644,7 @@
  - **RadioJavan**
  - **Rai**
  - **RaiPlay**
+ - **RaiPlayLive**
  - **RBMARadio**
  - **RDS**: RDS.ca
  - **RedBullTV**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index dfb69ab8d..b6d378896 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.06.23'
+__version__ = '2017.06.25'

From 4a87de72dfbc7a0ad9496b225a865907715a7bad Mon Sep 17 00:00:00 2001
From: Luca Steeb <contact@luca-steeb.com>
Date: Sun, 25 Jun 2017 21:30:05 +0200
Subject: [PATCH 1035/1696] [niconico] fix sp subdomain links

---
 youtube_dl/extractor/niconico.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 8baac23e4..f268a72d5 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -83,9 +83,12 @@ class NiconicoIE(InfoExtractor):
             'uploader_id': '312',
         },
         'skip': 'The viewing period of the video you were searching for has expired.',
+    }, {
+        'url': 'http://sp.nicovideo.jp/watch/sm28964488?ss_pos=1&cp_in=wt_tg',
+        'only_matching': True,
     }]
 
-    _VALID_URL = r'https?://(?:www\.|secure\.)?nicovideo\.jp/watch/(?P<id>(?:[a-z]{2})?[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.|secure\.|sp\.)?nicovideo\.jp/watch/(?P<id>(?:[a-z]{2})?[0-9]+)'
     _NETRC_MACHINE = 'niconico'
 
     def _real_initialize(self):

From 136503e302995c4264df7cfccb62fdae94a2c680 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 26 Jun 2017 19:56:07 +0800
Subject: [PATCH 1036/1696] [ChangeLog] Update after #13494

---
 ChangeLog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index a0072ffe3..7b3c6c6cf 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
++ [niconico] Support sp subdomain (#13494)
+
+
 version 2017.06.25
 
 Core

From 35544690e4f4fb1b8184e57d9c6c54fe893e6a3b Mon Sep 17 00:00:00 2001
From: "Xuan Hu (Sean)" <i+github@huxuan.org>
Date: Mon, 26 Jun 2017 23:00:31 +0800
Subject: [PATCH 1037/1696] [pandatv] Add support for https URLs

---
 youtube_dl/extractor/pandatv.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/pandatv.py b/youtube_dl/extractor/pandatv.py
index 133cc9b88..a5ed49449 100644
--- a/youtube_dl/extractor/pandatv.py
+++ b/youtube_dl/extractor/pandatv.py
@@ -10,13 +10,13 @@ from ..utils import (
 
 class PandaTVIE(InfoExtractor):
     IE_DESC = '熊猫TV'
-    _VALID_URL = r'http://(?:www\.)?panda\.tv/(?P<id>[0-9]+)'
-    _TEST = {
-        'url': 'http://www.panda.tv/10091',
+    _VALID_URL = r'https?://(?:www\.)?panda\.tv/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://www.panda.tv/66666',
         'info_dict': {
-            'id': '10091',
+            'id': '66666',
             'title': 're:.+',
-            'uploader': '囚徒',
+            'uploader': '刘杀鸡',
             'ext': 'flv',
             'is_live': True,
         },
@@ -24,7 +24,10 @@ class PandaTVIE(InfoExtractor):
             'skip_download': True,
         },
         'skip': 'Live stream is offline',
-    }
+    }, {
+        'url': 'https://www.panda.tv/66666',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 33ffb645a6881744f8befcb813ef1f5b6958d385 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 26 Jun 2017 22:11:09 +0700
Subject: [PATCH 1038/1696] [pandatv] Switch to https for API and download URLs

---
 youtube_dl/extractor/pandatv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/pandatv.py b/youtube_dl/extractor/pandatv.py
index a5ed49449..c86d70771 100644
--- a/youtube_dl/extractor/pandatv.py
+++ b/youtube_dl/extractor/pandatv.py
@@ -33,7 +33,7 @@ class PandaTVIE(InfoExtractor):
         video_id = self._match_id(url)
 
         config = self._download_json(
-            'http://www.panda.tv/api_room?roomid=%s' % video_id, video_id)
+            'https://www.panda.tv/api_room?roomid=%s' % video_id, video_id)
 
         error_code = config.get('errno', 0)
         if error_code is not 0:
@@ -77,7 +77,7 @@ class PandaTVIE(InfoExtractor):
                 continue
             for pref, (ext, pl) in enumerate((('m3u8', '-hls'), ('flv', ''))):
                 formats.append({
-                    'url': 'http://pl%s%s.live.panda.tv/live_panda/%s%s%s.%s'
+                    'url': 'https://pl%s%s.live.panda.tv/live_panda/%s%s%s.%s'
                     % (pl, plflag1, room_key, live_panda, suffix[quality], ext),
                     'format_id': '%s-%s' % (k, ext),
                     'quality': quality,

From f1c051009bcf52525df22eb3a59797076551a579 Mon Sep 17 00:00:00 2001
From: Viktor Szakats <vszakats@users.noreply.github.com>
Date: Tue, 27 Jun 2017 09:20:18 -0500
Subject: [PATCH 1039/1696] [soundcloud] Switch to https for API requests

---
 youtube_dl/extractor/soundcloud.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 0ee4a8ff8..994c86145 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -136,7 +136,7 @@ class SoundcloudIE(InfoExtractor):
 
     @classmethod
     def _resolv_url(cls, url):
-        return 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=' + cls._CLIENT_ID
+        return 'https://api.soundcloud.com/resolve.json?url=' + url + '&client_id=' + cls._CLIENT_ID
 
     def _extract_info_dict(self, info, full_title=None, quiet=False, secret_token=None):
         track_id = compat_str(info['id'])
@@ -174,7 +174,7 @@ class SoundcloudIE(InfoExtractor):
 
         # We have to retrieve the url
         format_dict = self._download_json(
-            'http://api.soundcloud.com/i1/tracks/%s/streams' % track_id,
+            'https://api.soundcloud.com/i1/tracks/%s/streams' % track_id,
             track_id, 'Downloading track url', query={
                 'client_id': self._CLIENT_ID,
                 'secret_token': secret_token,
@@ -236,7 +236,7 @@ class SoundcloudIE(InfoExtractor):
         track_id = mobj.group('track_id')
 
         if track_id is not None:
-            info_json_url = 'http://api.soundcloud.com/tracks/' + track_id + '.json?client_id=' + self._CLIENT_ID
+            info_json_url = 'https://api.soundcloud.com/tracks/' + track_id + '.json?client_id=' + self._CLIENT_ID
             full_title = track_id
             token = mobj.group('secret_token')
             if token:
@@ -261,7 +261,7 @@ class SoundcloudIE(InfoExtractor):
 
             self.report_resolve(full_title)
 
-            url = 'http://soundcloud.com/%s' % resolve_title
+            url = 'https://soundcloud.com/%s' % resolve_title
             info_json_url = self._resolv_url(url)
         info = self._download_json(info_json_url, full_title, 'Downloading info JSON')
 
@@ -304,7 +304,7 @@ class SoundcloudSetIE(SoundcloudPlaylistBaseIE):
         # extract simple title (uploader + slug of song title)
         slug_title = mobj.group('slug_title')
         full_title = '%s/sets/%s' % (uploader, slug_title)
-        url = 'http://soundcloud.com/%s/sets/%s' % (uploader, slug_title)
+        url = 'https://soundcloud.com/%s/sets/%s' % (uploader, slug_title)
 
         token = mobj.group('token')
         if token:
@@ -410,7 +410,7 @@ class SoundcloudUserIE(SoundcloudPlaylistBaseIE):
         mobj = re.match(self._VALID_URL, url)
         uploader = mobj.group('user')
 
-        url = 'http://soundcloud.com/%s/' % uploader
+        url = 'https://soundcloud.com/%s/' % uploader
         resolv_url = self._resolv_url(url)
         user = self._download_json(
             resolv_url, uploader, 'Downloading user info')
@@ -473,7 +473,7 @@ class SoundcloudPlaylistIE(SoundcloudPlaylistBaseIE):
     _VALID_URL = r'https?://api\.soundcloud\.com/playlists/(?P<id>[0-9]+)(?:/?\?secret_token=(?P<token>[^&]+?))?$'
     IE_NAME = 'soundcloud:playlist'
     _TESTS = [{
-        'url': 'http://api.soundcloud.com/playlists/4110309',
+        'url': 'https://api.soundcloud.com/playlists/4110309',
         'info_dict': {
             'id': '4110309',
             'title': 'TILT Brass - Bowery Poetry Club, August \'03 [Non-Site SCR 02]',

From bf2dc9cc6e1ca38955f646b67f0f6ec40aec139d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 27 Jun 2017 21:26:46 +0700
Subject: [PATCH 1040/1696] [soundcloud] Fix tests

---
 youtube_dl/extractor/soundcloud.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 994c86145..3f1a46bb2 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -290,7 +290,7 @@ class SoundcloudSetIE(SoundcloudPlaylistBaseIE):
             'id': '2284613',
             'title': 'The Royal Concept EP',
         },
-        'playlist_mincount': 6,
+        'playlist_mincount': 5,
     }, {
         'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep/token',
         'only_matching': True,
@@ -380,7 +380,7 @@ class SoundcloudUserIE(SoundcloudPlaylistBaseIE):
         'url': 'https://soundcloud.com/grynpyret/spotlight',
         'info_dict': {
             'id': '7098329',
-            'title': 'GRYNPYRET (Spotlight)',
+            'title': 'Grynpyret (Spotlight)',
         },
         'playlist_mincount': 1,
     }]

From 0646e34c7d511a02d8d93e840bceaa3521c2204e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 27 Jun 2017 22:25:34 +0700
Subject: [PATCH 1041/1696] [facebook] Add support for plugin video embeds and
 multiple embeds (closes #13493)

---
 youtube_dl/extractor/buzzfeed.py |  7 ++++---
 youtube_dl/extractor/facebook.py | 20 ++++++++++----------
 youtube_dl/extractor/generic.py  |  6 +++---
 3 files changed, 17 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/buzzfeed.py b/youtube_dl/extractor/buzzfeed.py
index 75fa92d7c..ec411091e 100644
--- a/youtube_dl/extractor/buzzfeed.py
+++ b/youtube_dl/extractor/buzzfeed.py
@@ -84,9 +84,10 @@ class BuzzFeedIE(InfoExtractor):
                 continue
             entries.append(self.url_result(video['url']))
 
-        facebook_url = FacebookIE._extract_url(webpage)
-        if facebook_url:
-            entries.append(self.url_result(facebook_url))
+        facebook_urls = FacebookIE._extract_urls(webpage)
+        entries.extend([
+            self.url_result(facebook_url)
+            for facebook_url in facebook_urls])
 
         return {
             '_type': 'playlist',
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index b69c1ede0..4b3f6cc86 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -203,19 +203,19 @@ class FacebookIE(InfoExtractor):
     }]
 
     @staticmethod
-    def _extract_url(webpage):
-        mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>https://www\.facebook\.com/video/embed.+?)\1', webpage)
-        if mobj is not None:
-            return mobj.group('url')
-
+    def _extract_urls(webpage):
+        urls = []
+        for mobj in re.finditer(
+                r'<iframe[^>]+?src=(["\'])(?P<url>https?://www\.facebook\.com/(?:video/embed|plugins/video\.php).+?)\1',
+                webpage):
+            urls.append(mobj.group('url'))
         # Facebook API embed
         # see https://developers.facebook.com/docs/plugins/embedded-video-player
-        mobj = re.search(r'''(?x)<div[^>]+
+        for mobj in re.finditer(r'''(?x)<div[^>]+
                 class=(?P<q1>[\'"])[^\'"]*\bfb-(?:video|post)\b[^\'"]*(?P=q1)[^>]+
-                data-href=(?P<q2>[\'"])(?P<url>(?:https?:)?//(?:www\.)?facebook.com/.+?)(?P=q2)''', webpage)
-        if mobj is not None:
-            return mobj.group('url')
+                data-href=(?P<q2>[\'"])(?P<url>(?:https?:)?//(?:www\.)?facebook.com/.+?)(?P=q2)''', webpage):
+            urls.append(mobj.group('url'))
+        return urls
 
     def _login(self):
         (useremail, password) = self._get_login_info()
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8ef1a2980..760a7f9c2 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2222,9 +2222,9 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'))
 
         # Look for embedded Facebook player
-        facebook_url = FacebookIE._extract_url(webpage)
-        if facebook_url is not None:
-            return self.url_result(facebook_url, 'Facebook')
+        facebook_urls = FacebookIE._extract_urls(webpage)
+        if facebook_urls:
+            return self.playlist_from_matches(facebook_urls, video_id, video_title)
 
         # Look for embedded VK player
         mobj = re.search(r'<iframe[^>]+?src=(["\'])(?P<url>https?://vk\.com/video_ext\.php.+?)\1', webpage)

From fd1c5fba6bee4de36cc3276d86c84c0abc054a2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 27 Jun 2017 22:36:54 +0700
Subject: [PATCH 1042/1696] [facebook] Add test for plugin video embed (#13493)

---
 youtube_dl/extractor/generic.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 760a7f9c2..2792ea3cf 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1522,6 +1522,21 @@ class GenericIE(InfoExtractor):
                 'title': 'Facebook video #599637780109885',
             },
         },
+        # Facebook <iframe> embed, plugin video
+        {
+            'url': 'http://5pillarsuk.com/2017/06/07/tariq-ramadan-disagrees-with-pr-exercise-by-imams-refusing-funeral-prayers-for-london-attackers/',
+            'info_dict': {
+                'id': '1754168231264132',
+                'ext': 'mp4',
+                'title': 'About the Imams and Religious leaders refusing to perform funeral prayers for...',
+                'uploader': 'Tariq Ramadan (official)',
+                'timestamp': 1496758379,
+                'upload_date': '20170606',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
         # Facebook API embed
         {
             'url': 'http://www.lothype.com/blue-stars-2016-preview-standstill-full-show/',

From 88d6b7c2bde239ad3593de54ab894ff43020142d Mon Sep 17 00:00:00 2001
From: Parmjit Virk <pvirk@mts.net>
Date: Wed, 28 Jun 2017 10:21:35 -0500
Subject: [PATCH 1043/1696] [cbsnews] Relax video info regex (fixes #13284)

---
 youtube_dl/extractor/cbsnews.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 5b61c3f52..ef5911bd6 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -52,6 +52,21 @@ class CBSNewsIE(CBSIE):
                 'skip_download': True,
             },
         },
+        {
+            'url': 'http://www.cbsnews.com/news/maria-ridulph-murder-will-the-nations-oldest-cold-case-to-go-to-trial-ever-get-solved/',
+            'info_dict': {
+                'id': 'QpM5BJjBVEAUFi7ydR9LusS69DPLqPJ1',
+                'ext': 'mp4',
+                'title': 'Cold as Ice',
+                'description': 'Can a childhood memory of a friend\'s murder solve a 1957 cold case? "48 Hours" correspondent Erin Moriarty has the latest.',
+                'upload_date': '20170604',
+                'timestamp': 1496538000,
+                'uploader': 'CBSI-NEW',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
     ]
 
     def _real_extract(self, url):
@@ -60,7 +75,7 @@ class CBSNewsIE(CBSIE):
         webpage = self._download_webpage(url, video_id)
 
         video_info = self._parse_json(self._html_search_regex(
-            r'(?:<ul class="media-list items" id="media-related-items"><li data-video-info|<div id="cbsNewsVideoPlayer" data-video-player-options)=\'({.+?})\'',
+            r'(?:<ul class="media-list items" id="media-related-items"[^>]*><li data-video-info|<div id="cbsNewsVideoPlayer" data-video-player-options)=\'({.+?})\'',
             webpage, 'video JSON info', default='{}'), video_id, fatal=False)
 
         if video_info:

From a919ca0ad60c4ef6165fceadeb9f6ad70b53e7a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 28 Jun 2017 22:30:12 +0700
Subject: [PATCH 1044/1696] [cbsnews] Actualize test

---
 youtube_dl/extractor/cbsnews.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index ef5911bd6..51df15fac 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -15,19 +15,23 @@ class CBSNewsIE(CBSIE):
 
     _TESTS = [
         {
-            'url': 'http://www.cbsnews.com/news/tesla-and-spacex-elon-musks-industrial-empire/',
+            # 60 minutes
+            'url': 'http://www.cbsnews.com/news/artificial-intelligence-positioned-to-be-a-game-changer/',
             'info_dict': {
-                'id': 'tesla-and-spacex-elon-musks-industrial-empire',
-                'ext': 'flv',
-                'title': 'Tesla and SpaceX: Elon Musk\'s industrial empire',
-                'thumbnail': 'http://beta.img.cbsnews.com/i/2014/03/30/60147937-2f53-4565-ad64-1bdd6eb64679/60-0330-pelley-640x360.jpg',
-                'duration': 791,
+                'id': '_B6Ga3VJrI4iQNKsir_cdFo9Re_YJHE_',
+                'ext': 'mp4',
+                'title': 'Artificial Intelligence',
+                'description': 'md5:8818145f9974431e0fb58a1b8d69613c',
+                'thumbnail': r're:^https?://.*\.jpg$',
+                'duration': 1606,
+                'uploader': 'CBSI-NEW',
+                'timestamp': 1498431900,
+                'upload_date': '20170625',
             },
             'params': {
-                # rtmp download
+                # m3u8 download
                 'skip_download': True,
             },
-            'skip': 'Subscribers only',
         },
         {
             'url': 'http://www.cbsnews.com/videos/fort-hood-shooting-army-downplays-mental-illness-as-cause-of-attack/',
@@ -53,6 +57,7 @@ class CBSNewsIE(CBSIE):
             },
         },
         {
+            # 48 hours
             'url': 'http://www.cbsnews.com/news/maria-ridulph-murder-will-the-nations-oldest-cold-case-to-go-to-trial-ever-get-solved/',
             'info_dict': {
                 'id': 'QpM5BJjBVEAUFi7ydR9LusS69DPLqPJ1',

From b2ed954fc6f19a00c7c8c12f45a6a44bd1722867 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 29 Jun 2017 22:27:53 +0700
Subject: [PATCH 1045/1696] [bbccouk] Capture and output error message (closes
 #13518)

---
 youtube_dl/extractor/bbc.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 7cd4446b4..6f5f9aa3a 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -523,6 +523,12 @@ class BBCCoUkIE(InfoExtractor):
 
         webpage = self._download_webpage(url, group_id, 'Downloading video page')
 
+        error = self._search_regex(
+            r'<div\b[^>]+\bclass=["\']smp__message delta["\'][^>]*>([^<]+)<',
+            webpage, 'error', default=None)
+        if error:
+            raise ExtractorError(error, expected=True)
+
         programme_id = None
         duration = None
 

From 72d256c434169b9dbfbc1c2fbce2a14f0195a0bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 29 Jun 2017 22:29:28 +0700
Subject: [PATCH 1046/1696] [bbccouk] Extend _VALID_URL

---
 youtube_dl/extractor/bbc.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 6f5f9aa3a..79ded6ba1 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -36,7 +36,7 @@ class BBCCoUkIE(InfoExtractor):
                         (?:
                             programmes/(?!articles/)|
                             iplayer(?:/[^/]+)?/(?:episode/|playlist/)|
-                            music/clips[/#]|
+                            music/(?:clips|audiovideo/popular)[/#]|
                             radio/player/
                         )
                         (?P<id>%s)(?!/(?:episodes|broadcasts|clips))
@@ -229,8 +229,10 @@ class BBCCoUkIE(InfoExtractor):
         }, {
             'url': 'http://www.bbc.co.uk/radio/player/p03cchwf',
             'only_matching': True,
-        }
-    ]
+        }, {
+            'url': 'https://www.bbc.co.uk/music/audiovideo/popular#p055bc55',
+            'only_matching': True,
+        }]
 
     _USP_RE = r'/([^/]+?)\.ism(?:\.hlsv2\.ism)?/[^/]+\.m3u8'
 

From b311b0ead22f13f7cb10a3c2802f58e0692addcc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jun 2017 21:41:05 +0700
Subject: [PATCH 1047/1696] [generic] Extract more generic metadata (closes
 #13527)

---
 youtube_dl/extractor/generic.py | 35 ++++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 2792ea3cf..f9bff433c 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2048,6 +2048,13 @@ class GenericIE(InfoExtractor):
         video_description = self._og_search_description(webpage, default=None)
         video_thumbnail = self._og_search_thumbnail(webpage, default=None)
 
+        info_dict.update({
+            'title': video_title,
+            'description': video_description,
+            'thumbnail': video_thumbnail,
+            'age_limit': age_limit,
+        })
+
         # Look for Brightcove Legacy Studio embeds
         bc_urls = BrightcoveLegacyIE._extract_brightcove_urls(webpage)
         if bc_urls:
@@ -2684,18 +2691,26 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 mediaset_urls, video_id, video_title, ie=MediasetIE.ie_key())
 
+        def merge_dicts(dict1, dict2):
+            merged = {}
+            for k, v in dict1.items():
+                if v is not None:
+                    merged[k] = v
+            for k, v in dict2.items():
+                if v is None:
+                    continue
+                if (k not in merged or
+                        (isinstance(v, compat_str) and v and
+                            isinstance(merged[k], compat_str) and
+                            not merged[k])):
+                    merged[k] = v
+            return merged
+
         # Looking for http://schema.org/VideoObject
         json_ld = self._search_json_ld(
             webpage, video_id, default={}, expected_type='VideoObject')
         if json_ld.get('url'):
-            info_dict.update({
-                'title': video_title or info_dict['title'],
-                'description': video_description,
-                'thumbnail': video_thumbnail,
-                'age_limit': age_limit
-            })
-            info_dict.update(json_ld)
-            return info_dict
+            return merge_dicts(json_ld, info_dict)
 
         # Look for HTML5 media
         entries = self._parse_html5_media_entries(url, webpage, video_id, m3u8_id='hls')
@@ -2713,9 +2728,7 @@ class GenericIE(InfoExtractor):
         if jwplayer_data:
             info = self._parse_jwplayer_data(
                 jwplayer_data, video_id, require_title=False, base_url=url)
-            if not info.get('title'):
-                info['title'] = video_title
-            return info
+            return merge_dicts(info, info_dict)
 
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):

From d4f8ce6e91aff0ba0a6a4464d5525ba66720ae86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jun 2017 21:55:45 +0700
Subject: [PATCH 1048/1696] [dplayit] Relax video id regex (closes #13524)

---
 youtube_dl/extractor/dplay.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dplay.py b/youtube_dl/extractor/dplay.py
index 87c5dd63e..1a41760f8 100644
--- a/youtube_dl/extractor/dplay.py
+++ b/youtube_dl/extractor/dplay.py
@@ -184,7 +184,7 @@ class DPlayItIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         info_url = self._search_regex(
-            r'url\s*:\s*["\']((?:https?:)?//[^/]+/playback/videoPlaybackInfo/\d+)',
+            r'url\s*[:=]\s*["\']((?:https?:)?//[^/]+/playback/videoPlaybackInfo/\d+)',
             webpage, 'video id')
 
         title = remove_end(self._og_search_title(webpage), ' | Dplay')

From c69701c6ab94ca2fce4fedf6390515acb55e9086 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jun 2017 22:19:06 +0700
Subject: [PATCH 1049/1696] [extractor/common] Improve _json_ld

---
 youtube_dl/extractor/common.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9751ab021..afeb4c5da 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1002,17 +1002,17 @@ class InfoExtractor(object):
                 item_type = e.get('@type')
                 if expected_type is not None and expected_type != item_type:
                     return info
-                if item_type == 'TVEpisode':
+                if item_type in ('TVEpisode', 'Episode'):
                     info.update({
                         'episode': unescapeHTML(e.get('name')),
                         'episode_number': int_or_none(e.get('episodeNumber')),
                         'description': unescapeHTML(e.get('description')),
                     })
                     part_of_season = e.get('partOfSeason')
-                    if isinstance(part_of_season, dict) and part_of_season.get('@type') == 'TVSeason':
+                    if isinstance(part_of_season, dict) and part_of_season.get('@type') in ('TVSeason', 'Season', 'CreativeWorkSeason'):
                         info['season_number'] = int_or_none(part_of_season.get('seasonNumber'))
                     part_of_series = e.get('partOfSeries') or e.get('partOfTVSeries')
-                    if isinstance(part_of_series, dict) and part_of_series.get('@type') == 'TVSeries':
+                    if isinstance(part_of_series, dict) and part_of_series.get('@type') in ('TVSeries', 'Series', 'CreativeWorkSeries'):
                         info['series'] = unescapeHTML(part_of_series.get('name'))
                 elif item_type == 'Article':
                     info.update({
@@ -1022,10 +1022,10 @@ class InfoExtractor(object):
                     })
                 elif item_type == 'VideoObject':
                     extract_video_object(e)
-                elif item_type == 'WebPage':
-                    video = e.get('video')
-                    if isinstance(video, dict) and video.get('@type') == 'VideoObject':
-                        extract_video_object(video)
+                    continue
+                video = e.get('video')
+                if isinstance(video, dict) and video.get('@type') == 'VideoObject':
+                    extract_video_object(video)
                 break
         return dict((k, v) for k, v in info.items() if v is not None)
 

From 54faac223505758e48c1f7ce1c7ed8c110f9bac4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jun 2017 22:20:30 +0700
Subject: [PATCH 1050/1696] [tastytrade] Add extractor (closes #13521)

---
 youtube_dl/extractor/tastytrade.py | 43 ++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)
 create mode 100644 youtube_dl/extractor/tastytrade.py

diff --git a/youtube_dl/extractor/tastytrade.py b/youtube_dl/extractor/tastytrade.py
new file mode 100644
index 000000000..7fe96bd5f
--- /dev/null
+++ b/youtube_dl/extractor/tastytrade.py
@@ -0,0 +1,43 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from .ooyala import OoyalaIE
+
+
+class TastyTradeIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?tastytrade\.com/tt/shows/[^/]+/episodes/(?P<id>[^/?#&]+)'
+
+    _TESTS = [{
+        'url': 'https://www.tastytrade.com/tt/shows/market-measures/episodes/correlation-in-short-volatility-06-28-2017',
+        'info_dict': {
+            'id': 'F3bnlzbToeI6pLEfRyrlfooIILUjz4nM',
+            'ext': 'mp4',
+            'title': 'A History of Teaming',
+            'description': 'md5:2a9033db8da81f2edffa4c99888140b3',
+            'duration': 422.255,
+        },
+        'params': {
+            'skip_download': True,
+        },
+        'add_ie': ['Ooyala'],
+    }, {
+        'url': 'https://www.tastytrade.com/tt/shows/daily-dose/episodes/daily-dose-06-30-2017',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        ooyala_code = self._search_regex(
+            r'data-media-id=(["\'])(?P<code>(?:(?!\1).)+)\1',
+            webpage, 'ooyala code', group='code')
+
+        info = self._search_json_ld(webpage, display_id, fatal=False)
+        info.update({
+            '_type': 'url_transparent',
+            'ie_key': OoyalaIE.ie_key(),
+            'url': 'ooyala:%s' % ooyala_code,
+            'display_id': display_id,
+        })
+        return info

From 49174788035038b8f12f971ec04ef897df60f435 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 Jul 2017 18:39:01 +0700
Subject: [PATCH 1051/1696] [ted] Fix extraction (closes #13535))

---
 youtube_dl/extractor/ted.py | 49 +++++++++++++++++++++++++------------
 1 file changed, 34 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 3f3c681ae..f27d0e313 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -6,7 +6,10 @@ import re
 from .common import InfoExtractor
 
 from ..compat import compat_str
-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    try_get,
+)
 
 
 class TEDIE(InfoExtractor):
@@ -113,8 +116,9 @@ class TEDIE(InfoExtractor):
     }
 
     def _extract_info(self, webpage):
-        info_json = self._search_regex(r'q\("\w+.init",({.+})\)</script>',
-                                       webpage, 'info json')
+        info_json = self._search_regex(
+            r'(?s)q\(\s*"\w+.init"\s*,\s*({.+})\)\s*</script>',
+            webpage, 'info json')
         return json.loads(info_json)
 
     def _real_extract(self, url):
@@ -136,11 +140,16 @@ class TEDIE(InfoExtractor):
         webpage = self._download_webpage(url, name,
                                          'Downloading playlist webpage')
         info = self._extract_info(webpage)
-        playlist_info = info['playlist']
+
+        playlist_info = try_get(
+            info, lambda x: x['__INITIAL_DATA__']['playlist'],
+            dict) or info['playlist']
 
         playlist_entries = [
             self.url_result('http://www.ted.com/talks/' + talk['slug'], self.ie_key())
-            for talk in info['talks']
+            for talk in try_get(
+                info, lambda x: x['__INITIAL_DATA__']['talks'],
+                dict) or info['talks']
         ]
         return self.playlist_result(
             playlist_entries,
@@ -149,9 +158,14 @@ class TEDIE(InfoExtractor):
 
     def _talk_info(self, url, video_name):
         webpage = self._download_webpage(url, video_name)
-        self.report_extraction(video_name)
 
-        talk_info = self._extract_info(webpage)['talks'][0]
+        info = self._extract_info(webpage)
+
+        talk_info = try_get(
+            info, lambda x: x['__INITIAL_DATA__']['talks'][0],
+            dict) or info['talks'][0]
+
+        title = talk_info['title'].strip()
 
         external = talk_info.get('external')
         if external:
@@ -165,19 +179,27 @@ class TEDIE(InfoExtractor):
                 'url': ext_url or external['uri'],
             }
 
+        native_downloads = try_get(
+            talk_info, lambda x: x['downloads']['nativeDownloads'],
+            dict) or talk_info['nativeDownloads']
+
         formats = [{
             'url': format_url,
             'format_id': format_id,
             'format': format_id,
-        } for (format_id, format_url) in talk_info['nativeDownloads'].items() if format_url is not None]
+        } for (format_id, format_url) in native_downloads.items() if format_url is not None]
         if formats:
             for f in formats:
                 finfo = self._NATIVE_FORMATS.get(f['format_id'])
                 if finfo:
                     f.update(finfo)
 
+        player_talk = talk_info['player_talks'][0]
+
+        resources_ = player_talk.get('resources') or talk_info.get('resources')
+
         http_url = None
-        for format_id, resources in talk_info['resources'].items():
+        for format_id, resources in resources_.items():
             if format_id == 'h264':
                 for resource in resources:
                     h264_url = resource.get('file')
@@ -237,14 +259,11 @@ class TEDIE(InfoExtractor):
 
         video_id = compat_str(talk_info['id'])
 
-        thumbnail = talk_info['thumb']
-        if not thumbnail.startswith('http'):
-            thumbnail = 'http://' + thumbnail
         return {
             'id': video_id,
-            'title': talk_info['title'].strip(),
-            'uploader': talk_info['speaker'],
-            'thumbnail': thumbnail,
+            'title': title,
+            'uploader': player_talk.get('speaker') or talk_info.get('speaker'),
+            'thumbnail': player_talk.get('thumb') or talk_info.get('thumb'),
             'description': self._og_search_description(webpage),
             'subtitles': self._get_subtitles(video_id, talk_info),
             'formats': formats,

From 9b35297be1b7e5fa7372bac664ab91a7fa364760 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 Jul 2017 18:39:29 +0700
Subject: [PATCH 1052/1696] [extractors] Add import for tastytrade

---
 youtube_dl/extractor/extractors.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index a263c88b3..bbdb4a2fe 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -973,6 +973,7 @@ from .tagesschau import (
     TagesschauIE,
 )
 from .tass import TassIE
+from .tastytrade import TastyTradeIE
 from .tbs import TBSIE
 from .tdslifeway import TDSLifewayIE
 from .teachertube import (

From 02d61a65e27c9f186ea24bce8e9f0c65f4ec508a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Jul 2017 08:00:22 +0700
Subject: [PATCH 1053/1696] [xfileshare] Extend format regex (closes #13536)

---
 youtube_dl/extractor/xfileshare.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index 74aac85a3..ad747978d 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -157,7 +157,7 @@ class XFileShareIE(InfoExtractor):
         def extract_formats(default=NO_DEFAULT):
             urls = []
             for regex in (
-                    r'file\s*:\s*(["\'])(?P<url>http(?:(?!\1).)+\.(?:m3u8|mp4|flv)(?:(?!\1).)*)\1',
+                    r'(?:file|src)\s*:\s*(["\'])(?P<url>http(?:(?!\1).)+\.(?:m3u8|mp4|flv)(?:(?!\1).)*)\1',
                     r'file_link\s*=\s*(["\'])(?P<url>http(?:(?!\1).)+)\1',
                     r'addVariable\((\\?["\'])file\1\s*,\s*(\\?["\'])(?P<url>http(?:(?!\2).)+)\2\)',
                     r'<embed[^>]+src=(["\'])(?P<url>http(?:(?!\1).)+\.(?:m3u8|mp4|flv)(?:(?!\1).)*)\1'):

From a3a6d01a9613566347ac39bac9cef3c4b8cd692f Mon Sep 17 00:00:00 2001
From: Parmjit Virk <pvirk@mts.net>
Date: Sat, 1 Jul 2017 21:09:57 -0500
Subject: [PATCH 1054/1696] [thisoldhouse] Fix video id extraction (closes
 #13540)

---
 youtube_dl/extractor/thisoldhouse.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/thisoldhouse.py b/youtube_dl/extractor/thisoldhouse.py
index 197258df1..6a3efb9df 100644
--- a/youtube_dl/extractor/thisoldhouse.py
+++ b/youtube_dl/extractor/thisoldhouse.py
@@ -31,5 +31,5 @@ class ThisOldHouseIE(InfoExtractor):
         drupal_settings = self._parse_json(self._search_regex(
             r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);',
             webpage, 'drupal settings'), display_id)
-        video_id = drupal_settings['jwplatform']['video_id']
+        video_id = list(drupal_settings['comScore'])[0]
         return self.url_result('jwplatform:' + video_id, 'JWPlatform', video_id)

From 99a7e76240f3e79d770bff0029dc3321e14b2a97 Mon Sep 17 00:00:00 2001
From: Parmjit Virk <pvirk@mts.net>
Date: Sat, 1 Jul 2017 21:11:58 -0500
Subject: [PATCH 1055/1696] [thisoldhouse] Update test

---
 youtube_dl/extractor/thisoldhouse.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/thisoldhouse.py b/youtube_dl/extractor/thisoldhouse.py
index 6a3efb9df..0c321354f 100644
--- a/youtube_dl/extractor/thisoldhouse.py
+++ b/youtube_dl/extractor/thisoldhouse.py
@@ -8,7 +8,7 @@ class ThisOldHouseIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?thisoldhouse\.com/(?:watch|how-to|tv-episode)/(?P<id>[^/?#]+)'
     _TESTS = [{
         'url': 'https://www.thisoldhouse.com/how-to/how-to-build-storage-bench',
-        'md5': '946f05bbaa12a33f9ae35580d2dfcfe3',
+        'md5': '568acf9ca25a639f0c4ff905826b662f',
         'info_dict': {
             'id': '2REGtUDQ',
             'ext': 'mp4',

From 50ae3f646e6c7eec187078dd43f1cb3777ccfb4c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Jul 2017 20:04:08 +0700
Subject: [PATCH 1056/1696] [thisoldhouse] Add more fallbacks for video id
 (closes #13541)

---
 youtube_dl/extractor/thisoldhouse.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/thisoldhouse.py b/youtube_dl/extractor/thisoldhouse.py
index 0c321354f..6ab147ad7 100644
--- a/youtube_dl/extractor/thisoldhouse.py
+++ b/youtube_dl/extractor/thisoldhouse.py
@@ -2,6 +2,8 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import try_get
 
 
 class ThisOldHouseIE(InfoExtractor):
@@ -28,8 +30,15 @@ class ThisOldHouseIE(InfoExtractor):
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
-        drupal_settings = self._parse_json(self._search_regex(
-            r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);',
-            webpage, 'drupal settings'), display_id)
-        video_id = list(drupal_settings['comScore'])[0]
+        video_id = self._search_regex(
+            (r'data-mid=(["\'])(?P<id>(?:(?!\1).)+)\1',
+             r'id=(["\'])inline-video-player-(?P<id>(?:(?!\1).)+)\1'),
+            webpage, 'video id', default=None, group='id')
+        if not video_id:
+            drupal_settings = self._parse_json(self._search_regex(
+                r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);',
+                webpage, 'drupal settings'), display_id)
+            video_id = try_get(
+                drupal_settings, lambda x: x['jwplatform']['video_id'],
+                compat_str) or list(drupal_settings['comScore'])[0]
         return self.url_result('jwplatform:' + video_id, 'JWPlatform', video_id)

From 4d9ba27bba916d4e305d41fbef0685170eb1991f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Jul 2017 20:12:40 +0700
Subject: [PATCH 1057/1696] [ChangeLog] Actualize

---
 ChangeLog | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 7b3c6c6cf..c17acf45e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,22 @@
 version <unreleased>
 
+Core
+* [extractor/common] Improve _json_ld
+
 Extractors
++ [thisoldhouse] Add more fallbacks for video id
+* [thisoldhouse] Fix video id extraction (#13540, #13541)
+* [xfileshare] Extend format regular expression (#13536)
+* [ted] Fix extraction (#13535)
++ [tastytrade] Add support for tastytrade.com (#13521)
+* [dplayit] Relax video id regular expression (#13524)
++ [generic] Extract more generic metadata (#13527)
++ [bbccouk] Capture and output error message (#13501, #13518)
+* [cbsnews] Relax video info regular expression (#13284, #13503)
++ [facebook] Add support for plugin video embeds and multiple embeds (#13493)
+* [soundcloud] Switch to https for API requests (#13502)
+* [pandatv] Switch to https for API and download URLs
++ [pandatv] Add support for https URLs (#13491)
 + [niconico] Support sp subdomain (#13494)
 
 

From b6c9fe416243373bcb59eb8aa5ef0baca8f3c97c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Jul 2017 20:17:10 +0700
Subject: [PATCH 1058/1696] release 2017.07.02

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 1 +
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 82bbbdaa6..974603507 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.06.25*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.06.25**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.07.02*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.07.02**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.06.25
+[debug] youtube-dl version 2017.07.02
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index c17acf45e..4c2011326 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.07.02
 
 Core
 * [extractor/common] Improve _json_ld
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 010ff762c..db2e2bac9 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -768,6 +768,7 @@
  - **Tagesschau**
  - **tagesschau:player**
  - **Tass**
+ - **TastyTrade**
  - **TBS**
  - **TDSLifeway**
  - **teachertube**: teachertube.com videos
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b6d378896..0db974f97 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.06.25'
+__version__ = '2017.07.02'

From 609ff8ca19f1c4c168a81121074b91cc0f0d4c47 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 5 Jul 2017 23:23:35 +0800
Subject: [PATCH 1059/1696] [utils] Support attributes with no values in
 get_elements_by_attribute()

---
 test/test_utils.py  | 6 ++++++
 youtube_dl/utils.py | 4 ++--
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 2b93b3604..30738e7cf 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -1228,6 +1228,12 @@ part 3</font></u>
         self.assertEqual(get_element_by_attribute('class', 'foo', html), None)
         self.assertEqual(get_element_by_attribute('class', 'no-such-foo', html), None)
 
+        html = '''
+            <div itemprop="author" itemscope>foo</div>
+        '''
+
+        self.assertEqual(get_element_by_attribute('itemprop', 'author', html), 'foo')
+
     def test_get_elements_by_class(self):
         html = '''
             <span class="foo bar">nice</span><span class="foo bar">also nice</span>
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 39860e9d1..fdf5e29e7 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -365,9 +365,9 @@ def get_elements_by_attribute(attribute, value, html, escape_value=True):
     retlist = []
     for m in re.finditer(r'''(?xs)
         <([a-zA-Z0-9:._-]+)
-         (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'))*?
+         (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'|))*?
          \s+%s=['"]?%s['"]?
-         (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'))*?
+         (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'|))*?
         \s*>
         (?P<content>.*?)
         </\1>

From babbc04d459a6e8b6a2e65c522107cce1ca2cbbb Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 13 Jun 2017 16:21:26 +0800
Subject: [PATCH 1060/1696] [xuite] Move to the new HTML5 API and reduce # of
 requests

---
 youtube_dl/extractor/xuite.py | 88 ++++++++++++++---------------------
 1 file changed, 35 insertions(+), 53 deletions(-)

diff --git a/youtube_dl/extractor/xuite.py b/youtube_dl/extractor/xuite.py
index e0818201a..0276c0dbb 100644
--- a/youtube_dl/extractor/xuite.py
+++ b/youtube_dl/extractor/xuite.py
@@ -1,14 +1,13 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import base64
-
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     ExtractorError,
+    float_or_none,
+    get_element_by_attribute,
     parse_iso8601,
-    parse_duration,
+    remove_end,
 )
 
 
@@ -24,6 +23,7 @@ class XuiteIE(InfoExtractor):
             'id': '3860914',
             'ext': 'mp3',
             'title': '孤單南半球-歐德陽',
+            'description': '孤單南半球-歐德陽',
             'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 247.246,
             'timestamp': 1314932940,
@@ -44,7 +44,7 @@ class XuiteIE(InfoExtractor):
             'duration': 596.458,
             'timestamp': 1454242500,
             'upload_date': '20160131',
-            'uploader': 'yan12125',
+            'uploader': '屁姥',
             'uploader_id': '12158353',
             'categories': ['個人短片'],
             'description': 'http://download.blender.org/peach/bigbuckbunny_movies/BigBuckBunny_320x180.mp4',
@@ -72,10 +72,10 @@ class XuiteIE(InfoExtractor):
         # from http://forgetfulbc.blogspot.com/2016/06/date.html
         'url': 'http://vlog.xuite.net/embed/cE1xbENoLTI3NDQ3MzM2LmZsdg==?ar=0&as=0',
         'info_dict': {
-            'id': 'cE1xbENoLTI3NDQ3MzM2LmZsdg==',
+            'id': '27447336',
             'ext': 'mp4',
             'title': '男女平權只是口號？專家解釋約會時男生是否該幫女生付錢 (中字)',
-            'description': 'md5:f0abdcb69df300f522a5442ef3146f2a',
+            'description': 'md5:1223810fa123b179083a3aed53574706',
             'timestamp': 1466160960,
             'upload_date': '20160617',
             'uploader': 'B.C. & Lowy',
@@ -86,29 +86,9 @@ class XuiteIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    @staticmethod
-    def base64_decode_utf8(data):
-        return base64.b64decode(data.encode('utf-8')).decode('utf-8')
-
-    @staticmethod
-    def base64_encode_utf8(data):
-        return base64.b64encode(data.encode('utf-8')).decode('utf-8')
-
-    def _extract_flv_config(self, encoded_media_id):
-        flv_config = self._download_xml(
-            'http://vlog.xuite.net/flash/player?media=%s' % encoded_media_id,
-            'flv config')
-        prop_dict = {}
-        for prop in flv_config.findall('./property'):
-            prop_id = self.base64_decode_utf8(prop.attrib['id'])
-            # CDATA may be empty in flv config
-            if not prop.text:
-                continue
-            encoded_content = self.base64_decode_utf8(prop.text)
-            prop_dict[prop_id] = compat_urllib_parse_unquote(encoded_content)
-        return prop_dict
-
     def _real_extract(self, url):
+        # /play/ URLs provide embedded video URL and more metadata
+        url = url.replace('/embed/', '/play/')
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
@@ -121,51 +101,53 @@ class XuiteIE(InfoExtractor):
                 '%s returned error: %s' % (self.IE_NAME, error_msg),
                 expected=True)
 
-        encoded_media_id = self._search_regex(
-            r'attributes\.name\s*=\s*"([^"]+)"', webpage,
-            'encoded media id', default=None)
-        if encoded_media_id is None:
-            video_id = self._html_search_regex(
-                r'data-mediaid="(\d+)"', webpage, 'media id')
-            encoded_media_id = self.base64_encode_utf8(video_id)
-        flv_config = self._extract_flv_config(encoded_media_id)
+        media_info = self._parse_json(self._search_regex(
+            r'var\s+mediaInfo\s*=\s*({.*});', webpage, 'media info'), video_id)
 
-        FORMATS = {
-            'audio': 'mp3',
-            'video': 'mp4',
-        }
+        video_id = media_info['MEDIA_ID']
 
         formats = []
-        for format_tag in ('src', 'hq_src'):
-            video_url = flv_config.get(format_tag)
+        for key in ('html5Url', 'html5HQUrl'):
+            video_url = media_info.get(key)
             if not video_url:
                 continue
             format_id = self._search_regex(
-                r'\bq=(.+?)\b', video_url, 'format id', default=format_tag)
+                r'\bq=(.+?)\b', video_url, 'format id', default=None)
             formats.append({
                 'url': video_url,
-                'ext': FORMATS.get(flv_config['type'], 'mp4'),
+                'ext': 'mp4' if format_id.isnumeric() else format_id,
                 'format_id': format_id,
                 'height': int(format_id) if format_id.isnumeric() else None,
             })
         self._sort_formats(formats)
 
-        timestamp = flv_config.get('publish_datetime')
+        timestamp = media_info.get('PUBLISH_DATETIME')
         if timestamp:
             timestamp = parse_iso8601(timestamp + ' +0800', ' ')
 
-        category = flv_config.get('category')
+        category = media_info.get('catName')
         categories = [category] if category else []
 
+        uploader = media_info.get('NICKNAME')
+        uploader_url = None
+
+        author_div = get_element_by_attribute('itemprop', 'author', webpage)
+        if author_div:
+            uploader = uploader or self._html_search_meta('name', author_div)
+            uploader_url = self._html_search_regex(
+                r'<link[^>]+itemprop="url"[^>]+href="([^"]+)"', author_div,
+                'uploader URL', fatal=False)
+
         return {
             'id': video_id,
-            'title': flv_config['title'],
-            'description': flv_config.get('description'),
-            'thumbnail': flv_config.get('thumb'),
+            'title': media_info['TITLE'],
+            'description': remove_end(media_info.get('metaDesc'), ' (Xuite 影音)'),
+            'thumbnail': media_info.get('ogImageUrl'),
             'timestamp': timestamp,
-            'uploader': flv_config.get('author_name'),
-            'uploader_id': flv_config.get('author_id'),
-            'duration': parse_duration(flv_config.get('duration')),
+            'uploader': uploader,
+            'uploader_id': media_info.get('MEMBER_ID'),
+            'uploader_url': uploader_url,
+            'duration': float_or_none(media_info.get('MEDIA_DURATION'), 1000000),
             'categories': categories,
             'formats': formats,
         }

From 0a2e1b2e30045de7834aca880d35253c5e8a3812 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Jul 2017 22:13:47 +0700
Subject: [PATCH 1061/1696] [vier] Adapt extraction to redesign (#13575)

---
 youtube_dl/extractor/vier.py | 45 +++++++++++++++++++++++++++++-------
 1 file changed, 37 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py
index 3e67eb8c2..dbd5ba9ba 100644
--- a/youtube_dl/extractor/vier.py
+++ b/youtube_dl/extractor/vier.py
@@ -15,7 +15,21 @@ from ..utils import (
 class VierIE(InfoExtractor):
     IE_NAME = 'vier'
     IE_DESC = 'vier.be and vijf.be'
-    _VALID_URL = r'https?://(?:www\.)?(?P<site>vier|vijf)\.be/(?:[^/]+/videos/(?P<display_id>[^/]+)(?:/(?P<id>\d+))?|video/v3/embed/(?P<embed_id>\d+))'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:www\.)?(?P<site>vier|vijf)\.be/
+                        (?:
+                            (?:
+                                [^/]+/videos|
+                                video(?:/[^/]+)*
+                            )/
+                            (?P<display_id>[^/]+)(?:/(?P<id>\d+))?|
+                            (?:
+                                video/v3/embed|
+                                embed/video/public
+                            )/(?P<embed_id>\d+)
+                        )
+                    '''
     _NETRC_MACHINE = 'vier'
     _TESTS = [{
         'url': 'http://www.vier.be/planb/videos/het-wordt-warm-de-moestuin/16129',
@@ -83,6 +97,15 @@ class VierIE(InfoExtractor):
     }, {
         'url': 'http://www.vier.be/video/v3/embed/16129',
         'only_matching': True,
+    }, {
+        'url': 'https://www.vijf.be/embed/video/public/4093',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.vier.be/video/blockbusters/in-juli-en-augustus-summer-classics',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.vier.be/video/achter-de-rug/2017/achter-de-rug-seizoen-1-aflevering-6',
+        'only_matching': True,
     }]
 
     def _real_initialize(self):
@@ -133,14 +156,20 @@ class VierIE(InfoExtractor):
         video_id = self._search_regex(
             [r'data-nid="(\d+)"', r'"nid"\s*:\s*"(\d+)"'],
             webpage, 'video id', default=video_id or display_id)
-        application = self._search_regex(
-            [r'data-application="([^"]+)"', r'"application"\s*:\s*"([^"]+)"'],
-            webpage, 'application', default=site + '_vod')
-        filename = self._search_regex(
-            [r'data-filename="([^"]+)"', r'"filename"\s*:\s*"([^"]+)"'],
-            webpage, 'filename')
 
-        playlist_url = 'http://vod.streamcloud.be/%s/_definst_/mp4:%s.mp4/playlist.m3u8' % (application, filename)
+        playlist_url = self._search_regex(
+            r'data-file=(["\'])(?P<url>(?:https?:)?//[^/]+/.+?\.m3u8.*?)\1',
+            webpage, 'm3u8 url', default=None, group='url')
+
+        if not playlist_url:
+            application = self._search_regex(
+                [r'data-application="([^"]+)"', r'"application"\s*:\s*"([^"]+)"'],
+                webpage, 'application', default=site + '_vod')
+            filename = self._search_regex(
+                [r'data-filename="([^"]+)"', r'"filename"\s*:\s*"([^"]+)"'],
+                webpage, 'filename')
+            playlist_url = 'http://vod.streamcloud.be/%s/_definst_/mp4:%s.mp4/playlist.m3u8' % (application, filename)
+
         formats = self._extract_wowza_formats(
             playlist_url, display_id, skip_protocols=['dash'])
         self._sort_formats(formats)

From 655470825231eaa03b4b82cbc1314d551e72a01e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Jul 2017 23:20:50 +0700
Subject: [PATCH 1062/1696] [kaltura] Fix typo in subtitles extraction (closes
 #13569)

---
 youtube_dl/extractor/kaltura.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 41c1f3d96..138d4844d 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -324,7 +324,7 @@ class KalturaIE(InfoExtractor):
         if captions:
             for caption in captions.get('objects', []):
                 # Continue if caption is not ready
-                if f.get('status') != 2:
+                if caption.get('status') != 2:
                     continue
                 if not caption.get('id'):
                     continue

From dee2ff1d818bebd74990b7cebbc698f22163a43a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 6 Jul 2017 00:25:37 +0700
Subject: [PATCH 1063/1696] [test_utils] Fix tests under Windows

---
 test/test_utils.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 30738e7cf..7803e5bc7 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -98,6 +98,7 @@ from youtube_dl.compat import (
     compat_chr,
     compat_etree_fromstring,
     compat_getenv,
+    compat_os_name,
     compat_setenv,
     compat_urlparse,
     compat_parse_qs,
@@ -448,7 +449,9 @@ class TestUtil(unittest.TestCase):
 
     def test_shell_quote(self):
         args = ['ffmpeg', '-i', encodeFilename('ñ€ß\'.mp4')]
-        self.assertEqual(shell_quote(args), """ffmpeg -i 'ñ€ß'"'"'.mp4'""")
+        self.assertEqual(
+            shell_quote(args),
+            """ffmpeg -i 'ñ€ß'"'"'.mp4'""" if compat_os_name != 'nt' else '''ffmpeg -i "ñ€ß'.mp4"''')
 
     def test_str_to_int(self):
         self.assertEqual(str_to_int('123,456'), 123456)
@@ -932,7 +935,7 @@ class TestUtil(unittest.TestCase):
     def test_args_to_str(self):
         self.assertEqual(
             args_to_str(['foo', 'ba/r', '-baz', '2 be', '']),
-            'foo ba/r -baz \'2 be\' \'\''
+            'foo ba/r -baz \'2 be\' \'\'' if compat_os_name != 'nt' else 'foo ba/r -baz "2 be" ""'
         )
 
     def test_parse_filesize(self):

From 60d4401c5e14d94574e2418d229dcc0c067d3559 Mon Sep 17 00:00:00 2001
From: Parmjit Virk <pvirk@mts.net>
Date: Thu, 6 Jul 2017 10:55:59 -0500
Subject: [PATCH 1064/1696] [espn] Extend _VALID_URL (fixes #13244)

---
 youtube_dl/extractor/espn.py | 48 ++++++++++++++++++++++++++++++------
 1 file changed, 40 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/espn.py b/youtube_dl/extractor/espn.py
index 8795e0ddf..7a7436068 100644
--- a/youtube_dl/extractor/espn.py
+++ b/youtube_dl/extractor/espn.py
@@ -10,7 +10,25 @@ from ..utils import (
 
 
 class ESPNIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:espn\.go|(?:www\.)?espn)\.com/video/clip(?:\?.*?\bid=|/_/id/)(?P<id>\d+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:
+                            (?:(?:\w+\.)+)?espn\.go|
+                            (?:www\.)?espn
+                        )\.com/
+                        (?:
+                            (?:
+                                video/clip|
+                                watch/player
+                            )
+                            (?:
+                                \?.*?\bid=|
+                                /_/id/
+                            )
+                        )
+                        (?P<id>\d+)
+                    '''
+
     _TESTS = [{
         'url': 'http://espn.go.com/video/clip?id=10365079',
         'info_dict': {
@@ -25,20 +43,34 @@ class ESPNIE(InfoExtractor):
             'skip_download': True,
         },
     }, {
-        # intl video, from http://www.espnfc.us/video/mls-highlights/150/video/2743663/must-see-moments-best-of-the-mls-season
-        'url': 'http://espn.go.com/video/clip?id=2743663',
+        'url': 'https://broadband.espn.go.com/video/clip?id=18910086',
         'info_dict': {
-            'id': '2743663',
+            'id': '18910086',
             'ext': 'mp4',
-            'title': 'Must-See Moments: Best of the MLS season',
-            'description': 'md5:4c2d7232beaea572632bec41004f0aeb',
-            'timestamp': 1449446454,
-            'upload_date': '20151207',
+            'title': 'Kyrie spins around defender for two',
+            'description': 'md5:2b0f5bae9616d26fba8808350f0d2b9b',
+            'timestamp': 1489539155,
+            'upload_date': '20170315',
         },
         'params': {
             'skip_download': True,
         },
         'expected_warnings': ['Unable to download f4m manifest'],
+    }, {
+        'url': 'http://nonredline.sports.espn.go.com/video/clip?id=19744672',
+        'only_matching': True,
+    }, {
+        'url': 'https://cdn.espn.go.com/video/clip/_/id/19771774',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.espn.com/watch/player?id=19141491',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.espn.com/watch/player?bucketId=257&id=19505875',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.espn.com/watch/player/_/id/19141491',
+        'only_matching': True,
     }, {
         'url': 'http://www.espn.com/video/clip?id=10365079',
         'only_matching': True,

From ddeff4be3fb491a82d775fcd4140fc49a443864c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 6 Jul 2017 23:03:51 +0700
Subject: [PATCH 1065/1696] Credit @gfabiano for #13382, #13385, #13415

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index e2bdebe5e..2f828adb5 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -220,3 +220,4 @@ gritstub
 Adam Voss
 Mike Fährmann
 Jan Kundrát
+Giuseppe Fabiano

From ab328411d53eee079a66f373bc83cf09d207dae4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 6 Jul 2017 23:05:27 +0700
Subject: [PATCH 1066/1696] Credit @orng for ruv (#13396)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 2f828adb5..ec58744cf 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -221,3 +221,4 @@ Adam Voss
 Mike Fährmann
 Jan Kundrát
 Giuseppe Fabiano
+Örn Guðjónsson

From 8a04ade86bf3a7f0a321232850a2d03a52c47819 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 6 Jul 2017 23:06:53 +0700
Subject: [PATCH 1067/1696] Credit @parmjitv for #13322, #13503, #13541, #13549

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index ec58744cf..053159cc3 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -222,3 +222,4 @@ Mike Fährmann
 Jan Kundrát
 Giuseppe Fabiano
 Örn Guðjónsson
+Parmjit Virk

From 00e5c363159f7771dc3a2e33b1a38af520782372 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 7 Jul 2017 22:22:29 +0700
Subject: [PATCH 1068/1696] [xhamster] Add support for new URL schema (closes
 #13593)

---
 youtube_dl/extractor/xhamster.py | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 6987b2e3a..de9ec9b9e 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -14,7 +14,15 @@ from ..utils import (
 
 
 class XHamsterIE(InfoExtractor):
-    _VALID_URL = r'(?P<proto>https?)://(?:.+?\.)?xhamster\.com/movies/(?P<id>[0-9]+)/(?P<seo>.*?)\.html(?:\?.*)?'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:.+?\.)?xhamster\.com/
+                        (?:
+                            movies/(?P<id>\d+)/(?P<display_id>[^/]*)\.html|
+                            videos/(?P<display_id_2>[^/]*)-(?P<id_2>\d+)
+                        )
+                    '''
+
     _TESTS = [{
         'url': 'http://xhamster.com/movies/1509445/femaleagent_shy_beauty_takes_the_bait.html',
         'md5': '8281348b8d3c53d39fffb377d24eac4e',
@@ -66,6 +74,10 @@ class XHamsterIE(InfoExtractor):
         # This video is visible for marcoalfa123456's friends only
         'url': 'https://it.xhamster.com/movies/7263980/la_mia_vicina.html',
         'only_matching': True,
+    }, {
+        # new URL schema
+        'url': 'https://pt.xhamster.com/videos/euro-pedal-pumping-7937821',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -81,11 +93,10 @@ class XHamsterIE(InfoExtractor):
 
         mobj = re.match(self._VALID_URL, url)
 
-        video_id = mobj.group('id')
-        seo = mobj.group('seo')
-        proto = mobj.group('proto')
-        mrss_url = '%s://xhamster.com/movies/%s/%s.html' % (proto, video_id, seo)
-        webpage = self._download_webpage(mrss_url, video_id)
+        video_id = mobj.group('id') or mobj.group('id_2')
+        display_id = mobj.group('display_id') or mobj.group('display_id_2')
+
+        webpage = self._download_webpage(url, video_id)
 
         error = self._html_search_regex(
             r'<div[^>]+id=["\']videoClosed["\'][^>]*>(.+?)</div>',

From d852c6bc5995444096a6c07b66ea075269412cf7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 7 Jul 2017 22:49:11 +0700
Subject: [PATCH 1069/1696] [xhamster] Extract all formats and fix duration
 extraction (#13593)

---
 youtube_dl/extractor/xhamster.py | 75 +++++++++++++++++---------------
 1 file changed, 39 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index de9ec9b9e..c42b59e51 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     clean_html,
     dict_get,
@@ -28,6 +29,7 @@ class XHamsterIE(InfoExtractor):
         'md5': '8281348b8d3c53d39fffb377d24eac4e',
         'info_dict': {
             'id': '1509445',
+            'display_id': 'femaleagent_shy_beauty_takes_the_bait',
             'ext': 'mp4',
             'title': 'FemaleAgent Shy beauty takes the bait',
             'upload_date': '20121014',
@@ -40,6 +42,7 @@ class XHamsterIE(InfoExtractor):
         'url': 'http://xhamster.com/movies/2221348/britney_spears_sexy_booty.html?hd',
         'info_dict': {
             'id': '2221348',
+            'display_id': 'britney_spears_sexy_booty',
             'ext': 'mp4',
             'title': 'Britney Spears  Sexy Booty',
             'upload_date': '20130914',
@@ -81,18 +84,7 @@ class XHamsterIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        def extract_video_url(webpage, name):
-            return self._search_regex(
-                [r'''file\s*:\s*(?P<q>["'])(?P<mp4>.+?)(?P=q)''',
-                 r'''<a\s+href=(?P<q>["'])(?P<mp4>.+?)(?P=q)\s+class=["']mp4Thumb''',
-                 r'''<video[^>]+file=(?P<q>["'])(?P<mp4>.+?)(?P=q)[^>]*>'''],
-                webpage, name, group='mp4')
-
-        def is_hd(webpage):
-            return '<div class=\'icon iconHD\'' in webpage
-
         mobj = re.match(self._VALID_URL, url)
-
         video_id = mobj.group('id') or mobj.group('id_2')
         display_id = mobj.group('display_id') or mobj.group('display_id_2')
 
@@ -110,6 +102,39 @@ class XHamsterIE(InfoExtractor):
              r'<title[^>]*>(.+?)(?:,\s*[^,]*?\s*Porn\s*[^,]*?:\s*xHamster[^<]*| - xHamster\.com)</title>'],
             webpage, 'title')
 
+        formats = []
+        format_urls = set()
+
+        sources = self._parse_json(
+            self._search_regex(
+                r'sources\s*:\s*({.+?})\s*,?\s*\n', webpage, 'sources',
+                default='{}'),
+            video_id, fatal=False)
+        for format_id, format_url in sources.items():
+            if not isinstance(format_url, compat_str):
+                continue
+            if format_url in format_urls:
+                continue
+            format_urls.add(format_url)
+            formats.append({
+                'format_id': format_id,
+                'url': format_url,
+                'height': int_or_none(self._search_regex(
+                    r'^(\d+)[pP]', format_id, 'height', default=None))
+            })
+
+        video_url = self._search_regex(
+            [r'''file\s*:\s*(?P<q>["'])(?P<mp4>.+?)(?P=q)''',
+             r'''<a\s+href=(?P<q>["'])(?P<mp4>.+?)(?P=q)\s+class=["']mp4Thumb''',
+             r'''<video[^>]+file=(?P<q>["'])(?P<mp4>.+?)(?P=q)[^>]*>'''],
+            webpage, 'video url', group='mp4', default=None)
+        if video_url and video_url not in format_urls:
+            formats.append({
+                'url': video_url,
+            })
+
+        self._sort_formats(formats)
+
         # Only a few videos have an description
         mobj = re.search(r'<span>Description: </span>([^<]+)', webpage)
         description = mobj.group(1) if mobj else None
@@ -128,7 +153,8 @@ class XHamsterIE(InfoExtractor):
             webpage, 'thumbnail', fatal=False, group='thumbnail')
 
         duration = parse_duration(self._search_regex(
-            r'Runtime:\s*</span>\s*([\d:]+)', webpage,
+            [r'<[^<]+\bitemprop=["\']duration["\'][^<]+\bcontent=["\'](.+?)["\']',
+             r'Runtime:\s*</span>\s*([\d:]+)'], webpage,
             'duration', fatal=False))
 
         view_count = int_or_none(self._search_regex(
@@ -143,30 +169,6 @@ class XHamsterIE(InfoExtractor):
 
         age_limit = self._rta_search(webpage)
 
-        hd = is_hd(webpage)
-
-        format_id = 'hd' if hd else 'sd'
-
-        video_url = extract_video_url(webpage, format_id)
-        formats = [{
-            'url': video_url,
-            'format_id': 'hd' if hd else 'sd',
-            'preference': 1,
-        }]
-
-        if not hd:
-            mrss_url = self._search_regex(r'<link rel="canonical" href="([^"]+)', webpage, 'mrss_url')
-            webpage = self._download_webpage(mrss_url + '?hd', video_id, note='Downloading HD webpage')
-            if is_hd(webpage):
-                video_url = extract_video_url(webpage, 'hd')
-                formats.append({
-                    'url': video_url,
-                    'format_id': 'hd',
-                    'preference': 2,
-                })
-
-        self._sort_formats(formats)
-
         categories_html = self._search_regex(
             r'(?s)<table.+?(<span>Categories:.+?)</table>', webpage,
             'categories', default=None)
@@ -175,6 +177,7 @@ class XHamsterIE(InfoExtractor):
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': title,
             'description': description,
             'upload_date': upload_date,

From eadd313321f4fbacac2ec10f2f382197e401d7f7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 8 Jul 2017 15:48:05 +0800
Subject: [PATCH 1070/1696] [yam] Remove extractor

mymedia.yam.com is dead. An wikipedia user also pointed out that Yam's
blog service is no longer available. [1]

[1] https://zh.wikipedia.org/zh-tw/%E5%A4%A9%E7%A9%BA%E9%83%A8%E8%90%BD
---
 ChangeLog                          |   6 ++
 youtube_dl/extractor/extractors.py |   1 -
 youtube_dl/extractor/yam.py        | 123 -----------------------------
 3 files changed, 6 insertions(+), 124 deletions(-)
 delete mode 100644 youtube_dl/extractor/yam.py

diff --git a/ChangeLog b/ChangeLog
index 4c2011326..5d07c12cb 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+- [yam] Remove extractor
+
+
 version 2017.07.02
 
 Core
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index bbdb4a2fe..b83c3aba5 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1280,7 +1280,6 @@ from .yahoo import (
     YahooIE,
     YahooSearchIE,
 )
-from .yam import YamIE
 from .yandexmusic import (
     YandexMusicTrackIE,
     YandexMusicAlbumIE,
diff --git a/youtube_dl/extractor/yam.py b/youtube_dl/extractor/yam.py
deleted file mode 100644
index ef5535547..000000000
--- a/youtube_dl/extractor/yam.py
+++ /dev/null
@@ -1,123 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..compat import compat_urlparse
-from ..utils import (
-    float_or_none,
-    month_by_abbreviation,
-    ExtractorError,
-    get_element_by_attribute,
-)
-
-
-class YamIE(InfoExtractor):
-    IE_DESC = '蕃薯藤yam天空部落'
-    _VALID_URL = r'https?://mymedia\.yam\.com/m/(?P<id>\d+)'
-
-    _TESTS = [{
-        # An audio hosted on Yam
-        'url': 'http://mymedia.yam.com/m/2283921',
-        'md5': 'c011b8e262a52d5473d9c2e3c9963b9c',
-        'info_dict': {
-            'id': '2283921',
-            'ext': 'mp3',
-            'title': '發現 - 趙薇 京華煙雲主題曲',
-            'description': '發現 - 趙薇 京華煙雲主題曲',
-            'uploader_id': 'princekt',
-            'upload_date': '20080807',
-            'duration': 313.0,
-        }
-    }, {
-        # An external video hosted on YouTube
-        'url': 'http://mymedia.yam.com/m/3599430',
-        'md5': '03127cf10d8f35d120a9e8e52e3b17c6',
-        'info_dict': {
-            'id': 'CNpEoQlrIgA',
-            'ext': 'mp4',
-            'upload_date': '20150306',
-            'uploader': '新莊社大瑜伽社',
-            'description': 'md5:11e2e405311633ace874f2e6226c8b17',
-            'uploader_id': '2323agoy',
-            'title': '20090412陽明山二子坪-1',
-        },
-        'skip': 'Video does not exist',
-    }, {
-        'url': 'http://mymedia.yam.com/m/3598173',
-        'info_dict': {
-            'id': '3598173',
-            'ext': 'mp4',
-        },
-        'skip': 'cause Yam system error',
-    }, {
-        'url': 'http://mymedia.yam.com/m/3599437',
-        'info_dict': {
-            'id': '3599437',
-            'ext': 'mp4',
-        },
-        'skip': 'invalid YouTube URL',
-    }, {
-        'url': 'http://mymedia.yam.com/m/2373534',
-        'md5': '7ff74b91b7a817269d83796f8c5890b1',
-        'info_dict': {
-            'id': '2373534',
-            'ext': 'mp3',
-            'title': '林俊傑&蔡卓妍-小酒窩',
-            'description': 'md5:904003395a0fcce6cfb25028ff468420',
-            'upload_date': '20080928',
-            'uploader_id': 'onliner2',
-        }
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        page = self._download_webpage(url, video_id)
-
-        # Check for errors
-        system_msg = self._html_search_regex(
-            r'系統訊息(?:<br>|\n|\r)*([^<>]+)<br>', page, 'system message',
-            default=None)
-        if system_msg:
-            raise ExtractorError(system_msg, expected=True)
-
-        # Is it hosted externally on YouTube?
-        youtube_url = self._html_search_regex(
-            r'<embed src="(http://www.youtube.com/[^"]+)"',
-            page, 'YouTube url', default=None)
-        if youtube_url:
-            return self.url_result(youtube_url, 'Youtube')
-
-        title = self._html_search_regex(
-            r'<h1[^>]+class="heading"[^>]*>\s*(.+)\s*</h1>', page, 'title')
-
-        api_page = self._download_webpage(
-            'http://mymedia.yam.com/api/a/?pID=' + video_id, video_id,
-            note='Downloading API page')
-        api_result_obj = compat_urlparse.parse_qs(api_page)
-
-        info_table = get_element_by_attribute('class', 'info', page)
-        uploader_id = self._html_search_regex(
-            r'<!-- 發表作者 -->：[\n ]+<a href="/([a-z0-9]+)"',
-            info_table, 'uploader id', fatal=False)
-        mobj = re.search(r'<!-- 發表於 -->(?P<mon>[A-Z][a-z]{2})\s+' +
-                         r'(?P<day>\d{1,2}), (?P<year>\d{4})', page)
-        if mobj:
-            upload_date = '%s%02d%02d' % (
-                mobj.group('year'),
-                month_by_abbreviation(mobj.group('mon')),
-                int(mobj.group('day')))
-        else:
-            upload_date = None
-        duration = float_or_none(api_result_obj['totaltime'][0], scale=1000)
-
-        return {
-            'id': video_id,
-            'url': api_result_obj['mp3file'][0],
-            'title': title,
-            'description': self._html_search_meta('description', page),
-            'duration': duration,
-            'uploader_id': uploader_id,
-            'upload_date': upload_date,
-        }

From a49804816c0246d81b9d34d9f89f99fae06da887 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Jul 2017 18:12:15 +0700
Subject: [PATCH 1071/1696] [dailymotion] Add support for new layout (close
 #13580)

---
 youtube_dl/extractor/dailymotion.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index f8db76c18..74e991331 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -147,7 +147,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
         view_count_str = self._search_regex(
             (r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserPlays:([\s\d,.]+)"',
              r'video_views_count[^>]+>\s+([\s\d\,.]+)'),
-            webpage, 'view count', fatal=False)
+            webpage, 'view count', default=None)
         if view_count_str:
             view_count_str = re.sub(r'\s', '', view_count_str)
         view_count = str_to_int(view_count_str)
@@ -159,7 +159,9 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
             [r'buildPlayer\(({.+?})\);\n',  # See https://github.com/rg3/youtube-dl/issues/7826
              r'playerV5\s*=\s*dmp\.create\([^,]+?,\s*({.+?})\);',
              r'buildPlayer\(({.+?})\);',
-             r'var\s+config\s*=\s*({.+?});'],
+             r'var\s+config\s*=\s*({.+?});',
+             # New layout regex (see https://github.com/rg3/youtube-dl/issues/13580)
+             r'__PLAYER_CONFIG__\s*=\s*({.+?});'],
             webpage, 'player v5', default=None)
         if player_v5:
             player = self._parse_json(player_v5, video_id)

From 8b347a389eaa6d545ada901c2e236a5eb2272960 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 00:26:13 +0700
Subject: [PATCH 1072/1696] [googledrive] Fix height extraction (closes #13603)

---
 youtube_dl/extractor/googledrive.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
index 9705cfadd..c40da85c5 100644
--- a/youtube_dl/extractor/googledrive.py
+++ b/youtube_dl/extractor/googledrive.py
@@ -92,7 +92,7 @@ class GoogleDriveIE(InfoExtractor):
             if resolution:
                 f.update({
                     'width': resolution[0],
-                    'height': resolution[0],
+                    'height': resolution[1],
                 })
             formats.append(f)
         self._sort_formats(formats)

From 7a5773090789bec38a3f58dfb09039155919a540 Mon Sep 17 00:00:00 2001
From: rrooij <rrooij@users.noreply.github.com>
Date: Sun, 9 Jul 2017 09:21:40 +0200
Subject: [PATCH 1073/1696] [npo:live] Fix live stream id extraction (closes
 #13568)

---
 youtube_dl/extractor/npo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 5f8b6def1..516b1e941 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -341,7 +341,7 @@ class NPOLiveIE(NPOBaseIE):
         webpage = self._download_webpage(url, display_id)
 
         live_id = self._search_regex(
-            r'data-prid="([^"]+)"', webpage, 'live id')
+            [r'media-id="([^"]+)"', r'data-prid="([^"]+)"'], webpage, 'live id')
 
         return {
             '_type': 'url_transparent',

From 15237fcd51dca192103f08a910660616e3b241b8 Mon Sep 17 00:00:00 2001
From: mlindner <mlindner@users.noreply.github.com>
Date: Sun, 9 Jul 2017 00:54:52 -0700
Subject: [PATCH 1074/1696] [veoh] Extend _VALID_URL

---
 youtube_dl/extractor/veoh.py | 73 ++++++++++++++++++------------------
 1 file changed, 36 insertions(+), 37 deletions(-)

diff --git a/youtube_dl/extractor/veoh.py b/youtube_dl/extractor/veoh.py
index 0f5d68738..b20dddc5c 100644
--- a/youtube_dl/extractor/veoh.py
+++ b/youtube_dl/extractor/veoh.py
@@ -12,47 +12,46 @@ from ..utils import (
 
 
 class VeohIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?veoh\.com/(?:watch|iphone/#_Watch)/(?P<id>(?:v|yapi-)[\da-zA-Z]+)'
+    _VALID_URL = r'https?://(?:www\.)?veoh\.com/(?:watch|iphone/#_Watch)/(?P<id>(?:v|e|yapi-)[\da-zA-Z]+)'
 
-    _TESTS = [
-        {
-            'url': 'http://www.veoh.com/watch/v56314296nk7Zdmz3',
-            'md5': '620e68e6a3cff80086df3348426c9ca3',
-            'info_dict': {
-                'id': '56314296',
-                'ext': 'mp4',
-                'title': 'Straight Backs Are Stronger',
-                'uploader': 'LUMOback',
-                'description': 'At LUMOback, we believe straight backs are stronger.  The LUMOback Posture & Movement Sensor:  It gently vibrates when you slouch, inspiring improved posture and mobility.  Use the app to track your data and improve your posture over time. ',
-            },
+    _TESTS = [{
+        'url': 'http://www.veoh.com/watch/v56314296nk7Zdmz3',
+        'md5': '620e68e6a3cff80086df3348426c9ca3',
+        'info_dict': {
+            'id': '56314296',
+            'ext': 'mp4',
+            'title': 'Straight Backs Are Stronger',
+            'uploader': 'LUMOback',
+            'description': 'At LUMOback, we believe straight backs are stronger.  The LUMOback Posture & Movement Sensor:  It gently vibrates when you slouch, inspiring improved posture and mobility.  Use the app to track your data and improve your posture over time. ',
         },
-        {
-            'url': 'http://www.veoh.com/watch/v27701988pbTc4wzN?h1=Chile+workers+cover+up+to+avoid+skin+damage',
-            'md5': '4a6ff84b87d536a6a71e6aa6c0ad07fa',
-            'info_dict': {
-                'id': '27701988',
-                'ext': 'mp4',
-                'title': 'Chile workers cover up to avoid skin damage',
-                'description': 'md5:2bd151625a60a32822873efc246ba20d',
-                'uploader': 'afp-news',
-                'duration': 123,
-            },
-            'skip': 'This video has been deleted.',
+    }, {
+        'url': 'http://www.veoh.com/watch/v27701988pbTc4wzN?h1=Chile+workers+cover+up+to+avoid+skin+damage',
+        'md5': '4a6ff84b87d536a6a71e6aa6c0ad07fa',
+        'info_dict': {
+            'id': '27701988',
+            'ext': 'mp4',
+            'title': 'Chile workers cover up to avoid skin damage',
+            'description': 'md5:2bd151625a60a32822873efc246ba20d',
+            'uploader': 'afp-news',
+            'duration': 123,
         },
-        {
-            'url': 'http://www.veoh.com/watch/v69525809F6Nc4frX',
-            'md5': '4fde7b9e33577bab2f2f8f260e30e979',
-            'note': 'Embedded ooyala video',
-            'info_dict': {
-                'id': '69525809',
-                'ext': 'mp4',
-                'title': 'Doctors Alter Plan For Preteen\'s Weight Loss Surgery',
-                'description': 'md5:f5a11c51f8fb51d2315bca0937526891',
-                'uploader': 'newsy-videos',
-            },
-            'skip': 'This video has been deleted.',
+        'skip': 'This video has been deleted.',
+    }, {
+        'url': 'http://www.veoh.com/watch/v69525809F6Nc4frX',
+        'md5': '4fde7b9e33577bab2f2f8f260e30e979',
+        'note': 'Embedded ooyala video',
+        'info_dict': {
+            'id': '69525809',
+            'ext': 'mp4',
+            'title': 'Doctors Alter Plan For Preteen\'s Weight Loss Surgery',
+            'description': 'md5:f5a11c51f8fb51d2315bca0937526891',
+            'uploader': 'newsy-videos',
         },
-    ]
+        'skip': 'This video has been deleted.',
+    }, {
+        'url': 'http://www.veoh.com/watch/e152215AJxZktGS',
+        'only_matching': True,
+    }]
 
     def _extract_formats(self, source):
         formats = []

From 5af2fd7fa02734c2a23f917fb60f1c14da149d3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 15:55:04 +0700
Subject: [PATCH 1075/1696] [eagleplatform] Add support for another embed
 pattern (#13557)

---
 youtube_dl/extractor/eagleplatform.py | 36 ++++++++++++++++++++++-----
 1 file changed, 30 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index 76d39adac..5e1de04a1 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -60,16 +60,40 @@ class EaglePlatformIE(InfoExtractor):
             webpage)
         if mobj is not None:
             return mobj.group('url')
-        # Basic usage embedding (see http://dultonmedia.github.io/eplayer/)
+        PLAYER_JS_RE = r'''
+                        <script[^>]+
+                            src=(?P<qjs>["\'])(?:https?:)?//(?P<host>(?:(?!(?P=qjs)).)+\.media\.eagleplatform\.com)/player/player\.js(?P=qjs)
+                        .+?
+                    '''
+        # "Basic usage" embedding (see http://dultonmedia.github.io/eplayer/)
         mobj = re.search(
             r'''(?xs)
-                    <script[^>]+
-                        src=(?P<q1>["\'])(?:https?:)?//(?P<host>.+?\.media\.eagleplatform\.com)/player/player\.js(?P=q1)
-                    .+?
+                    %s
                     <div[^>]+
-                        class=(?P<q2>["\'])eagleplayer(?P=q2)[^>]+
+                        class=(?P<qclass>["\'])eagleplayer(?P=qclass)[^>]+
                         data-id=["\'](?P<id>\d+)
-            ''', webpage)
+            ''' % PLAYER_JS_RE, webpage)
+        if mobj is not None:
+            return 'eagleplatform:%(host)s:%(id)s' % mobj.groupdict()
+        # Generalization of "Javascript code usage", "Combined usage" and
+        # "Usage without attaching to DOM" embeddings (see
+        # http://dultonmedia.github.io/eplayer/)
+        mobj = re.search(
+            r'''(?xs)
+                    %s
+                    <script>
+                    .+?
+                    new\s+EaglePlayer\(
+                        (?:[^,]+\s*,\s*)?
+                        {
+                            .+?
+                            \bid\s*:\s*["\']?(?P<id>\d+)
+                            .+?
+                        }
+                    \s*\)
+                    .+?
+                    </script>
+            ''' % PLAYER_JS_RE, webpage)
         if mobj is not None:
             return 'eagleplatform:%(host)s:%(id)s' % mobj.groupdict()
 

From 665e9452461abaff7127653265c78bd585acea6c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 15:57:33 +0700
Subject: [PATCH 1076/1696] [eagleplatform] Add support for referrer protected
 videos (closes #13557)

---
 youtube_dl/extractor/eagleplatform.py | 25 ++++++++++++++++++++++---
 youtube_dl/extractor/generic.py       | 10 +++++-----
 2 files changed, 27 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index 5e1de04a1..34891a362 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -11,6 +11,7 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     int_or_none,
+    unsmuggle_url,
 )
 
 
@@ -50,6 +51,10 @@ class EaglePlatformIE(InfoExtractor):
             'view_count': int,
         },
         'skip': 'Georestricted',
+    }, {
+        # referrer protected video (https://tvrain.ru/lite/teleshow/kak_vse_nachinalos/namin-418921/)
+        'url': 'tvrainru.media.eagleplatform.com:582306',
+        'only_matching': True,
     }]
 
     @staticmethod
@@ -103,9 +108,10 @@ class EaglePlatformIE(InfoExtractor):
         if status != 200:
             raise ExtractorError(' '.join(response['errors']), expected=True)
 
-    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata', *args, **kwargs):
+    def _download_json(self, url_or_request, video_id, *args, **kwargs):
         try:
-            response = super(EaglePlatformIE, self)._download_json(url_or_request, video_id, note)
+            response = super(EaglePlatformIE, self)._download_json(
+                url_or_request, video_id, *args, **kwargs)
         except ExtractorError as ee:
             if isinstance(ee.cause, compat_HTTPError):
                 response = self._parse_json(ee.cause.read().decode('utf-8'), video_id)
@@ -117,11 +123,24 @@ class EaglePlatformIE(InfoExtractor):
         return self._download_json(url_or_request, video_id, note)['data'][0]
 
     def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
+
         mobj = re.match(self._VALID_URL, url)
         host, video_id = mobj.group('custom_host') or mobj.group('host'), mobj.group('id')
 
+        headers = {}
+        query = {
+            'id': video_id,
+        }
+
+        referrer = smuggled_data.get('referrer')
+        if referrer:
+            headers['Referer'] = referrer
+            query['referrer'] = referrer
+
         player_data = self._download_json(
-            'http://%s/api/player_data?id=%s' % (host, video_id), video_id)
+            'http://%s/api/player_data' % host, video_id,
+            headers=headers, query=query)
 
         media = player_data['data']['playlist']['viewports'][0]['medialist'][0]
 
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index f9bff433c..7232f39db 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1185,7 +1185,7 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['Kaltura'],
         },
-        # Eagle.Platform embed (generic URL)
+        # EaglePlatform embed (generic URL)
         {
             'url': 'http://lenta.ru/news/2015/03/06/navalny/',
             # Not checking MD5 as sometimes the direct HTTP link results in 404 and HLS is used
@@ -1200,7 +1200,7 @@ class GenericIE(InfoExtractor):
                 'age_limit': 0,
             },
         },
-        # ClipYou (Eagle.Platform) embed (custom URL)
+        # ClipYou (EaglePlatform) embed (custom URL)
         {
             'url': 'http://muz-tv.ru/play/7129/',
             # Not checking MD5 as sometimes the direct HTTP link results in 404 and HLS is used
@@ -2443,12 +2443,12 @@ class GenericIE(InfoExtractor):
         if kaltura_url:
             return self.url_result(smuggle_url(kaltura_url, {'source_url': url}), KalturaIE.ie_key())
 
-        # Look for Eagle.Platform embeds
+        # Look for EaglePlatform embeds
         eagleplatform_url = EaglePlatformIE._extract_url(webpage)
         if eagleplatform_url:
-            return self.url_result(eagleplatform_url, EaglePlatformIE.ie_key())
+            return self.url_result(smuggle_url(eagleplatform_url, {'referrer': url}), EaglePlatformIE.ie_key())
 
-        # Look for ClipYou (uses Eagle.Platform) embeds
+        # Look for ClipYou (uses EaglePlatform) embeds
         mobj = re.search(
             r'<iframe[^>]+src="https?://(?P<host>media\.clipyou\.ru)/index/player\?.*\brecord_id=(?P<id>\d+).*"', webpage)
         if mobj is not None:

From 250b042c7e71a6e8bbff534aa41c2b92dae1acf7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 16:02:38 +0700
Subject: [PATCH 1077/1696] [generic] Add tests for #13557

---
 youtube_dl/extractor/generic.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 7232f39db..95c38698d 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1199,6 +1199,24 @@ class GenericIE(InfoExtractor):
                 'view_count': int,
                 'age_limit': 0,
             },
+            'params': {
+                'skip_download': True,
+            },
+        },
+        # referrer protected EaglePlatform embed
+        {
+            'url': 'https://tvrain.ru/lite/teleshow/kak_vse_nachinalos/namin-418921/',
+            'info_dict': {
+                'id': '582306',
+                'ext': 'mp4',
+                'title': 'Стас Намин: «Мы нарушили девственность Кремля»',
+                'thumbnail': r're:^https?://.*\.jpg$',
+                'duration': 3382,
+                'view_count': int,
+            },
+            'params': {
+                'skip_download': True,
+            },
         },
         # ClipYou (EaglePlatform) embed (custom URL)
         {
@@ -1212,6 +1230,9 @@ class GenericIE(InfoExtractor):
                 'duration': 216,
                 'view_count': int,
             },
+            'params': {
+                'skip_download': True,
+            },
         },
         # Pladform embed
         {

From 4328ddf82b812420ffc120b4150251f751bff08c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 16:29:52 +0700
Subject: [PATCH 1078/1696] [extractor/common] Add support for AMP tags in
 _parse_html5_media_entries

---
 youtube_dl/extractor/common.py  |  7 +++++--
 youtube_dl/extractor/generic.py | 10 ++++++++++
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index afeb4c5da..daa10885f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2132,15 +2132,18 @@ class InfoExtractor(object):
             return is_plain_url, formats
 
         entries = []
+        # amp-video and amp-audio are very similar to their HTML5 counterparts
+        # so we wll include them right here (see
+        # https://www.ampproject.org/docs/reference/components/amp-video)
         media_tags = [(media_tag, media_type, '')
                       for media_tag, media_type
-                      in re.findall(r'(?s)(<(video|audio)[^>]*/>)', webpage)]
+                      in re.findall(r'(?s)(<(?:amp-)?(video|audio)[^>]*/>)', webpage)]
         media_tags.extend(re.findall(
             # We only allow video|audio followed by a whitespace or '>'.
             # Allowing more characters may end up in significant slow down (see
             # https://github.com/rg3/youtube-dl/issues/11979, example URL:
             # http://www.porntrex.com/maps/videositemap.xml).
-            r'(?s)(<(?P<tag>video|audio)(?:\s+[^>]*)?>)(.*?)</(?P=tag)>', webpage))
+            r'(?s)(<(?P<tag>(?:amp-)?(?:video|audio))(?:\s+[^>]*)?>)(.*?)</(?P=tag)>', webpage))
         for media_tag, media_type, media_content in media_tags:
             media_info = {
                 'formats': [],
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 95c38698d..919f4f987 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1770,6 +1770,16 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': [MediasetIE.ie_key()],
         },
+        {
+            # AMP embed (see https://www.ampproject.org/docs/reference/components/amp-video)
+            'url': 'https://tvrain.ru/amp/418921/',
+            'md5': 'cc00413936695987e8de148b67d14f1d',
+            'info_dict': {
+                'id': '418921',
+                'ext': 'mp4',
+                'title': 'Стас Намин: «Мы нарушили девственность Кремля»',
+            },
+        },
         # {
         #     # TODO: find another test
         #     # http://schema.org/VideoObject

From d2b9f362fabad8f9490825456d8ed679d7159271 Mon Sep 17 00:00:00 2001
From: Christopher Smith <ctsmi@ucalgary.ca>
Date: Thu, 29 Jun 2017 13:10:45 -0600
Subject: [PATCH 1079/1696] [cjsw] Add extractor

---
 youtube_dl/extractor/cjsw.py       | 41 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  1 +
 2 files changed, 42 insertions(+)
 create mode 100644 youtube_dl/extractor/cjsw.py

diff --git a/youtube_dl/extractor/cjsw.py b/youtube_dl/extractor/cjsw.py
new file mode 100644
index 000000000..087cac9bc
--- /dev/null
+++ b/youtube_dl/extractor/cjsw.py
@@ -0,0 +1,41 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class CJSWIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?cjsw\.com/program/\S+/(?P<id>[0-9]+)'
+    IE_NAME = 'cjsw'
+    _TEST = {
+        'url': 'http://cjsw.com/program/freshly-squeezed/episode/20170620',
+        'md5': 'cee14d40f1e9433632c56e3d14977120',
+        'info_dict': {
+            'id': '20170620',
+            'ext': 'mp3',
+            'title': 'Freshly Squeezed',
+            'description': 'Sled Island artists featured // Live session with Phi Pho, followed by a live session with Sinzere & The Late Nights! // Stay Fresh Y\'all!!',
+        }
+    }
+
+    def _real_extract(self, url):
+        episode_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, episode_id)
+
+        title = self._search_regex(
+            r'<button[^>]+data-showname=(["\'])(?P<title>(?!\1).+?)\1[^>]*>', webpage, 'title', group='title')
+        description = self._html_search_regex(
+            r'<p>(?P<description>.+?)</p>', webpage, 'description', fatal=False)
+        formats = [{
+            'url': self._search_regex(
+                r'<button[^>]+data-audio-src=(["\'])(?P<audio_url>(?!\1).+?)\1[^>]*>', webpage, 'audio_url', group='audio_url'),
+            'ext': 'mp3',
+            'vcodec': 'none',
+        }]
+        return {
+            'id': episode_id,
+            'title': title,
+            'description': description,
+            'formats': formats,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index b83c3aba5..4524fa687 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -185,6 +185,7 @@ from .chirbit import (
     ChirbitProfileIE,
 )
 from .cinchcast import CinchcastIE
+from .cjsw import CJSWIE
 from .clipfish import ClipfishIE
 from .cliphunter import CliphunterIE
 from .cliprs import ClipRsIE

From c319d1c4833f89df818fe39f4c99cdc5c9a8bf01 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 17:00:45 +0700
Subject: [PATCH 1080/1696] [csjw] Fix issues and improve extraction (closes
 #13525)

---
 youtube_dl/extractor/cjsw.py | 57 ++++++++++++++++++++++++++----------
 1 file changed, 41 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/cjsw.py b/youtube_dl/extractor/cjsw.py
index 087cac9bc..aab6ea535 100644
--- a/youtube_dl/extractor/cjsw.py
+++ b/youtube_dl/extractor/cjsw.py
@@ -1,41 +1,66 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    unescapeHTML,
+)
 
 
 class CJSWIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?cjsw\.com/program/\S+/(?P<id>[0-9]+)'
-    IE_NAME = 'cjsw'
+    _VALID_URL = r'https?://(?:www\.)?cjsw\.com/program/(?P<program>[^/]+)/episode/(?P<id>\d+)'
     _TEST = {
         'url': 'http://cjsw.com/program/freshly-squeezed/episode/20170620',
         'md5': 'cee14d40f1e9433632c56e3d14977120',
         'info_dict': {
-            'id': '20170620',
+            'id': '91d9f016-a2e7-46c5-8dcb-7cbcd7437c41',
             'ext': 'mp3',
-            'title': 'Freshly Squeezed',
-            'description': 'Sled Island artists featured // Live session with Phi Pho, followed by a live session with Sinzere & The Late Nights! // Stay Fresh Y\'all!!',
-        }
+            'title': 'Freshly Squeezed – Episode June 20, 2017',
+            'description': 'md5:c967d63366c3898a80d0c7b0ff337202',
+            'series': 'Freshly Squeezed',
+            'episode_id': '20170620',
+        },
     }
 
     def _real_extract(self, url):
-        episode_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        program, episode_id = mobj.group('program', 'id')
+        audio_id = '%s/%s' % (program, episode_id)
 
         webpage = self._download_webpage(url, episode_id)
 
-        title = self._search_regex(
-            r'<button[^>]+data-showname=(["\'])(?P<title>(?!\1).+?)\1[^>]*>', webpage, 'title', group='title')
-        description = self._html_search_regex(
-            r'<p>(?P<description>.+?)</p>', webpage, 'description', fatal=False)
+        title = unescapeHTML(self._search_regex(
+            (r'<h1[^>]+class=["\']episode-header__title["\'][^>]*>(?P<title>[^<]+)',
+             r'data-audio-title=(["\'])(?P<title>(?:(?!\1).)+)\1'),
+            webpage, 'title', group='title'))
+
+        audio_url = self._search_regex(
+            r'<button[^>]+data-audio-src=(["\'])(?P<url>(?:(?!\1).)+)\1',
+            webpage, 'audio url', group='url')
+
+        audio_id = self._search_regex(
+            r'/([\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})\.mp3',
+            audio_url, 'audio id', default=audio_id)
+
         formats = [{
-            'url': self._search_regex(
-                r'<button[^>]+data-audio-src=(["\'])(?P<audio_url>(?!\1).+?)\1[^>]*>', webpage, 'audio_url', group='audio_url'),
-            'ext': 'mp3',
+            'url': audio_url,
+            'ext': determine_ext(audio_url, 'mp3'),
             'vcodec': 'none',
         }]
+
+        description = self._html_search_regex(
+            r'<p>(?P<description>.+?)</p>', webpage, 'description', fatal=False)
+        series = self._search_regex(
+            r'data-showname=(["\'])(?P<name>(?:(?!\1).)+)\1', webpage,
+            'series', default=program, group='name')
+
         return {
-            'id': episode_id,
+            'id': audio_id,
             'title': title,
             'description': description,
             'formats': formats,
+            'series': series,
+            'episode_id': episode_id,
         }

From 0d2f0b0357325823782884327a158aeccf4f9b49 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 17:05:11 +0700
Subject: [PATCH 1081/1696] [csjw] Make description optional

---
 youtube_dl/extractor/cjsw.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/cjsw.py b/youtube_dl/extractor/cjsw.py
index aab6ea535..dd271586f 100644
--- a/youtube_dl/extractor/cjsw.py
+++ b/youtube_dl/extractor/cjsw.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 class CJSWIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?cjsw\.com/program/(?P<program>[^/]+)/episode/(?P<id>\d+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://cjsw.com/program/freshly-squeezed/episode/20170620',
         'md5': 'cee14d40f1e9433632c56e3d14977120',
         'info_dict': {
@@ -22,7 +22,11 @@ class CJSWIE(InfoExtractor):
             'series': 'Freshly Squeezed',
             'episode_id': '20170620',
         },
-    }
+    }, {
+        # no description
+        'url': 'http://cjsw.com/program/road-pops/episode/20170707/',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -51,7 +55,8 @@ class CJSWIE(InfoExtractor):
         }]
 
         description = self._html_search_regex(
-            r'<p>(?P<description>.+?)</p>', webpage, 'description', fatal=False)
+            r'<p>(?P<description>.+?)</p>', webpage, 'description',
+            default=None)
         series = self._search_regex(
             r'data-showname=(["\'])(?P<name>(?:(?!\1).)+)\1', webpage,
             'series', default=program, group='name')

From a02682fd13ce5ba88d2508c90559eaa7f43b65d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 17:09:44 +0700
Subject: [PATCH 1082/1696] Keep in sync with ffmpeg's current malformed AAC
 bitstream wording (closes #13587)

---
 youtube_dl/YoutubeDL.py            | 4 ++--
 youtube_dl/postprocessor/ffmpeg.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b3a6d4d3b..60ee4b7d8 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1890,7 +1890,7 @@ class YoutubeDL(object):
                         info_dict.get('protocol') == 'm3u8' and
                         self.params.get('hls_prefer_native')):
                     if fixup_policy == 'warn':
-                        self.report_warning('%s: malformated aac bitstream.' % (
+                        self.report_warning('%s: malformed AAC bitstream detected.' % (
                             info_dict['id']))
                     elif fixup_policy == 'detect_or_warn':
                         fixup_pp = FFmpegFixupM3u8PP(self)
@@ -1899,7 +1899,7 @@ class YoutubeDL(object):
                             info_dict['__postprocessors'].append(fixup_pp)
                         else:
                             self.report_warning(
-                                '%s: malformated aac bitstream. %s'
+                                '%s: malformed AAC bitstream detected. %s'
                                 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
                     else:
                         assert fixup_policy in ('ignore', 'never')
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index f021ea8fd..51256a3fb 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -542,7 +542,7 @@ class FFmpegFixupM3u8PP(FFmpegPostProcessor):
             temp_filename = prepend_extension(filename, 'temp')
 
             options = ['-c', 'copy', '-f', 'mp4', '-bsf:a', 'aac_adtstoasc']
-            self._downloader.to_screen('[ffmpeg] Fixing malformated aac bitstream in "%s"' % filename)
+            self._downloader.to_screen('[ffmpeg] Fixing malformed AAC bitstream in "%s"' % filename)
             self.run_ffmpeg(filename, temp_filename, options)
 
             os.remove(encodeFilename(filename))

From ed84454d358f3cbfdc43dab31328b165f9c72c68 Mon Sep 17 00:00:00 2001
From: Santiago Calcagno <santicalcagno@gmail.com>
Date: Tue, 13 Jun 2017 12:32:04 -0300
Subject: [PATCH 1083/1696] [egghead:course] Fix extraction

---
 youtube_dl/extractor/egghead.py | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/egghead.py b/youtube_dl/extractor/egghead.py
index db921465e..01fcdb6cf 100644
--- a/youtube_dl/extractor/egghead.py
+++ b/youtube_dl/extractor/egghead.py
@@ -1,8 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -22,18 +20,18 @@ class EggheadCourseIE(InfoExtractor):
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
-        webpage = self._download_webpage(url, playlist_id)
+        api_url = 'https://egghead.io/api/v1/series/' + playlist_id
+        course = self._download_json(api_url, playlist_id)
+        title = course.get('title')
+        description = course.get('description')
 
-        title = self._html_search_regex(r'<h1 class="title">([^<]+)</h1>', webpage, 'title')
-        ul = self._search_regex(r'(?s)<ul class="series-lessons-list">(.*?)</ul>', webpage, 'session list')
-
-        found = re.findall(r'(?s)<a class="[^"]*"\s*href="([^"]+)">\s*<li class="item', ul)
-        entries = [self.url_result(m) for m in found]
+        lessons = course.get('lessons')
+        entries = [{'_type': 'url', 'ie_key': 'Wistia', 'url': 'wistia:' + l.get('wistia_id')} for l in lessons]
 
         return {
             '_type': 'playlist',
             'id': playlist_id,
             'title': title,
-            'description': self._og_search_description(webpage),
+            'description': description,
             'entries': entries,
         }

From 485cb375766df8f2ef79b7fe2915ead4ef61a01e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 17:28:42 +0700
Subject: [PATCH 1084/1696] [egghead:course] Improve (closes #13370)

---
 youtube_dl/extractor/egghead.py | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/egghead.py b/youtube_dl/extractor/egghead.py
index 01fcdb6cf..c86f52319 100644
--- a/youtube_dl/extractor/egghead.py
+++ b/youtube_dl/extractor/egghead.py
@@ -7,7 +7,7 @@ from .common import InfoExtractor
 class EggheadCourseIE(InfoExtractor):
     IE_DESC = 'egghead.io course'
     IE_NAME = 'egghead:course'
-    _VALID_URL = r'https://egghead\.io/courses/(?P<id>[a-zA-Z_0-9-]+)'
+    _VALID_URL = r'https://egghead\.io/courses/(?P<id>[^/?#&]+)'
     _TEST = {
         'url': 'https://egghead.io/courses/professor-frisby-introduces-composable-functional-javascript',
         'playlist_count': 29,
@@ -20,18 +20,16 @@ class EggheadCourseIE(InfoExtractor):
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
-        api_url = 'https://egghead.io/api/v1/series/' + playlist_id
-        course = self._download_json(api_url, playlist_id)
-        title = course.get('title')
-        description = course.get('description')
 
-        lessons = course.get('lessons')
-        entries = [{'_type': 'url', 'ie_key': 'Wistia', 'url': 'wistia:' + l.get('wistia_id')} for l in lessons]
+        course = self._download_json(
+            'https://egghead.io/api/v1/series/%s' % playlist_id, playlist_id)
 
-        return {
-            '_type': 'playlist',
-            'id': playlist_id,
-            'title': title,
-            'description': description,
-            'entries': entries,
-        }
+        entries = [
+            self.url_result(
+                'wistia:%s' % lesson['wistia_id'], ie='Wistia',
+                video_id=lesson['wistia_id'], video_title=lesson.get('title'))
+            for lesson in course['lessons'] if lesson.get('wistia_id')]
+
+        return self.playlist_result(
+            entries, playlist_id, course.get('title'),
+            course.get('description'))

From 58179eb7d96ebef26a0083e80a2022fab4ca1558 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 17:55:40 +0700
Subject: [PATCH 1085/1696] [abc.net.au:iview] Extract more formats (closes
 #13492, closes #13489)

---
 youtube_dl/extractor/abc.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py
index 0247cabf9..60f753b95 100644
--- a/youtube_dl/extractor/abc.py
+++ b/youtube_dl/extractor/abc.py
@@ -3,11 +3,13 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     ExtractorError,
     js_to_json,
     int_or_none,
     parse_iso8601,
+    try_get,
 )
 
 
@@ -124,7 +126,20 @@ class ABCIViewIE(InfoExtractor):
         title = video_params.get('title') or video_params['seriesTitle']
         stream = next(s for s in video_params['playlist'] if s.get('type') == 'program')
 
-        formats = self._extract_akamai_formats(stream['hds-unmetered'], video_id)
+        format_urls = [
+            try_get(stream, lambda x: x['hds-unmetered'], compat_str)]
+
+        # May have higher quality video
+        sd_url = try_get(
+            stream, lambda x: x['streams']['hds']['sd'], compat_str)
+        if sd_url:
+            format_urls.append(sd_url.replace('metered', 'um'))
+
+        formats = []
+        for format_url in format_urls:
+            if format_url:
+                formats.extend(
+                    self._extract_akamai_formats(format_url, video_id))
         self._sort_formats(formats)
 
         subtitles = {}

From 256a746d21634eccad07a1e6dcafedcdf8b6181b Mon Sep 17 00:00:00 2001
From: luboss <lubos.katrinec@gmail.com>
Date: Fri, 2 Jun 2017 22:44:39 +0200
Subject: [PATCH 1086/1696] [joj] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/joj.py        | 56 ++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+)
 create mode 100755 youtube_dl/extractor/joj.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 4524fa687..9ee080895 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -470,6 +470,7 @@ from .jamendo import (
 )
 from .jeuxvideo import JeuxVideoIE
 from .jove import JoveIE
+from .joj import JojIE
 from .jwplatform import JWPlatformIE
 from .jpopsukitv import JpopsukiIE
 from .kaltura import KalturaIE
diff --git a/youtube_dl/extractor/joj.py b/youtube_dl/extractor/joj.py
new file mode 100755
index 000000000..2ebfec902
--- /dev/null
+++ b/youtube_dl/extractor/joj.py
@@ -0,0 +1,56 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+import re
+
+
+class JojIE(InfoExtractor):
+    _VALID_URL = r'https?://[a-z0-9]+\.joj\.sk/([^/]+/)*(?P<title_query>(?P<release_date>[0-9]{4}(-[0-9]{2}){2}).*)' # noqa
+    _TESTS = [{
+        'url': 'https://www.joj.sk/nove-byvanie/archiv/2017-05-28-nove-byvanie', # noqa
+        'info_dict': {
+            'id': 'a388ec4c-6019-4a4a-9312-b1bee194e932',
+            'ext': 'mp4',
+            'title': 'Nové Bývanie',
+            'release_date': '20170528'
+        }
+    }, {
+        'url': 'http://nasi.joj.sk/epizody/2016-09-06-stari-rodicia',
+        'info_dict': {
+            'id': 'f18b2c5f-9ea8-4941-a164-a814c53306ad',
+            'ext': 'mp4',
+            'title': 'Starí Rodičia',
+            'release_date': '20160906'
+        }
+    }]
+
+    media_src_url = 'http://n16.joj.sk/storage/'
+    xml_source_url = 'https://media.joj.sk/services/Video.php?clip='
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        release_date = mobj.group('release_date').replace('-', '')
+        webpage = self._download_webpage(url, 'id')
+        video_id = self._html_search_regex(
+            r'https?://([a-z0-9]+\.)joj\.sk/embed/(?P<video_id>[a-f0-9\-]+)',
+            webpage, 'id', group='video_id')
+        xml_playlist_url = self.xml_source_url + video_id
+        xml_playlist_et = self._download_xml(xml_playlist_url, 'XML playlist')
+        formats = []
+        for file_el in xml_playlist_et.findall('files/file'):
+            try:
+                height = int(file_el.attrib['id'].replace('p', ''))
+            except ValueError:
+                height = 0
+            formats.append({'height': height,
+                            'url': self.media_src_url + file_el.attrib['path'].replace(  # noqa
+                                'dat/', '', 1)})
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage).title(),
+            'formats': formats,
+            'release_date': release_date
+        }

From 73cf76a93fe48240bf82b1685b1403f05b793ebf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 19:05:18 +0700
Subject: [PATCH 1087/1696] [joj] Rewrite and add support for generic embeds
 (closes #13268)

---
 youtube_dl/extractor/generic.py |  17 +++++
 youtube_dl/extractor/joj.py     | 108 ++++++++++++++++++++++----------
 2 files changed, 93 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 919f4f987..f2c577f98 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -91,6 +91,7 @@ from .anvato import AnvatoIE
 from .washingtonpost import WashingtonPostIE
 from .wistia import WistiaIE
 from .mediaset import MediasetIE
+from .joj import JojIE
 
 
 class GenericIE(InfoExtractor):
@@ -1770,6 +1771,16 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': [MediasetIE.ie_key()],
         },
+        {
+            # JOJ.sk embeds
+            'url': 'https://www.noviny.sk/slovensko/238543-slovenskom-sa-prehnala-vlna-silnych-burok',
+            'info_dict': {
+                'id': '238543-slovenskom-sa-prehnala-vlna-silnych-burok',
+                'title': 'Slovenskom sa prehnala vlna silných búrok',
+            },
+            'playlist_mincount': 5,
+            'add_ie': [JojIE.ie_key()],
+        },
         {
             # AMP embed (see https://www.ampproject.org/docs/reference/components/amp-video)
             'url': 'https://tvrain.ru/amp/418921/',
@@ -2722,6 +2733,12 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 mediaset_urls, video_id, video_title, ie=MediasetIE.ie_key())
 
+        # Look for JOJ.sk embeds
+        joj_urls = JojIE._extract_urls(webpage)
+        if joj_urls:
+            return self.playlist_from_matches(
+                joj_urls, video_id, video_title, ie=JojIE.ie_key())
+
         def merge_dicts(dict1, dict2):
             merged = {}
             for k, v in dict1.items():
diff --git a/youtube_dl/extractor/joj.py b/youtube_dl/extractor/joj.py
index 2ebfec902..a764023e9 100755
--- a/youtube_dl/extractor/joj.py
+++ b/youtube_dl/extractor/joj.py
@@ -1,56 +1,100 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
 import re
 
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    int_or_none,
+    js_to_json,
+    try_get,
+)
+
 
 class JojIE(InfoExtractor):
-    _VALID_URL = r'https?://[a-z0-9]+\.joj\.sk/([^/]+/)*(?P<title_query>(?P<release_date>[0-9]{4}(-[0-9]{2}){2}).*)' # noqa
+    _VALID_URL = r'''(?x)
+                    (?:
+                        joj:|
+                        https?://media\.joj\.sk/embed/
+                    )
+                    (?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})
+                '''
     _TESTS = [{
-        'url': 'https://www.joj.sk/nove-byvanie/archiv/2017-05-28-nove-byvanie', # noqa
+        'url': 'https://media.joj.sk/embed/a388ec4c-6019-4a4a-9312-b1bee194e932',
         'info_dict': {
             'id': 'a388ec4c-6019-4a4a-9312-b1bee194e932',
             'ext': 'mp4',
-            'title': 'Nové Bývanie',
-            'release_date': '20170528'
+            'title': 'NOVÉ BÝVANIE',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 3118,
         }
     }, {
-        'url': 'http://nasi.joj.sk/epizody/2016-09-06-stari-rodicia',
-        'info_dict': {
-            'id': 'f18b2c5f-9ea8-4941-a164-a814c53306ad',
-            'ext': 'mp4',
-            'title': 'Starí Rodičia',
-            'release_date': '20160906'
-        }
+        'url': 'joj:a388ec4c-6019-4a4a-9312-b1bee194e932',
+        'only_matching': True,
     }]
 
-    media_src_url = 'http://n16.joj.sk/storage/'
-    xml_source_url = 'https://media.joj.sk/services/Video.php?clip='
+    @staticmethod
+    def _extract_urls(webpage):
+        return re.findall(
+            r'<iframe\b[^>]+\bsrc=["\'](?P<url>(?:https?:)?//media\.joj\.sk/embed/[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})',
+            webpage)
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        release_date = mobj.group('release_date').replace('-', '')
-        webpage = self._download_webpage(url, 'id')
-        video_id = self._html_search_regex(
-            r'https?://([a-z0-9]+\.)joj\.sk/embed/(?P<video_id>[a-f0-9\-]+)',
-            webpage, 'id', group='video_id')
-        xml_playlist_url = self.xml_source_url + video_id
-        xml_playlist_et = self._download_xml(xml_playlist_url, 'XML playlist')
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(
+            'https://media.joj.sk/embed/%s' % video_id, video_id)
+
+        title = self._search_regex(
+            (r'videoTitle\s*:\s*(["\'])(?P<title>(?:(?!\1).)+)\1',
+             r'<title>(?P<title>[^<]+)'), webpage, 'title',
+            default=None, group='title') or self._og_search_title(webpage)
+
+        bitrates = self._parse_json(
+            self._search_regex(
+                r'(?s)bitrates\s*=\s*({.+?});', webpage, 'bitrates',
+                default='{}'),
+            video_id, transform_source=js_to_json, fatal=False)
+
         formats = []
-        for file_el in xml_playlist_et.findall('files/file'):
-            try:
-                height = int(file_el.attrib['id'].replace('p', ''))
-            except ValueError:
-                height = 0
-            formats.append({'height': height,
-                            'url': self.media_src_url + file_el.attrib['path'].replace(  # noqa
-                                'dat/', '', 1)})
+        for format_url in try_get(bitrates, lambda x: x['mp4'], list) or []:
+            if isinstance(format_url, compat_str):
+                height = self._search_regex(
+                    r'(\d+)[pP]\.', format_url, 'height', default=None)
+                formats.append({
+                    'url': format_url,
+                    'format_id': '%sp' % height if height else None,
+                    'height': int(height),
+                })
+        if not formats:
+            playlist = self._download_xml(
+                'https://media.joj.sk/services/Video.php?clip=%s' % video_id,
+                video_id)
+            for file_el in playlist.findall('./files/file'):
+                path = file_el.get('path')
+                if not path:
+                    continue
+                format_id = file_el.get('id') or file_el.get('label')
+                formats.append({
+                    'url': 'http://n16.joj.sk/storage/%s' % path.replace(
+                        'dat/', '', 1),
+                    'format_id': format_id,
+                    'height': int_or_none(self._search_regex(
+                        r'(\d+)[pP]', format_id or path, 'height',
+                        default=None)),
+                })
         self._sort_formats(formats)
 
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        duration = int_or_none(self._search_regex(
+            r'videoDuration\s*:\s*(\d+)', webpage, 'duration', fatal=False))
+
         return {
             'id': video_id,
-            'title': self._og_search_title(webpage).title(),
+            'title': title,
+            'thumbnail': thumbnail,
+            'duration': duration,
             'formats': formats,
-            'release_date': release_date
         }

From 6e925598d68f5d5216aa3e9abed5c7706a68c891 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 19:15:48 +0700
Subject: [PATCH 1088/1696] [csjw] Add coding cookie

---
 youtube_dl/extractor/cjsw.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/cjsw.py b/youtube_dl/extractor/cjsw.py
index dd271586f..505bdbe16 100644
--- a/youtube_dl/extractor/cjsw.py
+++ b/youtube_dl/extractor/cjsw.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

From 71a1db89198100a0e9bc5099aeed622264690203 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 20:06:24 +0700
Subject: [PATCH 1089/1696] [dailymail] Add support for embeds

---
 youtube_dl/extractor/dailymail.py | 17 ++++++++++++++---
 youtube_dl/extractor/generic.py   | 21 +++++++++++++++++++++
 2 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dailymail.py b/youtube_dl/extractor/dailymail.py
index 538565c66..af3978035 100644
--- a/youtube_dl/extractor/dailymail.py
+++ b/youtube_dl/extractor/dailymail.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
@@ -12,8 +14,8 @@ from ..utils import (
 
 
 class DailyMailIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?dailymail\.co\.uk/video/[^/]+/video-(?P<id>[0-9]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?dailymail\.co\.uk/(?:video/[^/]+/video-|embed/video/)(?P<id>[0-9]+)'
+    _TESTS = [{
         'url': 'http://www.dailymail.co.uk/video/tvshowbiz/video-1295863/The-Mountain-appears-sparkling-water-ad-Heavy-Bubbles.html',
         'md5': 'f6129624562251f628296c3a9ffde124',
         'info_dict': {
@@ -22,7 +24,16 @@ class DailyMailIE(InfoExtractor):
             'title': 'The Mountain appears in sparkling water ad for \'Heavy Bubbles\'',
             'description': 'md5:a93d74b6da172dd5dc4d973e0b766a84',
         }
-    }
+    }, {
+        'url': 'http://www.dailymail.co.uk/embed/video/1295863.html',
+        'only_matching': True,
+    }]
+
+    @staticmethod
+    def _extract_urls(webpage):
+        return re.findall(
+            r'<iframe\b[^>]+\bsrc=["\'](?P<url>(?:https?:)?//(?:www\.)?dailymail\.co\.uk/embed/video/\d+\.html)',
+            webpage)
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index f2c577f98..5e8890d41 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -57,6 +57,7 @@ from .dailymotion import (
     DailymotionIE,
     DailymotionCloudIE,
 )
+from .dailymail import DailyMailIE
 from .onionstudios import OnionStudiosIE
 from .viewlift import ViewLiftEmbedIE
 from .mtv import MTVServicesEmbeddedIE
@@ -760,6 +761,20 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['Dailymotion'],
         },
+        # DailyMail embed
+        {
+            'url': 'http://www.bumm.sk/krimi/2017/07/05/biztonsagi-kamera-buktatta-le-az-agg-ferfit-utlegelo-apolot',
+            'info_dict': {
+                'id': '1495629',
+                'ext': 'mp4',
+                'title': 'Care worker punches elderly dementia patient in head 11 times',
+                'description': 'md5:3a743dee84e57e48ec68bf67113199a5',
+            },
+            'add_ie': ['DailyMail'],
+            'params': {
+                'skip_download': True,
+            },
+        },
         # YouTube embed
         {
             'url': 'http://www.badzine.de/ansicht/datum/2014/06/09/so-funktioniert-die-neue-englische-badminton-liga.html',
@@ -2190,6 +2205,12 @@ class GenericIE(InfoExtractor):
                 return self.playlist_from_matches(
                     playlists, video_id, video_title, lambda p: '//dailymotion.com/playlist/%s' % p)
 
+        # Look for DailyMail embeds
+        dailymail_urls = DailyMailIE._extract_urls(webpage)
+        if dailymail_urls:
+            return self.playlist_from_matches(
+                dailymail_urls, video_id, video_title, ie=DailyMailIE.ie_key())
+
         # Look for embedded Wistia player
         wistia_url = WistiaIE._extract_url(webpage)
         if wistia_url:

From 207acd8465b51d9d00d2bdda22f10858eb7f1bb5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 20:15:15 +0700
Subject: [PATCH 1090/1696] [ChangeLog] Actualize

---
 ChangeLog | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 5d07c12cb..edfde8b6f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,27 @@
 version <unreleased>
 
+Core
++ [extractor/common] Add support for AMP tags in _parse_html5_media_entries
++ [utils] Support attributes with no values in get_elements_by_attribute
+
 Extractors
++ [dailymail] Add support for embeds
++ [joj] Add support for joj.sk (#13268)
+* [abc.net.au:iview] Extract more formats (#13492, #13489)
+* [egghead:course] Fix extraction (#6635, #13370)
++ [cjsw] Add support for cjsw.com (#13525)
++ [eagleplatform] Add support for referrer protected videos (#13557)
++ [eagleplatform] Add support for another embed pattern (#13557)
+* [veoh] Extend URL regular expression (#13601)
+* [npo:live] Fix live stream id extraction (#13568, #13605)
+* [googledrive] Fix height extraction (#13603)
++ [dailymotion] Add support for new layout (#13580)
 - [yam] Remove extractor
+* [xhamster] Extract all formats and fix duration extraction (#13593)
++ [xhamster] Add support for new URL schema (#13593)
+* [espn] Extend URL regular expression (#13244, #13549)
+* [kaltura] Fix typo in subtitles extraction (#13569)
+* [vier] Adapt extraction to redesign (#13575)
 
 
 version 2017.07.02

From 65c416dda896f8a0023f01547e6b707dd57ed30a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Jul 2017 20:16:38 +0700
Subject: [PATCH 1091/1696] release 2017.07.09

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 3 ++-
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 974603507..c4314855d 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.07.02*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.07.02**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.07.09*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.07.09**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.07.02
+[debug] youtube-dl version 2017.07.09
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index edfde8b6f..c379cae71 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.07.09
 
 Core
 + [extractor/common] Add support for AMP tags in _parse_html5_media_entries
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index db2e2bac9..b6a147faf 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -154,6 +154,7 @@
  - **chirbit**
  - **chirbit:profile**
  - **Cinchcast**
+ - **CJSW**
  - **Clipfish**
  - **cliphunter**
  - **ClipRs**
@@ -369,6 +370,7 @@
  - **Jamendo**
  - **JamendoAlbum**
  - **JeuxVideo**
+ - **Joj**
  - **Jove**
  - **jpopsuki.tv**
  - **JWPlatform**
@@ -996,7 +998,6 @@
  - **XVideos**
  - **XXXYMovies**
  - **Yahoo**: Yahoo screen and movies
- - **Yam**: 蕃薯藤yam天空部落
  - **yandexmusic:album**: Яндекс.Музыка - Альбом
  - **yandexmusic:playlist**: Яндекс.Музыка - Плейлист
  - **yandexmusic:track**: Яндекс.Музыка - Трек
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0db974f97..14358a74c 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.07.02'
+__version__ = '2017.07.09'

From 7bf539edcc3dc44481d5196fd01637698653ffc7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 10 Jul 2017 00:14:41 +0700
Subject: [PATCH 1092/1696] [eagleplatform] Fix test

---
 youtube_dl/extractor/eagleplatform.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index 34891a362..42789278e 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -53,7 +53,7 @@ class EaglePlatformIE(InfoExtractor):
         'skip': 'Georestricted',
     }, {
         # referrer protected video (https://tvrain.ru/lite/teleshow/kak_vse_nachinalos/namin-418921/)
-        'url': 'tvrainru.media.eagleplatform.com:582306',
+        'url': 'eagleplatform:tvrainru.media.eagleplatform.com:582306',
         'only_matching': True,
     }]
 

From b71c18b4343d54ce8373e9a11df882aca1ae82a0 Mon Sep 17 00:00:00 2001
From: coreynicholson <coreynicholson@users.noreply.github.com>
Date: Sun, 9 Jul 2017 22:24:04 +0100
Subject: [PATCH 1093/1696] [vlive:playlist] Add extractor

---
 youtube_dl/extractor/extractors.py |  3 +-
 youtube_dl/extractor/vlive.py      | 56 ++++++++++++++++++++++++++++++
 2 files changed, 58 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 9ee080895..eb1541729 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1206,7 +1206,8 @@ from .vk import (
 )
 from .vlive import (
     VLiveIE,
-    VLiveChannelIE
+    VLiveChannelIE,
+    VLivePlaylistIE
 )
 from .vodlocker import VodlockerIE
 from .vodpl import VODPlIE
diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index e58940607..f3825db5c 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -49,6 +49,10 @@ class VLiveIE(InfoExtractor):
         },
     }]
 
+    @classmethod
+    def suitable(cls, url):
+        return False if VLivePlaylistIE.suitable(url) else super(VLiveIE, cls).suitable(url)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
@@ -261,3 +265,55 @@ class VLiveChannelIE(InfoExtractor):
 
         return self.playlist_result(
             entries, channel_code, channel_name)
+
+
+class VLivePlaylistIE(InfoExtractor):
+    IE_NAME = 'vlive:playlist'
+    _VALID_URL = r'https?://(?:(?:www|m)\.)?vlive\.tv/video/(?P<video_id>[0-9]+)/playlist/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.vlive.tv/video/22867/playlist/22912',
+        'info_dict': {
+            'id': '22912',
+            'title': 'Valentine Day Message from TWICE'
+        },
+        'playlist_mincount': 9
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        video_id_match = re.match(self._VALID_URL, url)
+        assert video_id_match
+        video_id = compat_str(video_id_match.group('video_id'))
+
+        VIDEO_URL_TEMPLATE = 'http://www.vlive.tv/video/%s'
+        if self._downloader.params.get('noplaylist'):
+            self.to_screen(
+                'Downloading just video %s because of --no-playlist' % video_id)
+            return self.url_result(
+                VIDEO_URL_TEMPLATE % video_id,
+                ie=VLiveIE.ie_key(), video_id=video_id)
+
+        self.to_screen(
+            'Downloading playlist %s - add --no-playlist to just download video' % playlist_id)
+
+        webpage = self._download_webpage(
+            'http://www.vlive.tv/video/%s/playlist/%s' % (video_id, playlist_id), video_id)
+
+        playlist_name = self._html_search_regex(
+            r'<div[^>]+class="[^"]*multicam_playlist[^>]*>\s*<h3[^>]+>([^<]+)',
+            webpage, 'playlist name', fatal=False)
+
+        item_ids = self._search_regex(
+            r'\bvar\s+playlistVideoSeqs\s*=\s*(\[[^]]+\])',
+            webpage, 'playlist item ids')
+
+        entries = []
+        for item_id in self._parse_json(item_ids, playlist_id):
+            item_id = compat_str(item_id)
+            entries.append(
+                self.url_result(
+                    VIDEO_URL_TEMPLATE % item_id,
+                    ie=VLiveIE.ie_key(), video_id=item_id))
+
+        return self.playlist_result(
+            entries, playlist_id, playlist_name)

From e3cd1fcdd177613acae4198cafbff51fbbb912c2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 10 Jul 2017 04:32:24 +0700
Subject: [PATCH 1094/1696] [vlive:playlist] Relax and simplify

---
 youtube_dl/extractor/vlive.py | 41 +++++++++++++++++------------------
 1 file changed, 20 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index f3825db5c..77c120a57 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -280,10 +280,8 @@ class VLivePlaylistIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        playlist_id = self._match_id(url)
-        video_id_match = re.match(self._VALID_URL, url)
-        assert video_id_match
-        video_id = compat_str(video_id_match.group('video_id'))
+        mobj = re.match(self._VALID_URL, url)
+        video_id, playlist_id = mobj.group('video_id', 'id')
 
         VIDEO_URL_TEMPLATE = 'http://www.vlive.tv/video/%s'
         if self._downloader.params.get('noplaylist'):
@@ -294,26 +292,27 @@ class VLivePlaylistIE(InfoExtractor):
                 ie=VLiveIE.ie_key(), video_id=video_id)
 
         self.to_screen(
-            'Downloading playlist %s - add --no-playlist to just download video' % playlist_id)
+            'Downloading playlist %s - add --no-playlist to just download video'
+            % playlist_id)
 
         webpage = self._download_webpage(
-            'http://www.vlive.tv/video/%s/playlist/%s' % (video_id, playlist_id), video_id)
+            'http://www.vlive.tv/video/%s/playlist/%s'
+            % (video_id, playlist_id), playlist_id)
+
+        item_ids = self._parse_json(
+            self._search_regex(
+                r'playlistVideoSeqs\s*=\s*(\[[^]]+\])', webpage,
+                'playlist video seqs'),
+            playlist_id)
+
+        entries = [
+            self.url_result(
+                VIDEO_URL_TEMPLATE % item_id, ie=VLiveIE.ie_key(),
+                video_id=compat_str(item_id))
+            for item_id in item_ids]
 
         playlist_name = self._html_search_regex(
             r'<div[^>]+class="[^"]*multicam_playlist[^>]*>\s*<h3[^>]+>([^<]+)',
-            webpage, 'playlist name', fatal=False)
+            webpage, 'playlist title', fatal=False)
 
-        item_ids = self._search_regex(
-            r'\bvar\s+playlistVideoSeqs\s*=\s*(\[[^]]+\])',
-            webpage, 'playlist item ids')
-
-        entries = []
-        for item_id in self._parse_json(item_ids, playlist_id):
-            item_id = compat_str(item_id)
-            entries.append(
-                self.url_result(
-                    VIDEO_URL_TEMPLATE % item_id,
-                    ie=VLiveIE.ie_key(), video_id=item_id))
-
-        return self.playlist_result(
-            entries, playlist_id, playlist_name)
+        return self.playlist_result(entries, playlist_id, playlist_name)

From c3c94ca4a40504147fce387ffb7eb9cb43233550 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 10 Jul 2017 21:34:27 +0800
Subject: [PATCH 1095/1696] [giantbomb] Extract m3u8 formats (closes #13626)

---
 ChangeLog                         |  6 ++++++
 youtube_dl/extractor/giantbomb.py | 14 ++++++++++----
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index c379cae71..a37d621a9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [giantbomb] Extract m3u8 formats (#13626)
+
+
 version 2017.07.09
 
 Core
diff --git a/youtube_dl/extractor/giantbomb.py b/youtube_dl/extractor/giantbomb.py
index 29b684d35..6a1b1e96e 100644
--- a/youtube_dl/extractor/giantbomb.py
+++ b/youtube_dl/extractor/giantbomb.py
@@ -5,9 +5,10 @@ import json
 
 from .common import InfoExtractor
 from ..utils import (
-    unescapeHTML,
-    qualities,
+    determine_ext,
     int_or_none,
+    qualities,
+    unescapeHTML,
 )
 
 
@@ -15,7 +16,7 @@ class GiantBombIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?giantbomb\.com/videos/(?P<display_id>[^/]+)/(?P<id>\d+-\d+)'
     _TEST = {
         'url': 'http://www.giantbomb.com/videos/quick-look-destiny-the-dark-below/2300-9782/',
-        'md5': '57badeface303ecf6b98b812de1b9018',
+        'md5': 'c8ea694254a59246a42831155dec57ac',
         'info_dict': {
             'id': '2300-9782',
             'display_id': 'quick-look-destiny-the-dark-below',
@@ -51,11 +52,16 @@ class GiantBombIE(InfoExtractor):
         for format_id, video_url in video['videoStreams'].items():
             if format_id == 'f4m_stream':
                 continue
-            if video_url.endswith('.f4m'):
+            ext = determine_ext(video_url)
+            if ext == 'f4m':
                 f4m_formats = self._extract_f4m_formats(video_url + '?hdcore=3.3.1', display_id)
                 if f4m_formats:
                     f4m_formats[0]['quality'] = quality(format_id)
                     formats.extend(f4m_formats)
+            elif ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    video_url, display_id, ext='mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls', fatal=False))
             else:
                 formats.append({
                     'url': video_url,

From bb13949197458fc6bd888bbe9255c391927a997b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 7 Jun 2017 14:47:25 +0800
Subject: [PATCH 1096/1696] [niconico] Check login errors (#12486)

---
 youtube_dl/extractor/niconico.py | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index f268a72d5..695e32e59 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -1,23 +1,22 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
 import json
 import datetime
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_parse_qs,
     compat_urlparse,
 )
 from ..utils import (
+    determine_ext,
     ExtractorError,
     int_or_none,
     parse_duration,
     parse_iso8601,
-    sanitized_Request,
-    xpath_text,
-    determine_ext,
     urlencode_postdata,
+    xpath_text,
 )
 
 
@@ -101,19 +100,24 @@ class NiconicoIE(InfoExtractor):
             return True
 
         # Log in
+        login_ok = True
         login_form_strs = {
-            'mail': username,
+            'mail_tel': username,
             'password': password,
         }
-        login_data = urlencode_postdata(login_form_strs)
-        request = sanitized_Request(
-            'https://secure.nicovideo.jp/secure/login', login_data)
-        login_results = self._download_webpage(
-            request, None, note='Logging in', errnote='Unable to log in')
-        if re.search(r'(?i)<h1 class="mb8p4">Log in error</h1>', login_results) is not None:
+        urlh = self._request_webpage(
+            'https://account.nicovideo.jp/api/v1/login', None,
+            note='Logging in', errnote='Unable to log in',
+            data=urlencode_postdata(login_form_strs))
+        if urlh is False:
+            login_ok = False
+        else:
+            parts = compat_urlparse.urlparse(urlh.geturl())
+            if compat_parse_qs(parts.query).get('message', [None])[0] == 'cant_login':
+                login_ok = False
+        if not login_ok:
             self._downloader.report_warning('unable to log in: bad username or password')
-            return False
-        return True
+        return login_ok
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 708f6f511e1d03363c9dbac7ed42cbdebfbc7718 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 11 Jul 2017 15:04:45 +0800
Subject: [PATCH 1097/1696] [niconico] Fix authentication error handling
 (closes #12486)

---
 ChangeLog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChangeLog b/ChangeLog
index a37d621a9..db1776dce 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [niconico] Fix authentication error handling (#12486)
 * [giantbomb] Extract m3u8 formats (#13626)
 
 

From 2edfd745df4b7a764d7456e7536e47ef140de24d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 4 Jun 2017 00:41:55 +0800
Subject: [PATCH 1098/1696] [twitter] Extract mp4 urls via mobile API (closes
 #12726)

---
 youtube_dl/extractor/twitter.py | 123 +++++++++++++++++++++++---------
 1 file changed, 89 insertions(+), 34 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 37e3bc412..2041b4199 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -7,11 +7,13 @@ from .common import InfoExtractor
 from ..compat import compat_urlparse
 from ..utils import (
     determine_ext,
-    float_or_none,
-    xpath_text,
-    remove_end,
-    int_or_none,
+    dict_get,
     ExtractorError,
+    float_or_none,
+    int_or_none,
+    remove_end,
+    try_get,
+    xpath_text,
 )
 
 from .periscope import PeriscopeIE
@@ -22,6 +24,15 @@ class TwitterBaseIE(InfoExtractor):
         vmap_data = self._download_xml(vmap_url, video_id)
         return xpath_text(vmap_data, './/MediaFile').strip()
 
+    @staticmethod
+    def _search_dimensions_in_video_url(a_format, video_url):
+        m = re.search(r'/(?P<width>\d+)x(?P<height>\d+)/', video_url)
+        if m:
+            a_format.update({
+                'width': int(m.group('width')),
+                'height': int(m.group('height')),
+            })
+
 
 class TwitterCardIE(TwitterBaseIE):
     IE_NAME = 'twitter:card'
@@ -90,6 +101,59 @@ class TwitterCardIE(TwitterBaseIE):
         },
     ]
 
+    def _parse_media_info(self, media_info, video_id):
+        formats = []
+        for media_variant in media_info.get('variants', []):
+            media_url = media_variant['url']
+            if media_url.endswith('.m3u8'):
+                formats.extend(self._extract_m3u8_formats(media_url, video_id, ext='mp4', m3u8_id='hls'))
+            elif media_url.endswith('.mpd'):
+                formats.extend(self._extract_mpd_formats(media_url, video_id, mpd_id='dash'))
+            else:
+                vbr = int_or_none(dict_get(media_variant, ('bitRate', 'bitrate')), scale=1000)
+                a_format = {
+                    'url': media_url,
+                    'format_id': 'http-%d' % vbr if vbr else 'http',
+                    'vbr': vbr,
+                }
+                # Reported bitRate may be zero
+                if not a_format['vbr']:
+                    del a_format['vbr']
+
+                self._search_dimensions_in_video_url(a_format, media_url)
+
+                formats.append(a_format)
+        return formats
+
+    def _extract_mobile_formats(self, username, video_id):
+        webpage = self._download_webpage(
+            'https://mobile.twitter.com/%s/status/%s' % (username, video_id),
+            video_id, 'Downloading mobile webpage',
+            headers={
+                # A recent mobile UA is necessary for `gt` cookie
+                'User-Agent': 'Mozilla/5.0 (Android 6.0.1; Mobile; rv:54.0) Gecko/54.0 Firefox/54.0',
+            })
+        main_script_url = self._html_search_regex(
+            r'<script[^>]+src="([^"]+main\.[^"]+)"', webpage, 'main script URL')
+        main_script = self._download_webpage(
+            main_script_url, video_id, 'Downloading main script')
+        bearer_token = self._search_regex(
+            r'BEARER_TOKEN\s*:\s*"([^"]+)"',
+            main_script, 'bearer token')
+        guest_token = self._search_regex(
+            r'document\.cookie\s*=\s*decodeURIComponent\("gt=(\d+)',
+            webpage, 'guest token')
+        api_data = self._download_json(
+            'https://api.twitter.com/2/timeline/conversation/%s.json' % video_id,
+            video_id, 'Downloading mobile API data',
+            headers={
+                'Authorization': 'Bearer ' + bearer_token,
+                'x-guest-token': guest_token,
+            })
+        media_info = try_get(api_data, lambda o: o['globalObjects']['tweets'][video_id]
+                                                  ['extended_entities']['media'][0]['video_info']) or {}
+        return self._parse_media_info(media_info, video_id)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
@@ -117,14 +181,6 @@ class TwitterCardIE(TwitterBaseIE):
         if periscope_url:
             return self.url_result(periscope_url, PeriscopeIE.ie_key())
 
-        def _search_dimensions_in_video_url(a_format, video_url):
-            m = re.search(r'/(?P<width>\d+)x(?P<height>\d+)/', video_url)
-            if m:
-                a_format.update({
-                    'width': int(m.group('width')),
-                    'height': int(m.group('height')),
-                })
-
         video_url = config.get('video_url') or config.get('playlist', [{}])[0].get('source')
 
         if video_url:
@@ -135,7 +191,7 @@ class TwitterCardIE(TwitterBaseIE):
                     'url': video_url,
                 }
 
-                _search_dimensions_in_video_url(f, video_url)
+                self._search_dimensions_in_video_url(f, video_url)
 
                 formats.append(f)
 
@@ -152,29 +208,14 @@ class TwitterCardIE(TwitterBaseIE):
                 media_info = entity['mediaInfo']
 
         if media_info:
-            for media_variant in media_info['variants']:
-                media_url = media_variant['url']
-                if media_url.endswith('.m3u8'):
-                    formats.extend(self._extract_m3u8_formats(media_url, video_id, ext='mp4', m3u8_id='hls'))
-                elif media_url.endswith('.mpd'):
-                    formats.extend(self._extract_mpd_formats(media_url, video_id, mpd_id='dash'))
-                else:
-                    vbr = int_or_none(media_variant.get('bitRate'), scale=1000)
-                    a_format = {
-                        'url': media_url,
-                        'format_id': 'http-%d' % vbr if vbr else 'http',
-                        'vbr': vbr,
-                    }
-                    # Reported bitRate may be zero
-                    if not a_format['vbr']:
-                        del a_format['vbr']
-
-                    _search_dimensions_in_video_url(a_format, media_url)
-
-                    formats.append(a_format)
-
+            formats.extend(self._parse_media_info(media_info, video_id))
             duration = float_or_none(media_info.get('duration', {}).get('nanos'), scale=1e9)
 
+        username = config.get('user', {}).get('screen_name')
+        if username:
+            formats.extend(self._extract_mobile_formats(username, video_id))
+
+        self._remove_duplicate_formats(formats)
         self._sort_formats(formats)
 
         title = self._search_regex(r'<title>([^<]+)</title>', webpage, 'title')
@@ -301,6 +342,20 @@ class TwitterIE(InfoExtractor):
             'timestamp': 1474613214,
         },
         'add_ie': ['Periscope'],
+    }, {
+        # has mp4 formats via mobile API
+        'url': 'https://twitter.com/news_al3alm/status/852138619213144067',
+        'info_dict': {
+            'id': '852138619213144067',
+            'ext': 'mp4',
+            'title': 'عالم الأخبار - كلمة تاريخية بجلسة الجناسي التاريخية.. النائب خالد مؤنس العتيبي للمعارضين : اتقوا الله .. الظلم ظلمات يوم القيامة',
+            'description': 'عالم الأخبار on Twitter: "كلمة تاريخية بجلسة الجناسي التاريخية.. النائب خالد مؤنس العتيبي للمعارضين : اتقوا الله .. الظلم ظلمات يوم القيامة   https://t.co/xg6OhpyKfN"',
+            'uploader': 'عالم الأخبار',
+            'uploader_id': 'news_al3alm',
+        },
+        'params': {
+            'format': 'best[format_id^=http-]',
+        },
     }]
 
     def _real_extract(self, url):

From 7f176ac4775aed5e5a72d2f0c4b579d1b886d419 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 11 Jul 2017 15:35:19 +0800
Subject: [PATCH 1099/1696] [periscope] Support pscp.tv URLs in embedded frames

And fix a relevant twitter test
---
 ChangeLog                         | 1 +
 youtube_dl/extractor/periscope.py | 2 +-
 youtube_dl/extractor/twitter.py   | 5 +++--
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index db1776dce..ffb4b6994 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
++ [periscope] Support pscp.tv URLs in embedded frames
 * [niconico] Fix authentication error handling (#12486)
 * [giantbomb] Extract m3u8 formats (#13626)
 
diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 1add6b840..bfa12edc9 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -49,7 +49,7 @@ class PeriscopeIE(PeriscopeBaseIE):
     @staticmethod
     def _extract_url(webpage):
         mobj = re.search(
-            r'<iframe[^>]+src=([\'"])(?P<url>(?:https?:)?//(?:www\.)?periscope\.tv/(?:(?!\1).)+)\1', webpage)
+            r'<iframe[^>]+src=([\'"])(?P<url>(?:https?:)?//(?:www\.)?(?:periscope|pscp)\.tv/(?:(?!\1).)+)\1', webpage)
         if mobj:
             return mobj.group('url')
 
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 2041b4199..e4bc7e012 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -335,10 +335,11 @@ class TwitterIE(InfoExtractor):
         'info_dict': {
             'id': '1zqKVVlkqLaKB',
             'ext': 'mp4',
-            'title': 'Sgt Kerry Schmidt - Ontario Provincial Police - Road rage, mischief, assault, rollover and fire in one occurrence',
+            'title': 'Sgt Kerry Schmidt - LIVE on #Periscope: Road rage, mischief, assault, rollover and fire in one occurrence',
+            'description': 'Sgt Kerry Schmidt on Twitter: "LIVE on #Periscope: Road rage, mischief, assault, rollover and fire in one occurrence  https://t.co/EKrVgIXF3s"',
             'upload_date': '20160923',
             'uploader_id': 'OPP_HSD',
-            'uploader': 'Sgt Kerry Schmidt - Ontario Provincial Police',
+            'uploader': 'Sgt Kerry Schmidt',
             'timestamp': 1474613214,
         },
         'add_ie': ['Periscope'],

From 9be31e771cd9481ea690c01eed398645deadc1de Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 11 Jul 2017 15:48:34 +0800
Subject: [PATCH 1100/1696] [twitter] Support HLS streams in vmap URLs

---
 ChangeLog                       |  1 +
 youtube_dl/extractor/twitter.py | 23 ++++++++++++++---------
 2 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index ffb4b6994..8e6451144 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
++ [twitter] Support HLS streams in vmap URLs
 + [periscope] Support pscp.tv URLs in embedded frames
 * [niconico] Fix authentication error handling (#12486)
 * [giantbomb] Extract m3u8 formats (#13626)
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index e4bc7e012..89eabe77e 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -20,9 +20,16 @@ from .periscope import PeriscopeIE
 
 
 class TwitterBaseIE(InfoExtractor):
-    def _get_vmap_video_url(self, vmap_url, video_id):
+    def _extract_formats_from_vmap_url(self, vmap_url, video_id):
         vmap_data = self._download_xml(vmap_url, video_id)
-        return xpath_text(vmap_data, './/MediaFile').strip()
+        video_url = xpath_text(vmap_data, './/MediaFile').strip()
+        if determine_ext(video_url) == 'm3u8':
+            return self._extract_m3u8_formats(
+                video_url, video_id, ext='mp4', m3u8_id='hls',
+                entry_protocol='m3u8_native')
+        return [{
+            'url': video_url,
+        }]
 
     @staticmethod
     def _search_dimensions_in_video_url(a_format, video_url):
@@ -197,9 +204,8 @@ class TwitterCardIE(TwitterBaseIE):
 
         vmap_url = config.get('vmapUrl') or config.get('vmap_url')
         if vmap_url:
-            formats.append({
-                'url': self._get_vmap_video_url(vmap_url, video_id),
-            })
+            formats.extend(
+                self._extract_formats_from_vmap_url(vmap_url, video_id))
 
         media_info = None
 
@@ -449,7 +455,7 @@ class TwitterAmplifyIE(TwitterBaseIE):
 
         vmap_url = self._html_search_meta(
             'twitter:amplify:vmap', webpage, 'vmap url')
-        video_url = self._get_vmap_video_url(vmap_url, video_id)
+        formats = self._extract_formats_from_vmap_url(vmap_url, video_id)
 
         thumbnails = []
         thumbnail = self._html_search_meta(
@@ -471,11 +477,10 @@ class TwitterAmplifyIE(TwitterBaseIE):
             })
 
         video_w, video_h = _find_dimension('player')
-        formats = [{
-            'url': video_url,
+        formats[0].update({
             'width': video_w,
             'height': video_h,
-        }]
+        })
 
         return {
             'id': video_id,

From e8f20ffa032a791548a66bb7b694c424673537e6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 11 Jul 2017 16:05:15 +0800
Subject: [PATCH 1101/1696] [vine] Make sure the title won't be empty

And fix a relevant TwitterCard test case
---
 ChangeLog                       | 1 +
 youtube_dl/extractor/twitter.py | 1 +
 youtube_dl/extractor/vine.py    | 6 ++++--
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 8e6451144..a5de3c232 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [vine] Make sure the title won't be empty
 + [twitter] Support HLS streams in vmap URLs
 + [periscope] Support pscp.tv URLs in embedded frames
 * [niconico] Fix authentication error handling (#12486)
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 89eabe77e..2ff55412b 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -91,6 +91,7 @@ class TwitterCardIE(TwitterBaseIE):
                 'uploader_id': '1189339351084113920',
                 'uploader': 'ArsenalTerje',
                 'title': 'Vine by ArsenalTerje',
+                'timestamp': 1447451307,
             },
             'add_ie': ['Vine'],
         }, {
diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index 4957a07f7..46950d3a1 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -92,10 +92,12 @@ class VineIE(InfoExtractor):
 
         username = data.get('username')
 
+        alt_title = 'Vine by %s' % username if username else None
+
         return {
             'id': video_id,
-            'title': data.get('description'),
-            'alt_title': 'Vine by %s' % username if username else None,
+            'title': data.get('description') or alt_title or 'Vine video',
+            'alt_title': alt_title,
             'thumbnail': data.get('thumbnailUrl'),
             'timestamp': unified_timestamp(data.get('created')),
             'uploader': username,

From 3615bfe1b4b97cb35ddd63c455160a50c2a10961 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 11 Jul 2017 16:46:37 +0800
Subject: [PATCH 1102/1696] [twitter] Fix remaining tests

---
 youtube_dl/extractor/twitter.py | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 2ff55412b..6eaf360a6 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -54,7 +54,8 @@ class TwitterCardIE(TwitterBaseIE):
                 'title': 'Twitter Card',
                 'thumbnail': r're:^https?://.*\.jpg$',
                 'duration': 30.033,
-            }
+            },
+            'skip': 'Video gone',
         },
         {
             'url': 'https://twitter.com/i/cards/tfw/v1/623160978427936768',
@@ -66,6 +67,7 @@ class TwitterCardIE(TwitterBaseIE):
                 'thumbnail': r're:^https?://.*\.jpg',
                 'duration': 80.155,
             },
+            'skip': 'Video gone',
         },
         {
             'url': 'https://twitter.com/i/cards/tfw/v1/654001591733886977',
@@ -83,7 +85,7 @@ class TwitterCardIE(TwitterBaseIE):
         },
         {
             'url': 'https://twitter.com/i/cards/tfw/v1/665289828897005568',
-            'md5': 'ab2745d0b0ce53319a534fccaa986439',
+            'md5': '6dabeaca9e68cbb71c99c322a4b42a11',
             'info_dict': {
                 'id': 'iBb2x00UVlv',
                 'ext': 'mp4',
@@ -96,12 +98,12 @@ class TwitterCardIE(TwitterBaseIE):
             'add_ie': ['Vine'],
         }, {
             'url': 'https://twitter.com/i/videos/tweet/705235433198714880',
-            'md5': '3846d0a07109b5ab622425449b59049d',
+            'md5': '884812a2adc8aaf6fe52b15ccbfa3b88',
             'info_dict': {
                 'id': '705235433198714880',
                 'ext': 'mp4',
                 'title': 'Twitter web player',
-                'thumbnail': r're:^https?://.*\.jpg',
+                'thumbnail': r're:^https?://.*',
             },
         }, {
             'url': 'https://twitter.com/i/videos/752274308186120192',
@@ -303,10 +305,10 @@ class TwitterIE(InfoExtractor):
         'info_dict': {
             'id': '700207533655363584',
             'ext': 'mp4',
-            'title': 'JG - BEAT PROD: @suhmeduh #Damndaniel',
-            'description': 'JG on Twitter: "BEAT PROD: @suhmeduh  https://t.co/HBrQ4AfpvZ #Damndaniel https://t.co/byBooq2ejZ"',
+            'title': 'Donte - BEAT PROD: @suhmeduh #Damndaniel',
+            'description': 'Donte on Twitter: "BEAT PROD: @suhmeduh  https://t.co/HBrQ4AfpvZ #Damndaniel https://t.co/byBooq2ejZ"',
             'thumbnail': r're:^https?://.*\.jpg',
-            'uploader': 'JG',
+            'uploader': 'Donte',
             'uploader_id': 'jaydingeer',
         },
         'params': {
@@ -318,9 +320,11 @@ class TwitterIE(InfoExtractor):
         'info_dict': {
             'id': 'MIOxnrUteUd',
             'ext': 'mp4',
-            'title': 'Dr.Pepperの飲み方 #japanese #バカ #ドクペ #電動ガン',
-            'uploader': 'TAKUMA',
-            'uploader_id': '1004126642786242560',
+            'title': 'FilmDrunk - Vine of the day',
+            'description': 'FilmDrunk on Twitter: "Vine of the day https://t.co/xmTvRdqxWf"',
+            'uploader': 'FilmDrunk',
+            'uploader_id': 'Filmdrunk',
+            'timestamp': 1402826626,
             'upload_date': '20140615',
         },
         'add_ie': ['Vine'],

From f2bb33a9868e499f4582fee24a7b67d559d33575 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Jul 2017 21:36:45 +0700
Subject: [PATCH 1103/1696] [ted] Fix subtitles extraction (closes #13628,
 closes #13629)

---
 youtube_dl/extractor/ted.py | 30 ++++++++++++++++--------------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index f27d0e313..06a27fd04 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -271,20 +271,22 @@ class TEDIE(InfoExtractor):
         }
 
     def _get_subtitles(self, video_id, talk_info):
-        languages = [lang['languageCode'] for lang in talk_info.get('languages', [])]
-        if languages:
-            sub_lang_list = {}
-            for l in languages:
-                sub_lang_list[l] = [
-                    {
-                        'url': 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/%s' % (video_id, l, ext),
-                        'ext': ext,
-                    }
-                    for ext in ['ted', 'srt']
-                ]
-            return sub_lang_list
-        else:
-            return {}
+        sub_lang_list = {}
+        for language in try_get(
+                talk_info,
+                (lambda x: x['downloads']['languages'],
+                 lambda x: x['languages']), list):
+            lang_code = language.get('languageCode') or language.get('ianaCode')
+            if not lang_code:
+                continue
+            sub_lang_list[lang_code] = [
+                {
+                    'url': 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/%s' % (video_id, lang_code, ext),
+                    'ext': ext,
+                }
+                for ext in ['ted', 'srt']
+            ]
+        return sub_lang_list
 
     def _watch_info(self, url, name):
         webpage = self._download_webpage(url, name)

From 9a0942ad55bba714d6eaeb9ee4f66a138ec85e17 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Jul 2017 22:59:36 +0700
Subject: [PATCH 1104/1696] [drtv] Make HLS and HDS extraction non fatal

---
 youtube_dl/extractor/drtv.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index c84624f1e..69effba58 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -118,7 +118,7 @@ class DRTVIE(InfoExtractor):
                     if target == 'HDS':
                         f4m_formats = self._extract_f4m_formats(
                             uri + '?hdcore=3.3.0&plugin=aasp-3.3.0.99.43',
-                            video_id, preference, f4m_id=format_id)
+                            video_id, preference, f4m_id=format_id, fatal=False)
                         if kind == 'AudioResource':
                             for f in f4m_formats:
                                 f['vcodec'] = 'none'
@@ -126,7 +126,8 @@ class DRTVIE(InfoExtractor):
                     elif target == 'HLS':
                         formats.extend(self._extract_m3u8_formats(
                             uri, video_id, 'mp4', entry_protocol='m3u8_native',
-                            preference=preference, m3u8_id=format_id))
+                            preference=preference, m3u8_id=format_id,
+                            fatal=False))
                     else:
                         bitrate = link.get('Bitrate')
                         if bitrate:

From 15da37c7dc8cf14ba5ce880aa1805fceaa71fc44 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 14 Jul 2017 00:40:54 +0700
Subject: [PATCH 1105/1696] [YoutubeDL] Don't expand env variables in meta
 fields (closes #13637)

---
 test/test_YoutubeDL.py  |  6 ++++++
 youtube_dl/YoutubeDL.py | 15 ++++++++++++++-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 75945e38f..70989e232 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -527,6 +527,8 @@ class TestYoutubeDL(unittest.TestCase):
             'ext': 'mp4',
             'width': None,
             'height': 1080,
+            'title1': '$PATH',
+            'title2': '%PATH%',
         }
 
         def fname(templ):
@@ -545,10 +547,14 @@ class TestYoutubeDL(unittest.TestCase):
         self.assertEqual(fname('%(height)0 6d.%(ext)s'), ' 01080.mp4')
         self.assertEqual(fname('%(height)0   6d.%(ext)s'), ' 01080.mp4')
         self.assertEqual(fname('%(height)   0   6d.%(ext)s'), ' 01080.mp4')
+        self.assertEqual(fname('%%'), '%')
+        self.assertEqual(fname('%%%%'), '%%')
         self.assertEqual(fname('%%(height)06d.%(ext)s'), '%(height)06d.mp4')
         self.assertEqual(fname('%(width)06d.%(ext)s'), 'NA.mp4')
         self.assertEqual(fname('%(width)06d.%%(ext)s'), 'NA.%(ext)s')
         self.assertEqual(fname('%%(width)06d.%(ext)s'), '%(width)06d.mp4')
+        self.assertEqual(fname('Hello %(title1)s'), 'Hello $PATH')
+        self.assertEqual(fname('Hello %(title2)s'), 'Hello %PATH%')
 
     def test_format_note(self):
         ydl = YoutubeDL()
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 60ee4b7d8..8730d32ef 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -20,6 +20,7 @@ import re
 import shutil
 import subprocess
 import socket
+import string
 import sys
 import time
 import tokenize
@@ -674,7 +675,19 @@ class YoutubeDL(object):
                         FORMAT_RE.format(numeric_field),
                         r'%({0})s'.format(numeric_field), outtmpl)
 
-            filename = expand_path(outtmpl % template_dict)
+            # expand_path translates '%%' into '%' and '$$' into '$'
+            # correspondingly that is not what we want since we need to keep
+            # '%%' intact for template dict substitution step. Working around
+            # with boundary-alike separator hack.
+            sep = ''.join([random.choice(string.ascii_letters) for _ in range(32)])
+            outtmpl = outtmpl.replace('%%', '%{0}%'.format(sep)).replace('$$', '${0}$'.format(sep))
+
+            # outtmpl should be expand_path'ed before template dict substitution
+            # because meta fields may contain env variables we don't want to
+            # be expanded. For example, for outtmpl "%(title)s.%(ext)s" and
+            # title "Hello $PATH", we don't want `$PATH` to be expanded.
+            filename = expand_path(outtmpl).replace(sep, '') % template_dict
+
             # Temporary fix for #4787
             # 'Treat' all problem characters by passing filename through preferredencoding
             # to workaround encoding issues with subprocess on python2 @ Windows

From f354d8480700c5e6f288bfce497a363b4c6f0859 Mon Sep 17 00:00:00 2001
From: rrooij <rrooij@users.noreply.github.com>
Date: Fri, 14 Jul 2017 17:10:17 +0200
Subject: [PATCH 1106/1696] [5tv] Add another video URL pattern (closes #13354)

---
 youtube_dl/extractor/fivetv.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/fivetv.py b/youtube_dl/extractor/fivetv.py
index 15736c9fe..9f9863746 100644
--- a/youtube_dl/extractor/fivetv.py
+++ b/youtube_dl/extractor/fivetv.py
@@ -43,7 +43,7 @@ class FiveTVIE(InfoExtractor):
         'info_dict': {
             'id': 'glavnoe',
             'ext': 'mp4',
-            'title': 'Итоги недели с 8 по 14 июня 2015 года',
+            'title': r're:^Итоги недели с \d+ по \d+ \w+ \d{4} года$',
             'thumbnail': r're:^https?://.*\.jpg$',
         },
     }, {
@@ -70,7 +70,8 @@ class FiveTVIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         video_url = self._search_regex(
-            r'<a[^>]+?href="([^"]+)"[^>]+?class="videoplayer"',
+            [r'<div[^>]+?class="flowplayer[^>]+?data-href="([^"]+)"',
+             r'<a[^>]+?href="([^"]+)"[^>]+?class="videoplayer"'],
             webpage, 'video url')
 
         title = self._og_search_title(webpage, default=None) or self._search_regex(

From 00dbdfc1f741b919a0add36394065ce1aeccfda8 Mon Sep 17 00:00:00 2001
From: satunnainen <satunnainen@users.noreply.github.com>
Date: Fri, 14 Jul 2017 18:11:07 +0300
Subject: [PATCH 1107/1696] [slideshare] Fix extraction

---
 youtube_dl/extractor/slideshare.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/slideshare.py b/youtube_dl/extractor/slideshare.py
index 74a1dc672..e89ebebe7 100644
--- a/youtube_dl/extractor/slideshare.py
+++ b/youtube_dl/extractor/slideshare.py
@@ -31,7 +31,7 @@ class SlideshareIE(InfoExtractor):
         page_title = mobj.group('title')
         webpage = self._download_webpage(url, page_title)
         slideshare_obj = self._search_regex(
-            r'\$\.extend\(slideshare_object,\s*(\{.*?\})\);',
+            r'\$\.extend\(.*?slideshare_object,\s*(\{.*?\})\);',
             webpage, 'slideshare object')
         info = json.loads(slideshare_obj)
         if info['slideshow']['type'] != 'video':

From 7d02dcfaa2589453ee3cc6c88ee27f04c252f8a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 14 Jul 2017 22:37:04 +0700
Subject: [PATCH 1108/1696] [youtube] Don't capture YouTube Red ad for creator
 meta field (closes #13621)

---
 youtube_dl/extractor/youtube.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 77cd271ef..4597ccb3a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -673,6 +673,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             },
         },
         # video_info is None (https://github.com/rg3/youtube-dl/issues/4421)
+        # YouTube Red ad is not captured for creator
         {
             'url': '__2ABJjxzNo',
             'info_dict': {
@@ -1649,7 +1650,21 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             video_webpage, 'license', default=None)
 
         m_music = re.search(
-            r'<h4[^>]+class="title"[^>]*>\s*Music\s*</h4>\s*<ul[^>]*>\s*<li>(?P<title>.+?) by (?P<creator>.+?)(?:\(.+?\))?</li',
+            r'''(?x)
+                <h4[^>]+class="title"[^>]*>\s*Music\s*</h4>\s*
+                <ul[^>]*>\s*
+                <li>(?P<title>.+?)
+                by (?P<creator>.+?)
+                (?:
+                    \(.+?\)|
+                    <a[^>]*
+                        (?:
+                            \bhref=["\']/red[^>]*>|             # drop possible
+                            >\s*Listen ad-free with YouTube Red # YouTube Red ad 
+                        )
+                    .*?
+                )?</li
+            ''',
             video_webpage)
         if m_music:
             video_alt_title = remove_quotes(unescapeHTML(m_music.group('title')))

From 2583c0b54e56f6dbce85a079d91a05e9b13c2dce Mon Sep 17 00:00:00 2001
From: Robin Neatherway <robin.neatherway@gmail.com>
Date: Fri, 14 Jul 2017 17:08:32 +0100
Subject: [PATCH 1109/1696] Fix bugs caused by typos

---
 youtube_dl/downloader/ism.py           | 3 +--
 youtube_dl/extractor/audioboom.py      | 2 +-
 youtube_dl/extractor/generic.py        | 2 +-
 youtube_dl/extractor/karrierevideos.py | 2 +-
 4 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/downloader/ism.py b/youtube_dl/downloader/ism.py
index 5f6f9faef..9b001ecff 100644
--- a/youtube_dl/downloader/ism.py
+++ b/youtube_dl/downloader/ism.py
@@ -98,7 +98,7 @@ def write_piff_header(stream, params):
 
     if is_audio:
         smhd_payload = s88.pack(0)  # balance
-        smhd_payload = u16.pack(0)  # reserved
+        smhd_payload += u16.pack(0)  # reserved
         media_header_box = full_box(b'smhd', 0, 0, smhd_payload)  # Sound Media Header
     else:
         vmhd_payload = u16.pack(0)  # graphics mode
@@ -126,7 +126,6 @@ def write_piff_header(stream, params):
         if fourcc == 'AACL':
             sample_entry_box = box(b'mp4a', sample_entry_payload)
     else:
-        sample_entry_payload = sample_entry_payload
         sample_entry_payload += u16.pack(0)  # pre defined
         sample_entry_payload += u16.pack(0)  # reserved
         sample_entry_payload += u32.pack(0) * 3  # pre defined
diff --git a/youtube_dl/extractor/audioboom.py b/youtube_dl/extractor/audioboom.py
index e48bb8972..393f381c6 100644
--- a/youtube_dl/extractor/audioboom.py
+++ b/youtube_dl/extractor/audioboom.py
@@ -43,7 +43,7 @@ class AudioBoomIE(InfoExtractor):
 
         def from_clip(field):
             if clip:
-                clip.get(field)
+                return clip.get(field)
 
         audio_url = from_clip('clipURLPriorToLoading') or self._og_search_property(
             'audio', webpage, 'audio url')
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 5e8890d41..8c2ff39d5 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2740,7 +2740,7 @@ class GenericIE(InfoExtractor):
         rutube_urls = RutubeIE._extract_urls(webpage)
         if rutube_urls:
             return self.playlist_from_matches(
-                rutube_urls, ie=RutubeIE.ie_key())
+                rutube_urls, video_id, video_title, ie=RutubeIE.ie_key())
 
         # Look for WashingtonPost embeds
         wapo_urls = WashingtonPostIE._extract_urls(webpage)
diff --git a/youtube_dl/extractor/karrierevideos.py b/youtube_dl/extractor/karrierevideos.py
index 4e9eb67bf..f236a2f78 100644
--- a/youtube_dl/extractor/karrierevideos.py
+++ b/youtube_dl/extractor/karrierevideos.py
@@ -48,7 +48,7 @@ class KarriereVideosIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         title = (self._html_search_meta('title', webpage, default=None) or
-                 self._search_regex(r'<h1 class="title">([^<]+)</h1>'))
+                 self._search_regex(r'<h1 class="title">([^<]+)</h1>', webpage, 'video title'))
 
         video_id = self._search_regex(
             r'/config/video/(.+?)\.xml', webpage, 'video id')

From 4e826cd9aec383768a7b25aa3161efd4672f9310 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Jul 2017 06:48:12 +0700
Subject: [PATCH 1110/1696] [nexx] Add extractor (closes #10807, closes #13465)

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/generic.py    |  22 +++
 youtube_dl/extractor/nexx.py       | 221 +++++++++++++++++++++++++++++
 3 files changed, 244 insertions(+)
 create mode 100644 youtube_dl/extractor/nexx.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index eb1541729..9d34447a9 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -653,6 +653,7 @@ from .nextmedia import (
     AppleDailyIE,
     NextTVIE,
 )
+from .nexx import NexxIE
 from .nfb import NFBIE
 from .nfl import NFLIE
 from .nhk import NhkVodIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8c2ff39d5..123a21296 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -36,6 +36,7 @@ from .brightcove import (
     BrightcoveLegacyIE,
     BrightcoveNewIE,
 )
+from .nexx import NexxIE
 from .nbc import NBCSportsVPlayerIE
 from .ooyala import OoyalaIE
 from .rutv import RUTVIE
@@ -1549,6 +1550,22 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['BrightcoveLegacy'],
         },
+        # Nexx embed
+        {
+            'url': 'https://www.funk.net/serien/5940e15073f6120001657956/items/593efbb173f6120001657503',
+            'info_dict': {
+                'id': '247746',
+                'ext': 'mp4',
+                'title': "Yesterday's Jam (OV)",
+                'description': 'md5:09bc0984723fed34e2581624a84e05f0',
+                'timestamp': 1492594816,
+                'upload_date': '20170419',
+            },
+            'params': {
+                'format': 'bestvideo',
+                'skip_download': True,
+            },
+        },
         # Facebook <iframe> embed
         {
             'url': 'https://www.hostblogger.de/blog/archives/6181-Auto-jagt-Betonmischer.html',
@@ -2133,6 +2150,11 @@ class GenericIE(InfoExtractor):
         if bc_urls:
             return self.playlist_from_matches(bc_urls, video_id, video_title, ie='BrightcoveNew')
 
+        # Look for Nexx embeds
+        nexx_urls = NexxIE._extract_urls(webpage)
+        if nexx_urls:
+            return self.playlist_from_matches(nexx_urls, video_id, video_title, ie=NexxIE.ie_key())
+
         # Look for ThePlatform embeds
         tp_urls = ThePlatformIE._extract_urls(webpage)
         if tp_urls:
diff --git a/youtube_dl/extractor/nexx.py b/youtube_dl/extractor/nexx.py
new file mode 100644
index 000000000..60b42cb7d
--- /dev/null
+++ b/youtube_dl/extractor/nexx.py
@@ -0,0 +1,221 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import hashlib
+import random
+import re
+import time
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    parse_duration,
+    try_get,
+    urlencode_postdata,
+)
+
+
+class NexxIE(InfoExtractor):
+    _VALID_URL = r'https?://api\.nexx(?:\.cloud|cdn\.com)/v3/(?P<domain_id>\d+)/videos/byid/(?P<id>\d+)'
+    _TESTS = [{
+        # movie
+        'url': 'https://api.nexx.cloud/v3/748/videos/byid/128907',
+        'md5': '16746bfc28c42049492385c989b26c4a',
+        'info_dict': {
+            'id': '128907',
+            'ext': 'mp4',
+            'title': 'Stiftung Warentest',
+            'alt_title': 'Wie ein Test abläuft',
+            'description': 'md5:d1ddb1ef63de721132abd38639cc2fd2',
+            'release_year': 2013,
+            'creator': 'SPIEGEL TV',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 2509,
+            'timestamp': 1384264416,
+            'upload_date': '20131112',
+        },
+        'params': {
+            'format': 'bestvideo',
+        },
+    }, {
+        # episode
+        'url': 'https://api.nexx.cloud/v3/741/videos/byid/247858',
+        'info_dict': {
+            'id': '247858',
+            'ext': 'mp4',
+            'title': 'Return of the Golden Child (OV)',
+            'description': 'md5:5d969537509a92b733de21bae249dc63',
+            'release_year': 2017,
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 1397,
+            'timestamp': 1495033267,
+            'upload_date': '20170517',
+            'episode_number': 2,
+            'season_number': 2,
+        },
+        'params': {
+            'format': 'bestvideo',
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://api.nexxcdn.com/v3/748/videos/byid/128907',
+        'only_matching': True,
+    }]
+
+    @staticmethod
+    def _extract_urls(webpage):
+        # Reference:
+        # 1. https://nx-s.akamaized.net/files/201510/44.pdf
+
+        entries = []
+
+        # JavaScript Integration
+        for domain_id, video_id in re.findall(
+                r'''(?isx)
+                    <script\b[^>]+\bsrc=["\']https?://require\.nexx(?:\.cloud|cdn\.com)/(\d+).+?
+                    onPLAYReady.+?
+                    _play\.init\s*\(.+?\s*,\s*(\d+)\s*,\s*.+?\)
+                ''', webpage):
+            entries.append('https://api.nexx.cloud/v3/%s/videos/byid/%s' % (domain_id, video_id))
+
+        # TODO: support more embed formats
+
+        return entries
+
+    def _handle_error(self, response):
+        status = int_or_none(try_get(
+            response, lambda x: x['metadata']['status']) or 200)
+        if 200 <= status < 300:
+            return
+        raise ExtractorError(
+            '%s said: %s' % (self.IE_NAME, response['metadata']['errorhint']),
+            expected=True)
+
+    def _call_api(self, domain_id, path, video_id, data=None, headers={}):
+        headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8'
+        result = self._download_json(
+            'https://api.nexx.cloud/v3/%s/%s' % (domain_id, path), video_id,
+            'Downloading %s JSON' % path, data=urlencode_postdata(data),
+            headers=headers)
+        self._handle_error(result)
+        return result['result']
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        domain_id, video_id = mobj.group('domain_id', 'id')
+
+        # Reverse engineered from JS code (see getDeviceID function)
+        device_id = '%d:%d:%d%d' % (
+            random.randint(1, 4), int(time.time()),
+            random.randint(1e4, 99999), random.randint(1, 9))
+
+        result = self._call_api(domain_id, 'session/init', video_id, data={
+            'nxp_devh': device_id,
+            'nxp_userh': '',
+            'precid': '0',
+            'playlicense': '0',
+            'screenx': '1920',
+            'screeny': '1080',
+            'playerversion': '6.0.00',
+            'gateway': 'html5',
+            'adGateway': '',
+            'explicitlanguage': 'en-US',
+            'addTextTemplates': '1',
+            'addDomainData': '1',
+            'addAdModel': '1',
+        }, headers={
+            'X-Request-Enable-Auth-Fallback': '1',
+        })
+
+        cid = result['general']['cid']
+
+        # As described in [1] X-Request-Token generation algorithm is
+        # as follows:
+        #   md5( operation + domain_id + domain_secret )
+        # where domain_secret is a static value that will be given by nexx.tv
+        # as per [1]. Here is how this "secret" is generated (reversed
+        # from _play.api.init function, search for clienttoken). So it's
+        # actually not static and not that much of a secret.
+        # 1. https://nexxtvstorage.blob.core.windows.net/files/201610/27.pdf
+        secret = result['device']['clienttoken'][int(device_id[0]):]
+        secret = secret[0:len(secret) - int(device_id[-1])]
+
+        op = 'byid'
+
+        # Reversed from JS code for _play.api.call function (search for
+        # X-Request-Token)
+        request_token = hashlib.md5(
+            ''.join((op, domain_id, secret)).encode('utf-8')).hexdigest()
+
+        video = self._call_api(
+            domain_id, 'videos/%s/%s' % (op, video_id), video_id, data={
+                'additionalfields': 'language,channel,actors,studio,licenseby,slug,subtitle,teaser,description',
+                'addInteractionOptions': '1',
+                'addStatusDetails': '1',
+                'addStreamDetails': '1',
+                'addCaptions': '1',
+                'addScenes': '1',
+                'addHotSpots': '1',
+                'addBumpers': '1',
+                'captionFormat': 'data',
+            }, headers={
+                'X-Request-CID': cid,
+                'X-Request-Token': request_token,
+            })
+
+        general = video['general']
+        title = general['title']
+
+        stream_data = video['streamdata']
+        language = general.get('language_raw') or ''
+
+        # TODO: reverse more cdns and formats
+
+        cdn = stream_data['cdnType']
+        assert cdn == 'azure'
+
+        azure_locator = stream_data['azureLocator']
+
+        AZURE_URL = 'http://nx-p%02d.akamaized.net/'
+
+        for secure in ('s', ''):
+            cdn_shield = stream_data.get('cdnShieldHTTP%s' % secure.upper())
+            if cdn_shield:
+                azure_base = 'http%s://%s' % (secure, cdn_shield)
+                break
+        else:
+            azure_base = AZURE_URL % int(stream_data['azureAccount'].replace('nexxplayplus', ''))
+
+        is_ml = ',' in language
+        azure_m3u8_url = '%s%s/%s_src%s.ism/Manifest(format=m3u8-aapl)' % (
+            azure_base, azure_locator, video_id, ('_manifest' if is_ml else ''))
+
+        protection_token = try_get(
+            video, lambda x: x['protectiondata']['token'], compat_str)
+        if protection_token:
+            azure_m3u8_url += '?hdnts=%s' % protection_token
+
+        formats = self._extract_m3u8_formats(
+            azure_m3u8_url, video_id, 'mp4', entry_protocol='m3u8_native',
+            m3u8_id='%s-hls' % cdn)
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'alt_title': general.get('subtitle'),
+            'description': general.get('description'),
+            'release_year': int_or_none(general.get('year')),
+            'creator': general.get('studio') or general.get('studio_adref'),
+            'thumbnail': try_get(
+                video, lambda x: x['imagedata']['thumb'], compat_str),
+            'duration': parse_duration(general.get('runtime')),
+            'timestamp': int_or_none(general.get('uploaded')),
+            'episode_number': int_or_none(try_get(
+                video, lambda x: x['episodedata']['episode'])),
+            'season_number': int_or_none(try_get(
+                video, lambda x: x['episodedata']['season'])),
+            'formats': formats,
+        }

From c7604d79e9993e76845141a61fdf3c308af917e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Jul 2017 06:52:23 +0700
Subject: [PATCH 1111/1696] [spiegeltv] Delegate extraction to nexx (closes
 #13159)

---
 youtube_dl/extractor/spiegeltv.py | 113 +++---------------------------
 1 file changed, 8 insertions(+), 105 deletions(-)

diff --git a/youtube_dl/extractor/spiegeltv.py b/youtube_dl/extractor/spiegeltv.py
index e1cfb8698..6ccf4c342 100644
--- a/youtube_dl/extractor/spiegeltv.py
+++ b/youtube_dl/extractor/spiegeltv.py
@@ -1,114 +1,17 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlparse
-from ..utils import (
-    determine_ext,
-    float_or_none,
-)
+from .nexx import NexxIE
 
 
 class SpiegeltvIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?spiegel\.tv/(?:#/)?filme/(?P<id>[\-a-z0-9]+)'
-    _TESTS = [{
-        'url': 'http://www.spiegel.tv/filme/flug-mh370/',
-        'info_dict': {
-            'id': 'flug-mh370',
-            'ext': 'm4v',
-            'title': 'Flug MH370',
-            'description': 'Das Rätsel um die Boeing 777 der Malaysia-Airlines',
-            'thumbnail': r're:http://.*\.jpg$',
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        }
-    }, {
-        'url': 'http://www.spiegel.tv/#/filme/alleskino-die-wahrheit-ueber-maenner/',
+    _VALID_URL = r'https?://(?:www\.)?spiegel\.tv/videos/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.spiegel.tv/videos/161681-flug-mh370/',
         'only_matching': True,
-    }]
+    }
 
     def _real_extract(self, url):
-        if '/#/' in url:
-            url = url.replace('/#/', '/')
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        title = self._html_search_regex(r'<h1.*?>(.*?)</h1>', webpage, 'title')
-
-        apihost = 'http://spiegeltv-ivms2-restapi.s3.amazonaws.com'
-        version_json = self._download_json(
-            '%s/version.json' % apihost, video_id,
-            note='Downloading version information')
-        version_name = version_json['version_name']
-
-        slug_json = self._download_json(
-            '%s/%s/restapi/slugs/%s.json' % (apihost, version_name, video_id),
-            video_id,
-            note='Downloading object information')
-        oid = slug_json['object_id']
-
-        media_json = self._download_json(
-            '%s/%s/restapi/media/%s.json' % (apihost, version_name, oid),
-            video_id, note='Downloading media information')
-        uuid = media_json['uuid']
-        is_wide = media_json['is_wide']
-
-        server_json = self._download_json(
-            'http://spiegeltv-prod-static.s3.amazonaws.com/projectConfigs/projectConfig.json',
-            video_id, note='Downloading server information')
-
-        format = '16x9' if is_wide else '4x3'
-
-        formats = []
-        for streamingserver in server_json['streamingserver']:
-            endpoint = streamingserver.get('endpoint')
-            if not endpoint:
-                continue
-            play_path = 'mp4:%s_spiegeltv_0500_%s.m4v' % (uuid, format)
-            if endpoint.startswith('rtmp'):
-                formats.append({
-                    'url': endpoint,
-                    'format_id': 'rtmp',
-                    'app': compat_urllib_parse_urlparse(endpoint).path[1:],
-                    'play_path': play_path,
-                    'player_path': 'http://prod-static.spiegel.tv/frontend-076.swf',
-                    'ext': 'flv',
-                    'rtmp_live': True,
-                })
-            elif determine_ext(endpoint) == 'm3u8':
-                formats.append({
-                    'url': endpoint.replace('[video]', play_path),
-                    'ext': 'm4v',
-                    'format_id': 'hls',  # Prefer hls since it allows to workaround georestriction
-                    'protocol': 'm3u8',
-                    'preference': 1,
-                    'http_headers': {
-                        'Accept-Encoding': 'deflate',  # gzip causes trouble on the server side
-                    },
-                })
-            else:
-                formats.append({
-                    'url': endpoint,
-                })
-        self._check_formats(formats, video_id)
-
-        thumbnails = []
-        for image in media_json['images']:
-            thumbnails.append({
-                'url': image['url'],
-                'width': image['width'],
-                'height': image['height'],
-            })
-
-        description = media_json['subtitle']
-        duration = float_or_none(media_json.get('duration_in_ms'), scale=1000)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'duration': duration,
-            'thumbnails': thumbnails,
-            'formats': formats,
-        }
+        return self.url_result(
+            'https://api.nexx.cloud/v3/748/videos/byid/%s'
+            % self._match_id(url), ie=NexxIE.ie_key())

From ea3f20494f64c18123c61f722a7864e3dbdde566 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Jul 2017 07:02:05 +0700
Subject: [PATCH 1112/1696] [youtube] PEP 8

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4597ccb3a..2e71795e7 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1660,7 +1660,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     <a[^>]*
                         (?:
                             \bhref=["\']/red[^>]*>|             # drop possible
-                            >\s*Listen ad-free with YouTube Red # YouTube Red ad 
+                            >\s*Listen ad-free with YouTube Red # YouTube Red ad
                         )
                     .*?
                 )?</li

From 961ea474b6c6965d49a58d0400d0368fa0300b19 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Jul 2017 07:02:14 +0700
Subject: [PATCH 1113/1696] [YoutubeDL] PEP 8

---
 youtube_dl/YoutubeDL.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 8730d32ef..89c07be29 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -20,13 +20,14 @@ import re
 import shutil
 import subprocess
 import socket
-import string
 import sys
 import time
 import tokenize
 import traceback
 import random
 
+from string import ascii_letters
+
 from .compat import (
     compat_basestring,
     compat_cookiejar,
@@ -679,7 +680,7 @@ class YoutubeDL(object):
             # correspondingly that is not what we want since we need to keep
             # '%%' intact for template dict substitution step. Working around
             # with boundary-alike separator hack.
-            sep = ''.join([random.choice(string.ascii_letters) for _ in range(32)])
+            sep = ''.join([random.choice(ascii_letters) for _ in range(32)])
             outtmpl = outtmpl.replace('%%', '%{0}%'.format(sep)).replace('$$', '${0}$'.format(sep))
 
             # outtmpl should be expand_path'ed before template dict substitution

From ef78563e9c148d8a24382c282d74a960e244569e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Jul 2017 07:33:26 +0700
Subject: [PATCH 1114/1696] [ChangeLog] Actualize

---
 ChangeLog | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index a5de3c232..5f3ea6b5f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,11 +1,25 @@
 version <unreleased>
 
+Core
+* [YoutubeDL] Don't expand environment variables in meta fields (#13637)
+
 Extractors
+* [spiegeltv] Delegate extraction to nexx extractor (#13159)
++ [nexx] Add support for nexx.cloud (#10807, #13465)
+* [generic] Fix rutube embeds extraction (#13641)
+* [karrierevideos] Fix title extraction (#13641)
+* [youtube] Don't capture YouTube Red ad for creator meta field (#13621)
+* [slideshare] Fix extraction (#13617)
++ [5tv] Add another video URL pattern (#13354, #13606)
+* [drtv] Make HLS and HDS extraction non fatal
+* [ted] Fix subtitles extraction (#13628, #13629)
 * [vine] Make sure the title won't be empty
 + [twitter] Support HLS streams in vmap URLs
 + [periscope] Support pscp.tv URLs in embedded frames
+* [twitter] Extract mp4 urls via mobile API (#12726)
 * [niconico] Fix authentication error handling (#12486)
 * [giantbomb] Extract m3u8 formats (#13626)
++ [vlive:playlist] Add support for playlists (#13613)
 
 
 version 2017.07.09

From cea931a9e5cf682eafe5b7cdacbbe22630b7a9e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Jul 2017 07:36:05 +0700
Subject: [PATCH 1115/1696] release 2017.07.15

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 ++
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index c4314855d..0f20d0485 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.07.09*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.07.09**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.07.15*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.07.15**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.07.09
+[debug] youtube-dl version 2017.07.15
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 5f3ea6b5f..7d71fc5e1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.07.15
 
 Core
 * [YoutubeDL] Don't expand environment variables in meta fields (#13637)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index b6a147faf..d7304ba06 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -521,6 +521,7 @@
  - **NextMedia**: 蘋果日報
  - **NextMediaActionNews**: 蘋果日報 - 動新聞
  - **NextTV**: 壹電視
+ - **Nexx**
  - **nfb**: National Film Board of Canada
  - **nfl.com**
  - **NhkVod**
@@ -942,6 +943,7 @@
  - **vk:wallpost**
  - **vlive**
  - **vlive:channel**
+ - **vlive:playlist**
  - **Vodlocker**
  - **VODPl**
  - **VODPlatform**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 14358a74c..82e166fef 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.07.09'
+__version__ = '2017.07.15'

From 94b817edebb63c3d8485e1ae27cc394dd9e21f9d Mon Sep 17 00:00:00 2001
From: troywith77 <ruitang307@gmail.com>
Date: Tue, 9 May 2017 13:10:18 +0800
Subject: [PATCH 1116/1696] [pearvideo] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/pear.py       | 34 ++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)
 create mode 100644 youtube_dl/extractor/pear.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 9d34447a9..75c1a3d0e 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -762,6 +762,7 @@ from .pandoratv import PandoraTVIE
 from .parliamentliveuk import ParliamentLiveUKIE
 from .patreon import PatreonIE
 from .pbs import PBSIE
+from .pear import PearIE
 from .people import PeopleIE
 from .periscope import (
     PeriscopeIE,
diff --git a/youtube_dl/extractor/pear.py b/youtube_dl/extractor/pear.py
new file mode 100644
index 000000000..77fd46852
--- /dev/null
+++ b/youtube_dl/extractor/pear.py
@@ -0,0 +1,34 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class PearIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?pearvideo\.com/video_(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.pearvideo.com/video_1076290',
+        'info_dict': {
+            'id': '1076290',
+            'ext': 'mp4',
+            'title': '小浣熊在主人家玻璃上滚石头：没砸',
+            'description': '小浣熊找到一个小石头，仿佛发现了一个宝贝。它不停地用石头按在玻璃上，滚来滚去，吸引主人注意。',
+            'url': 'http://video.pearvideo.com/mp4/short/20170508/cont-1076290-10438018-hd.mp4'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(r'<h1[^>]+class="video-tt">(.+)</h1>', webpage, 'title', fatal=False)
+        description = self._html_search_regex(r'<div[^>]+class="summary"[^>]*>([^<]+)<', webpage, 'description', fatal=False)
+        url = self._html_search_regex(r'hdUrl="(.*?)"', webpage, 'url', fatal=False)
+
+        return {
+            'id': video_id,
+            'ext': 'mp4',
+            'title': title,
+            'description': description,
+            'url': url
+        }

From decf86044d17a8ec04e43a4805a0092622d976ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Jul 2017 03:06:04 +0700
Subject: [PATCH 1117/1696] [pearvideo] Improve (closes #13031)

---
 youtube_dl/extractor/extractors.py |  2 +-
 youtube_dl/extractor/pear.py       | 34 ----------------
 youtube_dl/extractor/pearvideo.py  | 63 ++++++++++++++++++++++++++++++
 3 files changed, 64 insertions(+), 35 deletions(-)
 delete mode 100644 youtube_dl/extractor/pear.py
 create mode 100644 youtube_dl/extractor/pearvideo.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 75c1a3d0e..28f0d3f0d 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -762,7 +762,7 @@ from .pandoratv import PandoraTVIE
 from .parliamentliveuk import ParliamentLiveUKIE
 from .patreon import PatreonIE
 from .pbs import PBSIE
-from .pear import PearIE
+from .pearvideo import PearVideoIE
 from .people import PeopleIE
 from .periscope import (
     PeriscopeIE,
diff --git a/youtube_dl/extractor/pear.py b/youtube_dl/extractor/pear.py
deleted file mode 100644
index 77fd46852..000000000
--- a/youtube_dl/extractor/pear.py
+++ /dev/null
@@ -1,34 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-
-
-class PearIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?pearvideo\.com/video_(?P<id>[0-9]+)'
-    _TEST = {
-        'url': 'http://www.pearvideo.com/video_1076290',
-        'info_dict': {
-            'id': '1076290',
-            'ext': 'mp4',
-            'title': '小浣熊在主人家玻璃上滚石头：没砸',
-            'description': '小浣熊找到一个小石头，仿佛发现了一个宝贝。它不停地用石头按在玻璃上，滚来滚去，吸引主人注意。',
-            'url': 'http://video.pearvideo.com/mp4/short/20170508/cont-1076290-10438018-hd.mp4'
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        title = self._html_search_regex(r'<h1[^>]+class="video-tt">(.+)</h1>', webpage, 'title', fatal=False)
-        description = self._html_search_regex(r'<div[^>]+class="summary"[^>]*>([^<]+)<', webpage, 'description', fatal=False)
-        url = self._html_search_regex(r'hdUrl="(.*?)"', webpage, 'url', fatal=False)
-
-        return {
-            'id': video_id,
-            'ext': 'mp4',
-            'title': title,
-            'description': description,
-            'url': url
-        }
diff --git a/youtube_dl/extractor/pearvideo.py b/youtube_dl/extractor/pearvideo.py
new file mode 100644
index 000000000..1d777221c
--- /dev/null
+++ b/youtube_dl/extractor/pearvideo.py
@@ -0,0 +1,63 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    qualities,
+    unified_timestamp,
+)
+
+
+class PearVideoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?pearvideo\.com/video_(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.pearvideo.com/video_1076290',
+        'info_dict': {
+            'id': '1076290',
+            'ext': 'mp4',
+            'title': '小浣熊在主人家玻璃上滚石头：没砸',
+            'description': 'md5:01d576b747de71be0ee85eb7cac25f9d',
+            'timestamp': 1494275280,
+            'upload_date': '20170508',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        quality = qualities(
+            ('ldflv', 'ld', 'sdflv', 'sd', 'hdflv', 'hd', 'src'))
+
+        formats = [{
+            'url': mobj.group('url'),
+            'format_id': mobj.group('id'),
+            'quality': quality(mobj.group('id')),
+        } for mobj in re.finditer(
+            r'(?P<id>[a-zA-Z]+)Url\s*=\s*(["\'])(?P<url>(?:https?:)?//.+?)\2',
+            webpage)]
+        self._sort_formats(formats)
+
+        title = self._search_regex(
+            (r'<h1[^>]+\bclass=(["\'])video-tt\1[^>]*>(?P<value>[^<]+)',
+             r'<[^>]+\bdata-title=(["\'])(?P<value>(?:(?!\1).)+)\1'),
+            webpage, 'title', group='value')
+        description = self._search_regex(
+            (r'<div[^>]+\bclass=(["\'])summary\1[^>]*>(?P<value>[^<]+)',
+             r'<[^>]+\bdata-summary=(["\'])(?P<value>(?:(?!\1).)+)\1'),
+            webpage, 'description', default=None,
+            group='value') or self._html_search_meta('Description', webpage)
+        timestamp = unified_timestamp(self._search_regex(
+            r'<div[^>]+\bclass=["\']date["\'][^>]*>([^<]+)',
+            webpage, 'timestamp', fatal=False))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'timestamp': timestamp,
+            'formats': formats,
+        }

From 089b97cfee8553886d33cd52b7ede178cebd7034 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Jul 2017 04:30:48 +0700
Subject: [PATCH 1118/1696] [nexx] Improve JS embed extraction

---
 youtube_dl/extractor/nexx.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/nexx.py b/youtube_dl/extractor/nexx.py
index 60b42cb7d..12450d4c5 100644
--- a/youtube_dl/extractor/nexx.py
+++ b/youtube_dl/extractor/nexx.py
@@ -72,13 +72,17 @@ class NexxIE(InfoExtractor):
         entries = []
 
         # JavaScript Integration
-        for domain_id, video_id in re.findall(
-                r'''(?isx)
-                    <script\b[^>]+\bsrc=["\']https?://require\.nexx(?:\.cloud|cdn\.com)/(\d+).+?
-                    onPLAYReady.+?
-                    _play\.init\s*\(.+?\s*,\s*(\d+)\s*,\s*.+?\)
-                ''', webpage):
-            entries.append('https://api.nexx.cloud/v3/%s/videos/byid/%s' % (domain_id, video_id))
+        mobj = re.search(
+            r'<script\b[^>]+\bsrc=["\']https?://require\.nexx(?:\.cloud|cdn\.com)/(?P<id>\d+)',
+            webpage)
+        if mobj:
+            domain_id = mobj.group('id')
+            for video_id in re.findall(
+                    r'(?is)onPLAYReady.+?_play\.init\s*\(.+?\s*,\s*["\']?(\d+)',
+                    webpage):
+                entries.append(
+                    'https://api.nexx.cloud/v3/%s/videos/byid/%s'
+                    % (domain_id, video_id))
 
         # TODO: support more embed formats
 

From 3f59b0154a8b6dc85425edfbb3dfdc64f41a6ecb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Jul 2017 04:32:37 +0700
Subject: [PATCH 1119/1696] [nexx:embed] Add extractor for iframe embeds

---
 youtube_dl/extractor/extractors.py |  5 +++-
 youtube_dl/extractor/generic.py    | 31 +++++++++++++++++++-
 youtube_dl/extractor/nexx.py       | 46 ++++++++++++++++++++++++++++++
 3 files changed, 80 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 28f0d3f0d..e8a066b83 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -653,7 +653,10 @@ from .nextmedia import (
     AppleDailyIE,
     NextTVIE,
 )
-from .nexx import NexxIE
+from .nexx import (
+    NexxIE,
+    NexxEmbedIE,
+)
 from .nfb import NFBIE
 from .nfl import NFLIE
 from .nhk import NhkVodIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 123a21296..0ab2ef2d6 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -36,7 +36,10 @@ from .brightcove import (
     BrightcoveLegacyIE,
     BrightcoveNewIE,
 )
-from .nexx import NexxIE
+from .nexx import (
+    NexxIE,
+    NexxEmbedIE,
+)
 from .nbc import NBCSportsVPlayerIE
 from .ooyala import OoyalaIE
 from .rutv import RUTVIE
@@ -1566,6 +1569,27 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        # Nexx iFrame embed
+        {
+            'url': 'http://www.spiegel.de/sptv/spiegeltv/spiegel-tv-ueber-schnellste-katapult-achterbahn-der-welt-taron-a-1137884.html',
+            'info_dict': {
+                'id': '161464',
+                'ext': 'mp4',
+                'title': 'Nervenkitzel Achterbahn',
+                'alt_title': 'Karussellbauer in Deutschland',
+                'description': 'md5:ffe7b1cc59a01f585e0569949aef73cc',
+                'release_year': 2005,
+                'creator': 'SPIEGEL TV',
+                'thumbnail': r're:^https?://.*\.jpg$',
+                'duration': 2761,
+                'timestamp': 1394021479,
+                'upload_date': '20140305',
+            },
+            'params': {
+                'format': 'bestvideo',
+                'skip_download': True,
+            },
+        },
         # Facebook <iframe> embed
         {
             'url': 'https://www.hostblogger.de/blog/archives/6181-Auto-jagt-Betonmischer.html',
@@ -2155,6 +2179,11 @@ class GenericIE(InfoExtractor):
         if nexx_urls:
             return self.playlist_from_matches(nexx_urls, video_id, video_title, ie=NexxIE.ie_key())
 
+        # Look for Nexx iFrame embeds
+        nexx_embed_urls = NexxEmbedIE._extract_urls(webpage)
+        if nexx_embed_urls:
+            return self.playlist_from_matches(nexx_embed_urls, video_id, video_title, ie=NexxEmbedIE.ie_key())
+
         # Look for ThePlatform embeds
         tp_urls = ThePlatformIE._extract_urls(webpage)
         if tp_urls:
diff --git a/youtube_dl/extractor/nexx.py b/youtube_dl/extractor/nexx.py
index 12450d4c5..e2960273e 100644
--- a/youtube_dl/extractor/nexx.py
+++ b/youtube_dl/extractor/nexx.py
@@ -88,6 +88,10 @@ class NexxIE(InfoExtractor):
 
         return entries
 
+    @staticmethod
+    def _extract_url(webpage):
+        return NexxIE._extract_urls(webpage)[0]
+
     def _handle_error(self, response):
         status = int_or_none(try_get(
             response, lambda x: x['metadata']['status']) or 200)
@@ -223,3 +227,45 @@ class NexxIE(InfoExtractor):
                 video, lambda x: x['episodedata']['season'])),
             'formats': formats,
         }
+
+
+class NexxEmbedIE(InfoExtractor):
+    _VALID_URL = r'https?://embed\.nexx(?:\.cloud|cdn\.com)/\d+/(?P<id>[^/?#&]+)'
+    _TEST = {
+        'url': 'http://embed.nexx.cloud/748/KC1614647Z27Y7T?autoplay=1',
+        'md5': '16746bfc28c42049492385c989b26c4a',
+        'info_dict': {
+            'id': '161464',
+            'ext': 'mp4',
+            'title': 'Nervenkitzel Achterbahn',
+            'alt_title': 'Karussellbauer in Deutschland',
+            'description': 'md5:ffe7b1cc59a01f585e0569949aef73cc',
+            'release_year': 2005,
+            'creator': 'SPIEGEL TV',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 2761,
+            'timestamp': 1394021479,
+            'upload_date': '20140305',
+        },
+        'params': {
+            'format': 'bestvideo',
+            'skip_download': True,
+        },
+    }
+
+    @staticmethod
+    def _extract_urls(webpage):
+        # Reference:
+        # 1. https://nx-s.akamaized.net/files/201510/44.pdf
+
+        # iFrame Embed Integration
+        return [mobj.group('url') for mobj in re.finditer(
+                r'<iframe[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//embed\.nexx(?:\.cloud|cdn\.com)/\d+/(?:(?!\1).)+)\1',
+            webpage)]
+
+    def _real_extract(self, url):
+        embed_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, embed_id)
+
+        return self.url_result(NexxIE._extract_url(webpage), ie=NexxIE.ie_key())

From 749ca5eced0b9a8ed1ef79f4ee80908e0ac242c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Jul 2017 04:33:14 +0700
Subject: [PATCH 1120/1696] [extractor/common] Fix playlist_from_matches

---
 youtube_dl/extractor/common.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index daa10885f..748b4d59f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -730,12 +730,12 @@ class InfoExtractor(object):
             video_info['title'] = video_title
         return video_info
 
-    def playlist_from_matches(self, matches, video_id, video_title, getter=None, ie=None):
-        urlrs = orderedSet(
+    def playlist_from_matches(self, matches, playlist_id=None, playlist_title=None, getter=None, ie=None):
+        urls = orderedSet(
             self.url_result(self._proto_relative_url(getter(m) if getter else m), ie)
             for m in matches)
         return self.playlist_result(
-            urlrs, playlist_id=video_id, playlist_title=video_title)
+            urls, playlist_id=playlist_id, playlist_title=playlist_title)
 
     @staticmethod
     def playlist_result(entries, playlist_id=None, playlist_title=None, playlist_description=None):

From 00d06e3cfcb7bfffa3b585694525f05a6ce36af9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Jul 2017 04:38:20 +0700
Subject: [PATCH 1121/1696] [spiegel:article] Add support for nexx iframe
 embeds (closes #13029)

---
 youtube_dl/extractor/spiegel.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dl/extractor/spiegel.py
index ec1b60388..8598377b0 100644
--- a/youtube_dl/extractor/spiegel.py
+++ b/youtube_dl/extractor/spiegel.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from .nexx import NexxEmbedIE
 from .spiegeltv import SpiegeltvIE
 from ..compat import compat_urlparse
 from ..utils import (
@@ -143,6 +144,9 @@ class SpiegelArticleIE(InfoExtractor):
         entries = [
             self.url_result(compat_urlparse.urljoin(
                 self.http_scheme() + '//spiegel.de/', embed_path))
-            for embed_path in embeds
-        ]
-        return self.playlist_result(entries)
+            for embed_path in embeds]
+        if embeds:
+            return self.playlist_result(entries)
+
+        return self.playlist_from_matches(
+            NexxEmbedIE._extract_urls(webpage), ie=NexxEmbedIE.ie_key())

From 13eb526f111dc9b98421f3e05287980b88766409 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Jul 2017 05:23:19 +0700
Subject: [PATCH 1122/1696] [nexx:embed] PEP 8

---
 youtube_dl/extractor/nexx.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nexx.py b/youtube_dl/extractor/nexx.py
index e2960273e..d0235fdfe 100644
--- a/youtube_dl/extractor/nexx.py
+++ b/youtube_dl/extractor/nexx.py
@@ -260,7 +260,7 @@ class NexxEmbedIE(InfoExtractor):
 
         # iFrame Embed Integration
         return [mobj.group('url') for mobj in re.finditer(
-                r'<iframe[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//embed\.nexx(?:\.cloud|cdn\.com)/\d+/(?:(?!\1).)+)\1',
+            r'<iframe[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//embed\.nexx(?:\.cloud|cdn\.com)/\d+/(?:(?!\1).)+)\1',
             webpage)]
 
     def _real_extract(self, url):

From 7abed4e06c8935026873a9c01821d5a5b2d80d4f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Jul 2017 12:40:45 +0700
Subject: [PATCH 1123/1696] [crunchyroll] Relax series and season regex (closes
 #13659)

---
 youtube_dl/extractor/crunchyroll.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 2ffa4a7f8..8bdaf0c2c 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -510,7 +510,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
 
         # webpage provide more accurate data than series_title from XML
         series = self._html_search_regex(
-            r'id=["\']showmedia_about_episode_num[^>]+>\s*<a[^>]+>([^<]+)',
+            r'(?s)<h\d[^>]+\bid=["\']showmedia_about_episode_num[^>]+>(.+?)</h\d',
             webpage, 'series', fatal=False)
         season = xpath_text(metadata, 'series_title')
 
@@ -518,7 +518,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         episode_number = int_or_none(xpath_text(metadata, 'episode_number'))
 
         season_number = int_or_none(self._search_regex(
-            r'(?s)<h4[^>]+id=["\']showmedia_about_episode_num[^>]+>.+?</h4>\s*<h4>\s*Season (\d+)',
+            r'(?s)<h\d[^>]+id=["\']showmedia_about_episode_num[^>]+>.+?</h\d>\s*<h4>\s*Season (\d+)',
             webpage, 'season number', default=None))
 
         return {

From 83d00044c1c9aef487f1c30bb50246e9ad039636 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Jul 2017 20:48:09 +0700
Subject: [PATCH 1124/1696] [adn] Improve error reporting (#13663)

---
 youtube_dl/extractor/adn.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/adn.py b/youtube_dl/extractor/adn.py
index 39f80b23f..cffdab6ca 100644
--- a/youtube_dl/extractor/adn.py
+++ b/youtube_dl/extractor/adn.py
@@ -107,11 +107,13 @@ class ADNIE(InfoExtractor):
         metas = options.get('metas') or {}
         title = metas.get('title') or video_info['title']
         links = player_config.get('links') or {}
+        error = None
         if not links:
             links_url = player_config['linksurl']
             links_data = self._download_json(urljoin(
                 self._BASE_URL, links_url), video_id)
             links = links_data.get('links') or {}
+            error = links_data.get('error')
 
         formats = []
         for format_id, qualities in links.items():
@@ -130,7 +132,8 @@ class ADNIE(InfoExtractor):
                     for f in m3u8_formats:
                         f['language'] = 'fr'
                 formats.extend(m3u8_formats)
-        error = options.get('error')
+        if not error:
+            error = options.get('error')
         if not formats and error:
             raise ExtractorError('%s said: %s' % (self.IE_NAME, error), expected=True)
         self._sort_formats(formats)

From bb176df3bbef62dcf958dcc542f778686e44db63 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Jul 2017 22:19:40 +0700
Subject: [PATCH 1125/1696] [spiegel:article] Move test

---
 youtube_dl/extractor/generic.py | 21 ---------------------
 youtube_dl/extractor/spiegel.py | 20 ++++++++++++++++++++
 2 files changed, 20 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 0ab2ef2d6..36c81eda9 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1569,27 +1569,6 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
         },
-        # Nexx iFrame embed
-        {
-            'url': 'http://www.spiegel.de/sptv/spiegeltv/spiegel-tv-ueber-schnellste-katapult-achterbahn-der-welt-taron-a-1137884.html',
-            'info_dict': {
-                'id': '161464',
-                'ext': 'mp4',
-                'title': 'Nervenkitzel Achterbahn',
-                'alt_title': 'Karussellbauer in Deutschland',
-                'description': 'md5:ffe7b1cc59a01f585e0569949aef73cc',
-                'release_year': 2005,
-                'creator': 'SPIEGEL TV',
-                'thumbnail': r're:^https?://.*\.jpg$',
-                'duration': 2761,
-                'timestamp': 1394021479,
-                'upload_date': '20140305',
-            },
-            'params': {
-                'format': 'bestvideo',
-                'skip_download': True,
-            },
-        },
         # Facebook <iframe> embed
         {
             'url': 'https://www.hostblogger.de/blog/archives/6181-Auto-jagt-Betonmischer.html',
diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dl/extractor/spiegel.py
index 8598377b0..84298fee4 100644
--- a/youtube_dl/extractor/spiegel.py
+++ b/youtube_dl/extractor/spiegel.py
@@ -122,6 +122,26 @@ class SpiegelArticleIE(InfoExtractor):
 
         },
         'playlist_count': 6,
+    }, {
+        # Nexx iFrame embed
+        'url': 'http://www.spiegel.de/sptv/spiegeltv/spiegel-tv-ueber-schnellste-katapult-achterbahn-der-welt-taron-a-1137884.html',
+        'info_dict': {
+            'id': '161464',
+            'ext': 'mp4',
+            'title': 'Nervenkitzel Achterbahn',
+            'alt_title': 'Karussellbauer in Deutschland',
+            'description': 'md5:ffe7b1cc59a01f585e0569949aef73cc',
+            'release_year': 2005,
+            'creator': 'SPIEGEL TV',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 2761,
+            'timestamp': 1394021479,
+            'upload_date': '20140305',
+        },
+        'params': {
+            'format': 'bestvideo',
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):

From d20b1c6725fce956b44413cced449b7d09b12de2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 20 Jul 2017 18:14:14 +0800
Subject: [PATCH 1126/1696] [dispeak] Recognize sevt subdomain (closes #13276)

---
 ChangeLog                       | 6 ++++++
 youtube_dl/extractor/dispeak.py | 6 +++++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 7d71fc5e1..8e442d159 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
++ [dispeak] Recognize sevt subdomain (#13276)
+
+
 version 2017.07.15
 
 Core
diff --git a/youtube_dl/extractor/dispeak.py b/youtube_dl/extractor/dispeak.py
index a78cb8a2a..c05f601e2 100644
--- a/youtube_dl/extractor/dispeak.py
+++ b/youtube_dl/extractor/dispeak.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class DigitallySpeakingIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:evt\.dispeak|events\.digitallyspeaking)\.com/(?:[^/]+/)+xml/(?P<id>[^.]+)\.xml'
+    _VALID_URL = r'https?://(?:s?evt\.dispeak|events\.digitallyspeaking)\.com/(?:[^/]+/)+xml/(?P<id>[^.]+)\.xml'
 
     _TESTS = [{
         # From http://gdcvault.com/play/1023460/Tenacious-Design-and-The-Interface
@@ -28,6 +28,10 @@ class DigitallySpeakingIE(InfoExtractor):
         # From http://www.gdcvault.com/play/1014631/Classic-Game-Postmortem-PAC
         'url': 'http://events.digitallyspeaking.com/gdc/sf11/xml/12396_1299111843500GMPX.xml',
         'only_matching': True,
+    }, {
+        # From http://www.gdcvault.com/play/1013700/Advanced-Material
+        'url': 'http://sevt.dispeak.com/ubm/gdc/eur10/xml/11256_1282118587281VNIT.xml',
+        'only_matching': True,
     }]
 
     def _parse_mp4(self, metadata):

From 85f5a74b6cf44b0c8b612c264c36eaabd958f501 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 20 Jul 2017 21:19:09 +0800
Subject: [PATCH 1127/1696] [tbs] Mark as broken and skip invalid tests

---
 youtube_dl/extractor/tbs.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tbs.py b/youtube_dl/extractor/tbs.py
index bf93eb868..e9474533f 100644
--- a/youtube_dl/extractor/tbs.py
+++ b/youtube_dl/extractor/tbs.py
@@ -8,6 +8,9 @@ from ..utils import extract_attributes
 
 
 class TBSIE(TurnerBaseIE):
+    # https://github.com/rg3/youtube-dl/issues/13658
+    _WORKING = False
+
     _VALID_URL = r'https?://(?:www\.)?(?P<site>tbs|tntdrama)\.com/videos/(?:[^/]+/)+(?P<id>[^/?#]+)\.html'
     _TESTS = [{
         'url': 'http://www.tbs.com/videos/people-of-earth/season-1/extras/2007318/theatrical-trailer.html',
@@ -17,7 +20,8 @@ class TBSIE(TurnerBaseIE):
             'ext': 'mp4',
             'title': 'Theatrical Trailer',
             'description': 'Catch the latest comedy from TBS, People of Earth, premiering Halloween night--Monday, October 31, at 9/8c.',
-        }
+        },
+        'skip': 'TBS videos are deleted after a while',
     }, {
         'url': 'http://www.tntdrama.com/videos/good-behavior/season-1/extras/1538823/you-better-run.html',
         'md5': 'ce53c6ead5e9f3280b4ad2031a6fab56',
@@ -26,7 +30,8 @@ class TBSIE(TurnerBaseIE):
             'ext': 'mp4',
             'title': 'You Better Run',
             'description': 'Letty Raines must figure out what she\'s running toward while running away from her past. Good Behavior premieres November 15 at 9/8c.',
-        }
+        },
+        'skip': 'TBS videos are deleted after a while',
     }]
 
     def _real_extract(self, url):

From fa63cf6c2301972b7d0ae76fb7a11c7d1a2786a9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 20 Jul 2017 22:57:51 +0800
Subject: [PATCH 1128/1696] [youku:show] Fix playlist extraction (closes
 #13248)

---
 ChangeLog                     |  1 +
 youtube_dl/extractor/youku.py | 66 ++++++++++++++++++++---------------
 2 files changed, 39 insertions(+), 28 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 8e442d159..a83523cb9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [youku:show] Fix playlist extraction (#13248)
 + [dispeak] Recognize sevt subdomain (#13276)
 
 
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index dcce15d77..4ae9adb51 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -1,7 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import itertools
 import random
 import re
 import string
@@ -222,50 +221,61 @@ class YoukuShowIE(InfoExtractor):
     _VALID_URL = r'https?://list\.youku\.com/show/id_(?P<id>[0-9a-z]+)\.html'
     IE_NAME = 'youku:show'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://list.youku.com/show/id_zc7c670be07ff11e48b3f.html',
         'info_dict': {
             'id': 'zc7c670be07ff11e48b3f',
-            'title': '花千骨 未删减版',
+            'title': '花千骨 DVD版',
             'description': 'md5:a1ae6f5618571bbeb5c9821f9c81b558',
         },
         'playlist_count': 50,
-    }
+    }, {
+        # Episode number not starting from 1
+        'url': 'http://list.youku.com/show/id_zefbfbd70efbfbd780bef.html',
+        'info_dict': {
+            'id': 'zefbfbd70efbfbd780bef',
+            'title': '超级飞侠3',
+            'description': 'md5:275715156abebe5ccc2a1992e9d56b98',
+        },
+        'playlist_count': 24,
+    }]
 
-    _PAGE_SIZE = 40
+    def _extract_entries(self, playlist_data_url, show_id, idx, query, url):
+        query['callback'] = 'cb'
+        playlist_data = self._download_json(
+            playlist_data_url, show_id, query=query,
+            note='Downloading playlist data page %d' % (idx + 1),
+            transform_source=lambda s: js_to_json(strip_jsonp(s)))['html']
+        drama_list = (get_element_by_class('p-drama-grid', playlist_data) or
+                      get_element_by_class('p-drama-half-row', playlist_data))
+        if drama_list is None:
+            raise ExtractorError('No episodes found')
+        video_urls = re.findall(r'<a[^>]+href="([^"]+)"', drama_list)
+        return playlist_data, [
+            self.url_result(urljoin(url, video_url), YoukuIE.ie_key())
+            for video_url in video_urls]
 
     def _real_extract(self, url):
         show_id = self._match_id(url)
         webpage = self._download_webpage(url, show_id)
 
-        entries = []
         page_config = self._parse_json(self._search_regex(
             r'var\s+PageConfig\s*=\s*({.+});', webpage, 'page config'),
             show_id, transform_source=js_to_json)
-        for idx in itertools.count(0):
-            if idx == 0:
-                playlist_data_url = 'http://list.youku.com/show/module'
-                query = {'id': page_config['showid'], 'tab': 'point'}
-            else:
-                playlist_data_url = 'http://list.youku.com/show/point'
-                query = {
+        first_page, entries = self._extract_entries(
+            'http://list.youku.com/show/module', show_id, 0, {
+                'id': page_config['showid'],
+                'tab': 'showInfo',
+            }, url)
+        # The first reload_id has the same items as first_page
+        reload_ids = re.findall('<li[^>]+data-id="([^"]+)">', first_page)[1:]
+        for idx, reload_id in enumerate(reload_ids):
+            _, new_entries = self._extract_entries(
+                'http://list.youku.com/show/episode', show_id, idx + 1, {
                     'id': page_config['showid'],
-                    'stage': 'reload_%d' % (self._PAGE_SIZE * idx + 1),
-                }
-            query['callback'] = 'cb'
-            playlist_data = self._download_json(
-                playlist_data_url, show_id, query=query,
-                note='Downloading playlist data page %d' % (idx + 1),
-                transform_source=lambda s: js_to_json(strip_jsonp(s)))['html']
-            video_urls = re.findall(
-                r'<div[^>]+class="p-thumb"[^<]+<a[^>]+href="([^"]+)"',
-                playlist_data)
-            new_entries = [
-                self.url_result(urljoin(url, video_url), YoukuIE.ie_key())
-                for video_url in video_urls]
+                    'stage': reload_id,
+                }, url)
             entries.extend(new_entries)
-            if len(new_entries) < self._PAGE_SIZE:
-                break
 
         desc = self._html_search_meta('description', webpage, fatal=False)
         playlist_title = desc.split(',')[0] if desc else None

From 3fcf346ac16e6fe1963a3eab861d6bd9c32ce6db Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 20 Jul 2017 23:20:46 +0800
Subject: [PATCH 1129/1696] [youku:show] Refine playlist extraction

Handle playlists that the initial page is not the first page
---
 youtube_dl/extractor/youku.py | 34 +++++++++++++++++++++++-----------
 1 file changed, 23 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 4ae9adb51..0c4bc2eda 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -13,7 +13,6 @@ from ..utils import (
     js_to_json,
     str_or_none,
     strip_jsonp,
-    urljoin,
 )
 
 
@@ -238,13 +237,16 @@ class YoukuShowIE(InfoExtractor):
             'description': 'md5:275715156abebe5ccc2a1992e9d56b98',
         },
         'playlist_count': 24,
+    }, {
+        # Ongoing playlist. The initial page is the last one
+        'url': 'http://list.youku.com/show/id_za7c275ecd7b411e1a19e.html',
+        'only_matchine': True,
     }]
 
-    def _extract_entries(self, playlist_data_url, show_id, idx, query, url):
+    def _extract_entries(self, playlist_data_url, show_id, note, query):
         query['callback'] = 'cb'
         playlist_data = self._download_json(
-            playlist_data_url, show_id, query=query,
-            note='Downloading playlist data page %d' % (idx + 1),
+            playlist_data_url, show_id, query=query, note=note,
             transform_source=lambda s: js_to_json(strip_jsonp(s)))['html']
         drama_list = (get_element_by_class('p-drama-grid', playlist_data) or
                       get_element_by_class('p-drama-half-row', playlist_data))
@@ -252,29 +254,39 @@ class YoukuShowIE(InfoExtractor):
             raise ExtractorError('No episodes found')
         video_urls = re.findall(r'<a[^>]+href="([^"]+)"', drama_list)
         return playlist_data, [
-            self.url_result(urljoin(url, video_url), YoukuIE.ie_key())
+            self.url_result(self._proto_relative_url(video_url, 'http:'), YoukuIE.ie_key())
             for video_url in video_urls]
 
     def _real_extract(self, url):
         show_id = self._match_id(url)
         webpage = self._download_webpage(url, show_id)
 
+        entries = []
         page_config = self._parse_json(self._search_regex(
             r'var\s+PageConfig\s*=\s*({.+});', webpage, 'page config'),
             show_id, transform_source=js_to_json)
-        first_page, entries = self._extract_entries(
-            'http://list.youku.com/show/module', show_id, 0, {
+        first_page, initial_entries = self._extract_entries(
+            'http://list.youku.com/show/module', show_id,
+            note='Downloading initial playlist data page',
+            query={
                 'id': page_config['showid'],
                 'tab': 'showInfo',
-            }, url)
+            })
+        first_page_reload_id = self._html_search_regex(
+            r'<div[^>]+id="(reload_\d+)', first_page, 'first page reload id')
         # The first reload_id has the same items as first_page
-        reload_ids = re.findall('<li[^>]+data-id="([^"]+)">', first_page)[1:]
+        reload_ids = re.findall('<li[^>]+data-id="([^"]+)">', first_page)
         for idx, reload_id in enumerate(reload_ids):
+            if reload_id == first_page_reload_id:
+                entries.extend(initial_entries)
+                continue
             _, new_entries = self._extract_entries(
-                'http://list.youku.com/show/episode', show_id, idx + 1, {
+                'http://list.youku.com/show/episode', show_id,
+                note='Downloading playlist data page %d' % (idx + 1),
+                query={
                     'id': page_config['showid'],
                     'stage': reload_id,
-                }, url)
+                })
             entries.extend(new_entries)
 
         desc = self._html_search_meta('description', webpage, fatal=False)

From c653326a1425f4c271f387fde7a706bf4b52a7a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 20 Jul 2017 22:49:52 +0700
Subject: [PATCH 1130/1696] [funnyordie] Extract more metadata (closes #13677)

---
 youtube_dl/extractor/funnyordie.py | 64 ++++++++++++++++++++++++++----
 1 file changed, 56 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/funnyordie.py b/youtube_dl/extractor/funnyordie.py
index 49409369c..f85e7de14 100644
--- a/youtube_dl/extractor/funnyordie.py
+++ b/youtube_dl/extractor/funnyordie.py
@@ -1,10 +1,14 @@
 from __future__ import unicode_literals
 
-import json
 import re
 
 from .common import InfoExtractor
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    float_or_none,
+    int_or_none,
+    unified_timestamp,
+)
 
 
 class FunnyOrDieIE(InfoExtractor):
@@ -18,6 +22,10 @@ class FunnyOrDieIE(InfoExtractor):
             'title': 'Heart-Shaped Box: Literal Video Version',
             'description': 'md5:ea09a01bc9a1c46d9ab696c01747c338',
             'thumbnail': r're:^http:.*\.jpg$',
+            'uploader': 'DASjr',
+            'timestamp': 1317904928,
+            'upload_date': '20111006',
+            'duration': 318.3,
         },
     }, {
         'url': 'http://www.funnyordie.com/embed/e402820827',
@@ -27,6 +35,8 @@ class FunnyOrDieIE(InfoExtractor):
             'title': 'Please Use This Song (Jon Lajoie)',
             'description': 'Please use this to sell something.  www.jonlajoie.com',
             'thumbnail': r're:^http:.*\.jpg$',
+            'timestamp': 1398988800,
+            'upload_date': '20140502',
         },
         'params': {
             'skip_download': True,
@@ -100,15 +110,53 @@ class FunnyOrDieIE(InfoExtractor):
                 'url': 'http://www.funnyordie.com%s' % src,
             }]
 
-        post_json = self._search_regex(
-            r'fb_post\s*=\s*(\{.*?\});', webpage, 'post details')
-        post = json.loads(post_json)
+        timestamp = unified_timestamp(self._html_search_meta(
+            'uploadDate', webpage, 'timestamp', default=None))
+
+        uploader = self._html_search_regex(
+            r'<h\d[^>]+\bclass=["\']channel-preview-name[^>]+>(.+?)</h',
+            webpage, 'uploader', default=None)
+
+        title, description, thumbnail, duration = [None] * 4
+
+        medium = self._parse_json(
+            self._search_regex(
+                r'jsonMedium\s*=\s*({.+?});', webpage, 'JSON medium',
+                default='{}'),
+            video_id, fatal=False)
+        if medium:
+            title = medium.get('title')
+            duration = float_or_none(medium.get('duration'))
+            if not timestamp:
+                timestamp = unified_timestamp(medium.get('publishDate'))
+
+        post = self._parse_json(
+            self._search_regex(
+                r'fb_post\s*=\s*(\{.*?\});', webpage, 'post details',
+                default='{}'),
+            video_id, fatal=False)
+        if post:
+            if not title:
+                title = post.get('name')
+            description = post.get('description')
+            thumbnail = post.get('picture')
+
+        if not title:
+            title = self._og_search_title(webpage)
+        if not description:
+            description = self._og_search_description(webpage)
+        if not duration:
+            duration = int_or_none(self._html_search_meta(
+                ('video:duration', 'duration'), webpage, 'duration', default=False))
 
         return {
             'id': video_id,
-            'title': post['name'],
-            'description': post.get('description'),
-            'thumbnail': post.get('picture'),
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'uploader': uploader,
+            'timestamp': timestamp,
+            'duration': duration,
             'formats': formats,
             'subtitles': subtitles,
         }

From dc6520aa3d1fe7afc52613e392f15dde90af4844 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 20 Jul 2017 23:22:36 +0700
Subject: [PATCH 1131/1696] [egghead:lesson] Add extractor (#6635)

---
 youtube_dl/extractor/egghead.py    | 49 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  5 ++-
 2 files changed, 53 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/egghead.py b/youtube_dl/extractor/egghead.py
index c86f52319..e4a3046af 100644
--- a/youtube_dl/extractor/egghead.py
+++ b/youtube_dl/extractor/egghead.py
@@ -2,6 +2,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    try_get,
+    unified_timestamp,
+)
 
 
 class EggheadCourseIE(InfoExtractor):
@@ -33,3 +38,47 @@ class EggheadCourseIE(InfoExtractor):
         return self.playlist_result(
             entries, playlist_id, course.get('title'),
             course.get('description'))
+
+
+class EggheadLessonIE(InfoExtractor):
+    IE_DESC = 'egghead.io lesson'
+    IE_NAME = 'egghead:lesson'
+    _VALID_URL = r'https://egghead\.io/lessons/(?P<id>[^/?#&]+)'
+    _TEST = {
+        'url': 'https://egghead.io/lessons/javascript-linear-data-flow-with-container-style-types-box',
+        'info_dict': {
+            'id': 'fv5yotjxcg',
+            'ext': 'mp4',
+            'title': 'Create linear data flow with container style types (Box)',
+            'description': 'md5:9aa2cdb6f9878ed4c39ec09e85a8150e',
+            'thumbnail': r're:^https?:.*\.jpg$',
+            'timestamp': 1481296768,
+            'upload_date': '20161209',
+            'duration': 304,
+            'view_count': 0,
+            'tags': ['javascript', 'free'],
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        lesson_id = self._match_id(url)
+
+        lesson = self._download_json(
+            'https://egghead.io/api/v1/lessons/%s' % lesson_id, lesson_id)
+
+        return {
+            '_type': 'url_transparent',
+            'ie_key': 'Wistia',
+            'url': 'wistia:%s' % lesson['wistia_id'],
+            'id': lesson['wistia_id'],
+            'title': lesson.get('title'),
+            'description': lesson.get('summary'),
+            'thumbnail': lesson.get('thumb_nail'),
+            'timestamp': unified_timestamp(lesson.get('published_at')),
+            'duration': int_or_none(lesson.get('duration')),
+            'view_count': int_or_none(lesson.get('plays_count')),
+            'tags': try_get(lesson, lambda x: x['tag_list'], list),
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index e8a066b83..db7616caa 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -298,7 +298,10 @@ from .dw import (
 from .eagleplatform import EaglePlatformIE
 from .ebaumsworld import EbaumsWorldIE
 from .echomsk import EchoMskIE
-from .egghead import EggheadCourseIE
+from .egghead import (
+    EggheadCourseIE,
+    EggheadLessonIE,
+)
 from .ehow import EHowIE
 from .eighttracks import EightTracksIE
 from .einthusan import EinthusanIE

From 0396806f671e5828c2abdeb8048acf8b654507b6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 21 Jul 2017 00:13:32 +0700
Subject: [PATCH 1132/1696] [YoutubeDL] Do not override id, extractor and
 extractor_key in url_transparent All these meta fields must be borrowed from
 final extractor that actually performs extraction. This commit fixes
 extractor id in download archives for url_transparent downloads. Previously,
 'transparent' extractor was erroneously used for extractor archive id, e.g.
 'eggheadlesson 4n8ugwwj5t' instead of 'wistia 4n8ugwwj5t'.

---
 test/test_YoutubeDL.py  | 7 ++++++-
 youtube_dl/YoutubeDL.py | 2 +-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 70989e232..e0decb81c 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -41,6 +41,7 @@ def _make_result(formats, **kwargs):
         'id': 'testid',
         'title': 'testttitle',
         'extractor': 'testex',
+        'extractor_key': 'TestEx',
     }
     res.update(**kwargs)
     return res
@@ -761,7 +762,8 @@ class TestYoutubeDL(unittest.TestCase):
                     '_type': 'url_transparent',
                     'url': 'foo2:',
                     'ie_key': 'Foo2',
-                    'title': 'foo1 title'
+                    'title': 'foo1 title',
+                    'id': 'foo1_id',
                 }
 
         class Foo2IE(InfoExtractor):
@@ -787,6 +789,9 @@ class TestYoutubeDL(unittest.TestCase):
         downloaded = ydl.downloaded_info_dicts[0]
         self.assertEqual(downloaded['url'], TEST_URL)
         self.assertEqual(downloaded['title'], 'foo1 title')
+        self.assertEqual(downloaded['id'], 'testid')
+        self.assertEqual(downloaded['extractor'], 'testex')
+        self.assertEqual(downloaded['extractor_key'], 'TestEx')
 
 
 if __name__ == '__main__':
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 89c07be29..f94836d06 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -860,7 +860,7 @@ class YoutubeDL(object):
 
             force_properties = dict(
                 (k, v) for k, v in ie_result.items() if v is not None)
-            for f in ('_type', 'url', 'ie_key'):
+            for f in ('_type', 'url', 'id', 'extractor', 'extractor_key', 'ie_key'):
                 if f in force_properties:
                     del force_properties[f]
             new_result = info.copy()

From 7d9a1db1110b13e8e6b65613ebb3daf7f0ff3c4c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 22 Jul 2017 11:40:46 +0700
Subject: [PATCH 1133/1696] [dramafever] Remove video id from title (closes
 #13699)

---
 youtube_dl/extractor/dramafever.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index e7abc8889..03fa3aabc 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -12,6 +12,7 @@ from ..utils import (
     ExtractorError,
     clean_html,
     int_or_none,
+    remove_end,
     sanitized_Request,
     urlencode_postdata
 )
@@ -73,7 +74,7 @@ class DramaFeverIE(DramaFeverBaseIE):
         'info_dict': {
             'id': '4512.1',
             'ext': 'mp4',
-            'title': 'Cooking with Shin 4512.1',
+            'title': 'Cooking with Shin',
             'description': 'md5:a8eec7942e1664a6896fcd5e1287bfd0',
             'episode': 'Episode 1',
             'episode_number': 1,
@@ -91,7 +92,7 @@ class DramaFeverIE(DramaFeverBaseIE):
         'info_dict': {
             'id': '4826.4',
             'ext': 'mp4',
-            'title': 'Mnet Asian Music Awards 2015 4826.4',
+            'title': 'Mnet Asian Music Awards 2015',
             'description': 'md5:3ff2ee8fedaef86e076791c909cf2e91',
             'episode': 'Mnet Asian Music Awards 2015 - Part 3',
             'episode_number': 4,
@@ -122,6 +123,10 @@ class DramaFeverIE(DramaFeverBaseIE):
                     countries=self._GEO_COUNTRIES)
             raise
 
+        # title is postfixed with video id for some reason, removing
+        if info.get('title'):
+            info['title'] = remove_end(info['title'], video_id).strip()
+
         series_id, episode_number = video_id.split('.')
         episode_info = self._download_json(
             # We only need a single episode info, so restricting page size to one episode

From f76c02c87b479310b0e090216895879257b1062a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 22 Jul 2017 11:41:40 +0700
Subject: [PATCH 1134/1696] [dramafever] Fix tests

---
 youtube_dl/extractor/dramafever.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 03fa3aabc..9a498d72a 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -73,7 +73,7 @@ class DramaFeverIE(DramaFeverBaseIE):
         'url': 'http://www.dramafever.com/drama/4512/1/Cooking_with_Shin/',
         'info_dict': {
             'id': '4512.1',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Cooking with Shin',
             'description': 'md5:a8eec7942e1664a6896fcd5e1287bfd0',
             'episode': 'Episode 1',
@@ -81,7 +81,7 @@ class DramaFeverIE(DramaFeverBaseIE):
             'thumbnail': r're:^https?://.*\.jpg',
             'timestamp': 1404336058,
             'upload_date': '20140702',
-            'duration': 343,
+            'duration': 344,
         },
         'params': {
             # m3u8 download
@@ -91,7 +91,7 @@ class DramaFeverIE(DramaFeverBaseIE):
         'url': 'http://www.dramafever.com/drama/4826/4/Mnet_Asian_Music_Awards_2015/?ap=1',
         'info_dict': {
             'id': '4826.4',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Mnet Asian Music Awards 2015',
             'description': 'md5:3ff2ee8fedaef86e076791c909cf2e91',
             'episode': 'Mnet Asian Music Awards 2015 - Part 3',
@@ -99,7 +99,7 @@ class DramaFeverIE(DramaFeverBaseIE):
             'thumbnail': r're:^https?://.*\.jpg',
             'timestamp': 1450213200,
             'upload_date': '20151215',
-            'duration': 5602,
+            'duration': 5359,
         },
         'params': {
             # m3u8 download

From 359aa2fdd145d11a29a04f620fed95acbf142f66 Mon Sep 17 00:00:00 2001
From: dubber0 <rexa.mose@gmail.com>
Date: Sat, 22 Jul 2017 14:15:55 +0200
Subject: [PATCH 1135/1696] [npo] Add support for npo3.nl URLs

---
 youtube_dl/extractor/npo.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 516b1e941..fa4ef20c5 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -28,7 +28,7 @@ class NPOBaseIE(InfoExtractor):
 
 class NPOIE(NPOBaseIE):
     IE_NAME = 'npo'
-    IE_DESC = 'npo.nl and ntr.nl'
+    IE_DESC = 'npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl'
     _VALID_URL = r'''(?x)
                     (?:
                         npo:|
@@ -38,7 +38,7 @@ class NPOIE(NPOBaseIE):
                                 npo\.nl/(?!(?:live|radio)/)(?:[^/]+/){2}|
                                 ntr\.nl/(?:[^/]+/){2,}|
                                 omroepwnl\.nl/video/fragment/[^/]+__|
-                                zapp\.nl/[^/]+/[^/]+/
+                                (?:zapp|npo3)\.nl/(?:[^/]+/){2}
                             )
                         )
                         (?P<id>[^/?#]+)
@@ -146,6 +146,9 @@ class NPOIE(NPOBaseIE):
     }, {
         'url': 'http://www.zapp.nl/beste-vrienden-quiz/extra-video-s/WO_NTR_1067990',
         'only_matching': True,
+    }, {
+        'url': 'https://www.npo3.nl/3onderzoekt/16-09-2015/VPWON_1239870',
+        'only_matching': True,
     }, {
         # live stream
         'url': 'npo:LI_NL1_4188102',

From 327c8364f11f23dd919e8009c6adb021c34054fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 22 Jul 2017 21:35:14 +0700
Subject: [PATCH 1136/1696] [sportbox:embed] Fix extraction

---
 youtube_dl/extractor/sportbox.py | 61 ++++++++++++++++++--------------
 1 file changed, 34 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index e7bd5bf91..54497c880 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -4,7 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import js_to_json
+from ..utils import (
+    determine_ext,
+    int_or_none,
+    js_to_json,
+)
 
 
 class SportBoxEmbedIE(InfoExtractor):
@@ -14,8 +18,10 @@ class SportBoxEmbedIE(InfoExtractor):
         'info_dict': {
             'id': '211355',
             'ext': 'mp4',
-            'title': 'В Новороссийске прошел детский турнир «Поле славы боевой»',
+            'title': '211355',
             'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 292,
+            'view_count': int,
         },
         'params': {
             # m3u8 download
@@ -24,6 +30,9 @@ class SportBoxEmbedIE(InfoExtractor):
     }, {
         'url': 'http://news.sportbox.ru/vdl/player?nid=370908&only_player=1&autostart=false&playeri=2&height=340&width=580',
         'only_matching': True,
+    }, {
+        'url': 'https://news.sportbox.ru/vdl/player/media/193095',
+        'only_matching': True,
     }]
 
     @staticmethod
@@ -37,36 +46,34 @@ class SportBoxEmbedIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
+        wjplayer_data = self._parse_json(
+            self._search_regex(
+                r'(?s)wjplayer\(({.+?})\);', webpage, 'wjplayer settings'),
+            video_id, transform_source=js_to_json)
+
         formats = []
-
-        def cleanup_js(code):
-            # desktop_advert_config contains complex Javascripts and we don't need it
-            return js_to_json(re.sub(r'desktop_advert_config.*', '', code))
-
-        jwplayer_data = self._parse_json(self._search_regex(
-            r'(?s)player\.setup\(({.+?})\);', webpage, 'jwplayer settings'), video_id,
-            transform_source=cleanup_js)
-
-        hls_url = jwplayer_data.get('hls_url')
-        if hls_url:
-            formats.extend(self._extract_m3u8_formats(
-                hls_url, video_id, ext='mp4', m3u8_id='hls'))
-
-        rtsp_url = jwplayer_data.get('rtsp_url')
-        if rtsp_url:
-            formats.append({
-                'url': rtsp_url,
-                'format_id': 'rtsp',
-            })
-
+        for source in wjplayer_data['sources']:
+            src = source.get('src')
+            if not src:
+                continue
+            if determine_ext(src) == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    src, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls', fatal=False))
+            else:
+                formats.append({
+                    'url': src,
+                })
         self._sort_formats(formats)
 
-        title = jwplayer_data['node_title']
-        thumbnail = jwplayer_data.get('image_url')
+        view_count = int_or_none(self._search_regex(
+            r'Просмотров\s*:\s*(\d+)', webpage, 'view count', default=None))
 
         return {
             'id': video_id,
-            'title': title,
-            'thumbnail': thumbnail,
+            'title': video_id,
+            'thumbnail': wjplayer_data.get('poster'),
+            'duration': int_or_none(wjplayer_data.get('duration')),
+            'view_count': view_count,
             'formats': formats,
         }

From 0017d9ad6de831384e74db14a821e4c94020c9ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Jul 2017 00:12:01 +0700
Subject: [PATCH 1137/1696] [YoutubeDL] Improve default format specification
 (closes #13704)

---
 test/test_YoutubeDL.py  | 11 +++++++++++
 youtube_dl/YoutubeDL.py | 31 +++++++++++++++++++++++--------
 2 files changed, 34 insertions(+), 8 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index e0decb81c..4af14f9db 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -449,6 +449,17 @@ class TestFormatSelection(unittest.TestCase):
             pass
         self.assertEqual(ydl.downloaded_info_dicts, [])
 
+    def test_default_format_spec(self):
+        ydl = YDL({'simulate': True})
+        self.assertEqual(ydl._default_format_spec({}), 'bestvideo+bestaudio/best')
+
+        ydl = YDL({'outtmpl': '-'})
+        self.assertEqual(ydl._default_format_spec({}), 'best')
+
+        ydl = YDL({})
+        self.assertEqual(ydl._default_format_spec({}, download=False), 'bestvideo+bestaudio/best')
+        self.assertEqual(ydl._default_format_spec({'is_live': True}), 'best')
+
 
 class TestYoutubeDL(unittest.TestCase):
     def test_subtitles(self):
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index f94836d06..367ae3533 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1064,6 +1064,25 @@ class YoutubeDL(object):
             return op(actual_value, comparison_value)
         return _filter
 
+    def _default_format_spec(self, info_dict, download=True):
+        req_format_list = []
+
+        def can_have_partial_formats():
+            if self.params.get('simulate', False):
+                return True
+            if not download:
+                return True
+            if self.params.get('outtmpl', DEFAULT_OUTTMPL) == '-':
+                return False
+            if info_dict.get('is_live'):
+                return False
+            merger = FFmpegMergerPP(self)
+            return merger.available and merger.can_merge()
+        if can_have_partial_formats():
+            req_format_list.append('bestvideo+bestaudio')
+        req_format_list.append('best')
+        return '/'.join(req_format_list)
+
     def build_format_selector(self, format_spec):
         def syntax_error(note, start):
             message = (
@@ -1534,14 +1553,10 @@ class YoutubeDL(object):
 
         req_format = self.params.get('format')
         if req_format is None:
-            req_format_list = []
-            if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
-                    not info_dict.get('is_live')):
-                merger = FFmpegMergerPP(self)
-                if merger.available and merger.can_merge():
-                    req_format_list.append('bestvideo+bestaudio')
-            req_format_list.append('best')
-            req_format = '/'.join(req_format_list)
+            req_format = self._default_format_spec(info_dict, download=download)
+            if self.params.get('verbose'):
+                self.to_stdout('[debug] Default format spec: %s' % req_format)
+
         format_selector = self.build_format_selector(req_format)
 
         # While in format selection we may need to have an access to the original

From e0f1fb0a27612c2398df59dd85194edfdf8cbc2a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Jul 2017 00:25:23 +0700
Subject: [PATCH 1138/1696] [mtv] Skip missing video parts (closes #13690)

---
 youtube_dl/extractor/mtv.py | 28 ++++++++++++++++++++++------
 youtube_dl/extractor/vh1.py | 12 ++++++++----
 2 files changed, 30 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 8acea1461..fc098cd13 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -83,7 +83,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
                 hls_url = rendition.find('./src').text
                 formats.extend(self._extract_m3u8_formats(
                     hls_url, video_id, ext='mp4', entry_protocol='m3u8_native',
-                    m3u8_id='hls'))
+                    m3u8_id='hls', fatal=False))
             else:
                 # fms
                 try:
@@ -106,7 +106,8 @@ class MTVServicesInfoExtractor(InfoExtractor):
                     }])
                 except (KeyError, TypeError):
                     raise ExtractorError('Invalid rendition field.')
-        self._sort_formats(formats)
+        if formats:
+            self._sort_formats(formats)
         return formats
 
     def _extract_subtitles(self, mdoc, mtvn_id):
@@ -133,8 +134,11 @@ class MTVServicesInfoExtractor(InfoExtractor):
             mediagen_url += 'acceptMethods='
             mediagen_url += 'hls' if use_hls else 'fms'
 
-        mediagen_doc = self._download_xml(mediagen_url, video_id,
-                                          'Downloading video urls')
+        mediagen_doc = self._download_xml(
+            mediagen_url, video_id, 'Downloading video urls', fatal=False)
+
+        if mediagen_doc is False:
+            return None
 
         item = mediagen_doc.find('./video/item')
         if item is not None and item.get('type') == 'text':
@@ -174,6 +178,13 @@ class MTVServicesInfoExtractor(InfoExtractor):
 
         formats = self._extract_video_formats(mediagen_doc, mtvn_id, video_id)
 
+        # Some parts of complete video may be missing (e.g. missing Act 3 in
+        # http://www.southpark.de/alle-episoden/s14e01-sexual-healing)
+        if not formats:
+            return None
+
+        self._sort_formats(formats)
+
         return {
             'title': title,
             'formats': formats,
@@ -205,9 +216,14 @@ class MTVServicesInfoExtractor(InfoExtractor):
         title = xpath_text(idoc, './channel/title')
         description = xpath_text(idoc, './channel/description')
 
+        entries = []
+        for item in idoc.findall('.//item'):
+            info = self._get_video_info(item, use_hls)
+            if info:
+                entries.append(info)
+
         return self.playlist_result(
-            [self._get_video_info(item, use_hls) for item in idoc.findall('.//item')],
-            playlist_title=title, playlist_description=description)
+            entries, playlist_title=title, playlist_description=description)
 
     def _extract_triforce_mgid(self, webpage, data_zone=None, video_id=None):
         triforce_feed = self._parse_json(self._search_regex(
diff --git a/youtube_dl/extractor/vh1.py b/youtube_dl/extractor/vh1.py
index 6be3774b7..570fa45ea 100644
--- a/youtube_dl/extractor/vh1.py
+++ b/youtube_dl/extractor/vh1.py
@@ -121,7 +121,11 @@ class VH1IE(MTVIE):
         idoc = self._download_xml(
             doc_url, video_id,
             'Downloading info', transform_source=fix_xml_ampersands)
-        return self.playlist_result(
-            [self._get_video_info(item) for item in idoc.findall('.//item')],
-            playlist_id=video_id,
-        )
+
+        entries = []
+        for item in idoc.findall('.//item'):
+            info = self._get_video_info(item)
+            if info:
+                entries.append(info)
+
+        return self.playlist_result(entries, playlist_id=video_id)

From 935d6c20c00536cf39cf2844295266e64492bb10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Jul 2017 00:44:50 +0700
Subject: [PATCH 1139/1696] [vidio] Make duration non fatal and fix typo

---
 youtube_dl/extractor/vidio.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vidio.py b/youtube_dl/extractor/vidio.py
index 701bb1d01..01da32f1c 100644
--- a/youtube_dl/extractor/vidio.py
+++ b/youtube_dl/extractor/vidio.py
@@ -56,7 +56,8 @@ class VidioIE(InfoExtractor):
         self._sort_formats(formats)
 
         duration = int_or_none(duration or self._search_regex(
-            r'data-video-duration=(["\'])(?P<duartion>\d+)\1', webpage, 'duration'))
+            r'data-video-duration=(["\'])(?P<duration>\d+)\1', webpage,
+            'duration', fatal=False, group='duration'))
         thumbnail = thumbnail or self._og_search_thumbnail(webpage)
 
         like_count = int_or_none(self._search_regex(

From 71dde5eecf07ae3a8871e5d4a05a944097e17fb9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Jul 2017 00:59:07 +0700
Subject: [PATCH 1140/1696] [itv] Fix production id extraction (closes #13671)

---
 youtube_dl/extractor/itv.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py
index f3156804d..26c48e4b8 100644
--- a/youtube_dl/extractor/itv.py
+++ b/youtube_dl/extractor/itv.py
@@ -59,12 +59,18 @@ class ITVIE(InfoExtractor):
         def _add_sub_element(element, name):
             return etree.SubElement(element, _add_ns(name))
 
+        production_id = (
+            params.get('data-video-autoplay-id') or
+            '%s#001' % (
+                params.get('data-video-episode-id') or
+                video_id.replace('a', '/')))
+
         req_env = etree.Element(_add_ns('soapenv:Envelope'))
         _add_sub_element(req_env, 'soapenv:Header')
         body = _add_sub_element(req_env, 'soapenv:Body')
         get_playlist = _add_sub_element(body, ('tem:GetPlaylist'))
         request = _add_sub_element(get_playlist, 'tem:request')
-        _add_sub_element(request, 'itv:ProductionId').text = params['data-video-id']
+        _add_sub_element(request, 'itv:ProductionId').text = production_id
         _add_sub_element(request, 'itv:RequestGuid').text = compat_str(uuid.uuid4()).upper()
         vodcrid = _add_sub_element(request, 'itv:Vodcrid')
         _add_sub_element(vodcrid, 'com:Id')

From 425f41319aec6940195818e980005cef4946eb75 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Jul 2017 01:06:08 +0700
Subject: [PATCH 1141/1696] [ChangeLog] Actualize

---
 ChangeLog | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index a83523cb9..cd8a34a86 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,8 +1,28 @@
 version <unreleased>
 
+Core
+* [YoutubeDL] Improve default format specification (#13704)
+* [YoutubeDL] Do not override id, extractor and extractor_key for
+  url_transparent entities
+* [extractor/common] Fix playlist_from_matches
+
 Extractors
+* [itv] Fix production id extraction (#13671, #13703)
+* [vidio] Make duration non fatal and fix typo
+* [mtv] Skip missing video parts (#13690)
+* [sportbox:embed] Fix extraction
++ [npo] Add support for npo3.nl URLs (#13695)
+* [dramafever] Remove video id from title (#13699)
++ [egghead:lesson] Add support for lessons (#6635)
+* [funnyordie] Extract more metadata (#13677)
 * [youku:show] Fix playlist extraction (#13248)
 + [dispeak] Recognize sevt subdomain (#13276)
+* [adn] Improve error reporting (#13663)
+* [crunchyroll] Relax series and season regex (#13659)
++ [spiegel:article] Add support for nexx iframe embeds (#13029)
++ [nexx:embed] Add support for iframe embeds
+* [nexx] Improve JS embed extraction
++ [pearvideo] Add support for pearvideo.com (#13031)
 
 
 version 2017.07.15

From 0db492c02a60dbfb44514833445bf267f5319ae2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Jul 2017 01:09:09 +0700
Subject: [PATCH 1142/1696] release 2017.07.23

---
 .github/ISSUE_TEMPLATE.md |  6 +++---
 ChangeLog                 |  2 +-
 docs/supportedsites.md    | 13 ++++++++-----
 youtube_dl/version.py     |  2 +-
 4 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 0f20d0485..37d09d796 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.07.15*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.07.15**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.07.23*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.07.23**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.07.15
+[debug] youtube-dl version 2017.07.23
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index cd8a34a86..302d32aab 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.07.23
 
 Core
 * [YoutubeDL] Improve default format specification (#13704)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index d7304ba06..eb09c470c 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -42,7 +42,7 @@
  - **Allocine**
  - **AlphaPorno**
  - **AMCNetworks**
- - **anderetijden**: npo.nl and ntr.nl
+ - **anderetijden**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
  - **AnimeOnDemand**
  - **anitube.se**
  - **Anvato**
@@ -238,6 +238,7 @@
  - **EbaumsWorld**
  - **EchoMsk**
  - **egghead:course**: egghead.io course
+ - **egghead:lesson**: egghead.io lesson
  - **eHow**
  - **Einthusan**
  - **eitb.tv**
@@ -522,6 +523,7 @@
  - **NextMediaActionNews**: 蘋果日報 - 動新聞
  - **NextTV**: 壹電視
  - **Nexx**
+ - **NexxEmbed**
  - **nfb**: National Film Board of Canada
  - **nfl.com**
  - **NhkVod**
@@ -552,7 +554,7 @@
  - **NowTVList**
  - **nowvideo**: NowVideo
  - **Noz**
- - **npo**: npo.nl and ntr.nl
+ - **npo**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
  - **npo.nl:live**
  - **npo.nl:radio**
  - **npo.nl:radio:fragment**
@@ -596,6 +598,7 @@
  - **Patreon**
  - **pbs**: Public Broadcasting Service (PBS) and member stations: PBS: Public Broadcasting Service, APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), Mississippi Public Broadcasting (WMPN), Nashville Public Television (WNPT), WFSU-TV (WFSU), WSRE (WSRE), WTCI (WTCI), WPBA/Channel 30 (WPBA), Alaska Public Media (KAKM), Arizona PBS (KAET), KNME-TV/Channel 5 (KNME), Vegas PBS (KLVX), AETN/ARKANSAS ETV NETWORK (KETS), KET (WKLE), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), OETA (KETA), Ozarks Public Television (KOZK), WSIU Public Broadcasting (WSIU), KEET TV (KEET), KIXE/Channel 9 (KIXE), KPBS San Diego (KPBS), KQED (KQED), KVIE Public Television (KVIE), PBS SoCal/KOCE (KOCE), ValleyPBS (KVPT), CONNECTICUT PUBLIC TELEVISION (WEDH), KNPB Channel 5 (KNPB), SOPTV (KSYS), Rocky Mountain PBS (KRMA), KENW-TV3 (KENW), KUED Channel 7 (KUED), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), Thirteen/WNET New York (WNET), WGBH/Channel 2 (WGBH), WGBY (WGBY), NJTV Public Media NJ (WNJT), WLIW21 (WLIW), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), WHYY (WHYY), PBS 39 (WLVT), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WEDU PBS (WEDU), WGCU Public Media (WGCU), WPBT2 (WPBT), WUCF TV (WUCF), WUFT/Channel 5 (WUFT), WXEL/Channel 42 (WXEL), WLRN/Channel 17 (WLRN), WUSF Public Broadcasting (WUSF), ETV (WRLK), UNC-TV (WUNC), PBS Hawaii - Oceanic Cable Channel 10 (KHET), Idaho Public Television (KAID), KSPS (KSPS), OPB (KOPB), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WTTW11 (WTTW), Iowa Public Television/IPTV (KDIN), Nine Network (KETC), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), Milwaukee Public Television (WMVS), WNIN (WNIN), WNIT Public Television (WNIT), WPT (WPNE), WVUT/Channel 22 (WVUT), WEIU/Channel 51 (WEIU), WQPT-TV (WQPT), WYCC PBS Chicago (WYCC), WIPB-TV (WIPB), WTIU (WTIU), CET  (WCET), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGVU TV (WGVU), NET1 (KUON), Pioneer Public Television (KWCM), SDPB Television (KUSD), TPT (KTCA), KSMQ (KSMQ), KPTS/Channel 8 (KPTS), KTWU/Channel 11 (KTWU), East Tennessee PBS (WSJK), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WOSU TV (WOSU), WOUB/WOUC (WOUB), WVPB (WVPB), WKYU-PBS (WKYU), KERA 13 (KERA), MPBN (WCBB), Mountain Lake PBS (WCFE), NHPTV (WENH), Vermont PBS (WETK), witf (WITF), WQED Multimedia (WQED), WMHT Educational Telecommunications (WMHT), Q-TV (WDCQ), WTVS Detroit Public TV (WTVS), CMU Public Television (WCMU), WKAR-TV (WKAR), WNMU-TV Public TV 13 (WNMU), WDSE - WRPT (WDSE), WGTE TV (WGTE), Lakeland Public Television (KAWE), KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS), MontanaPBS (KUSM), KRWG/Channel 22 (KRWG), KACV (KACV), KCOS/Channel 13 (KCOS), WCNY/Channel 24 (WCNY), WNED (WNED), WPBS (WPBS), WSKG Public TV (WSKG), WXXI (WXXI), WPSU (WPSU), WVIA Public Media Studios (WVIA), WTVI (WTVI), Western Reserve PBS (WNEO), WVIZ/PBS ideastream (WVIZ), KCTS 9 (KCTS), Basin PBS (KPBT), KUHT / Channel 8 (KUHT), KLRN (KLRN), KLRU (KLRU), WTJX Channel 12 (WTJX), WCVE PBS (WCVE), KBTC Public Television (KBTC)
  - **pcmag**
+ - **PearVideo**
  - **People**
  - **periscope**: Periscope
  - **periscope:user**: Periscope user videos
@@ -772,7 +775,7 @@
  - **tagesschau:player**
  - **Tass**
  - **TastyTrade**
- - **TBS**
+ - **TBS** (Currently broken)
  - **TDSLifeway**
  - **teachertube**: teachertube.com videos
  - **teachertube:user:collection**: teachertube.com user and collection videos
@@ -950,7 +953,7 @@
  - **VoiceRepublic**
  - **VoxMedia**
  - **Vporn**
- - **vpro**: npo.nl and ntr.nl
+ - **vpro**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
  - **Vrak**
  - **VRT**: deredactie.be, sporza.be, cobra.be and cobra.canvas.be
  - **vrv**
@@ -976,7 +979,7 @@
  - **wholecloud**: WholeCloud
  - **Wimp**
  - **Wistia**
- - **wnl**: npo.nl and ntr.nl
+ - **wnl**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
  - **WorldStarHipHop**
  - **wrzuta.pl**
  - **wrzuta.pl:playlist**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 82e166fef..a8dbb93e3 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.07.15'
+__version__ = '2017.07.23'

From 905d18a7aa42263c66f311ac0cdf46b2caa2f4d6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 23 Jul 2017 16:21:35 +0800
Subject: [PATCH 1143/1696] [options] Correctly hide login info from debug
 outputs (#13696)

Iterate over opts instead of PRIVATE_OPTS for both performance and
correctness
---
 ChangeLog             |  6 ++++++
 test/test_options.py  | 26 ++++++++++++++++++++++++++
 youtube_dl/options.py | 38 ++++++++++++++++++--------------------
 3 files changed, 50 insertions(+), 20 deletions(-)
 create mode 100644 test/test_options.py

diff --git a/ChangeLog b/ChangeLog
index 302d32aab..8e63b5c11 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Core
+* [options] Correctly hide login info from debug outputs (#13696)
+
+
 version 2017.07.23
 
 Core
diff --git a/test/test_options.py b/test/test_options.py
new file mode 100644
index 000000000..785281fe3
--- /dev/null
+++ b/test/test_options.py
@@ -0,0 +1,26 @@
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.options import _hide_login_info
+
+
+class TestOptions(unittest.TestCase):
+    def test_hide_login_inf(self):
+        self.assertEqual(_hide_login_info(['-u', 'foo', '-p', 'bar']),
+                         ['-u', 'PRIVATE', '-p', 'PRIVATE'])
+        self.assertEqual(_hide_login_info(['-u']), ['-u'])
+        self.assertEqual(_hide_login_info(['-u', 'foo', '-u', 'bar']),
+                         ['-u', 'PRIVATE', '-u', 'PRIVATE'])
+        self.assertEqual(_hide_login_info(['--username=foo']),
+                         ['--username=PRIVATE'])
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 79e9fd12c..38439c971 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -20,6 +20,24 @@ from .utils import (
 from .version import __version__
 
 
+def _hide_login_info(opts):
+    PRIVATE_OPTS = set(['-p', '--password', '-u', '--username', '--video-password', '--ap-password', '--ap-username'])
+    eqre = re.compile('^(?P<key>' + ('|'.join(re.escape(po) for po in PRIVATE_OPTS)) + ')=.+$')
+
+    def _scrub_eq(o):
+        m = eqre.match(o)
+        if m:
+            return m.group('key') + '=PRIVATE'
+        else:
+            return o
+
+    opts = list(map(_scrub_eq, opts))
+    for idx, opt in enumerate(opts):
+        if opt in PRIVATE_OPTS and idx + 1 < len(opts):
+            opts[idx + 1] = 'PRIVATE'
+    return opts
+
+
 def parseOpts(overrideArguments=None):
     def _readOptions(filename_bytes, default=[]):
         try:
@@ -93,26 +111,6 @@ def parseOpts(overrideArguments=None):
     def _comma_separated_values_options_callback(option, opt_str, value, parser):
         setattr(parser.values, option.dest, value.split(','))
 
-    def _hide_login_info(opts):
-        PRIVATE_OPTS = ['-p', '--password', '-u', '--username', '--video-password', '--ap-password', '--ap-username']
-        eqre = re.compile('^(?P<key>' + ('|'.join(re.escape(po) for po in PRIVATE_OPTS)) + ')=.+$')
-
-        def _scrub_eq(o):
-            m = eqre.match(o)
-            if m:
-                return m.group('key') + '=PRIVATE'
-            else:
-                return o
-
-        opts = list(map(_scrub_eq, opts))
-        for private_opt in PRIVATE_OPTS:
-            try:
-                i = opts.index(private_opt)
-                opts[i + 1] = 'PRIVATE'
-            except ValueError:
-                pass
-        return opts
-
     # No need to wrap help messages if we're on a wide console
     columns = compat_get_terminal_size().columns
     max_width = columns if columns else 80

From 73095e013fb1bc4a1e676d7be77a103f0013a227 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 23 Jul 2017 16:24:18 +0800
Subject: [PATCH 1144/1696] [options] Typo

---
 test/test_options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_options.py b/test/test_options.py
index 785281fe3..3a25a6ba3 100644
--- a/test/test_options.py
+++ b/test/test_options.py
@@ -12,7 +12,7 @@ from youtube_dl.options import _hide_login_info
 
 
 class TestOptions(unittest.TestCase):
-    def test_hide_login_inf(self):
+    def test_hide_login_info(self):
         self.assertEqual(_hide_login_info(['-u', 'foo', '-p', 'bar']),
                          ['-u', 'PRIVATE', '-p', 'PRIVATE'])
         self.assertEqual(_hide_login_info(['-u']), ['-u'])

From e3ce912c3d767fcb1a1225d05ac64da1acab94aa Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 23 Jul 2017 16:25:17 +0800
Subject: [PATCH 1145/1696] [niconico] improve error reporting (#13696)

---
 youtube_dl/extractor/niconico.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 695e32e59..79b9952c3 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -147,6 +147,9 @@ class NiconicoIE(InfoExtractor):
             elif 'closed' in flv_info:
                 raise ExtractorError('Niconico videos now require logging in',
                                      expected=True)
+            elif 'error' in flv_info:
+                raise ExtractorError('%s reports error: %s' % (
+                    self.IE_NAME, flv_info['error'][0]), expected=True)
             else:
                 raise ExtractorError('Unable to find video URL')
 

From 3150976669ef2ffc9f4eee9e99a6e70730bc22fb Mon Sep 17 00:00:00 2001
From: nyuszika7h <nyuszika7h@gmail.com>
Date: Sun, 23 Jul 2017 15:33:18 +0200
Subject: [PATCH 1146/1696] [ISSUE_TEMPLATE_tmpl.md] Minor improvements

---
 .github/ISSUE_TEMPLATE_tmpl.md | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE_tmpl.md b/.github/ISSUE_TEMPLATE_tmpl.md
index df79503d3..26f61d3b4 100644
--- a/.github/ISSUE_TEMPLATE_tmpl.md
+++ b/.github/ISSUE_TEMPLATE_tmpl.md
@@ -1,16 +1,16 @@
 ## Please follow the guide below
 
 - You will be asked some questions and requested to provide some information, please read them **carefully** and answer honestly
-- Put an `x` into all the boxes [ ] relevant to your *issue* (like that [x])
-- Use *Preview* tab to see how your issue will actually look like
+- Put an `x` into all the boxes [ ] relevant to your *issue* (like this: `[x]`)
+- Use the *Preview* tab to see what your issue will actually look like
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *%(version)s*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *%(version)s*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
 - [ ] I've **verified** and **I assure** that I'm running youtube-dl **%(version)s**
 
 ### Before submitting an *issue* make sure you have:
-- [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
+- [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
 - [ ] [Searched](https://github.com/rg3/youtube-dl/search?type=Issues) the bugtracker for similar issues including closed ones
 
 ### What is the purpose of your *issue*?
@@ -28,9 +28,9 @@
 
 ### If the purpose of this *issue* is a *bug report*, *site support request* or you are not completely sure provide the full verbose output as follows:
 
-Add `-v` flag to **your command line** you run youtube-dl with, copy the **whole** output and insert it here. It should look similar to one below (replace it with **your** log inserted between triple ```):
+Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl -v <your command line>`), copy the **whole** output and insert it here. It should look similar to one below (replace it with **your** log inserted between triple ```):
+
 ```
-$ youtube-dl -v <your command line>
 [debug] System config: []
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']

From f0e31e32c940d8529353f40bd2426163c3199216 Mon Sep 17 00:00:00 2001
From: nyuszika7h <nyuszika7h@gmail.com>
Date: Sun, 23 Jul 2017 15:40:04 +0200
Subject: [PATCH 1147/1696] [nick] Automate geo-restriction bypass (#13711)

---
 youtube_dl/extractor/nick.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
index 08a75929e..1fa19cde4 100644
--- a/youtube_dl/extractor/nick.py
+++ b/youtube_dl/extractor/nick.py
@@ -12,6 +12,7 @@ class NickIE(MTVServicesInfoExtractor):
     IE_NAME = 'nick.com'
     _VALID_URL = r'https?://(?:(?:www|beta)\.)?nick(?:jr)?\.com/(?:[^/]+/)?(?:videos/clip|[^/]+/videos)/(?P<id>[^/?#.]+)'
     _FEED_URL = 'http://udat.mtvnservices.com/service1/dispatch.htm'
+    _GEO_COUNTRIES = ['US']
     _TESTS = [{
         'url': 'http://www.nick.com/videos/clip/alvinnn-and-the-chipmunks-112-full-episode.html',
         'playlist': [

From 70bfab0e9ac8ddb7da67d71633c8c4b0704054cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Jul 2017 21:00:19 +0700
Subject: [PATCH 1148/1696] [mtv] Improve thumbnal extraction

---
 youtube_dl/extractor/mtv.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index fc098cd13..25af5ddfd 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -50,8 +50,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
         thumb_node = itemdoc.find(search_path)
         if thumb_node is None:
             return None
-        else:
-            return thumb_node.attrib['url']
+        return thumb_node.get('url') or thumb_node.text or None
 
     def _extract_mobile_video_formats(self, mtvn_id):
         webpage_url = self._MOBILE_TEMPLATE % mtvn_id

From c99d6890cb46626870474e5c1092d9772096c4b6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Jul 2017 21:00:56 +0700
Subject: [PATCH 1149/1696] [nickru] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/nick.py       | 18 ++++++++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index db7616caa..2513f2587 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -673,6 +673,7 @@ from .nick import (
     NickIE,
     NickDeIE,
     NickNightIE,
+    NickRuIE,
 )
 from .niconico import NiconicoIE, NiconicoPlaylistIE
 from .ninecninemedia import (
diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
index 1fa19cde4..b688637bc 100644
--- a/youtube_dl/extractor/nick.py
+++ b/youtube_dl/extractor/nick.py
@@ -125,3 +125,21 @@ class NickNightIE(NickDeIE):
         return self._search_regex(
             r'mrss\s*:\s*(["\'])(?P<url>http.+?)\1', webpage,
             'mrss url', group='url')
+
+
+class NickRuIE(MTVServicesInfoExtractor):
+    IE_NAME = 'nickelodeonru'
+    _VALID_URL = r'https?://(?:www\.)nickelodeon\.ru/(?:playlist|shows|videos)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'http://www.nickelodeon.ru/shows/henrydanger/videos/episodes/3-sezon-15-seriya-licenziya-na-polyot/pmomfb#playlist/7airc6',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.nickelodeon.ru/videos/smotri-na-nickelodeon-v-iyule/g9hvh7',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        mgid = self._extract_mgid(webpage)
+        return self.url_result('http://media.mtvnservices.com/embed/%s' % mgid)

From f9c48d895b5600c82e9b55f703e68b060f25de07 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Jul 2017 23:12:43 +0700
Subject: [PATCH 1150/1696] [cloudy] Fix extraction (closes #13737)

---
 youtube_dl/extractor/cloudy.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cloudy.py b/youtube_dl/extractor/cloudy.py
index 9bc8dbea4..85ca20ecc 100644
--- a/youtube_dl/extractor/cloudy.py
+++ b/youtube_dl/extractor/cloudy.py
@@ -30,7 +30,11 @@ class CloudyIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(
-            'http://www.cloudy.ec/embed.php?id=%s' % video_id, video_id)
+            'https://www.cloudy.ec/embed.php', video_id, query={
+                'id': video_id,
+                'playerPage': 1,
+                'autoplay': 1,
+            })
 
         info = self._parse_html5_media_entries(url, webpage, video_id)[0]
 

From 9682666bdadec955fb8600fa3721f59b2a4b8099 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 27 Jul 2017 02:04:51 +0700
Subject: [PATCH 1151/1696] [amcnetworks] Make rating optional (closes #12453)

---
 youtube_dl/extractor/amcnetworks.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/amcnetworks.py b/youtube_dl/extractor/amcnetworks.py
index 3a0ec6776..dd3b18d72 100644
--- a/youtube_dl/extractor/amcnetworks.py
+++ b/youtube_dl/extractor/amcnetworks.py
@@ -3,9 +3,10 @@ from __future__ import unicode_literals
 
 from .theplatform import ThePlatformIE
 from ..utils import (
-    update_url_query,
-    parse_age_limit,
     int_or_none,
+    parse_age_limit,
+    try_get,
+    update_url_query,
 )
 
 
@@ -68,7 +69,8 @@ class AMCNetworksIE(ThePlatformIE):
         info = self._parse_theplatform_metadata(theplatform_metadata)
         video_id = theplatform_metadata['pid']
         title = theplatform_metadata['title']
-        rating = theplatform_metadata['ratings'][0]['rating']
+        rating = try_get(
+            theplatform_metadata, lambda x: x['ratings'][0]['rating'])
         auth_required = self._search_regex(
             r'window\.authRequired\s*=\s*(true|false);',
             webpage, 'auth required')

From 24e966e8dab954136dabbc497064ac63b252495b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 28 Jul 2017 12:13:19 +0200
Subject: [PATCH 1152/1696] [megaphone] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/generic.py    |  8 +++++
 youtube_dl/extractor/megaphone.py  | 55 ++++++++++++++++++++++++++++++
 3 files changed, 64 insertions(+)
 create mode 100644 youtube_dl/extractor/megaphone.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 2513f2587..668248648 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -558,6 +558,7 @@ from .matchtv import MatchTVIE
 from .mdr import MDRIE
 from .mediaset import MediasetIE
 from .medici import MediciIE
+from .megaphone import MegaphoneIE
 from .meipai import MeipaiIE
 from .melonvod import MelonVODIE
 from .meta import METAIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 36c81eda9..9678c32c4 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -97,6 +97,7 @@ from .washingtonpost import WashingtonPostIE
 from .wistia import WistiaIE
 from .mediaset import MediasetIE
 from .joj import JojIE
+from .megaphone import MegaphoneIE
 
 
 class GenericIE(InfoExtractor):
@@ -2790,6 +2791,13 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 joj_urls, video_id, video_title, ie=JojIE.ie_key())
 
+        # Look for megaphone.fm embeds
+        mpfn_urls = MegaphoneIE._extract_urls(webpage)
+        if mpfn_urls:
+            return self.playlist_from_matches(
+                mpfn_urls, video_id, video_title, ie=MegaphoneIE.ie_key())
+
+
         def merge_dicts(dict1, dict2):
             merged = {}
             for k, v in dict1.items():
diff --git a/youtube_dl/extractor/megaphone.py b/youtube_dl/extractor/megaphone.py
new file mode 100644
index 000000000..60e3caf0d
--- /dev/null
+++ b/youtube_dl/extractor/megaphone.py
@@ -0,0 +1,55 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import js_to_json
+
+
+class MegaphoneIE(InfoExtractor):
+    IE_NAME = 'megaphone.fm'
+    IE_DESC = 'megaphone.fm embedded players'
+    _VALID_URL = r'https://player\.megaphone\.fm/(?P<id>[A-Z0-9]+)'
+    _TEST = {
+        'url': 'https://player.megaphone.fm/GLT9749789991?"',
+        'md5': '4816a0de523eb3e972dc0dda2c191f96',
+        'info_dict': {
+            'id': 'GLT9749789991',
+            'ext': 'mp3',
+            'title': '#97 What Kind Of Idiot Gets Phished?',
+            'thumbnail': 're:^https://.*\.png.*$',
+            'duration': 1776.26375,
+            'author': 'Reply All',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._og_search_property('audio:title', webpage)
+        author = self._og_search_property('audio:artist', webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        episode_json = self._search_regex(r'(?s)var\s+episode\s*=\s*(\{.+?\});', webpage, 'episode JSON')
+        episode_data = self._parse_json(episode_json, video_id, js_to_json)
+        video_url = self._proto_relative_url(episode_data['mediaUrl'], 'https:')
+
+        formats = [{
+            'url': video_url,
+        }]
+
+        return {
+            'id': video_id,
+            'thumbnail': thumbnail,
+            'title': title,
+            'author': author,
+            'duration': episode_data['duration'],
+            'formats': formats,
+        }
+
+    @classmethod
+    def _extract_urls(cls, webpage):
+        return [m[0] for m in re.findall(
+            r'<iframe[^>]*?\ssrc=["\'](%s)' % cls._VALID_URL, webpage)]

From c5a49ff08413411174837f1034ef439b79ff774b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Jul 2017 15:02:41 +0700
Subject: [PATCH 1153/1696] [downloader/hls] Use redirect URL as manifest base
 (#13755)

---
 youtube_dl/downloader/hls.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 0e29c8a2a..46308cf07 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -59,9 +59,9 @@ class HlsFD(FragmentFD):
         man_url = info_dict['url']
         self.to_screen('[%s] Downloading m3u8 manifest' % self.FD_NAME)
 
-        manifest = self.ydl.urlopen(self._prepare_url(info_dict, man_url)).read()
-
-        s = manifest.decode('utf-8', 'ignore')
+        urlh = self.ydl.urlopen(self._prepare_url(info_dict, man_url))
+        man_url = urlh.geturl()
+        s = urlh.read().decode('utf-8', 'ignore')
 
         if not self.can_download(s, info_dict):
             if info_dict.get('extra_param_to_segment_url'):

From cbbe66635f3c23316f04a6f56ad57e025bc47263 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Jul 2017 15:10:19 +0700
Subject: [PATCH 1154/1696] [yandexdisk] Add extractor (closes #13755)

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/yandexdisk.py | 115 +++++++++++++++++++++++++++++
 2 files changed, 116 insertions(+)
 create mode 100644 youtube_dl/extractor/yandexdisk.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 668248648..852942e0d 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1298,6 +1298,7 @@ from .yandexmusic import (
     YandexMusicAlbumIE,
     YandexMusicPlaylistIE,
 )
+from .yandexdisk import YandexDiskIE
 from .yesjapan import YesJapanIE
 from .yinyuetai import YinYueTaiIE
 from .ynet import YnetIE
diff --git a/youtube_dl/extractor/yandexdisk.py b/youtube_dl/extractor/yandexdisk.py
new file mode 100644
index 000000000..11729f0f7
--- /dev/null
+++ b/youtube_dl/extractor/yandexdisk.py
@@ -0,0 +1,115 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    determine_ext,
+    float_or_none,
+    int_or_none,
+    try_get,
+    urlencode_postdata,
+)
+
+
+class YandexDiskIE(InfoExtractor):
+    _VALID_URL = r'https?://yadi\.sk/i/(?P<id>[^/?#&]+)'
+
+    _TEST = {
+        'url': 'https://yadi.sk/i/VdOeDou8eZs6Y',
+        'md5': '33955d7ae052f15853dc41f35f17581c',
+        'info_dict': {
+            'id': 'VdOeDou8eZs6Y',
+            'ext': 'mp4',
+            'title': '4.mp4',
+            'duration': 168.6,
+            'uploader': 'y.botova',
+            'uploader_id': '300043621',
+            'view_count': int,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        status = self._download_webpage(
+            'https://disk.yandex.com/auth/status', video_id, query={
+                'urlOrigin': url,
+                'source': 'public',
+                'md5': 'false',
+            })
+
+        sk = self._search_regex(
+            r'(["\'])sk(?:External)?\1\s*:\s*(["\'])(?P<value>(?:(?!\2).)+)\2',
+            status, 'sk', group='value')
+
+        webpage = self._download_webpage(url, video_id)
+
+        models = self._parse_json(
+            self._search_regex(
+                r'<script[^>]+id=["\']models-client[^>]+>\s*(\[.+?\])\s*</script',
+                webpage, 'video JSON'),
+            video_id)
+
+        data = next(
+            model['data'] for model in models
+            if model.get('model') == 'resource')
+
+        video_hash = data['id']
+        title = data['name']
+
+        models = self._download_json(
+            'https://disk.yandex.com/models/', video_id,
+            data=urlencode_postdata({
+                '_model.0': 'videoInfo',
+                'id.0': video_hash,
+                '_model.1': 'do-get-resource-url',
+                'id.1': video_hash,
+                'version': '13.6',
+                'sk': sk,
+            }), query={'_m': 'videoInfo'})['models']
+
+        videos = try_get(models, lambda x: x[0]['data']['videos'], list) or []
+        source_url = try_get(
+            models, lambda x: x[1]['data']['file'], compat_str)
+
+        formats = []
+        if source_url:
+            formats.append({
+                'url': source_url,
+                'format_id': 'source',
+                'ext': determine_ext(title, 'mp4'),
+                'quality': 1,
+            })
+        for video in videos:
+            format_url = video.get('url')
+            if not format_url:
+                continue
+            if determine_ext(format_url) == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls', fatal=False))
+            else:
+                formats.append({
+                    'url': format_url,
+                })
+        self._sort_formats(formats)
+
+        duration = float_or_none(try_get(
+            models, lambda x: x[0]['data']['duration']), 1000)
+        uploader = try_get(
+            data, lambda x: x['user']['display_name'], compat_str)
+        uploader_id = try_get(
+            data, lambda x: x['user']['uid'], compat_str)
+        view_count = int_or_none(try_get(
+            data, lambda x: x['meta']['views_counter']))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'duration': duration,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'view_count': view_count,
+            'formats': formats,
+        }

From 95908ce45382a72be26e918f8db1809b0ce81190 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Jul 2017 15:13:12 +0700
Subject: [PATCH 1155/1696] [extractor/generic] PEP 8

---
 youtube_dl/extractor/generic.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 9678c32c4..4b83e861b 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2797,7 +2797,6 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 mpfn_urls, video_id, video_title, ie=MegaphoneIE.ie_key())
 
-
         def merge_dicts(dict1, dict2):
             merged = {}
             for k, v in dict1.items():

From 2a7a82321135bc59364c91caddde4211f378785b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Jul 2017 15:25:32 +0700
Subject: [PATCH 1156/1696] [svtplay] Update API URL (closes #13767)

---
 youtube_dl/extractor/svt.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/svt.py b/youtube_dl/extractor/svt.py
index 1b5afb73e..38a505f9c 100644
--- a/youtube_dl/extractor/svt.py
+++ b/youtube_dl/extractor/svt.py
@@ -181,7 +181,7 @@ class SVTPlayIE(SVTBaseIE):
 
         if video_id:
             data = self._download_json(
-                'http://www.svt.se/videoplayer-api/video/%s' % video_id, video_id)
+                'https://api.svt.se/videoplayer-api/video/%s' % video_id, video_id)
             info_dict = self._extract_video(data, video_id)
             if not info_dict.get('title'):
                 info_dict['title'] = re.sub(

From c04017519da74a375d6c1c95733d921e96d8ee82 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Jul 2017 15:30:53 +0700
Subject: [PATCH 1157/1696] [svtplay] Use geo verification proxy for API
 request

---
 youtube_dl/extractor/svt.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/svt.py b/youtube_dl/extractor/svt.py
index 38a505f9c..48bc4529e 100644
--- a/youtube_dl/extractor/svt.py
+++ b/youtube_dl/extractor/svt.py
@@ -181,7 +181,8 @@ class SVTPlayIE(SVTBaseIE):
 
         if video_id:
             data = self._download_json(
-                'https://api.svt.se/videoplayer-api/video/%s' % video_id, video_id)
+                'https://api.svt.se/videoplayer-api/video/%s' % video_id,
+                video_id, headers=self.geo_verification_headers())
             info_dict = self._extract_video(data, video_id)
             if not info_dict.get('title'):
                 info_dict['title'] = re.sub(

From 836ef2648613f4ca565b319af4769c02e35f60f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Jul 2017 18:41:42 +0700
Subject: [PATCH 1158/1696] [soundcloud:trackstation] Add extractor (closes
 #13733)

---
 youtube_dl/extractor/extractors.py |   3 +-
 youtube_dl/extractor/soundcloud.py | 141 ++++++++++++++++++-----------
 2 files changed, 89 insertions(+), 55 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 852942e0d..d2c5e8030 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -935,8 +935,9 @@ from .soundcloud import (
     SoundcloudIE,
     SoundcloudSetIE,
     SoundcloudUserIE,
+    SoundcloudTrackStationIE,
     SoundcloudPlaylistIE,
-    SoundcloudSearchIE
+    SoundcloudSearchIE,
 )
 from .soundgasm import (
     SoundgasmIE,
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 3f1a46bb2..2f1b2978c 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -31,6 +31,7 @@ class SoundcloudIE(InfoExtractor):
 
     _VALID_URL = r'''(?x)^(?:https?://)?
                     (?:(?:(?:www\.|m\.)?soundcloud\.com/
+                            (?!stations/track)
                             (?P<uploader>[\w\d-]+)/
                             (?!(?:tracks|sets(?:/.+?)?|reposts|likes|spotlight)/?(?:$|[?#]))
                             (?P<title>[\w\d-]+)/?
@@ -330,7 +331,63 @@ class SoundcloudSetIE(SoundcloudPlaylistBaseIE):
         }
 
 
-class SoundcloudUserIE(SoundcloudPlaylistBaseIE):
+class SoundcloudPagedPlaylistBaseIE(SoundcloudPlaylistBaseIE):
+    _API_BASE = 'https://api.soundcloud.com'
+    _API_V2_BASE = 'https://api-v2.soundcloud.com'
+
+    def _extract_playlist(self, base_url, playlist_id, playlist_title):
+        COMMON_QUERY = {
+            'limit': 50,
+            'client_id': self._CLIENT_ID,
+            'linked_partitioning': '1',
+        }
+
+        query = COMMON_QUERY.copy()
+        query['offset'] = 0
+
+        next_href = base_url + '?' + compat_urllib_parse_urlencode(query)
+
+        entries = []
+        for i in itertools.count():
+            response = self._download_json(
+                next_href, playlist_id, 'Downloading track page %s' % (i + 1))
+
+            collection = response['collection']
+            if not collection:
+                break
+
+            def resolve_permalink_url(candidates):
+                for cand in candidates:
+                    if isinstance(cand, dict):
+                        permalink_url = cand.get('permalink_url')
+                        entry_id = self._extract_id(cand)
+                        if permalink_url and permalink_url.startswith('http'):
+                            return permalink_url, entry_id
+
+            for e in collection:
+                permalink_url, entry_id = resolve_permalink_url((e, e.get('track'), e.get('playlist')))
+                if permalink_url:
+                    entries.append(self.url_result(permalink_url, video_id=entry_id))
+
+            next_href = response.get('next_href')
+            if not next_href:
+                break
+
+            parsed_next_href = compat_urlparse.urlparse(response['next_href'])
+            qs = compat_urlparse.parse_qs(parsed_next_href.query)
+            qs.update(COMMON_QUERY)
+            next_href = compat_urlparse.urlunparse(
+                parsed_next_href._replace(query=compat_urllib_parse_urlencode(qs, True)))
+
+        return {
+            '_type': 'playlist',
+            'id': playlist_id,
+            'title': playlist_title,
+            'entries': entries,
+        }
+
+
+class SoundcloudUserIE(SoundcloudPagedPlaylistBaseIE):
     _VALID_URL = r'''(?x)
                         https?://
                             (?:(?:www|m)\.)?soundcloud\.com/
@@ -385,16 +442,13 @@ class SoundcloudUserIE(SoundcloudPlaylistBaseIE):
         'playlist_mincount': 1,
     }]
 
-    _API_BASE = 'https://api.soundcloud.com'
-    _API_V2_BASE = 'https://api-v2.soundcloud.com'
-
     _BASE_URL_MAP = {
-        'all': '%s/profile/soundcloud:users:%%s' % _API_V2_BASE,
-        'tracks': '%s/users/%%s/tracks' % _API_BASE,
-        'sets': '%s/users/%%s/playlists' % _API_V2_BASE,
-        'reposts': '%s/profile/soundcloud:users:%%s/reposts' % _API_V2_BASE,
-        'likes': '%s/users/%%s/likes' % _API_V2_BASE,
-        'spotlight': '%s/users/%%s/spotlight' % _API_V2_BASE,
+        'all': '%s/profile/soundcloud:users:%%s' % SoundcloudPagedPlaylistBaseIE._API_V2_BASE,
+        'tracks': '%s/users/%%s/tracks' % SoundcloudPagedPlaylistBaseIE._API_BASE,
+        'sets': '%s/users/%%s/playlists' % SoundcloudPagedPlaylistBaseIE._API_V2_BASE,
+        'reposts': '%s/profile/soundcloud:users:%%s/reposts' % SoundcloudPagedPlaylistBaseIE._API_V2_BASE,
+        'likes': '%s/users/%%s/likes' % SoundcloudPagedPlaylistBaseIE._API_V2_BASE,
+        'spotlight': '%s/users/%%s/spotlight' % SoundcloudPagedPlaylistBaseIE._API_V2_BASE,
     }
 
     _TITLE_MAP = {
@@ -416,57 +470,36 @@ class SoundcloudUserIE(SoundcloudPlaylistBaseIE):
             resolv_url, uploader, 'Downloading user info')
 
         resource = mobj.group('rsrc') or 'all'
-        base_url = self._BASE_URL_MAP[resource] % user['id']
 
-        COMMON_QUERY = {
-            'limit': 50,
-            'client_id': self._CLIENT_ID,
-            'linked_partitioning': '1',
-        }
+        return self._extract_playlist(
+            self._BASE_URL_MAP[resource] % user['id'], compat_str(user['id']),
+            '%s (%s)' % (user['username'], self._TITLE_MAP[resource]))
 
-        query = COMMON_QUERY.copy()
-        query['offset'] = 0
 
-        next_href = base_url + '?' + compat_urllib_parse_urlencode(query)
+class SoundcloudTrackStationIE(SoundcloudPagedPlaylistBaseIE):
+    _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/stations/track/[^/]+/(?P<id>[^/?#&]+)'
+    IE_NAME = 'soundcloud:trackstation'
+    _TESTS = [{
+        'url': 'https://soundcloud.com/stations/track/officialsundial/your-text',
+        'info_dict': {
+            'id': '286017854',
+            'title': 'Track station: your-text',
+        },
+        'playlist_mincount': 47,
+    }]
 
-        entries = []
-        for i in itertools.count():
-            response = self._download_json(
-                next_href, uploader, 'Downloading track page %s' % (i + 1))
+    def _real_extract(self, url):
+        track_name = self._match_id(url)
 
-            collection = response['collection']
-            if not collection:
-                break
+        webpage = self._download_webpage(url, track_name)
 
-            def resolve_permalink_url(candidates):
-                for cand in candidates:
-                    if isinstance(cand, dict):
-                        permalink_url = cand.get('permalink_url')
-                        entry_id = self._extract_id(cand)
-                        if permalink_url and permalink_url.startswith('http'):
-                            return permalink_url, entry_id
+        track_id = self._search_regex(
+            r'soundcloud:track-stations:(\d+)', webpage, 'track id')
 
-            for e in collection:
-                permalink_url, entry_id = resolve_permalink_url((e, e.get('track'), e.get('playlist')))
-                if permalink_url:
-                    entries.append(self.url_result(permalink_url, video_id=entry_id))
-
-            next_href = response.get('next_href')
-            if not next_href:
-                break
-
-            parsed_next_href = compat_urlparse.urlparse(response['next_href'])
-            qs = compat_urlparse.parse_qs(parsed_next_href.query)
-            qs.update(COMMON_QUERY)
-            next_href = compat_urlparse.urlunparse(
-                parsed_next_href._replace(query=compat_urllib_parse_urlencode(qs, True)))
-
-        return {
-            '_type': 'playlist',
-            'id': compat_str(user['id']),
-            'title': '%s (%s)' % (user['username'], self._TITLE_MAP[resource]),
-            'entries': entries,
-        }
+        return self._extract_playlist(
+            '%s/stations/soundcloud:track-stations:%s/tracks'
+            % (self._API_V2_BASE, track_id),
+            track_id, 'Track station: %s' % track_name)
 
 
 class SoundcloudPlaylistIE(SoundcloudPlaylistBaseIE):

From e445850e69990502b171765343fc38317e932aca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Jul 2017 18:45:57 +0700
Subject: [PATCH 1159/1696] [soundcloud] Update client id

---
 youtube_dl/extractor/soundcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 2f1b2978c..2e52e092b 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -122,7 +122,7 @@ class SoundcloudIE(InfoExtractor):
         },
     ]
 
-    _CLIENT_ID = '2t9loNQH90kzJcsFCODdigxfp325aq4z'
+    _CLIENT_ID = 'JlZIsxg2hY5WnBgtn3jfS0UYCl0K8DOg'
     _IPHONE_CLIENT_ID = '376f225bf427445fc4bfb6b99b72e0bf'
 
     @staticmethod

From ca127ab2c174cdee4428eb4e192393c6ca942ac8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Jul 2017 23:07:28 +0700
Subject: [PATCH 1160/1696] [ard] Add support for lives (closes #13771)

---
 youtube_dl/extractor/ard.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 2d5599456..3f248b147 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -93,6 +93,7 @@ class ARDMediathekIE(InfoExtractor):
 
         duration = int_or_none(media_info.get('_duration'))
         thumbnail = media_info.get('_previewImage')
+        is_live = media_info.get('_isLive') is True
 
         subtitles = {}
         subtitle_url = media_info.get('_subtitleUrl')
@@ -106,6 +107,7 @@ class ARDMediathekIE(InfoExtractor):
             'id': video_id,
             'duration': duration,
             'thumbnail': thumbnail,
+            'is_live': is_live,
             'formats': formats,
             'subtitles': subtitles,
         }
@@ -166,9 +168,11 @@ class ARDMediathekIE(InfoExtractor):
         # determine video id from url
         m = re.match(self._VALID_URL, url)
 
+        document_id = None
+
         numid = re.search(r'documentId=([0-9]+)', url)
         if numid:
-            video_id = numid.group(1)
+            document_id = video_id = numid.group(1)
         else:
             video_id = m.group('video_id')
 
@@ -228,12 +232,16 @@ class ARDMediathekIE(InfoExtractor):
                 'formats': formats,
             }
         else:  # request JSON file
+            if not document_id:
+                video_id = self._search_regex(
+                    r'/play/(?:config|media)/(\d+)', webpage, 'media id')
             info = self._extract_media_info(
-                'http://www.ardmediathek.de/play/media/%s' % video_id, webpage, video_id)
+                'http://www.ardmediathek.de/play/media/%s' % video_id,
+                webpage, video_id)
 
         info.update({
             'id': video_id,
-            'title': title,
+            'title': self._live_title(title) if info.get('is_live') else title,
             'description': description,
             'thumbnail': thumbnail,
         })

From 198d4cb40ce9d819e8e4079058642ee96dae213b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Grzegorz=20Ruci=C5=84ski?= <grucin@gmail.com>
Date: Sat, 29 Jul 2017 20:30:04 +0200
Subject: [PATCH 1161/1696] [generic] Add support for another ooyala embed
 pattern (closes #13727)

---
 youtube_dl/extractor/generic.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 4b83e861b..34e814988 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -575,6 +575,19 @@ class GenericIE(InfoExtractor):
             },
             'skip': 'movie expired',
         },
+        # ooyala video embedded with http://player.ooyala.com/static/v4/production/latest/core.min.js
+        {
+            'url': 'http://wnep.com/2017/07/22/steampunk-fest-comes-to-honesdale/',
+            'info_dict': {
+                'id': 'lwYWYxYzE6V5uJMjNGyKtwwiw9ZJD7t2',
+                'ext': 'mp4',
+                'title': 'Steampunk Fest Comes to Honesdale',
+                'duration': 43.276,
+            },
+            'params': {
+                'skip_download': True,
+            }
+        },
         # embed.ly video
         {
             'url': 'http://www.tested.com/science/weird/460206-tested-grinding-coffee-2000-frames-second/',
@@ -2293,6 +2306,7 @@ class GenericIE(InfoExtractor):
         # Look for Ooyala videos
         mobj = (re.search(r'player\.ooyala\.com/[^"?]+[?#][^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
                 re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage) or
+                re.search(r'OO\.Player\.create\.apply\(\s*OO\.Player\s*,\s*op\(\s*\[\s*[\'"][^\'"]*[\'"]\s*,\s*[\'"](?P<ec>.{32})[\'"]', webpage) or
                 re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P<ec>.{32})[\'"]\)', webpage) or
                 re.search(r'data-ooyala-video-id\s*=\s*[\'"](?P<ec>.{32})[\'"]', webpage))
         if mobj is not None:

From a0a477b885dc1dd688058924357c4935f3c935cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Jul 2017 15:48:22 +0700
Subject: [PATCH 1162/1696] [youjizz] Fix extraction (closes #13744)

---
 youtube_dl/extractor/youjizz.py | 80 ++++++++++++++++++++++++++++-----
 1 file changed, 68 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/youjizz.py b/youtube_dl/extractor/youjizz.py
index b50f34e9b..f33fabe19 100644
--- a/youtube_dl/extractor/youjizz.py
+++ b/youtube_dl/extractor/youjizz.py
@@ -1,39 +1,95 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    determine_ext,
+    int_or_none,
+    parse_duration,
+)
 
 
 class YouJizzIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:\w+\.)?youjizz\.com/videos/(?:[^/#?]+)?-(?P<id>[0-9]+)\.html(?:$|[?#])'
+    _VALID_URL = r'https?://(?:\w+\.)?youjizz\.com/videos/(?:[^/#?]*-(?P<id>\d+)\.html|embed/(?P<embed_id>\d+))'
     _TESTS = [{
         'url': 'http://www.youjizz.com/videos/zeichentrick-1-2189178.html',
-        'md5': '78fc1901148284c69af12640e01c6310',
+        'md5': 'b1e1dfaa8bb9537d8b84eeda9cf4acf4',
         'info_dict': {
             'id': '2189178',
             'ext': 'mp4',
             'title': 'Zeichentrick 1',
             'age_limit': 18,
+            'duration': 2874,
         }
     }, {
         'url': 'http://www.youjizz.com/videos/-2189178.html',
         'only_matching': True,
+    }, {
+        'url': 'https://www.youjizz.com/videos/embed/31991001',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        # YouJizz's HTML5 player has invalid HTML
-        webpage = webpage.replace('"controls', '" controls')
-        age_limit = self._rta_search(webpage)
-        video_title = self._html_search_regex(
-            r'<title>\s*(.*)\s*</title>', webpage, 'title')
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id') or mobj.group('embed_id')
 
-        info_dict = self._parse_html5_media_entries(url, webpage, video_id)[0]
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(
+            r'<title>(.+?)</title>', webpage, 'title')
+
+        formats = []
+
+        encodings = self._parse_json(
+            self._search_regex(
+                r'encodings\s*=\s*(\[.+?\]);\n', webpage, 'encodings',
+                default='[]'),
+            video_id, fatal=False)
+        for encoding in encodings:
+            if not isinstance(encoding, dict):
+                continue
+            format_url = encoding.get('filename')
+            if not isinstance(format_url, compat_str):
+                continue
+            if determine_ext(format_url) == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls', fatal=False))
+            else:
+                format_id = encoding.get('name') or encoding.get('quality')
+                height = int_or_none(self._search_regex(
+                    r'^(\d+)[pP]', format_id, 'height', default=None))
+                formats.append({
+                    'url': format_url,
+                    'format_id': format_id,
+                    'height': height,
+                })
+
+        if formats:
+            info_dict = {
+                'formats': formats,
+            }
+        else:
+            # YouJizz's HTML5 player has invalid HTML
+            webpage = webpage.replace('"controls', '" controls')
+            info_dict = self._parse_html5_media_entries(
+                url, webpage, video_id)[0]
+
+        duration = parse_duration(self._search_regex(
+            r'<strong>Runtime:</strong>([^<]+)', webpage, 'duration',
+            default=None))
+        uploader = self._search_regex(
+            r'<strong>Uploaded By:.*?<a[^>]*>([^<]+)', webpage, 'uploader',
+            default=None)
 
         info_dict.update({
             'id': video_id,
-            'title': video_title,
-            'age_limit': age_limit,
+            'title': title,
+            'age_limit': self._rta_search(webpage),
+            'duration': duration,
+            'uploader': uploader,
         })
 
         return info_dict

From 0ed4758023ddfb4d9630ba9114ef70ef7e6ac09d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Jul 2017 19:08:44 +0700
Subject: [PATCH 1163/1696] [clipfish] Remove extractor

---
 youtube_dl/extractor/clipfish.py   | 67 ------------------------------
 youtube_dl/extractor/extractors.py |  1 -
 2 files changed, 68 deletions(-)
 delete mode 100644 youtube_dl/extractor/clipfish.py

diff --git a/youtube_dl/extractor/clipfish.py b/youtube_dl/extractor/clipfish.py
deleted file mode 100644
index 0920f6219..000000000
--- a/youtube_dl/extractor/clipfish.py
+++ /dev/null
@@ -1,67 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    unified_strdate,
-)
-
-
-class ClipfishIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?clipfish\.de/(?:[^/]+/)+video/(?P<id>[0-9]+)'
-    _TEST = {
-        'url': 'http://www.clipfish.de/special/ugly-americans/video/4343170/s01-e01-ugly-americans-date-in-der-hoelle/',
-        'md5': 'b9a5dc46294154c1193e2d10e0c95693',
-        'info_dict': {
-            'id': '4343170',
-            'ext': 'mp4',
-            'title': 'S01 E01 - Ugly Americans - Date in der Hölle',
-            'description': 'Mark Lilly arbeitet im Sozialdienst der Stadt New York und soll Immigranten bei ihrer Einbürgerung in die USA zur Seite stehen.',
-            'upload_date': '20161005',
-            'duration': 1291,
-            'view_count': int,
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        video_info = self._download_json(
-            'http://www.clipfish.de/devapi/id/%s?format=json&apikey=hbbtv' % video_id,
-            video_id)['items'][0]
-
-        formats = []
-
-        m3u8_url = video_info.get('media_videourl_hls')
-        if m3u8_url:
-            formats.append({
-                'url': m3u8_url.replace('de.hls.fra.clipfish.de', 'hls.fra.clipfish.de'),
-                'ext': 'mp4',
-                'format_id': 'hls',
-            })
-
-        mp4_url = video_info.get('media_videourl')
-        if mp4_url:
-            formats.append({
-                'url': mp4_url,
-                'format_id': 'mp4',
-                'width': int_or_none(video_info.get('width')),
-                'height': int_or_none(video_info.get('height')),
-                'tbr': int_or_none(video_info.get('bitrate')),
-            })
-
-        descr = video_info.get('descr')
-        if descr:
-            descr = descr.strip()
-
-        return {
-            'id': video_id,
-            'title': video_info['title'],
-            'description': descr,
-            'formats': formats,
-            'thumbnail': video_info.get('media_content_thumbnail_large') or video_info.get('media_thumbnail'),
-            'duration': int_or_none(video_info.get('media_length')),
-            'upload_date': unified_strdate(video_info.get('pubDate')),
-            'view_count': int_or_none(video_info.get('media_views'))
-        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d2c5e8030..bdc7370cd 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -186,7 +186,6 @@ from .chirbit import (
 )
 from .cinchcast import CinchcastIE
 from .cjsw import CJSWIE
-from .clipfish import ClipfishIE
 from .cliphunter import CliphunterIE
 from .cliprs import ClipRsIE
 from .clipsyndicate import ClipsyndicateIE

From 8b9f50d7cb4cfab5d505f4233c3e176a8106d6db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Jul 2017 19:09:44 +0700
Subject: [PATCH 1164/1696] [watchbox] Add extractor (#13739)

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/watchbox.py   | 151 +++++++++++++++++++++++++++++
 2 files changed, 152 insertions(+)
 create mode 100644 youtube_dl/extractor/watchbox.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index bdc7370cd..3489e86f0 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1244,6 +1244,7 @@ from .washingtonpost import (
     WashingtonPostArticleIE,
 )
 from .wat import WatIE
+from .watchbox import WatchBoxIE
 from .watchindianporn import WatchIndianPornIE
 from .wdr import (
     WDRIE,
diff --git a/youtube_dl/extractor/watchbox.py b/youtube_dl/extractor/watchbox.py
new file mode 100644
index 000000000..b382338fa
--- /dev/null
+++ b/youtube_dl/extractor/watchbox.py
@@ -0,0 +1,151 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    int_or_none,
+    js_to_json,
+    strip_or_none,
+    try_get,
+    unified_timestamp,
+)
+
+
+class WatchBoxIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?watchbox\.de/(?P<kind>serien|filme)/(?:[^/]+/)*[^/]+-(?P<id>\d+)'
+    _TESTS = [{
+        # film
+        'url': 'https://www.watchbox.de/filme/free-jimmy-12325.html',
+        'info_dict': {
+            'id': '341368',
+            'ext': 'mp4',
+            'title': 'Free Jimmy',
+            'description': 'md5:bcd8bafbbf9dc0ef98063d344d7cc5f6',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 4890,
+            'age_limit': 16,
+            'release_year': 2009,
+        },
+        'params': {
+            'format': 'bestvideo',
+            'skip_download': True,
+        },
+        'expected_warnings': ['Failed to download m3u8 information'],
+    }, {
+        # episode
+        'url': 'https://www.watchbox.de/serien/ugly-americans-12231/staffel-1/date-in-der-hoelle-328286.html',
+        'info_dict': {
+            'id': '328286',
+            'ext': 'mp4',
+            'title': 'S01 E01 - Date in der Hölle',
+            'description': 'md5:2f31c74a8186899f33cb5114491dae2b',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 1291,
+            'age_limit': 12,
+            'release_year': 2010,
+            'series': 'Ugly Americans',
+            'season_number': 1,
+            'episode': 'Date in der Hölle',
+            'episode_number': 1,
+        },
+        'params': {
+            'format': 'bestvideo',
+            'skip_download': True,
+        },
+        'expected_warnings': ['Failed to download m3u8 information'],
+    }, {
+        'url': 'https://www.watchbox.de/serien/ugly-americans-12231/staffel-2/der-ring-des-powers-328270',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        kind, video_id = mobj.group('kind', 'id')
+
+        webpage = self._download_webpage(url, video_id)
+
+        source = self._parse_json(
+            self._search_regex(
+                r'(?s)source\s*:\s*({.+?})\s*,\s*\n', webpage, 'source',
+                default='{}'),
+            video_id, transform_source=js_to_json, fatal=False) or {}
+
+        video_id = compat_str(source.get('videoId') or video_id)
+
+        devapi = self._download_json(
+            'http://api.watchbox.de/devapi/id/%s' % video_id, video_id, query={
+                'format': 'json',
+                'apikey': 'hbbtv',
+            }, fatal=False)
+
+        item = try_get(devapi, lambda x: x['items'][0], dict) or {}
+
+        title = item.get('title') or try_get(
+            item, lambda x: x['movie']['headline_movie'],
+            compat_str) or source['title']
+
+        formats = []
+        hls_url = item.get('media_videourl_hls') or source.get('hls')
+        if hls_url:
+            formats.extend(self._extract_m3u8_formats(
+                hls_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                m3u8_id='hls', fatal=False))
+        dash_url = item.get('media_videourl_wv') or source.get('dash')
+        if dash_url:
+            formats.extend(self._extract_mpd_formats(
+                dash_url, video_id, mpd_id='dash', fatal=False))
+        mp4_url = item.get('media_videourl')
+        if mp4_url:
+            formats.append({
+                'url': mp4_url,
+                'format_id': 'mp4',
+                'width': int_or_none(item.get('width')),
+                'height': int_or_none(item.get('height')),
+                'tbr': int_or_none(item.get('bitrate')),
+            })
+        self._sort_formats(formats)
+
+        description = strip_or_none(item.get('descr'))
+        thumbnail = item.get('media_content_thumbnail_large') or source.get('poster') or item.get('media_thumbnail')
+        duration = int_or_none(item.get('media_length') or source.get('length'))
+        timestamp = unified_timestamp(item.get('pubDate'))
+        view_count = int_or_none(item.get('media_views'))
+        age_limit = int_or_none(try_get(item, lambda x: x['movie']['fsk']))
+        release_year = int_or_none(try_get(item, lambda x: x['movie']['rel_year']))
+
+        info = {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'timestamp': timestamp,
+            'view_count': view_count,
+            'age_limit': age_limit,
+            'release_year': release_year,
+            'formats': formats,
+        }
+
+        if kind.lower() == 'serien':
+            series = try_get(
+                item, lambda x: x['special']['title'],
+                compat_str) or source.get('format')
+            season_number = int_or_none(self._search_regex(
+                r'^S(\d{1,2})\s*E\d{1,2}', title, 'season number',
+                default=None) or self._search_regex(
+                    r'/staffel-(\d+)/', url, 'season number', default=None))
+            episode = source.get('title')
+            episode_number = int_or_none(self._search_regex(
+                r'^S\d{1,2}\s*E(\d{1,2})', title, 'episode number',
+                default=None))
+            info.update({
+                'series': series,
+                'season_number': season_number,
+                'episode': episode,
+                'episode_number': episode_number,
+            })
+
+        return info

From f701827e319963ca783d012f3647aa44fc0efcd6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Jul 2017 19:43:09 +0700
Subject: [PATCH 1165/1696] [ChangeLog] Actualize

---
 ChangeLog | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 8e63b5c11..ca3ee8a93 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,8 +1,28 @@
 version <unreleased>
 
 Core
+* [downloader/hls] Use redirect URL as manifest base (#13755)
 * [options] Correctly hide login info from debug outputs (#13696)
 
+Extractors
++ [watchbox] Add support for watchbox.de (#13739)
+- [clipfish] Remove extractor
++ [youjizz] Fix extraction (#13744)
++ [generic] Add support for another ooyala embed pattern (#13727)
++ [ard] Add support for lives (#13771)
+* [soundcloud] Update client id
++ [soundcloud:trackstation] Add support for track stations (#13733)
+* [svtplay] Use geo verification proxy for API request
+* [svtplay] Update API URL (#13767)
++ [yandexdisk] Add support for yadi.sk (#13755)
++ [megaphone] Add support for megaphone.fm
+* [amcnetworks] Make rating optional (#12453)
+* [cloudy] Fix extraction (#13737)
++ [nickru] Add support for nickelodeon.ru
+* [mtv] Improve thumbnal extraction
+* [nick] Automate geo-restriction bypass (#13711)
+* [niconico] Improve error reporting (#13696)
+
 
 version 2017.07.23
 

From 5c9ea67bc0dedd15a0ed9ad05d8fcf09946ca461 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Jul 2017 20:47:31 +0700
Subject: [PATCH 1166/1696] release 2017.07.30.1

---
 .github/ISSUE_TEMPLATE.md | 16 ++++++++--------
 ChangeLog                 |  2 +-
 docs/supportedsites.md    |  6 +++++-
 youtube_dl/version.py     |  2 +-
 4 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 37d09d796..0421de755 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -1,16 +1,16 @@
 ## Please follow the guide below
 
 - You will be asked some questions and requested to provide some information, please read them **carefully** and answer honestly
-- Put an `x` into all the boxes [ ] relevant to your *issue* (like that [x])
-- Use *Preview* tab to see how your issue will actually look like
+- Put an `x` into all the boxes [ ] relevant to your *issue* (like this: `[x]`)
+- Use the *Preview* tab to see what your issue will actually look like
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.07.23*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.07.23**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.07.30.1*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.07.30.1**
 
 ### Before submitting an *issue* make sure you have:
-- [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
+- [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
 - [ ] [Searched](https://github.com/rg3/youtube-dl/search?type=Issues) the bugtracker for similar issues including closed ones
 
 ### What is the purpose of your *issue*?
@@ -28,14 +28,14 @@
 
 ### If the purpose of this *issue* is a *bug report*, *site support request* or you are not completely sure provide the full verbose output as follows:
 
-Add `-v` flag to **your command line** you run youtube-dl with, copy the **whole** output and insert it here. It should look similar to one below (replace it with **your** log inserted between triple ```):
+Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl -v <your command line>`), copy the **whole** output and insert it here. It should look similar to one below (replace it with **your** log inserted between triple ```):
+
 ```
-$ youtube-dl -v <your command line>
 [debug] System config: []
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.07.23
+[debug] youtube-dl version 2017.07.30.1
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index ca3ee8a93..4f03ef064 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.07.30.1
 
 Core
 * [downloader/hls] Use redirect URL as manifest base (#13755)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index eb09c470c..77aac8249 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -155,7 +155,6 @@
  - **chirbit:profile**
  - **Cinchcast**
  - **CJSW**
- - **Clipfish**
  - **cliphunter**
  - **ClipRs**
  - **Clipsyndicate**
@@ -440,6 +439,7 @@
  - **Medialaan**
  - **Mediaset**
  - **Medici**
+ - **megaphone.fm**: megaphone.fm embedded players
  - **Meipai**: 美拍
  - **MelonVOD**
  - **META**
@@ -533,6 +533,7 @@
  - **nhl.com:videocenter:category**: NHL videocenter category
  - **nick.com**
  - **nick.de**
+ - **nickelodeonru**
  - **nicknight**
  - **niconico**: ニコニコ動画
  - **NiconicoPlaylist**
@@ -734,6 +735,7 @@
  - **soundcloud:playlist**
  - **soundcloud:search**: Soundcloud search
  - **soundcloud:set**
+ - **soundcloud:trackstation**
  - **soundcloud:user**
  - **soundgasm**
  - **soundgasm:profile**
@@ -968,6 +970,7 @@
  - **washingtonpost**
  - **washingtonpost:article**
  - **wat.tv**
+ - **WatchBox**
  - **WatchIndianPorn**: Watch Indian Porn
  - **WDR**
  - **wdr:mobile**
@@ -1003,6 +1006,7 @@
  - **XVideos**
  - **XXXYMovies**
  - **Yahoo**: Yahoo screen and movies
+ - **YandexDisk**
  - **yandexmusic:album**: Яндекс.Музыка - Альбом
  - **yandexmusic:playlist**: Яндекс.Музыка - Плейлист
  - **yandexmusic:track**: Яндекс.Музыка - Трек
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a8dbb93e3..38162157d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.07.23'
+__version__ = '2017.07.30.1'

From 9118c9f18a43a2f3e4814fcc02ac8e5180077df3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Aug 2017 05:20:14 +0700
Subject: [PATCH 1167/1696] [nrktv] Update API host (closes #13796)

---
 youtube_dl/extractor/nrk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 3b4f51f61..18ead9426 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -237,7 +237,7 @@ class NRKTVIE(NRKBaseIE):
                             (?:/\d{2}-\d{2}-\d{4})?
                             (?:\#del=(?P<part_id>\d+))?
                     ''' % _EPISODE_RE
-    _API_HOST = 'psapi-we.nrk.no'
+    _API_HOST = 'psapi-ne.nrk.no'
 
     _TESTS = [{
         'url': 'https://tv.nrk.no/serie/20-spoersmaal-tv/MUHH48000314/23-05-2014',

From 8cda78ef72c52c0424ddf90c22105dbc3b1d16f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Aug 2017 23:12:34 +0700
Subject: [PATCH 1168/1696] [test_YoutubeDL] Add a test for #10083

---
 test/test_YoutubeDL.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 4af14f9db..e70cbcd37 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -371,6 +371,19 @@ class TestFormatSelection(unittest.TestCase):
         ydl = YDL({'format': 'best[height>360]'})
         self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
 
+    def test_format_selection_issue_10083(self):
+        # See https://github.com/rg3/youtube-dl/issues/10083
+        formats = [
+            {'format_id': 'regular', 'height': 360, 'url': TEST_URL},
+            {'format_id': 'video', 'height': 720, 'acodec': 'none', 'url': TEST_URL},
+            {'format_id': 'audio', 'vcodec': 'none', 'url': TEST_URL},
+        ]
+        info_dict = _make_result(formats)
+
+        ydl = YDL({'format': 'best[height>360]/bestvideo[height>360]+bestaudio'})
+        ydl.process_ie_result(info_dict.copy())
+        self.assertEqual(ydl.downloaded_info_dicts[0]['format_id'], 'video+audio')
+
     def test_invalid_format_specs(self):
         def assert_syntax_error(format_spec):
             ydl = YDL({'format': format_spec})

From feee8d32e45c9521426cf4a089c70f37542f0065 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Thu, 3 Aug 2017 14:17:25 +0200
Subject: [PATCH 1169/1696] [phantomjs] add exe version to debug info

---
 youtube_dl/YoutubeDL.py | 2 ++
 youtube_dl/utils.py     | 6 +++++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index eb465c425..033b50702 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -86,6 +86,7 @@ from .utils import (
     write_string,
     YoutubeDLCookieProcessor,
     YoutubeDLHandler,
+    PhantomJSwrapper,
 )
 from .cache import Cache
 from .extractor import get_info_extractor, gen_extractor_classes, _LAZY_LOADER
@@ -2146,6 +2147,7 @@ class YoutubeDL(object):
 
         exe_versions = FFmpegPostProcessor.get_versions(self)
         exe_versions['rtmpdump'] = rtmpdump_version()
+        exe_versions['phantomjs'] = PhantomJSwrapper._version()
         exe_str = ', '.join(
             '%s %s' % (exe, v)
             for exe, v in sorted(exe_versions.items())
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index c67f95ac9..4d0685d83 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3895,6 +3895,10 @@ class PhantomJSwrapper(object):
 
     _TMP_FILE_NAMES = ['script', 'html', 'cookies']
 
+    @staticmethod
+    def _version():
+        return get_exe_version('phantomjs', version_re=r'([0-9.]+)')
+
     def __init__(self, extractor, required_version=None, timeout=10000):
         self.exe = check_executable('phantomjs', ['-v'])
         if not self.exe:
@@ -3905,7 +3909,7 @@ class PhantomJSwrapper(object):
         self.extractor = extractor
 
         if required_version:
-            version = get_exe_version(self.exe, version_re=r'([0-9.]+)')
+            version = self._version()
             if is_outdated_version(version, required_version):
                 self.extractor._downloader.report_warning(
                     'Your copy of PhantomJS is outdated, update it to version '

From 183062a4ab2f698f5096e69602fb2b5c861c01a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 3 Aug 2017 23:19:59 +0700
Subject: [PATCH 1170/1696] [pbs] Add support for new URL schema (closes
 #13801)

---
 youtube_dl/extractor/pbs.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 16cc667d0..8889e4a1a 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -189,7 +189,7 @@ class PBSIE(InfoExtractor):
            # Direct video URL
            (?:%s)/(?:viralplayer|video)/(?P<id>[0-9]+)/? |
            # Article with embedded player (or direct video)
-           (?:www\.)?pbs\.org/(?:[^/]+/){2,5}(?P<presumptive_id>[^/]+?)(?:\.html)?/?(?:$|[?\#]) |
+           (?:www\.)?pbs\.org/(?:[^/]+/){1,5}(?P<presumptive_id>[^/]+?)(?:\.html)?/?(?:$|[?\#]) |
            # Player
            (?:video|player)\.pbs\.org/(?:widget/)?partnerplayer/(?P<player_id>[^/]+)/
         )
@@ -345,6 +345,21 @@ class PBSIE(InfoExtractor):
                 'formats': 'mincount:8',
             },
         },
+        {
+            # https://github.com/rg3/youtube-dl/issues/13801
+            'url': 'https://www.pbs.org/video/pbs-newshour-full-episode-july-31-2017-1501539057/',
+            'info_dict': {
+                'id': '3003333873',
+                'ext': 'mp4',
+                'title': 'PBS NewsHour - full episode July 31, 2017',
+                'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
+                'duration': 3265,
+                'thumbnail': r're:^https?://.*\.jpg$',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
         {
             'url': 'http://player.pbs.org/widget/partnerplayer/2365297708/?start=0&end=0&chapterbar=false&endscreen=false&topbar=true',
             'only_matching': True,
@@ -433,6 +448,9 @@ class PBSIE(InfoExtractor):
                 if url:
                     break
 
+            if not url:
+                url = self._og_search_url(webpage)
+
             mobj = re.match(self._VALID_URL, url)
 
         player_id = mobj.group('player_id')

From 1f03fef994e076a827cfd818eb4d76fe2eb85130 Mon Sep 17 00:00:00 2001
From: Justin Quan <jnicquan@gmail.com>
Date: Fri, 4 Aug 2017 08:43:44 -0700
Subject: [PATCH 1171/1696] [README.md] Improve grammar

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index fe2bebc2a..0067184be 100644
--- a/README.md
+++ b/README.md
@@ -584,7 +584,7 @@ If you are using an output template inside a Windows batch file then you must es
 
 #### Output template examples
 
-Note on Windows you may need to use double quotes instead of single.
+Note that on Windows you may need to use double quotes instead of single.
 
 ```bash
 $ youtube-dl --get-filename -o '%(title)s.%(ext)s' BaW_jenozKc
@@ -671,7 +671,7 @@ If you want to preserve the old format selection behavior (prior to youtube-dl 2
 
 #### Format selection examples
 
-Note on Windows you may need to use double quotes instead of single.
+Note that on Windows you may need to use double quotes instead of single.
 
 ```bash
 # Download best mp4 format available or any other best if no mp4 available

From 11a6793f8013f37045d769e5b166f75e17f275d1 Mon Sep 17 00:00:00 2001
From: Matt Crupi <its.matthew.c@gmail.com>
Date: Fri, 4 Aug 2017 08:46:54 -0700
Subject: [PATCH 1172/1696] [mlb] Extend _VALID_URL (closes #13740)

---
 youtube_dl/extractor/mlb.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mlb.py b/youtube_dl/extractor/mlb.py
index 59cd4b838..4d45f960e 100644
--- a/youtube_dl/extractor/mlb.py
+++ b/youtube_dl/extractor/mlb.py
@@ -15,7 +15,7 @@ class MLBIE(InfoExtractor):
                         (?:[\da-z_-]+\.)*mlb\.com/
                         (?:
                             (?:
-                                (?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|
+                                (?:.*?/)?video/(?:topic/[\da-z_-]+/)?(?:v|.*?/c-)|
                                 (?:
                                     shared/video/embed/(?:embed|m-internal-embed)\.html|
                                     (?:[^/]+/)+(?:play|index)\.jsp|
@@ -94,6 +94,10 @@ class MLBIE(InfoExtractor):
                 'upload_date': '20150415',
             }
         },
+        {
+            'url': 'https://www.mlb.com/video/hargrove-homers-off-caldwell/c-1352023483?tid=67793694',
+            'only_matching': True,
+        },
         {
             'url': 'http://m.mlb.com/shared/video/embed/embed.html?content_id=35692085&topic_id=6479266&width=400&height=224&property=mlb',
             'only_matching': True,

From 57a38a38c32ea2eb1ca54ee4ba3fcd31a9b7f328 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 4 Aug 2017 23:44:07 +0700
Subject: [PATCH 1173/1696] [udemy] Fix subtitles extraction (closes #13812)

---
 youtube_dl/extractor/udemy.py | 47 ++++++++++++++++++++++-------------
 1 file changed, 30 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 160be1b1b..3b02f43e3 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -15,6 +15,7 @@ from ..utils import (
     ExtractorError,
     float_or_none,
     int_or_none,
+    js_to_json,
     sanitized_Request,
     unescapeHTML,
     urlencode_postdata,
@@ -268,6 +269,25 @@ class UdemyIE(InfoExtractor):
                     f = add_output_format_meta(f, format_id)
                 formats.append(f)
 
+        def extract_subtitles(track_list):
+            if not isinstance(track_list, list):
+                return
+            for track in track_list:
+                if not isinstance(track, dict):
+                    continue
+                if track.get('kind') != 'captions':
+                    continue
+                src = track.get('src')
+                if not src or not isinstance(src, compat_str):
+                    continue
+                lang = track.get('language') or track.get(
+                    'srclang') or track.get('label')
+                sub_dict = automatic_captions if track.get(
+                    'autogenerated') is True else subtitles
+                sub_dict.setdefault(lang, []).append({
+                    'url': src,
+                })
+
         download_urls = asset.get('download_urls')
         if isinstance(download_urls, dict):
             extract_formats(download_urls.get('Video'))
@@ -315,23 +335,16 @@ class UdemyIE(InfoExtractor):
                 extract_formats(data.get('sources'))
                 if not duration:
                     duration = int_or_none(data.get('duration'))
-                tracks = data.get('tracks')
-                if isinstance(tracks, list):
-                    for track in tracks:
-                        if not isinstance(track, dict):
-                            continue
-                        if track.get('kind') != 'captions':
-                            continue
-                        src = track.get('src')
-                        if not src or not isinstance(src, compat_str):
-                            continue
-                        lang = track.get('language') or track.get(
-                            'srclang') or track.get('label')
-                        sub_dict = automatic_captions if track.get(
-                            'autogenerated') is True else subtitles
-                        sub_dict.setdefault(lang, []).append({
-                            'url': src,
-                        })
+                extract_subtitles(data.get('tracks'))
+
+            if not subtitles and not automatic_captions:
+                text_tracks = self._parse_json(
+                    self._search_regex(
+                        r'text-tracks=(["\'])(?P<data>\[.+?\])\1', view_html,
+                        'text tracks', default='{}', group='data'), video_id,
+                    transform_source=lambda s: js_to_json(unescapeHTML(s)),
+                    fatal=False)
+                extract_subtitles(text_tracks)
 
         self._sort_formats(formats, field_preference=('height', 'width', 'tbr', 'format_id'))
 

From b3b5870cba46d84b7482f120f550822d3b64c3d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 4 Aug 2017 23:51:03 +0700
Subject: [PATCH 1174/1696] [pornhd] Fix extraction (closes #13783)

---
 youtube_dl/extractor/pornhd.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhd.py b/youtube_dl/extractor/pornhd.py
index 842317e6c..36761788d 100644
--- a/youtube_dl/extractor/pornhd.py
+++ b/youtube_dl/extractor/pornhd.py
@@ -54,7 +54,7 @@ class PornHdIE(InfoExtractor):
              r'<title>(.+?) - .*?[Pp]ornHD.*?</title>'], webpage, 'title')
 
         sources = self._parse_json(js_to_json(self._search_regex(
-            r"(?s)'sources'\s*:\s*(\{.+?\})\s*\}[;,)]",
+            r"(?s)sources'?\s*:\s*(\{.+?\})\s*\}[;,)]",
             webpage, 'sources', default='{}')), video_id)
 
         if not sources:

From 799802f368012f579750b26db117b3a9dfdcbe05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 4 Aug 2017 23:54:28 +0700
Subject: [PATCH 1175/1696] [teamfour] Remove extractor (closes #13782) Now
 covered with generic extractor

---
 youtube_dl/extractor/extractors.py   |  1 -
 youtube_dl/extractor/teamfourstar.py | 48 ----------------------------
 2 files changed, 49 deletions(-)
 delete mode 100644 youtube_dl/extractor/teamfourstar.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 3489e86f0..d0e04dd7d 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -994,7 +994,6 @@ from .teachertube import (
 )
 from .teachingchannel import TeachingChannelIE
 from .teamcoco import TeamcocoIE
-from .teamfourstar import TeamFourStarIE
 from .techtalks import TechTalksIE
 from .ted import TEDIE
 from .tele13 import Tele13IE
diff --git a/youtube_dl/extractor/teamfourstar.py b/youtube_dl/extractor/teamfourstar.py
deleted file mode 100644
index a8c6ed7be..000000000
--- a/youtube_dl/extractor/teamfourstar.py
+++ /dev/null
@@ -1,48 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from .jwplatform import JWPlatformIE
-from ..utils import unified_strdate
-
-
-class TeamFourStarIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?teamfourstar\.com/(?P<id>[a-z0-9\-]+)'
-    _TEST = {
-        'url': 'http://teamfourstar.com/tfs-abridged-parody-episode-1-2/',
-        'info_dict': {
-            'id': '0WdZO31W',
-            'title': 'TFS Abridged Parody Episode 1',
-            'description': 'md5:d60bc389588ebab2ee7ad432bda953ae',
-            'ext': 'mp4',
-            'timestamp': 1394168400,
-            'upload_date': '20080508',
-        },
-    }
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-
-        jwplatform_url = JWPlatformIE._extract_url(webpage)
-
-        video_title = self._html_search_regex(
-            r'<h1[^>]+class="entry-title"[^>]*>(?P<title>.+?)</h1>',
-            webpage, 'title')
-        video_date = unified_strdate(self._html_search_regex(
-            r'<span[^>]+class="meta-date date updated"[^>]*>(?P<date>.+?)</span>',
-            webpage, 'date', fatal=False))
-        video_description = self._html_search_regex(
-            r'(?s)<div[^>]+class="content-inner"[^>]*>.*?(?P<description><p>.+?)</div>',
-            webpage, 'description', fatal=False)
-        video_thumbnail = self._og_search_thumbnail(webpage)
-
-        return {
-            '_type': 'url_transparent',
-            'display_id': display_id,
-            'title': video_title,
-            'description': video_description,
-            'upload_date': video_date,
-            'thumbnail': video_thumbnail,
-            'url': jwplatform_url,
-        }

From f31fd0693b674e73f9273f0afba2a54853e4ca35 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Aug 2017 00:00:21 +0700
Subject: [PATCH 1176/1696] [vidme] Extract DASH and HLS formats

---
 youtube_dl/extractor/vidme.py | 34 ++++++++++++++++++++++++++--------
 1 file changed, 26 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index e9ff336c4..a7971d72e 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -3,7 +3,10 @@ from __future__ import unicode_literals
 import itertools
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..compat import (
+    compat_HTTPError,
+    compat_str,
+)
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -161,13 +164,28 @@ class VidmeIE(InfoExtractor):
                 'or for violating the terms of use.',
                 expected=True)
 
-        formats = [{
-            'format_id': f.get('type'),
-            'url': f['uri'],
-            'width': int_or_none(f.get('width')),
-            'height': int_or_none(f.get('height')),
-            'preference': 0 if f.get('type', '').endswith('clip') else 1,
-        } for f in video.get('formats', []) if f.get('uri')]
+        formats = []
+        for f in video.get('formats', []):
+            format_url = f.get('uri')
+            if not format_url or not isinstance(format_url, compat_str):
+                continue
+            format_type = f.get('type')
+            if format_type == 'dash':
+                formats.extend(self._extract_mpd_formats(
+                    format_url, video_id, mpd_id='dash', fatal=False))
+            elif format_type == 'hls':
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls', fatal=False))
+            else:
+                formats.append({
+                    'format_id': f.get('type'),
+                    'url': format_url,
+                    'width': int_or_none(f.get('width')),
+                    'height': int_or_none(f.get('height')),
+                    'preference': 0 if f.get('type', '').endswith(
+                        'clip') else 1,
+                })
 
         if not formats and video.get('complete_url'):
             formats.append({

From bbbe1cebfce3cfb63e9c01d29105fcba693ba54c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Aug 2017 00:09:36 +0700
Subject: [PATCH 1177/1696] [mlb] Update test (closes #13777)

---
 youtube_dl/extractor/mlb.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mlb.py b/youtube_dl/extractor/mlb.py
index 4d45f960e..675ff6873 100644
--- a/youtube_dl/extractor/mlb.py
+++ b/youtube_dl/extractor/mlb.py
@@ -84,7 +84,7 @@ class MLBIE(InfoExtractor):
         },
         {
             'url': 'http://m.mlb.com/news/article/118550098/blue-jays-kevin-pillar-goes-spidey-up-the-wall-to-rob-tim-beckham-of-a-homer',
-            'md5': 'b190e70141fb9a1552a85426b4da1b5d',
+            'md5': 'aafaf5b0186fee8f32f20508092f8111',
             'info_dict': {
                 'id': '75609783',
                 'ext': 'mp4',

From 8519b88f67de9c0c11cd2edd8dc55b9a4f13d110 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Aug 2017 00:59:07 +0700
Subject: [PATCH 1178/1696] [yandexdisk] Relax _VALID_URL (closes #13824)

---
 youtube_dl/extractor/yandexdisk.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/yandexdisk.py b/youtube_dl/extractor/yandexdisk.py
index 11729f0f7..e8f6ae10f 100644
--- a/youtube_dl/extractor/yandexdisk.py
+++ b/youtube_dl/extractor/yandexdisk.py
@@ -13,9 +13,9 @@ from ..utils import (
 
 
 class YandexDiskIE(InfoExtractor):
-    _VALID_URL = r'https?://yadi\.sk/i/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://yadi\.sk/[di]/(?P<id>[^/?#&]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'https://yadi.sk/i/VdOeDou8eZs6Y',
         'md5': '33955d7ae052f15853dc41f35f17581c',
         'info_dict': {
@@ -27,7 +27,10 @@ class YandexDiskIE(InfoExtractor):
             'uploader_id': '300043621',
             'view_count': int,
         },
-    }
+    }, {
+        'url': 'https://yadi.sk/d/h3WAXvDS3Li3Ce',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 1141e9104bc0f8d577f18cf28a1af58adea1248e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Aug 2017 06:57:19 +0700
Subject: [PATCH 1179/1696] Use relative paths for DASH fragments (closes
 #12990) 10x reduced JSON size refs #13810

---
 youtube_dl/downloader/dash.py  | 14 ++++++++++----
 youtube_dl/extractor/common.py | 16 ++++++++++------
 2 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index 7491fdad8..576ece6db 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 from .fragment import FragmentFD
 from ..compat import compat_urllib_error
+from ..utils import urljoin
 
 
 class DashSegmentsFD(FragmentFD):
@@ -12,12 +13,13 @@ class DashSegmentsFD(FragmentFD):
     FD_NAME = 'dashsegments'
 
     def real_download(self, filename, info_dict):
-        segments = info_dict['fragments'][:1] if self.params.get(
+        fragment_base_url = info_dict.get('fragment_base_url')
+        fragments = info_dict['fragments'][:1] if self.params.get(
             'test', False) else info_dict['fragments']
 
         ctx = {
             'filename': filename,
-            'total_frags': len(segments),
+            'total_frags': len(fragments),
         }
 
         self._prepare_and_start_frag_download(ctx)
@@ -26,7 +28,7 @@ class DashSegmentsFD(FragmentFD):
         skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
 
         frag_index = 0
-        for i, segment in enumerate(segments):
+        for i, fragment in enumerate(fragments):
             frag_index += 1
             if frag_index <= ctx['fragment_index']:
                 continue
@@ -36,7 +38,11 @@ class DashSegmentsFD(FragmentFD):
             count = 0
             while count <= fragment_retries:
                 try:
-                    success, frag_content = self._download_fragment(ctx, segment['url'], info_dict)
+                    fragment_url = fragment.get('url')
+                    if not fragment_url:
+                        assert fragment_base_url
+                        fragment_url = urljoin(fragment_base_url, fragment['path'])
+                    success, frag_content = self._download_fragment(ctx, fragment_url, info_dict)
                     if not success:
                         return False
                     self._append_fragment(ctx, frag_content)
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 748b4d59f..459e7ffd6 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1892,9 +1892,13 @@ class InfoExtractor(object):
                                 'Bandwidth': bandwidth,
                             }
 
+                        def location_key(location):
+                            return 'url' if re.match(r'^https?://', location) else 'path'
+
                         if 'segment_urls' not in representation_ms_info and 'media' in representation_ms_info:
 
                             media_template = prepare_template('media', ('Number', 'Bandwidth', 'Time'))
+                            media_location_key = location_key(media_template)
 
                             # As per [1, 5.3.9.4.4, Table 16, page 55] $Number$ and $Time$
                             # can't be used at the same time
@@ -1904,7 +1908,7 @@ class InfoExtractor(object):
                                     segment_duration = float_or_none(representation_ms_info['segment_duration'], representation_ms_info['timescale'])
                                     representation_ms_info['total_number'] = int(math.ceil(float(period_duration) / segment_duration))
                                 representation_ms_info['fragments'] = [{
-                                    'url': media_template % {
+                                    media_location_key: media_template % {
                                         'Number': segment_number,
                                         'Bandwidth': bandwidth,
                                     },
@@ -1928,7 +1932,7 @@ class InfoExtractor(object):
                                         'Number': segment_number,
                                     }
                                     representation_ms_info['fragments'].append({
-                                        'url': segment_url,
+                                        media_location_key: segment_url,
                                         'duration': float_or_none(segment_d, representation_ms_info['timescale']),
                                     })
 
@@ -1952,8 +1956,9 @@ class InfoExtractor(object):
                             for s in representation_ms_info['s']:
                                 duration = float_or_none(s['d'], timescale)
                                 for r in range(s.get('r', 0) + 1):
+                                    segment_uri = representation_ms_info['segment_urls'][segment_index]
                                     fragments.append({
-                                        'url': representation_ms_info['segment_urls'][segment_index],
+                                        location_key(segment_uri): segment_uri,
                                         'duration': duration,
                                     })
                                     segment_index += 1
@@ -1962,6 +1967,7 @@ class InfoExtractor(object):
                         # No fragments key is present in this case.
                         if 'fragments' in representation_ms_info:
                             f.update({
+                                'fragment_base_url': base_url,
                                 'fragments': [],
                                 'protocol': 'http_dash_segments',
                             })
@@ -1969,10 +1975,8 @@ class InfoExtractor(object):
                                 initialization_url = representation_ms_info['initialization_url']
                                 if not f.get('url'):
                                     f['url'] = initialization_url
-                                f['fragments'].append({'url': initialization_url})
+                                f['fragments'].append({location_key(initialization_url): initialization_url})
                             f['fragments'].extend(representation_ms_info['fragments'])
-                            for fragment in f['fragments']:
-                                fragment['url'] = urljoin(base_url, fragment['url'])
                         try:
                             existing_format = next(
                                 fo for fo in formats

From c983cc3b71e3b2c80df920481dfa90bbc2ad7937 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Aug 2017 08:17:01 +0700
Subject: [PATCH 1180/1696] [cinchcast] Extend _VALID_URL

---
 youtube_dl/extractor/cinchcast.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/cinchcast.py b/youtube_dl/extractor/cinchcast.py
index 562c9bbbb..b861d54b0 100644
--- a/youtube_dl/extractor/cinchcast.py
+++ b/youtube_dl/extractor/cinchcast.py
@@ -9,12 +9,20 @@ from ..utils import (
 
 
 class CinchcastIE(InfoExtractor):
-    _VALID_URL = r'https?://player\.cinchcast\.com/.*?assetId=(?P<id>[0-9]+)'
-    _TEST = {
+    _VALID_URL = r'https?://player\.cinchcast\.com/.*?(?:assetId|show_id)=(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://player.cinchcast.com/?show_id=5258197&platformId=1&assetType=single',
+        'info_dict': {
+            'id': '5258197',
+            'ext': 'mp3',
+            'title': 'Train Your Brain to Up Your Game with Coach Mandy',
+            'upload_date': '20130816',
+        },
+    }, {
         # Actual test is run in generic, look for undergroundwellness
         'url': 'http://player.cinchcast.com/?platformId=1&#038;assetType=single&#038;assetId=7141703',
         'only_matching': True,
-    }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 1d5472290f8c426c13e3403cb95fd44cc71b5a9b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Aug 2017 08:28:12 +0700
Subject: [PATCH 1181/1696] [podomatic] Extend _VALID_URL (closes #13827)

---
 youtube_dl/extractor/podomatic.py | 63 +++++++++++++++++--------------
 1 file changed, 35 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/podomatic.py b/youtube_dl/extractor/podomatic.py
index f20946a2b..25fcebf9f 100644
--- a/youtube_dl/extractor/podomatic.py
+++ b/youtube_dl/extractor/podomatic.py
@@ -9,39 +9,46 @@ from ..utils import int_or_none
 
 class PodomaticIE(InfoExtractor):
     IE_NAME = 'podomatic'
-    _VALID_URL = r'^(?P<proto>https?)://(?P<channel>[^.]+)\.podomatic\.com/entry/(?P<id>[^?]+)'
+    _VALID_URL = r'''(?x)
+                    (?P<proto>https?)://
+                        (?:
+                            (?P<channel>[^.]+)\.podomatic\.com/entry|
+                            (?:www\.)?podomatic\.com/podcasts/(?P<channel_2>[^/]+)/episodes
+                        )/
+                        (?P<id>[^/?#&]+)
+                '''
 
-    _TESTS = [
-        {
-            'url': 'http://scienceteachingtips.podomatic.com/entry/2009-01-02T16_03_35-08_00',
-            'md5': '84bb855fcf3429e6bf72460e1eed782d',
-            'info_dict': {
-                'id': '2009-01-02T16_03_35-08_00',
-                'ext': 'mp3',
-                'uploader': 'Science Teaching Tips',
-                'uploader_id': 'scienceteachingtips',
-                'title': '64.  When the Moon Hits Your Eye',
-                'duration': 446,
-            }
-        },
-        {
-            'url': 'http://ostbahnhof.podomatic.com/entry/2013-11-15T16_31_21-08_00',
-            'md5': 'd2cf443931b6148e27638650e2638297',
-            'info_dict': {
-                'id': '2013-11-15T16_31_21-08_00',
-                'ext': 'mp3',
-                'uploader': 'Ostbahnhof / Techno Mix',
-                'uploader_id': 'ostbahnhof',
-                'title': 'Einunddreizig',
-                'duration': 3799,
-            }
-        },
-    ]
+    _TESTS = [{
+        'url': 'http://scienceteachingtips.podomatic.com/entry/2009-01-02T16_03_35-08_00',
+        'md5': '84bb855fcf3429e6bf72460e1eed782d',
+        'info_dict': {
+            'id': '2009-01-02T16_03_35-08_00',
+            'ext': 'mp3',
+            'uploader': 'Science Teaching Tips',
+            'uploader_id': 'scienceteachingtips',
+            'title': '64.  When the Moon Hits Your Eye',
+            'duration': 446,
+        }
+    }, {
+        'url': 'http://ostbahnhof.podomatic.com/entry/2013-11-15T16_31_21-08_00',
+        'md5': 'd2cf443931b6148e27638650e2638297',
+        'info_dict': {
+            'id': '2013-11-15T16_31_21-08_00',
+            'ext': 'mp3',
+            'uploader': 'Ostbahnhof / Techno Mix',
+            'uploader_id': 'ostbahnhof',
+            'title': 'Einunddreizig',
+            'duration': 3799,
+        }
+    }, {
+        'url': 'https://www.podomatic.com/podcasts/scienceteachingtips/episodes/2009-01-02T16_03_35-08_00',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
-        channel = mobj.group('channel')
+        channel = mobj.group('channel') or mobj.group('channel_2')
 
         json_url = (('%s://%s.podomatic.com/entry/embed_params/%s' +
                      '?permalink=true&rtmp=0') %

From f172c86dcdb46e484afc63732db56df5633028ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Aug 2017 21:17:55 +0700
Subject: [PATCH 1182/1696] [vlive:channel] Limit number of videos per page to
 100 (closes #13830)

---
 youtube_dl/extractor/vlive.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index 77c120a57..64d0224e6 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -236,7 +236,12 @@ class VLiveChannelIE(InfoExtractor):
                 query={
                     'app_id': app_id,
                     'channelSeq': channel_seq,
-                    'maxNumOfRows': 1000,
+                    # Large values of maxNumOfRows (~300 or above) may cause
+                    # empty responses (see [1]), e.g. this happens for [2] that
+                    # has more than 300 videos.
+                    # 1. https://github.com/rg3/youtube-dl/issues/13830
+                    # 2. http://channels.vlive.tv/EDBF.
+                    'maxNumOfRows': 100,
                     '_': int(time.time()),
                     'pageNo': page_num
                 }

From daaaf5f5942252e9fbc367957bd3b8a96d0dd5bb Mon Sep 17 00:00:00 2001
From: Ashutosh Chaudhary <ashutosh.chaudhary@gmail.com>
Date: Mon, 23 Jan 2017 05:12:52 +0530
Subject: [PATCH 1183/1696] [voot] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/voot.py       | 55 ++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 youtube_dl/extractor/voot.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d0e04dd7d..48dda8b8e 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1333,3 +1333,4 @@ from .zapiks import ZapiksIE
 from .zaq1 import Zaq1IE
 from .zdf import ZDFIE, ZDFChannelIE
 from .zingmp3 import ZingMp3IE
+from .voot import VootIE
diff --git a/youtube_dl/extractor/voot.py b/youtube_dl/extractor/voot.py
new file mode 100644
index 000000000..db5bda660
--- /dev/null
+++ b/youtube_dl/extractor/voot.py
@@ -0,0 +1,55 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class VootIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?voot\.com/shows/(?:.+?[/-]?)/1/(?:.+?[0-9]?)/(?:.+?[/-]?)/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'https://www.voot.com/shows/ishq-ka-rang-safed/1/360558/is-this-the-end-of-kamini-/441353',
+        'info_dict': {
+            'id': '441353',
+            'ext': 'mp4',
+            'title': 'Ishq Ka Rang Safed - Season 01 - Episode 340',
+            'thumbnail': r're:^https?://.*\.jpg$',
+        }
+    }
+
+    _GET_CONTENT_TEMPLATE = 'https://wapi.voot.com/ws/ott/getMediaInfo.json?platform=Web&pId=3&mediaId=%s'
+
+    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata', fatal=True):
+        json_data = super(VootIE, self)._download_json(url_or_request, video_id, note, fatal=fatal)
+        if json_data['status']['code'] != 0:
+            if fatal:
+                raise ExtractorError(json_data['status']['message'])
+            return None
+        return json_data['assets']
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_data = self._download_json(
+            self._GET_CONTENT_TEMPLATE % video_id,
+            video_id)
+
+        thumbnail = ''
+        formats = []
+
+        if video_data:
+            format_url = video_data.get('URL')
+            formats.extend(self._extract_m3u8_formats(format_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
+
+        if video_data['Pictures']:
+            for picture in video_data['Pictures']:
+                #Get only first available thumbnail
+                thumbnail = picture.get('URL')
+                break
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': video_data.get('MediaName'),
+            'thumbnail': thumbnail,
+            'formats':formats,
+        }

From e2b4808fd8ed49424deaa6d800daf0950e55ffff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Aug 2017 08:04:51 +0700
Subject: [PATCH 1184/1696] [voot] Improve extraction (#10255, closes #11814)

---
 youtube_dl/extractor/extractors.py |   2 +-
 youtube_dl/extractor/voot.py       | 111 ++++++++++++++++++++---------
 2 files changed, 78 insertions(+), 35 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 48dda8b8e..ebe414dae 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1222,6 +1222,7 @@ from .vodlocker import VodlockerIE
 from .vodpl import VODPlIE
 from .vodplatform import VODPlatformIE
 from .voicerepublic import VoiceRepublicIE
+from .voot import VootIE
 from .voxmedia import VoxMediaIE
 from .vporn import VpornIE
 from .vrt import VRTIE
@@ -1333,4 +1334,3 @@ from .zapiks import ZapiksIE
 from .zaq1 import Zaq1IE
 from .zdf import ZDFIE, ZDFChannelIE
 from .zingmp3 import ZingMp3IE
-from .voot import VootIE
diff --git a/youtube_dl/extractor/voot.py b/youtube_dl/extractor/voot.py
index db5bda660..5de3deb8c 100644
--- a/youtube_dl/extractor/voot.py
+++ b/youtube_dl/extractor/voot.py
@@ -2,54 +2,97 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from .kaltura import KalturaIE
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    try_get,
+    unified_timestamp,
+)
 
 
 class VootIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?voot\.com/shows/(?:.+?[/-]?)/1/(?:.+?[0-9]?)/(?:.+?[/-]?)/(?P<id>[0-9]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?voot\.com/(?:[^/]+/)+(?P<id>\d+)'
+    _GEO_COUNTRIES = ['IN']
+    _TESTS = [{
         'url': 'https://www.voot.com/shows/ishq-ka-rang-safed/1/360558/is-this-the-end-of-kamini-/441353',
         'info_dict': {
-            'id': '441353',
+            'id': '0_8ledb18o',
             'ext': 'mp4',
             'title': 'Ishq Ka Rang Safed - Season 01 - Episode 340',
-            'thumbnail': r're:^https?://.*\.jpg$',
-        }
-    }
-
-    _GET_CONTENT_TEMPLATE = 'https://wapi.voot.com/ws/ott/getMediaInfo.json?platform=Web&pId=3&mediaId=%s'
-
-    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata', fatal=True):
-        json_data = super(VootIE, self)._download_json(url_or_request, video_id, note, fatal=fatal)
-        if json_data['status']['code'] != 0:
-            if fatal:
-                raise ExtractorError(json_data['status']['message'])
-            return None
-        return json_data['assets']
+            'description': 'md5:06291fbbbc4dcbe21235c40c262507c1',
+            'uploader_id': 'batchUser',
+            'timestamp': 1472162937,
+            'upload_date': '20160825',
+            'duration': 1146,
+            'series': 'Ishq Ka Rang Safed',
+            'season_number': 1,
+            'episode': 'Is this the end of Kamini?',
+            'episode_number': 340,
+            'view_count': int,
+            'like_count': int,
+        },
+        'params': {
+            'skip_download': True,
+        },
+        'expected_warnings': ['Failed to download m3u8 information'],
+    }, {
+        'url': 'https://www.voot.com/kids/characters/mighty-cat-masked-niyander-e-/400478/school-bag-disappears/440925',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.voot.com/movies/pandavas-5/424627',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        video_data = self._download_json(
-            self._GET_CONTENT_TEMPLATE % video_id,
-            video_id)
 
-        thumbnail = ''
-        formats = []
+        media_info = self._download_json(
+            'https://wapi.voot.com/ws/ott/getMediaInfo.json', video_id,
+            query={
+                'platform': 'Web',
+                'pId': 2,
+                'mediaId': video_id,
+            })
 
-        if video_data:
-            format_url = video_data.get('URL')
-            formats.extend(self._extract_m3u8_formats(format_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
+        status_code = try_get(media_info, lambda x: x['status']['code'], int)
+        if status_code != 0:
+            raise ExtractorError(media_info['status']['message'], expected=True)
 
-        if video_data['Pictures']:
-            for picture in video_data['Pictures']:
-                #Get only first available thumbnail
-                thumbnail = picture.get('URL')
-                break
+        media = media_info['assets']
 
-        self._sort_formats(formats)
+        entry_id = media['EntryId']
+        title = media['MediaName']
+
+        description, series, season_number, episode, episode_number = [None] * 5
+
+        for meta in try_get(media, lambda x: x['Metas'], list) or []:
+            key, value = meta.get('Key'), meta.get('Value')
+            if not key or not value:
+                continue
+            if key == 'ContentSynopsis':
+                description = value
+            elif key == 'RefSeriesTitle':
+                series = value
+            elif key == 'RefSeriesSeason':
+                season_number = int_or_none(value)
+            elif key == 'EpisodeMainTitle':
+                episode = value
+            elif key == 'EpisodeNo':
+                episode_number = int_or_none(value)
 
         return {
-            'id': video_id,
-            'title': video_data.get('MediaName'),
-            'thumbnail': thumbnail,
-            'formats':formats,
+            '_type': 'url_transparent',
+            'url': 'kaltura:1982551:%s' % entry_id,
+            'ie_key': KalturaIE.ie_key(),
+            'title': title,
+            'description': description,
+            'series': series,
+            'season_number': season_number,
+            'episode': episode,
+            'episode_number': episode_number,
+            'timestamp': unified_timestamp(media.get('CreationDate')),
+            'duration': int_or_none(media.get('Duration')),
+            'view_count': int_or_none(media.get('ViewCounter')),
+            'like_count': int_or_none(media.get('like_counter')),
         }

From 16afce174ea71690844d37776d518ae374b896ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Aug 2017 08:18:16 +0700
Subject: [PATCH 1185/1696] [mpora] Remove extractor (closes #13826)

---
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/mpora.py      | 62 ------------------------------
 2 files changed, 63 deletions(-)
 delete mode 100644 youtube_dl/extractor/mpora.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index ebe414dae..897557f93 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -584,7 +584,6 @@ from .mixcloud import (
 )
 from .mlb import MLBIE
 from .mnet import MnetIE
-from .mpora import MporaIE
 from .moevideo import MoeVideoIE
 from .mofosex import MofosexIE
 from .mojvideo import MojvideoIE
diff --git a/youtube_dl/extractor/mpora.py b/youtube_dl/extractor/mpora.py
deleted file mode 100644
index 5a1bee5c8..000000000
--- a/youtube_dl/extractor/mpora.py
+++ /dev/null
@@ -1,62 +0,0 @@
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import int_or_none
-
-
-class MporaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?mpora\.(?:com|de)/videos/(?P<id>[^?#/]+)'
-    IE_NAME = 'MPORA'
-
-    _TEST = {
-        'url': 'http://mpora.de/videos/AAdo8okx4wiz/embed?locale=de',
-        'md5': 'a7a228473eedd3be741397cf452932eb',
-        'info_dict': {
-            'id': 'AAdo8okx4wiz',
-            'ext': 'mp4',
-            'title': 'Katy Curd -  Winter in the Forest',
-            'duration': 416,
-            'uploader': 'Peter Newman Media',
-        },
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        data_json = self._search_regex(
-            [r"new FM\.Player\('[^']+',\s*(\{.*?)\).player;",
-             r"new\s+FM\.Kaltura\.Player\('[^']+'\s*,\s*({.+?})\);"],
-            webpage, 'json')
-        data = self._parse_json(data_json, video_id)
-
-        uploader = data['info_overlay'].get('username')
-        duration = data['video']['duration'] // 1000
-        thumbnail = data['video']['encodings']['sd']['poster']
-        title = data['info_overlay']['title']
-
-        formats = []
-        for encoding_id, edata in data['video']['encodings'].items():
-            for src in edata['sources']:
-                width_str = self._search_regex(
-                    r'_([0-9]+)\.[a-zA-Z0-9]+$', src['src'],
-                    False, default=None)
-                vcodec = src['type'].partition('/')[2]
-
-                formats.append({
-                    'format_id': encoding_id + '-' + vcodec,
-                    'url': src['src'],
-                    'vcodec': vcodec,
-                    'width': int_or_none(width_str),
-                })
-
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'formats': formats,
-            'uploader': uploader,
-            'duration': duration,
-            'thumbnail': thumbnail,
-        }

From fac188c6954edcccf3104abc3ac0155125a7d427 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Aug 2017 08:44:28 +0700
Subject: [PATCH 1186/1696] [pluralsight] Fix format selection

---
 youtube_dl/extractor/pluralsight.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index e45d9fe55..d35f54ce8 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -224,6 +224,7 @@ class PluralsightIE(PluralsightBaseIE):
                 req_format_split = req_format.split('-', 1)
                 if len(req_format_split) > 1:
                     req_ext, req_quality = req_format_split
+                    req_quality = '-'.join(req_quality.split('-')[:2])
                     for allowed_quality in ALLOWED_QUALITIES:
                         if req_ext == allowed_quality.ext and req_quality in allowed_quality.qualities:
                             return (AllowedQuality(req_ext, (req_quality, )), )

From 92740e42414cb47f785daf257b9726fa361977b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Aug 2017 09:02:14 +0700
Subject: [PATCH 1187/1696] [ChangeLog] Actualize

---
 ChangeLog | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 4f03ef064..e2515866e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,25 @@
+version <unreleased>
+
+Core
+* Use relative paths for DASH fragments (#12990)
+
+Extractors
+* [pluralsight] Fix format selection
+- [mpora] Remove extractor (#13826)
++ [voot] Add support for voot.com (#10255, #11644, #11814, #12350, #13218)
+* [vlive:channel] Limit number of videos per page to 100 (#13830)
+* [podomatic] Extend URL regular expression (#13827)
+* [cinchcast] Extend URL regular expression
+* [yandexdisk] Relax URL regular expression (#13824)
+* [vidme] Extract DASH and HLS formats
+- [teamfour] Remove extractor (#13782)
+* [pornhd] Fix extraction (#13783)
+* [udemy] Fix subtitles extraction (#13812)
+* [mlb] Extend URL regular expression (#13740, #13773)
++ [pbs] Add support for new URL schema (#13801)
+* [nrktv] Update API host (#13796)
+
+
 version 2017.07.30.1
 
 Core

From 903a183b6adc60808f04294a7003b6d4bd250304 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Aug 2017 09:05:36 +0700
Subject: [PATCH 1188/1696] release 2017.08.06

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 3 +--
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 0421de755..5b72032bc 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.07.30.1*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.07.30.1**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.08.06*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.08.06**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.07.30.1
+[debug] youtube-dl version 2017.08.06
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index e2515866e..18893bba8 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.08.06
 
 Core
 * Use relative paths for DASH fragments (#12990)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 77aac8249..a3bd07726 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -472,7 +472,6 @@
  - **MovieFap**
  - **Moviezine**
  - **MovingImage**
- - **MPORA**
  - **MSN**
  - **mtg**: MTG services
  - **mtv**
@@ -783,7 +782,6 @@
  - **teachertube:user:collection**: teachertube.com user and collection videos
  - **TeachingChannel**
  - **Teamcoco**
- - **TeamFourStar**
  - **TechTalks**
  - **techtv.mit.edu**
  - **ted**
@@ -953,6 +951,7 @@
  - **VODPl**
  - **VODPlatform**
  - **VoiceRepublic**
+ - **Voot**
  - **VoxMedia**
  - **Vporn**
  - **vpro**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 38162157d..11d3bf29f 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.07.30.1'
+__version__ = '2017.08.06'

From 463e7216c87814edf1453aa3a5bfad89474ba6b1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 6 Aug 2017 23:07:06 +0800
Subject: [PATCH 1189/1696] [niconico] Support HTML5-only videos (closes
 #13806)

---
 ChangeLog                        |   6 ++
 youtube_dl/extractor/niconico.py | 124 +++++++++++++++++++++----------
 2 files changed, 90 insertions(+), 40 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 18893bba8..7cd385e68 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
++ [niconico] Support HTML5-only videos (#13806)
+
+
 version 2017.08.06
 
 Core
diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 79b9952c3..b13dc0035 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -11,10 +11,13 @@ from ..compat import (
 )
 from ..utils import (
     determine_ext,
+    dict_get,
     ExtractorError,
     int_or_none,
     parse_duration,
     parse_iso8601,
+    try_get,
+    unified_timestamp,
     urlencode_postdata,
     xpath_text,
 )
@@ -31,12 +34,15 @@ class NiconicoIE(InfoExtractor):
             'id': 'sm22312215',
             'ext': 'mp4',
             'title': 'Big Buck Bunny',
+            'thumbnail': r're:https?://.*',
             'uploader': 'takuya0301',
             'uploader_id': '2698420',
             'upload_date': '20131123',
             'timestamp': 1385182762,
             'description': '(c) copyright 2008, Blender Foundation / www.bigbuckbunny.org',
             'duration': 33,
+            'view_count': int,
+            'comment_count': int,
         },
         'skip': 'Requires an account',
     }, {
@@ -48,6 +54,7 @@ class NiconicoIE(InfoExtractor):
             'ext': 'swf',
             'title': '【鏡音リン】Dance on media【オリジナル】take2!',
             'description': 'md5:689f066d74610b3b22e0f1739add0f58',
+            'thumbnail': r're:https?://.*',
             'uploader': 'りょうた',
             'uploader_id': '18822557',
             'upload_date': '20110429',
@@ -64,9 +71,11 @@ class NiconicoIE(InfoExtractor):
             'ext': 'unknown_video',
             'description': 'deleted',
             'title': 'ドラえもんエターナル第3話「決戦第3新東京市」＜前編＞',
+            'thumbnail': r're:https?://.*',
             'upload_date': '20071224',
             'timestamp': int,  # timestamp field has different value if logged in
             'duration': 304,
+            'view_count': int,
         },
         'skip': 'Requires an account',
     }, {
@@ -76,12 +85,31 @@ class NiconicoIE(InfoExtractor):
             'ext': 'mp4',
             'title': '【第1回】RADIOアニメロミックス ラブライブ！～のぞえりRadio Garden～',
             'description': 'md5:b27d224bb0ff53d3c8269e9f8b561cf1',
+            'thumbnail': r're:https?://.*',
             'timestamp': 1388851200,
             'upload_date': '20140104',
             'uploader': 'アニメロチャンネル',
             'uploader_id': '312',
         },
         'skip': 'The viewing period of the video you were searching for has expired.',
+    }, {
+        # video not available via `getflv`
+        'url': 'http://www.nicovideo.jp/watch/sm1151009',
+        'info_dict': {
+            'id': 'sm1151009',
+            'ext': 'flv',
+            'title': 'マスターシステム本体内蔵のスペハリのメインテーマ（ＰＳＧ版）',
+            'description': 'md5:6ee077e0581ff5019773e2e714cdd0b7',
+            'thumbnail': r're:https?://.*',
+            'duration': 184,
+            'timestamp': 1190868283,
+            'upload_date': '20070927',
+            'uploader': 'denden2',
+            'uploader_id': '1392194',
+            'view_count': int,
+            'comment_count': int,
+        },
+        'skip': 'Requires an account',
     }, {
         'url': 'http://sp.nicovideo.jp/watch/sm28964488?ss_pos=1&cp_in=wt_tg',
         'only_matching': True,
@@ -130,33 +158,51 @@ class NiconicoIE(InfoExtractor):
         if video_id.startswith('so'):
             video_id = self._match_id(handle.geturl())
 
-        video_info = self._download_xml(
-            'http://ext.nicovideo.jp/api/getthumbinfo/' + video_id, video_id,
-            note='Downloading video info page')
+        api_data = self._parse_json(self._html_search_regex(
+            'data-api-data="([^"]+)"', webpage,
+            'API data', default='{}'), video_id)
+        video_real_url = try_get(
+            api_data, lambda x: x['video']['smileInfo']['url'])
 
-        # Get flv info
-        flv_info_webpage = self._download_webpage(
-            'http://flapi.nicovideo.jp/api/getflv/' + video_id + '?as3=1',
-            video_id, 'Downloading flv info')
+        if video_real_url:
+            def get_video_info(items):
+                return dict_get(api_data['video'], items)
+        else:
+            # Get flv info
+            flv_info_webpage = self._download_webpage(
+                'http://flapi.nicovideo.jp/api/getflv/' + video_id + '?as3=1',
+                video_id, 'Downloading flv info')
 
-        flv_info = compat_urlparse.parse_qs(flv_info_webpage)
-        if 'url' not in flv_info:
-            if 'deleted' in flv_info:
-                raise ExtractorError('The video has been deleted.',
-                                     expected=True)
-            elif 'closed' in flv_info:
-                raise ExtractorError('Niconico videos now require logging in',
-                                     expected=True)
-            elif 'error' in flv_info:
-                raise ExtractorError('%s reports error: %s' % (
-                    self.IE_NAME, flv_info['error'][0]), expected=True)
-            else:
-                raise ExtractorError('Unable to find video URL')
+            flv_info = compat_urlparse.parse_qs(flv_info_webpage)
+            if 'url' not in flv_info:
+                if 'deleted' in flv_info:
+                    raise ExtractorError('The video has been deleted.',
+                                         expected=True)
+                elif 'closed' in flv_info:
+                    raise ExtractorError('Niconico videos now require logging in',
+                                         expected=True)
+                elif 'error' in flv_info:
+                    raise ExtractorError('%s reports error: %s' % (
+                        self.IE_NAME, flv_info['error'][0]), expected=True)
+                else:
+                    raise ExtractorError('Unable to find video URL')
 
-        video_real_url = flv_info['url'][0]
+            video_real_url = flv_info['url'][0]
+
+            video_info_xml = self._download_xml(
+                'http://ext.nicovideo.jp/api/getthumbinfo/' + video_id,
+                video_id, note='Downloading video info page')
+
+            def get_video_info(items):
+                if not isinstance(items, list):
+                    items = [items]
+                for item in items:
+                    ret = xpath_text(video_info_xml, './/' + item)
+                    if ret:
+                        return ret
 
         # Start extracting information
-        title = xpath_text(video_info, './/title')
+        title = get_video_info('title')
         if not title:
             title = self._og_search_title(webpage, default=None)
         if not title:
@@ -170,18 +216,19 @@ class NiconicoIE(InfoExtractor):
         watch_api_data = self._parse_json(watch_api_data_string, video_id) if watch_api_data_string else {}
         video_detail = watch_api_data.get('videoDetail', {})
 
-        extension = xpath_text(video_info, './/movie_type')
+        extension = get_video_info(['movie_type', 'movieType'])
         if not extension:
             extension = determine_ext(video_real_url)
 
         thumbnail = (
-            xpath_text(video_info, './/thumbnail_url') or
+            get_video_info(['thumbnail_url', 'thumbnailURL']) or
             self._html_search_meta('image', webpage, 'thumbnail', default=None) or
             video_detail.get('thumbnail'))
 
-        description = xpath_text(video_info, './/description')
+        description = get_video_info('description')
 
-        timestamp = parse_iso8601(xpath_text(video_info, './/first_retrieve'))
+        timestamp = (parse_iso8601(get_video_info('first_retrieve')) or
+                     unified_timestamp(get_video_info('postedDateTime')))
         if not timestamp:
             match = self._html_search_meta('datePublished', webpage, 'date published', default=None)
             if match:
@@ -191,7 +238,7 @@ class NiconicoIE(InfoExtractor):
                 video_detail['postedAt'].replace('/', '-'),
                 delimiter=' ', timezone=datetime.timedelta(hours=9))
 
-        view_count = int_or_none(xpath_text(video_info, './/view_counter'))
+        view_count = int_or_none(get_video_info(['view_counter', 'viewCount']))
         if not view_count:
             match = self._html_search_regex(
                 r'>Views: <strong[^>]*>([^<]+)</strong>',
@@ -200,31 +247,28 @@ class NiconicoIE(InfoExtractor):
                 view_count = int_or_none(match.replace(',', ''))
         view_count = view_count or video_detail.get('viewCount')
 
-        comment_count = int_or_none(xpath_text(video_info, './/comment_num'))
+        comment_count = (int_or_none(get_video_info('comment_num')) or
+                         video_detail.get('commentCount') or
+                         try_get(api_data, lambda x: x['thread']['commentCount']))
         if not comment_count:
             match = self._html_search_regex(
                 r'>Comments: <strong[^>]*>([^<]+)</strong>',
                 webpage, 'comment count', default=None)
             if match:
                 comment_count = int_or_none(match.replace(',', ''))
-        comment_count = comment_count or video_detail.get('commentCount')
 
         duration = (parse_duration(
-            xpath_text(video_info, './/length') or
+            get_video_info('length') or
             self._html_search_meta(
                 'video:duration', webpage, 'video duration', default=None)) or
-            video_detail.get('length'))
+            video_detail.get('length') or
+            get_video_info('duration'))
 
-        webpage_url = xpath_text(video_info, './/watch_url') or url
+        webpage_url = get_video_info('watch_url') or url
 
-        if video_info.find('.//ch_id') is not None:
-            uploader_id = video_info.find('.//ch_id').text
-            uploader = video_info.find('.//ch_name').text
-        elif video_info.find('.//user_id') is not None:
-            uploader_id = video_info.find('.//user_id').text
-            uploader = video_info.find('.//user_nickname').text
-        else:
-            uploader_id = uploader = None
+        owner = api_data.get('owner', {})
+        uploader_id = get_video_info(['ch_id', 'user_id']) or owner.get('id')
+        uploader = get_video_info(['ch_name', 'user_nickname']) or owner.get('nickname')
 
         return {
             'id': video_id,

From ee6a611665a1ee8583ce84bd9d36d03b6f697895 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 7 Aug 2017 00:19:46 +0800
Subject: [PATCH 1190/1696] [niconico] Support videos with multiple formats
 (closes #13522)

---
 ChangeLog                        |   1 +
 youtube_dl/extractor/niconico.py | 161 +++++++++++++++++++++++++++----
 2 files changed, 145 insertions(+), 17 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 7cd385e68..084e98c0e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
++ [niconico] Support videos with multiple formats (#13522)
 + [niconico] Support HTML5-only videos (#13806)
 
 
diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index b13dc0035..026329d3e 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -14,8 +14,10 @@ from ..utils import (
     dict_get,
     ExtractorError,
     int_or_none,
+    float_or_none,
     parse_duration,
     parse_iso8601,
+    remove_start,
     try_get,
     unified_timestamp,
     urlencode_postdata,
@@ -93,11 +95,12 @@ class NiconicoIE(InfoExtractor):
         },
         'skip': 'The viewing period of the video you were searching for has expired.',
     }, {
-        # video not available via `getflv`
+        # video not available via `getflv`; "old" HTML5 video
         'url': 'http://www.nicovideo.jp/watch/sm1151009',
+        'md5': '8fa81c364eb619d4085354eab075598a',
         'info_dict': {
             'id': 'sm1151009',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'マスターシステム本体内蔵のスペハリのメインテーマ（ＰＳＧ版）',
             'description': 'md5:6ee077e0581ff5019773e2e714cdd0b7',
             'thumbnail': r're:https?://.*',
@@ -110,6 +113,25 @@ class NiconicoIE(InfoExtractor):
             'comment_count': int,
         },
         'skip': 'Requires an account',
+    }, {
+        # "New" HTML5 video
+        'url': 'http://www.nicovideo.jp/watch/sm31464864',
+        'md5': '351647b4917660986dc0fa8864085135',
+        'info_dict': {
+            'id': 'sm31464864',
+            'ext': 'mp4',
+            'title': '新作TVアニメ「戦姫絶唱シンフォギアAXZ」PV 最高画質',
+            'description': 'md5:e52974af9a96e739196b2c1ca72b5feb',
+            'timestamp': 1498514060,
+            'upload_date': '20170626',
+            'uploader': 'ゲス',
+            'uploader_id': '40826363',
+            'thumbnail': r're:https?://.*',
+            'duration': 198,
+            'view_count': int,
+            'comment_count': int,
+        },
+        'skip': 'Requires an account',
     }, {
         'url': 'http://sp.nicovideo.jp/watch/sm28964488?ss_pos=1&cp_in=wt_tg',
         'only_matching': True,
@@ -147,6 +169,84 @@ class NiconicoIE(InfoExtractor):
             self._downloader.report_warning('unable to log in: bad username or password')
         return login_ok
 
+    def _extract_format_for_quality(self, api_data, video_id, audio_quality, video_quality):
+        def yesno(boolean):
+            return 'yes' if boolean else 'no'
+
+        session_api_data = api_data['video']['dmcInfo']['session_api']
+        session_api_endpoint = session_api_data['urls'][0]
+
+        format_id = '-'.join(map(lambda s: remove_start(s['id'], 'archive_'), [video_quality, audio_quality]))
+
+        session_response = self._download_json(
+            session_api_endpoint['url'], video_id,
+            query={'_format': 'json'},
+            headers={'Content-Type': 'application/json'},
+            note='Downloading JSON metadata for %s' % format_id,
+            data=json.dumps({
+                'session': {
+                    'client_info': {
+                        'player_id': session_api_data['player_id'],
+                    },
+                    'content_auth': {
+                        'auth_type': session_api_data['auth_types'][session_api_data['protocols'][0]],
+                        'content_key_timeout': session_api_data['content_key_timeout'],
+                        'service_id': 'nicovideo',
+                        'service_user_id': session_api_data['service_user_id']
+                    },
+                    'content_id': session_api_data['content_id'],
+                    'content_src_id_sets': [{
+                        'content_src_ids': [{
+                            'src_id_to_mux': {
+                                'audio_src_ids': [audio_quality['id']],
+                                'video_src_ids': [video_quality['id']],
+                            }
+                        }]
+                    }],
+                    'content_type': 'movie',
+                    'content_uri': '',
+                    'keep_method': {
+                        'heartbeat': {
+                            'lifetime': session_api_data['heartbeat_lifetime']
+                        }
+                    },
+                    'priority': session_api_data['priority'],
+                    'protocol': {
+                        'name': 'http',
+                        'parameters': {
+                            'http_parameters': {
+                                'parameters': {
+                                    'http_output_download_parameters': {
+                                        'use_ssl': yesno(session_api_endpoint['is_ssl']),
+                                        'use_well_known_port': yesno(session_api_endpoint['is_well_known_port']),
+                                    }
+                                }
+                            }
+                        }
+                    },
+                    'recipe_id': session_api_data['recipe_id'],
+                    'session_operation_auth': {
+                        'session_operation_auth_by_signature': {
+                            'signature': session_api_data['signature'],
+                            'token': session_api_data['token'],
+                        }
+                    },
+                    'timing_constraint': 'unlimited'
+                }
+            }))
+
+        resolution = video_quality.get('resolution', {})
+
+        return {
+            'url': session_response['data']['session']['content_uri'],
+            'format_id': format_id,
+            'ext': 'mp4',  # Session API are used in HTML5, which always serves mp4
+            'abr': float_or_none(audio_quality.get('bitrate'), 1000),
+            'vbr': float_or_none(video_quality.get('bitrate'), 1000),
+            'height': resolution.get('height'),
+            'width': resolution.get('width'),
+        }
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
@@ -161,13 +261,13 @@ class NiconicoIE(InfoExtractor):
         api_data = self._parse_json(self._html_search_regex(
             'data-api-data="([^"]+)"', webpage,
             'API data', default='{}'), video_id)
-        video_real_url = try_get(
-            api_data, lambda x: x['video']['smileInfo']['url'])
 
-        if video_real_url:
-            def get_video_info(items):
-                return dict_get(api_data['video'], items)
-        else:
+        def _format_id_from_url(video_url):
+            return 'economy' if video_real_url.endswith('low') else 'normal'
+
+        try:
+            video_real_url = api_data['video']['smileInfo']['url']
+        except KeyError:  # Flash videos
             # Get flv info
             flv_info_webpage = self._download_webpage(
                 'http://flapi.nicovideo.jp/api/getflv/' + video_id + '?as3=1',
@@ -187,8 +287,6 @@ class NiconicoIE(InfoExtractor):
                 else:
                     raise ExtractorError('Unable to find video URL')
 
-            video_real_url = flv_info['url'][0]
-
             video_info_xml = self._download_xml(
                 'http://ext.nicovideo.jp/api/getthumbinfo/' + video_id,
                 video_id, note='Downloading video info page')
@@ -201,6 +299,41 @@ class NiconicoIE(InfoExtractor):
                     if ret:
                         return ret
 
+            video_real_url = flv_info['url'][0]
+
+            extension = get_video_info('movie_type')
+            if not extension:
+                extension = determine_ext(video_real_url)
+
+            formats = [{
+                'url': video_real_url,
+                'ext': extension,
+                'format_id': _format_id_from_url(video_real_url),
+            }]
+        else:
+            formats = []
+
+            dmc_info = api_data['video'].get('dmcInfo')
+            if dmc_info:  # "New" HTML5 videos
+                quality_info = dmc_info['quality']
+                for audio_quality in quality_info['audios']:
+                    for video_quality in quality_info['videos']:
+                        if not audio_quality['available'] or not video_quality['available']:
+                            continue
+                        formats.append(self._extract_format_for_quality(
+                            api_data, video_id, audio_quality, video_quality))
+
+                self._sort_formats(formats)
+            else:  # "Old" HTML5 videos
+                formats = [{
+                    'url': video_real_url,
+                    'ext': 'mp4',
+                    'format_id': _format_id_from_url(video_real_url),
+                }]
+
+            def get_video_info(items):
+                return dict_get(api_data['video'], items)
+
         # Start extracting information
         title = get_video_info('title')
         if not title:
@@ -216,10 +349,6 @@ class NiconicoIE(InfoExtractor):
         watch_api_data = self._parse_json(watch_api_data_string, video_id) if watch_api_data_string else {}
         video_detail = watch_api_data.get('videoDetail', {})
 
-        extension = get_video_info(['movie_type', 'movieType'])
-        if not extension:
-            extension = determine_ext(video_real_url)
-
         thumbnail = (
             get_video_info(['thumbnail_url', 'thumbnailURL']) or
             self._html_search_meta('image', webpage, 'thumbnail', default=None) or
@@ -272,10 +401,8 @@ class NiconicoIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'url': video_real_url,
             'title': title,
-            'ext': extension,
-            'format_id': 'economy' if video_real_url.endswith('low') else 'normal',
+            'formats': formats,
             'thumbnail': thumbnail,
             'description': description,
             'uploader': uploader,

From 15d1e8a23dbaa28635cae30ff6c5cfb095b4c7c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 7 Aug 2017 22:43:42 +0700
Subject: [PATCH 1191/1696] [dplayit] Fix extraction (closes #13851)

---
 youtube_dl/extractor/dplay.py | 66 ++++++++++++++++++++++-------------
 1 file changed, 42 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/dplay.py b/youtube_dl/extractor/dplay.py
index 1a41760f8..76e784105 100644
--- a/youtube_dl/extractor/dplay.py
+++ b/youtube_dl/extractor/dplay.py
@@ -7,16 +7,18 @@ import time
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urlparse,
     compat_HTTPError,
+    compat_str,
+    compat_urlparse,
 )
 from ..utils import (
-    USER_AGENTS,
     ExtractorError,
     int_or_none,
-    unified_strdate,
     remove_end,
+    try_get,
+    unified_strdate,
     update_url_query,
+    USER_AGENTS,
 )
 
 
@@ -183,28 +185,44 @@ class DPlayItIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
-        info_url = self._search_regex(
-            r'url\s*[:=]\s*["\']((?:https?:)?//[^/]+/playback/videoPlaybackInfo/\d+)',
-            webpage, 'video id')
-
         title = remove_end(self._og_search_title(webpage), ' | Dplay')
 
-        try:
-            info = self._download_json(
-                info_url, display_id, headers={
-                    'Authorization': 'Bearer %s' % self._get_cookies(url).get(
-                        'dplayit_token').value,
-                    'Referer': url,
-                })
-        except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code in (400, 403):
-                info = self._parse_json(e.cause.read().decode('utf-8'), display_id)
-                error = info['errors'][0]
-                if error.get('code') == 'access.denied.geoblocked':
-                    self.raise_geo_restricted(
-                        msg=error.get('detail'), countries=self._GEO_COUNTRIES)
-                raise ExtractorError(info['errors'][0]['detail'], expected=True)
-            raise
+        video_id = None
+
+        info = self._search_regex(
+            r'playback_json\s*:\s*JSON\.parse\s*\(\s*("(?:\\.|[^"\\])+?")',
+            webpage, 'playback JSON', default=None)
+        if info:
+            for _ in range(2):
+                info = self._parse_json(info, display_id, fatal=False)
+                if not info:
+                    break
+            else:
+                video_id = try_get(info, lambda x: x['data']['id'])
+
+        if not info:
+            info_url = self._search_regex(
+                r'url\s*[:=]\s*["\']((?:https?:)?//[^/]+/playback/videoPlaybackInfo/\d+)',
+                webpage, 'info url')
+
+            video_id = info_url.rpartition('/')[-1]
+
+            try:
+                info = self._download_json(
+                    info_url, display_id, headers={
+                        'Authorization': 'Bearer %s' % self._get_cookies(url).get(
+                            'dplayit_token').value,
+                        'Referer': url,
+                    })
+            except ExtractorError as e:
+                if isinstance(e.cause, compat_HTTPError) and e.cause.code in (400, 403):
+                    info = self._parse_json(e.cause.read().decode('utf-8'), display_id)
+                    error = info['errors'][0]
+                    if error.get('code') == 'access.denied.geoblocked':
+                        self.raise_geo_restricted(
+                            msg=error.get('detail'), countries=self._GEO_COUNTRIES)
+                    raise ExtractorError(info['errors'][0]['detail'], expected=True)
+                raise
 
         hls_url = info['data']['attributes']['streaming']['hls']['url']
 
@@ -230,7 +248,7 @@ class DPlayItIE(InfoExtractor):
             season_number = episode_number = upload_date = None
 
         return {
-            'id': info_url.rpartition('/')[-1],
+            'id': compat_str(video_id or display_id),
             'display_id': display_id,
             'title': title,
             'description': self._og_search_description(webpage),

From 4bf22f7a1014c55e3358b5a419945071b152eafc Mon Sep 17 00:00:00 2001
From: Alex Seiler <seileralex@gmail.com>
Date: Tue, 8 Aug 2017 00:41:38 +0200
Subject: [PATCH 1192/1696] [20min] Fix embeds extraction

---
 youtube_dl/extractor/twentymin.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twentymin.py b/youtube_dl/extractor/twentymin.py
index 4fd1aa4bf..a42977f39 100644
--- a/youtube_dl/extractor/twentymin.py
+++ b/youtube_dl/extractor/twentymin.py
@@ -50,7 +50,7 @@ class TwentyMinutenIE(InfoExtractor):
     @staticmethod
     def _extract_urls(webpage):
         return [m.group('url') for m in re.finditer(
-            r'<iframe[^>]+src=(["\'])(?P<url>(?:https?://)?(?:www\.)?20min\.ch/videoplayer/videoplayer.html\?.*?\bvideoId@\d+.*?)\1',
+            r'<iframe[^>]+src=(["\'])(?P<url>(?:(?:https?:)?//)?(?:www\.)?20min\.ch/videoplayer/videoplayer.html\?.*?\bvideoId@\d+.*?)\1',
             webpage)]
 
     def _real_extract(self, url):

From 5b232f46dcbdc805507c02edd4fd598f31d544d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 9 Aug 2017 22:28:19 +0700
Subject: [PATCH 1193/1696] [utils] Skip missing params in cli_bool_option
 (closes #13865)

---
 test/test_utils.py  | 4 ++++
 youtube_dl/utils.py | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 7803e5bc7..2aab16b97 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -1182,6 +1182,10 @@ part 3</font></u>
             cli_bool_option(
                 {'nocheckcertificate': False}, '--check-certificate', 'nocheckcertificate', 'false', 'true', '='),
             ['--check-certificate=true'])
+        self.assertEqual(
+            cli_bool_option(
+                {}, '--check-certificate', 'nocheckcertificate', 'false', 'true', '='),
+            [])
 
     def test_ohdave_rsa_encrypt(self):
         N = 0xab86b6371b5318aaa1d3c9e612a9f1264f372323c8c0f19875b5fc3b3fd3afcc1e5bec527aa94bfa85bffc157e4245aebda05389a5357b75115ac94f074aefcd
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index fdf5e29e7..c9cbd5842 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2733,6 +2733,8 @@ def cli_option(params, command_option, param):
 
 def cli_bool_option(params, command_option, param, true_value='true', false_value='false', separator=None):
     param = params.get(param)
+    if param is None:
+        return []
     assert isinstance(param, bool)
     if separator:
         return [command_option + separator + (true_value if param else false_value)]

From 5b3ddadcc3012c4ef390a7cf70dbcb8b83f07428 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 9 Aug 2017 22:55:13 +0700
Subject: [PATCH 1194/1696] [mixcloud] Fix play info decryption (closes #13867)

---
 youtube_dl/extractor/mixcloud.py | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 0efbe660a..40cd2e389 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -54,15 +54,22 @@ class MixcloudIE(InfoExtractor):
     }]
 
     # See https://www.mixcloud.com/media/js2/www_js_2.9e23256562c080482435196ca3975ab5.js
-    @staticmethod
-    def _decrypt_play_info(play_info):
-        KEY = 'pleasedontdownloadourmusictheartistswontgetpaid'
-
+    def _decrypt_play_info(self, play_info, video_id):
+        KEYS = (
+            'pleasedontdownloadourmusictheartistswontgetpaid',
+            '(function() { return new Date().toLocaleDateString(); })()'
+        )
         play_info = base64.b64decode(play_info.encode('ascii'))
-
-        return ''.join([
-            compat_chr(compat_ord(ch) ^ compat_ord(KEY[idx % len(KEY)]))
-            for idx, ch in enumerate(play_info)])
+        for num, key in enumerate(KEYS, start=1):
+            try:
+                return self._parse_json(
+                    ''.join([
+                        compat_chr(compat_ord(ch) ^ compat_ord(key[idx % len(key)]))
+                        for idx, ch in enumerate(play_info)]),
+                    video_id)
+            except ExtractorError:
+                if num == len(KEYS):
+                    raise
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -78,8 +85,8 @@ class MixcloudIE(InfoExtractor):
 
         encrypted_play_info = self._search_regex(
             r'm-play-info="([^"]+)"', webpage, 'play info')
-        play_info = self._parse_json(
-            self._decrypt_play_info(encrypted_play_info), track_id)
+
+        play_info = self._decrypt_play_info(encrypted_play_info, track_id)
 
         if message and 'stream_url' not in play_info:
             raise ExtractorError('%s said: %s' % (self.IE_NAME, message), expected=True)

From dee04d24a422c0ea5586d2f1d1f97f1e3e4ecf70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 9 Aug 2017 23:12:02 +0700
Subject: [PATCH 1195/1696] [nick] Add support for nick.com.pl (closes #13860)

---
 youtube_dl/extractor/nick.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
index b688637bc..510b1c41f 100644
--- a/youtube_dl/extractor/nick.py
+++ b/youtube_dl/extractor/nick.py
@@ -75,7 +75,7 @@ class NickIE(MTVServicesInfoExtractor):
 
 class NickDeIE(MTVServicesInfoExtractor):
     IE_NAME = 'nick.de'
-    _VALID_URL = r'https?://(?:www\.)?(?P<host>nick\.de|nickelodeon\.(?:nl|at))/(?:playlist|shows)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<host>nick\.(?:de|com\.pl)|nickelodeon\.(?:nl|at))/[^/]+/(?:[^/]+/)*(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://www.nick.de/playlist/3773-top-videos/videos/episode/17306-zu-wasser-und-zu-land-rauchende-erdnusse',
         'only_matching': True,
@@ -88,6 +88,9 @@ class NickDeIE(MTVServicesInfoExtractor):
     }, {
         'url': 'http://www.nickelodeon.at/playlist/3773-top-videos/videos/episode/77993-das-letzte-gefecht',
         'only_matching': True,
+    }, {
+        'url': 'http://www.nick.com.pl/seriale/474-spongebob-kanciastoporty/wideo/17412-teatr-to-jest-to-rodeo-oszolom',
+        'only_matching': True,
     }]
 
     def _extract_mrss_url(self, webpage, host):

From baba5f4d1daa29c42b2ad56c06e3880f10b7b03d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 9 Aug 2017 23:46:49 +0700
Subject: [PATCH 1196/1696] [xxxymovies] Fix title extraction (closes #13868)

---
 youtube_dl/extractor/xxxymovies.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/xxxymovies.py b/youtube_dl/extractor/xxxymovies.py
index 5c8f17eb2..e34ebe3a6 100644
--- a/youtube_dl/extractor/xxxymovies.py
+++ b/youtube_dl/extractor/xxxymovies.py
@@ -39,8 +39,8 @@ class XXXYMoviesIE(InfoExtractor):
             r"video_url\s*:\s*'([^']+)'", webpage, 'video URL')
 
         title = self._html_search_regex(
-            [r'<div class="block_header">\s*<h1>([^<]+)</h1>',
-             r'<title>(.*?)\s*-\s*XXXYMovies\.com</title>'],
+            [r'<div[^>]+\bclass="block_header"[^>]*>\s*<h1>([^<]+)<',
+             r'<title>(.*?)\s*-\s*(?:XXXYMovies\.com|XXX\s+Movies)</title>'],
             webpage, 'title')
 
         thumbnail = self._search_regex(

From 0e7dfa7d16e9e013bfaa085a59b9bbe4b4d1dfb1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 9 Aug 2017 23:49:53 +0700
Subject: [PATCH 1197/1696] [ChangeLog] Actualize

---
 ChangeLog | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 084e98c0e..7c8eb92aa 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,14 @@
 version <unreleased>
 
+Core
+* [utils] Skip missing params in cli_bool_option (#13865)
+
 Extractors
+* [xxxymovies] Fix title extraction (#13868)
++ [nick] Add support for nick.com.pl (#13860)
+* [mixcloud] Fix play info decryption (#13867)
+* [20min] Fix embeds extraction (#13852)
+* [dplayit] Fix extraction (#13851)
 + [niconico] Support videos with multiple formats (#13522)
 + [niconico] Support HTML5-only videos (#13806)
 

From 6ed99754bb6074454b5d4875cc7b8b442e763ec5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 9 Aug 2017 23:52:22 +0700
Subject: [PATCH 1198/1696] release 2017.08.09

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 5b72032bc..7ee704e48 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.08.06*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.08.06**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.08.09*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.08.09**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.08.06
+[debug] youtube-dl version 2017.08.09
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 7c8eb92aa..b28ade446 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.08.09
 
 Core
 * [utils] Skip missing params in cli_bool_option (#13865)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 11d3bf29f..022172375 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.08.06'
+__version__ = '2017.08.09'

From 41918eaa5ce1225f7e0a94882e7c77919342210d Mon Sep 17 00:00:00 2001
From: tetra-eder <30865771+tetra-eder@users.noreply.github.com>
Date: Fri, 11 Aug 2017 17:00:39 +0200
Subject: [PATCH 1199/1696] [generic] Add support for vzaar embeds

---
 youtube_dl/extractor/generic.py | 17 +++++++++++++++++
 youtube_dl/extractor/vzaar.py   |  8 ++++++++
 2 files changed, 25 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 34e814988..51acead66 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -98,6 +98,7 @@ from .wistia import WistiaIE
 from .mediaset import MediasetIE
 from .joj import JojIE
 from .megaphone import MegaphoneIE
+from .vzaar import VzaarIE
 
 
 class GenericIE(InfoExtractor):
@@ -1840,6 +1841,16 @@ class GenericIE(InfoExtractor):
                 'title': 'Стас Намин: «Мы нарушили девственность Кремля»',
             },
         },
+        {
+            # vzaar embed
+            'url': 'http://www.xruniversity.com/bdsm-lets-begin-melissa-moore/',
+            'md5': 'cddc9fb8a8644a0a7742149eee95080b',
+            'info_dict': {
+                'id': '11002506',
+                'ext': 'mp4',
+                'title': 'XR-U SHOW: Ready Player Fuck - EP. 61',
+            },
+        },
         # {
         #     # TODO: find another test
         #     # http://schema.org/VideoObject
@@ -2781,6 +2792,12 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 videopress_urls, video_id, video_title, ie=VideoPressIE.ie_key())
 
+        # Look for vzaar embeds
+        vzaar_urls = VzaarIE._extract_urls(webpage)
+        if vzaar_urls:
+            return self.playlist_from_matches(
+                vzaar_urls, video_id, video_title, ie=VzaarIE.ie_key())
+
         # Look for Rutube embeds
         rutube_urls = RutubeIE._extract_urls(webpage)
         if rutube_urls:
diff --git a/youtube_dl/extractor/vzaar.py b/youtube_dl/extractor/vzaar.py
index b270f08d1..02fcd52c7 100644
--- a/youtube_dl/extractor/vzaar.py
+++ b/youtube_dl/extractor/vzaar.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
@@ -28,6 +30,12 @@ class VzaarIE(InfoExtractor):
         },
     }]
 
+    @staticmethod
+    def _extract_urls(webpage):
+        return re.findall(
+            r'<iframe[^>]+src=["\']((?:https?:)?//(?:view\.vzaar\.com)/[0-9]+)',
+            webpage)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         video_data = self._download_json(

From 1663bd6e1c11bf6cbf290fcbbf12358207570faf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 11 Aug 2017 22:02:00 +0700
Subject: [PATCH 1200/1696] [generic] Replace vzaar embed test

---
 youtube_dl/extractor/generic.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 51acead66..8362d9a36 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1843,12 +1843,12 @@ class GenericIE(InfoExtractor):
         },
         {
             # vzaar embed
-            'url': 'http://www.xruniversity.com/bdsm-lets-begin-melissa-moore/',
-            'md5': 'cddc9fb8a8644a0a7742149eee95080b',
+            'url': 'http://help.vzaar.com/article/165-embedding-video',
+            'md5': '7e3919d9d2620b89e3e00bec7fe8c9d4',
             'info_dict': {
-                'id': '11002506',
+                'id': '8707641',
                 'ext': 'mp4',
-                'title': 'XR-U SHOW: Ready Player Fuck - EP. 61',
+                'title': 'Building A Business Online: Principal Chairs Q & A',
             },
         },
         # {
@@ -2792,12 +2792,6 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 videopress_urls, video_id, video_title, ie=VideoPressIE.ie_key())
 
-        # Look for vzaar embeds
-        vzaar_urls = VzaarIE._extract_urls(webpage)
-        if vzaar_urls:
-            return self.playlist_from_matches(
-                vzaar_urls, video_id, video_title, ie=VzaarIE.ie_key())
-
         # Look for Rutube embeds
         rutube_urls = RutubeIE._extract_urls(webpage)
         if rutube_urls:
@@ -2828,6 +2822,12 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 mpfn_urls, video_id, video_title, ie=MegaphoneIE.ie_key())
 
+        # Look for vzaar embeds
+        vzaar_urls = VzaarIE._extract_urls(webpage)
+        if vzaar_urls:
+            return self.playlist_from_matches(
+                vzaar_urls, video_id, video_title, ie=VzaarIE.ie_key())
+
         def merge_dicts(dict1, dict2):
             merged = {}
             for k, v in dict1.items():

From 92a5c415328953851d0a6b7893de5387a1b7b469 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 12 Aug 2017 16:30:50 +0700
Subject: [PATCH 1201/1696] [mixcloud] Fix play info decryption (closes #13885)

---
 youtube_dl/extractor/mixcloud.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 40cd2e389..52f7428e0 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -57,7 +57,8 @@ class MixcloudIE(InfoExtractor):
     def _decrypt_play_info(self, play_info, video_id):
         KEYS = (
             'pleasedontdownloadourmusictheartistswontgetpaid',
-            '(function() { return new Date().toLocaleDateString(); })()'
+            'window.addEventListener = window.addEventListener || function() {};',
+            '(function() { return new Date().toLocaleDateString(); })()',
         )
         play_info = base64.b64decode(play_info.encode('ascii'))
         for num, key in enumerate(KEYS, start=1):

From 82889d4ae517640df217f99e7744002e0deba47a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 12 Aug 2017 16:48:11 +0700
Subject: [PATCH 1202/1696] [extractor/common] Respect source's type attribute
 for HTML5 media (closes #13892)

---
 youtube_dl/extractor/common.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 459e7ffd6..4d61275fd 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2114,9 +2114,9 @@ class InfoExtractor(object):
                 return f
             return {}
 
-        def _media_formats(src, cur_media_type):
+        def _media_formats(src, type_info, cur_media_type):
             full_url = absolute_url(src)
-            ext = determine_ext(full_url)
+            ext = type_info.get('ext') or determine_ext(full_url)
             if ext == 'm3u8':
                 is_plain_url = False
                 formats = self._extract_m3u8_formats(
@@ -2165,9 +2165,9 @@ class InfoExtractor(object):
                     src = source_attributes.get('src')
                     if not src:
                         continue
-                    is_plain_url, formats = _media_formats(src, media_type)
+                    f = parse_content_type(source_attributes.get('type'))
+                    is_plain_url, formats = _media_formats(src, f, media_type)
                     if is_plain_url:
-                        f = parse_content_type(source_attributes.get('type'))
                         f.update(formats[0])
                         media_info['formats'].append(f)
                     else:

From ac8491fcca6f9c0f6c7904e1cf13953f912eeb39 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 12 Aug 2017 17:11:35 +0700
Subject: [PATCH 1203/1696] [extractor/common] Make _family_friendly_search
 optional

---
 youtube_dl/extractor/common.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 4d61275fd..e565901af 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -940,7 +940,8 @@ class InfoExtractor(object):
 
     def _family_friendly_search(self, html):
         # See http://schema.org/VideoObject
-        family_friendly = self._html_search_meta('isFamilyFriendly', html)
+        family_friendly = self._html_search_meta(
+            'isFamilyFriendly', html, default=None)
 
         if not family_friendly:
             return None

From e74e3b63e3cdb31a61af1fc21c703e912c029b96 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 12 Aug 2017 17:14:11 +0700
Subject: [PATCH 1204/1696] [YoutubeDL] Make sure format id is not empty

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 367ae3533..df7378f83 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1500,7 +1500,7 @@ class YoutubeDL(object):
             sanitize_string_field(format, 'format_id')
             sanitize_numeric_fields(format)
             format['url'] = sanitize_url(format['url'])
-            if format.get('format_id') is None:
+            if not format.get('format_id'):
                 format['format_id'] = compat_str(i)
             else:
                 # Sanitize format_id from characters used in format selector expression

From 70851a95c307880f016fcb6f37427a8eeae73cad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 12 Aug 2017 17:18:23 +0700
Subject: [PATCH 1205/1696] [aparat] Extract all formats (closes #13887)

---
 youtube_dl/extractor/aparat.py | 49 +++++++++++++++++++++-------------
 1 file changed, 30 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/aparat.py b/youtube_dl/extractor/aparat.py
index 025e29aa4..e394cb661 100644
--- a/youtube_dl/extractor/aparat.py
+++ b/youtube_dl/extractor/aparat.py
@@ -3,13 +3,13 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
-    ExtractorError,
-    HEADRequest,
+    int_or_none,
+    mimetype2ext,
 )
 
 
 class AparatIE(InfoExtractor):
-    _VALID_URL = r'^https?://(?:www\.)?aparat\.com/(?:v/|video/video/embed/videohash/)(?P<id>[a-zA-Z0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?aparat\.com/(?:v/|video/video/embed/videohash/)(?P<id>[a-zA-Z0-9]+)'
 
     _TEST = {
         'url': 'http://www.aparat.com/v/wP8On',
@@ -29,30 +29,41 @@ class AparatIE(InfoExtractor):
         # Note: There is an easier-to-parse configuration at
         # http://www.aparat.com/video/video/config/videohash/%video_id
         # but the URL in there does not work
-        embed_url = 'http://www.aparat.com/video/video/embed/vt/frame/showvideo/yes/videohash/' + video_id
-        webpage = self._download_webpage(embed_url, video_id)
-
-        file_list = self._parse_json(self._search_regex(
-            r'fileList\s*=\s*JSON\.parse\(\'([^\']+)\'\)', webpage, 'file list'), video_id)
-        for i, item in enumerate(file_list[0]):
-            video_url = item['file']
-            req = HEADRequest(video_url)
-            res = self._request_webpage(
-                req, video_id, note='Testing video URL %d' % i, errnote=False)
-            if res:
-                break
-        else:
-            raise ExtractorError('No working video URLs found')
+        webpage = self._download_webpage(
+            'http://www.aparat.com/video/video/embed/vt/frame/showvideo/yes/videohash/' + video_id,
+            video_id)
 
         title = self._search_regex(r'\s+title:\s*"([^"]+)"', webpage, 'title')
+
+        file_list = self._parse_json(
+            self._search_regex(
+                r'fileList\s*=\s*JSON\.parse\(\'([^\']+)\'\)', webpage,
+                'file list'),
+            video_id)
+
+        formats = []
+        for item in file_list[0]:
+            file_url = item.get('file')
+            if not file_url:
+                continue
+            ext = mimetype2ext(item.get('type'))
+            label = item.get('label')
+            formats.append({
+                'url': file_url,
+                'ext': ext,
+                'format_id': label or ext,
+                'height': int_or_none(self._search_regex(
+                    r'(\d+)[pP]', label or '', 'height', default=None)),
+            })
+        self._sort_formats(formats)
+
         thumbnail = self._search_regex(
             r'image:\s*"([^"]+)"', webpage, 'thumbnail', fatal=False)
 
         return {
             'id': video_id,
             'title': title,
-            'url': video_url,
-            'ext': 'mp4',
             'thumbnail': thumbnail,
             'age_limit': self._family_friendly_search(webpage),
+            'formats': formats,
         }

From 868f79db41a4d81a87ef12c8bd5ef73205c9c029 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 12 Aug 2017 19:24:26 +0700
Subject: [PATCH 1206/1696] [extractor/common] Fix _media_formats

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e565901af..7fe888462 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2115,7 +2115,7 @@ class InfoExtractor(object):
                 return f
             return {}
 
-        def _media_formats(src, type_info, cur_media_type):
+        def _media_formats(src, cur_media_type, type_info={}):
             full_url = absolute_url(src)
             ext = type_info.get('ext') or determine_ext(full_url)
             if ext == 'm3u8':
@@ -2167,7 +2167,7 @@ class InfoExtractor(object):
                     if not src:
                         continue
                     f = parse_content_type(source_attributes.get('type'))
-                    is_plain_url, formats = _media_formats(src, f, media_type)
+                    is_plain_url, formats = _media_formats(src, media_type, f)
                     if is_plain_url:
                         f.update(formats[0])
                         media_info['formats'].append(f)

From 0c43a481b91c657643eb42f72d293f245a410c52 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 12 Aug 2017 23:24:46 +0700
Subject: [PATCH 1207/1696] [reddit] Add extractors (closes #13847)

---
 youtube_dl/extractor/extractors.py |   4 +
 youtube_dl/extractor/reddit.py     | 114 +++++++++++++++++++++++++++++
 2 files changed, 118 insertions(+)
 create mode 100644 youtube_dl/extractor/reddit.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 897557f93..f1a9f6edf 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -840,6 +840,10 @@ from .rai import (
 from .rbmaradio import RBMARadioIE
 from .rds import RDSIE
 from .redbulltv import RedBullTVIE
+from .reddit import (
+    RedditIE,
+    RedditRIE,
+)
 from .redtube import RedTubeIE
 from .regiotv import RegioTVIE
 from .rentv import (
diff --git a/youtube_dl/extractor/reddit.py b/youtube_dl/extractor/reddit.py
new file mode 100644
index 000000000..01c85ee01
--- /dev/null
+++ b/youtube_dl/extractor/reddit.py
@@ -0,0 +1,114 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    float_or_none,
+)
+
+
+class RedditIE(InfoExtractor):
+    _VALID_URL = r'https?://v\.redd\.it/(?P<id>[^/?#&]+)'
+    _TEST = {
+        # from https://www.reddit.com/r/videos/comments/6rrwyj/that_small_heart_attack/
+        'url': 'https://v.redd.it/zv89llsvexdz',
+        'md5': '655d06ace653ea3b87bccfb1b27ec99d',
+        'info_dict': {
+            'id': 'zv89llsvexdz',
+            'ext': 'mp4',
+            'title': 'zv89llsvexdz',
+        },
+        'params': {
+            'format': 'bestvideo',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        formats = self._extract_m3u8_formats(
+            'https://v.redd.it/%s/HLSPlaylist.m3u8' % video_id, video_id,
+            'mp4', entry_protocol='m3u8_native', m3u8_id='hls', fatal=False)
+
+        formats.extend(self._extract_mpd_formats(
+            'https://v.redd.it/%s/DASHPlaylist.mpd' % video_id, video_id,
+            mpd_id='dash', fatal=False))
+
+        return {
+            'id': video_id,
+            'title': video_id,
+            'formats': formats,
+        }
+
+
+class RedditRIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?reddit\.com/r/[^/]+/comments/(?P<id>[^/]+)'
+    _TESTS = [{
+        'url': 'https://www.reddit.com/r/videos/comments/6rrwyj/that_small_heart_attack/',
+        'info_dict': {
+            'id': 'zv89llsvexdz',
+            'ext': 'mp4',
+            'title': 'That small heart attack.',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'timestamp': 1501941939,
+            'upload_date': '20170805',
+            'uploader': 'Antw87',
+            'like_count': int,
+            'dislike_count': int,
+            'comment_count': int,
+            'age_limit': 0,
+        },
+        'params': {
+            'format': 'bestvideo',
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://www.reddit.com/r/videos/comments/6rrwyj',
+        'only_matching': True,
+    }, {
+        # imgur
+        'url': 'https://www.reddit.com/r/MadeMeSmile/comments/6t7wi5/wait_for_it/',
+        'only_matching': True,
+    }, {
+        # streamable
+        'url': 'https://www.reddit.com/r/videos/comments/6t7sg9/comedians_hilarious_joke_about_the_guam_flag/',
+        'only_matching': True,
+    }, {
+        # youtube
+        'url': 'https://www.reddit.com/r/videos/comments/6t75wq/southern_man_tries_to_speak_without_an_accent/',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        data = self._download_json(
+            url + '.json', video_id)[0]['data']['children'][0]['data']
+
+        video_url = data['url']
+
+        # Avoid recursing into the same reddit URL
+        if 'reddit.com/' in video_url and '/%s/' % video_id in video_url:
+            raise ExtractorError('No media found', expected=True)
+
+        over_18 = data.get('over_18')
+        if over_18 is True:
+            age_limit = 18
+        elif over_18 is False:
+            age_limit = 0
+        else:
+            age_limit = None
+
+        return {
+            '_type': 'url_transparent',
+            'url': video_url,
+            'title': data.get('title'),
+            'thumbnail': data.get('thumbnail'),
+            'timestamp': float_or_none(data.get('created_utc')),
+            'uploader': data.get('author'),
+            'like_count': int_or_none(data.get('ups')),
+            'dislike_count': int_or_none(data.get('downs')),
+            'comment_count': int_or_none(data.get('num_comments')),
+            'age_limit': age_limit,
+        }

From 4ef9152428c4a000cb5fc76732fc579f1f4c1d69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Aug 2017 00:58:39 +0700
Subject: [PATCH 1208/1696] [limelight] Improve embeds detection (closes
 #13895)

---
 youtube_dl/extractor/limelight.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index 0a5a3956c..ad65b2759 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -26,14 +26,16 @@ class LimelightBaseIE(InfoExtractor):
             'Channel': 'channel',
             'ChannelList': 'channel_list',
         }
+
+        def smuggle(url):
+            return smuggle_url(url, {'source_url': source_url})
+
         entries = []
         for kind, video_id in re.findall(
                 r'LimelightPlayer\.doLoad(Media|Channel|ChannelList)\(["\'](?P<id>[a-z0-9]{32})',
                 webpage):
             entries.append(cls.url_result(
-                smuggle_url(
-                    'limelight:%s:%s' % (lm[kind], video_id),
-                    {'source_url': source_url}),
+                smuggle('limelight:%s:%s' % (lm[kind], video_id)),
                 'Limelight%s' % kind, video_id))
         for mobj in re.finditer(
                 # As per [1] class attribute should be exactly equal to
@@ -49,10 +51,15 @@ class LimelightBaseIE(InfoExtractor):
                 ''', webpage):
             kind, video_id = mobj.group('kind'), mobj.group('id')
             entries.append(cls.url_result(
-                smuggle_url(
-                    'limelight:%s:%s' % (kind, video_id),
-                    {'source_url': source_url}),
+                smuggle('limelight:%s:%s' % (kind, video_id)),
                 'Limelight%s' % kind.capitalize(), video_id))
+        # http://support.3playmedia.com/hc/en-us/articles/115009517327-Limelight-Embedding-the-Audio-Description-Plugin-with-the-Limelight-Player-on-Your-Web-Page)
+        for video_id in re.findall(
+                r'(?s)LimelightPlayerUtil\.embed\s*\(\s*{.*?\bmediaId["\']\s*:\s*["\'](?P<id>[a-z0-9]{32})',
+                webpage):
+            entries.append(cls.url_result(
+                smuggle('limelight:media:%s' % video_id),
+                LimelightMediaIE.ie_key(), video_id))
         return entries
 
     def _call_playlist_service(self, item_id, method, fatal=True, referer=None):

From eb02940cc7dc2233f2d7873c12165245a3c3c14e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Aug 2017 01:11:27 +0700
Subject: [PATCH 1209/1696] [generic] Add test for #13895

---
 youtube_dl/extractor/generic.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8362d9a36..eff5fbfe8 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1785,6 +1785,21 @@ class GenericIE(InfoExtractor):
             },
             'playlist_mincount': 5,
         },
+        {
+            # Limelight embed (LimelightPlayerUtil.embed)
+            'url': 'https://tv5.ca/videos?v=xuu8qowr291ri',
+            'info_dict': {
+                'id': '95d035dc5c8a401588e9c0e6bd1e9c92',
+                'ext': 'mp4',
+                'title': '07448641',
+                'timestamp': 1499890639,
+                'upload_date': '20170712',
+            },
+            'params': {
+                'skip_download': True,
+            },
+            'add_ie': ['LimelightMedia'],
+        },
         {
             'url': 'http://kron4.com/2017/04/28/standoff-with-walnut-creek-murder-suspect-ends-with-arrest/',
             'info_dict': {

From b3c6515365ed415bbf813c0c2e6c12585824b77a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Aug 2017 07:23:29 +0700
Subject: [PATCH 1210/1696] [fourtube] Add support for other sites (closes
 #6022, closes #7859, closes #13901)

---
 youtube_dl/extractor/extractors.py |   7 +-
 youtube_dl/extractor/fourtube.py   | 176 +++++++++++++++++++++++------
 2 files changed, 147 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index f1a9f6edf..fb79a1736 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -350,7 +350,12 @@ from .flipagram import FlipagramIE
 from .folketinget import FolketingetIE
 from .footyroom import FootyRoomIE
 from .formula1 import Formula1IE
-from .fourtube import FourTubeIE
+from .fourtube import (
+    FourTubeIE,
+    PornTubeIE,
+    PornerBrosIE,
+    FuxIE,
+)
 from .fox import FOXIE
 from .fox9 import FOX9IE
 from .foxgay import FoxgayIE
diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index e3fd08bcf..ad273a0e7 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -3,39 +3,22 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
     parse_duration,
     parse_iso8601,
-    sanitized_Request,
     str_to_int,
 )
 
 
-class FourTubeIE(InfoExtractor):
-    IE_NAME = '4tube'
-    _VALID_URL = r'https?://(?:www\.)?4tube\.com/videos/(?P<id>\d+)'
-
-    _TEST = {
-        'url': 'http://www.4tube.com/videos/209733/hot-babe-holly-michaels-gets-her-ass-stuffed-by-black',
-        'md5': '6516c8ac63b03de06bc8eac14362db4f',
-        'info_dict': {
-            'id': '209733',
-            'ext': 'mp4',
-            'title': 'Hot Babe Holly Michaels gets her ass stuffed by black',
-            'uploader': 'WCP Club',
-            'uploader_id': 'wcp-club',
-            'upload_date': '20131031',
-            'timestamp': 1383263892,
-            'duration': 583,
-            'view_count': int,
-            'like_count': int,
-            'categories': list,
-            'age_limit': 18,
-        }
-    }
-
+class FourTubeBaseIE(InfoExtractor):
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        kind, video_id, display_id = mobj.group('kind', 'id', 'display_id')
+
+        if kind == 'm' or not display_id:
+            url = self._URL_TEMPLATE % video_id
+
         webpage = self._download_webpage(url, video_id)
 
         title = self._html_search_meta('name', webpage)
@@ -43,10 +26,10 @@ class FourTubeIE(InfoExtractor):
             'uploadDate', webpage))
         thumbnail = self._html_search_meta('thumbnailUrl', webpage)
         uploader_id = self._html_search_regex(
-            r'<a class="item-to-subscribe" href="[^"]+/channels/([^/"]+)" title="Go to [^"]+ page">',
+            r'<a class="item-to-subscribe" href="[^"]+/(?:channel|user)s?/([^/"]+)" title="Go to [^"]+ page">',
             webpage, 'uploader id', fatal=False)
         uploader = self._html_search_regex(
-            r'<a class="item-to-subscribe" href="[^"]+/channels/[^/"]+" title="Go to ([^"]+) page">',
+            r'<a class="item-to-subscribe" href="[^"]+/(?:channel|user)s?/[^/"]+" title="Go to ([^"]+) page">',
             webpage, 'uploader', fatal=False)
 
         categories_html = self._search_regex(
@@ -60,10 +43,10 @@ class FourTubeIE(InfoExtractor):
 
         view_count = str_to_int(self._search_regex(
             r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserPlays:([0-9,]+)">',
-            webpage, 'view count', fatal=False))
+            webpage, 'view count', default=None))
         like_count = str_to_int(self._search_regex(
             r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserLikes:([0-9,]+)">',
-            webpage, 'like count', fatal=False))
+            webpage, 'like count', default=None))
         duration = parse_duration(self._html_search_meta('duration', webpage))
 
         media_id = self._search_regex(
@@ -87,12 +70,12 @@ class FourTubeIE(InfoExtractor):
 
         token_url = 'https://tkn.kodicdn.com/{0}/desktop/{1}'.format(
             media_id, '+'.join(sources))
-        headers = {
-            b'Content-Type': b'application/x-www-form-urlencoded',
-            b'Origin': b'https://www.4tube.com',
-        }
-        token_req = sanitized_Request(token_url, b'{}', headers)
-        tokens = self._download_json(token_req, video_id)
+
+        parsed_url = compat_urlparse.urlparse(url)
+        tokens = self._download_json(token_url, video_id, data=b'', headers={
+            'Origin': '%s://%s' % (parsed_url.scheme, parsed_url.hostname),
+            'Referer': url,
+        })
         formats = [{
             'url': tokens[format]['token'],
             'format_id': format + 'p',
@@ -115,3 +98,126 @@ class FourTubeIE(InfoExtractor):
             'duration': duration,
             'age_limit': 18,
         }
+
+
+class FourTubeIE(FourTubeBaseIE):
+    IE_NAME = '4tube'
+    _VALID_URL = r'https?://(?:(?P<kind>www|m)\.)?4tube\.com/(?:videos|embed)/(?P<id>\d+)(?:/(?P<display_id>[^/?#&]+))?'
+    _URL_TEMPLATE = 'https://www.4tube.com/videos/%s/video'
+    _TESTS = [{
+        'url': 'http://www.4tube.com/videos/209733/hot-babe-holly-michaels-gets-her-ass-stuffed-by-black',
+        'md5': '6516c8ac63b03de06bc8eac14362db4f',
+        'info_dict': {
+            'id': '209733',
+            'ext': 'mp4',
+            'title': 'Hot Babe Holly Michaels gets her ass stuffed by black',
+            'uploader': 'WCP Club',
+            'uploader_id': 'wcp-club',
+            'upload_date': '20131031',
+            'timestamp': 1383263892,
+            'duration': 583,
+            'view_count': int,
+            'like_count': int,
+            'categories': list,
+            'age_limit': 18,
+        },
+    }, {
+        'url': 'http://www.4tube.com/embed/209733',
+        'only_matching': True,
+    }, {
+        'url': 'http://m.4tube.com/videos/209733/hot-babe-holly-michaels-gets-her-ass-stuffed-by-black',
+        'only_matching': True,
+    }]
+
+
+class FuxIE(FourTubeBaseIE):
+    _VALID_URL = r'https?://(?:(?P<kind>www|m)\.)?fux\.com/(?:video|embed)/(?P<id>\d+)(?:/(?P<display_id>[^/?#&]+))?'
+    _URL_TEMPLATE = 'https://www.fux.com/video/%s/video'
+    _TESTS = [{
+        'url': 'https://www.fux.com/video/195359/awesome-fucking-kitchen-ends-cum-swallow',
+        'info_dict': {
+            'id': '195359',
+            'ext': 'mp4',
+            'title': 'Awesome fucking in the kitchen ends with cum swallow',
+            'uploader': 'alenci2342',
+            'uploader_id': 'alenci2342',
+            'upload_date': '20131230',
+            'timestamp': 1388361660,
+            'duration': 289,
+            'view_count': int,
+            'like_count': int,
+            'categories': list,
+            'age_limit': 18,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://www.fux.com/embed/195359',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.fux.com/video/195359/awesome-fucking-kitchen-ends-cum-swallow',
+        'only_matching': True,
+    }]
+
+
+class PornTubeIE(FourTubeBaseIE):
+    _VALID_URL = r'https?://(?:(?P<kind>www|m)\.)?porntube\.com/(?:videos/(?P<display_id>[^/]+)_|embed/)(?P<id>\d+)'
+    _URL_TEMPLATE = 'https://www.porntube.com/videos/video_%s'
+    _TESTS = [{
+        'url': 'https://www.porntube.com/videos/teen-couple-doing-anal_7089759',
+        'info_dict': {
+            'id': '7089759',
+            'ext': 'mp4',
+            'title': 'Teen couple doing anal',
+            'uploader': 'Alexy',
+            'uploader_id': 'Alexy',
+            'upload_date': '20150606',
+            'timestamp': 1433595647,
+            'duration': 5052,
+            'view_count': int,
+            'like_count': int,
+            'categories': list,
+            'age_limit': 18,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://www.porntube.com/embed/7089759',
+        'only_matching': True,
+    }, {
+        'url': 'https://m.porntube.com/videos/teen-couple-doing-anal_7089759',
+        'only_matching': True,
+    }]
+
+
+class PornerBrosIE(FourTubeBaseIE):
+    _VALID_URL = r'https?://(?:(?P<kind>www|m)\.)?pornerbros\.com/(?:videos/(?P<display_id>[^/]+)_|embed/)(?P<id>\d+)'
+    _URL_TEMPLATE = 'https://www.pornerbros.com/videos/video_%s'
+    _TESTS = [{
+        'url': 'https://www.pornerbros.com/videos/skinny-brunette-takes-big-cock-down-her-anal-hole_181369',
+        'md5': '6516c8ac63b03de06bc8eac14362db4f',
+        'info_dict': {
+            'id': '181369',
+            'ext': 'mp4',
+            'title': 'Skinny brunette takes big cock down her anal hole',
+            'uploader': 'PornerBros HD',
+            'uploader_id': 'pornerbros-hd',
+            'upload_date': '20130130',
+            'timestamp': 1359527401,
+            'duration': 1224,
+            'view_count': int,
+            'categories': list,
+            'age_limit': 18,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://www.pornerbros.com/embed/181369',
+        'only_matching': True,
+    }, {
+        'url': 'https://m.pornerbros.com/videos/skinny-brunette-takes-big-cock-down-her-anal-hole_181369',
+        'only_matching': True,
+    }]

From 475bcb225f6046e38b47594c504da6ec15bac113 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Aug 2017 07:53:02 +0700
Subject: [PATCH 1211/1696] [pornhub:playlistbase] Skip videos from drop-down
 menu for all playlists (closes #12819, closes #13902)

---
 youtube_dl/extractor/pornhub.py | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index e032817f2..f6777cd26 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -227,20 +227,6 @@ class PornHubIE(InfoExtractor):
 
 class PornHubPlaylistBaseIE(InfoExtractor):
     def _extract_entries(self, webpage):
-        return [
-            self.url_result(
-                'http://www.pornhub.com/%s' % video_url,
-                PornHubIE.ie_key(), video_title=title)
-            for video_url, title in orderedSet(re.findall(
-                r'href="/?(view_video\.php\?.*\bviewkey=[\da-z]+[^"]*)"[^>]*\s+title="([^"]+)"',
-                webpage))
-        ]
-
-    def _real_extract(self, url):
-        playlist_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, playlist_id)
-
         # Only process container div with main playlist content skipping
         # drop-down menu that uses similar pattern for videos (see
         # https://github.com/rg3/youtube-dl/issues/11594).
@@ -248,7 +234,21 @@ class PornHubPlaylistBaseIE(InfoExtractor):
             r'(?s)(<div[^>]+class=["\']container.+)', webpage,
             'container', default=webpage)
 
-        entries = self._extract_entries(container)
+        return [
+            self.url_result(
+                'http://www.pornhub.com/%s' % video_url,
+                PornHubIE.ie_key(), video_title=title)
+            for video_url, title in orderedSet(re.findall(
+                r'href="/?(view_video\.php\?.*\bviewkey=[\da-z]+[^"]*)"[^>]*\s+title="([^"]+)"',
+                container))
+        ]
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        entries = self._extract_entries(webpage)
 
         playlist = self._parse_json(
             self._search_regex(

From 4f049e4aa866aef89d0f4b735fd89eb2ba84e809 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Aug 2017 08:00:15 +0700
Subject: [PATCH 1212/1696] [ChangeLog] Actualize

---
 ChangeLog | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index b28ade446..daa3601ba 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,22 @@
+version <unreleased>
+
+Core
+* [YoutubeDL] Make sure format id is not empty
+* [extractor/common] Make _family_friendly_search optional
+* [extractor/common] Respect source's type attribute for HTML5 media (#13892)
+
+Extractors
+* [pornhub:playlistbase] Skip videos from drop-down menu (#12819, #13902)
++ [fourtube] Add support pornerbros.com (#6022)
++ [fourtube] Add support porntube.com (#7859, #13901)
++ [fourtube] Add support fux.com
+* [limelight] Improve embeds detection (#13895)
++ [reddit] Add support for v.redd.it and reddit.com (#13847)
+* [aparat] Extract all formats (#13887)
+* [mixcloud] Fix play info decryption (#13885)
++ [generic] Add support for vzaar embeds (#13876)
+
+
 version 2017.08.09
 
 Core

From 16393d65355cdb1118e528d6dcb6d82f5f1c2b6e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Aug 2017 08:58:30 +0700
Subject: [PATCH 1213/1696] release 2017.08.13

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 5 +++++
 youtube_dl/version.py     | 2 +-
 4 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 7ee704e48..3bd61e0a6 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.08.09*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.08.09**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.08.13*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.08.13**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.08.09
+[debug] youtube-dl version 2017.08.13
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index daa3601ba..6bafb1e8f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.08.13
 
 Core
 * [YoutubeDL] Make sure format id is not empty
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index a3bd07726..cc442742f 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -294,6 +294,7 @@
  - **Funimation**
  - **FunnyOrDie**
  - **Fusion**
+ - **Fux**
  - **FXNetworks**
  - **GameInformer**
  - **GameOne**
@@ -621,6 +622,7 @@
  - **PolskieRadio**
  - **PolskieRadioCategory**
  - **PornCom**
+ - **PornerBros**
  - **PornFlip**
  - **PornHd**
  - **PornHub**: PornHub and Thumbzilla
@@ -629,6 +631,7 @@
  - **Pornotube**
  - **PornoVoisines**
  - **PornoXO**
+ - **PornTube**
  - **PressTV**
  - **PrimeShareTV**
  - **PromptFile**
@@ -654,6 +657,8 @@
  - **RBMARadio**
  - **RDS**: RDS.ca
  - **RedBullTV**
+ - **Reddit**
+ - **RedditR**
  - **RedTube**
  - **RegioTV**
  - **RENTV**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 022172375..da855a602 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.08.09'
+__version__ = '2017.08.13'

From da20951a57bddd4a0102cd776ff93a2adc6db77d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 14 Aug 2017 22:39:05 +0700
Subject: [PATCH 1214/1696] [mixcloud] Extract decrypt key

---
 youtube_dl/extractor/mixcloud.py | 32 +++++++++++++++++++++++++-------
 1 file changed, 25 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 52f7428e0..fcf7beeb2 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -9,6 +9,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_chr,
     compat_ord,
+    compat_str,
     compat_urllib_parse_unquote,
     compat_urlparse,
 )
@@ -53,15 +54,18 @@ class MixcloudIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    _keys = [
+        'return { requestAnimationFrame: function(callback) { callback(); }, innerHeight: 500 };',
+        'pleasedontdownloadourmusictheartistswontgetpaid',
+        'window.addEventListener = window.addEventListener || function() {};',
+        '(function() { return new Date().toLocaleDateString(); })()'
+    ]
+    _current_key = None
+
     # See https://www.mixcloud.com/media/js2/www_js_2.9e23256562c080482435196ca3975ab5.js
     def _decrypt_play_info(self, play_info, video_id):
-        KEYS = (
-            'pleasedontdownloadourmusictheartistswontgetpaid',
-            'window.addEventListener = window.addEventListener || function() {};',
-            '(function() { return new Date().toLocaleDateString(); })()',
-        )
         play_info = base64.b64decode(play_info.encode('ascii'))
-        for num, key in enumerate(KEYS, start=1):
+        for num, key in enumerate(self._keys, start=1):
             try:
                 return self._parse_json(
                     ''.join([
@@ -69,7 +73,7 @@ class MixcloudIE(InfoExtractor):
                         for idx, ch in enumerate(play_info)]),
                     video_id)
             except ExtractorError:
-                if num == len(KEYS):
+                if num == len(self._keys):
                     raise
 
     def _real_extract(self, url):
@@ -80,6 +84,20 @@ class MixcloudIE(InfoExtractor):
 
         webpage = self._download_webpage(url, track_id)
 
+        if not self._current_key:
+            js_url = self._search_regex(
+                r'<script[^>]+\bsrc=["\"](https://(?:www\.)?mixcloud\.com/media/js2/www_js_4\.[^>]+\.js)',
+                webpage, 'js url', default=None)
+            if js_url:
+                js = self._download_webpage(js_url, track_id, fatal=False)
+                if js:
+                    key = self._search_regex(
+                        r'player\s*:\s*{.*?\bvalue\s*:\s*(["\'])(?P<key>(?:(?!\1).)+)\1',
+                        js, 'key', default=None, group='key')
+                    if key and isinstance(key, compat_str):
+                        self._keys.insert(0, key)
+                        self._current_key = key
+
         message = self._html_search_regex(
             r'(?s)<div[^>]+class="global-message cloudcast-disabled-notice-light"[^>]*>(.+?)<(?:a|/div)',
             webpage, 'error message', default=None)

From 19ada898dc80a04ae1a2590c8886c9ec13958b03 Mon Sep 17 00:00:00 2001
From: forDream <gam2046@gmail.com>
Date: Wed, 2 Aug 2017 11:12:17 +0800
Subject: [PATCH 1215/1696] fix QQ Music Url changed

---
 youtube_dl/extractor/qqmusic.py | 43 +++++++++++++++++----------------
 1 file changed, 22 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 17c27da46..6bff6baa8 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -2,8 +2,8 @@
 from __future__ import unicode_literals
 
 import random
-import time
 import re
+import time
 
 from .common import InfoExtractor
 from ..utils import (
@@ -18,9 +18,9 @@ from ..utils import (
 class QQMusicIE(InfoExtractor):
     IE_NAME = 'qqmusic'
     IE_DESC = 'QQ音乐'
-    _VALID_URL = r'https?://y\.qq\.com/#type=song&mid=(?P<id>[0-9A-Za-z]+)'
+    _VALID_URL = r'https?://y\.qq\.com/n/yqq/song/(?P<id>[0-9A-Za-z]+)\.html'
     _TESTS = [{
-        'url': 'http://y.qq.com/#type=song&mid=004295Et37taLD',
+        'url': 'https://y.qq.com/n/yqq/song/004295Et37taLD.html',
         'md5': '9ce1c1c8445f561506d2e3cfb0255705',
         'info_dict': {
             'id': '004295Et37taLD',
@@ -33,7 +33,7 @@ class QQMusicIE(InfoExtractor):
         }
     }, {
         'note': 'There is no mp3-320 version of this song.',
-        'url': 'http://y.qq.com/#type=song&mid=004MsGEo3DdNxV',
+        'url': 'https://y.qq.com/n/yqq/song/004MsGEo3DdNxV.html',
         'md5': 'fa3926f0c585cda0af8fa4f796482e3e',
         'info_dict': {
             'id': '004MsGEo3DdNxV',
@@ -46,7 +46,7 @@ class QQMusicIE(InfoExtractor):
         }
     }, {
         'note': 'lyrics not in .lrc format',
-        'url': 'http://y.qq.com/#type=song&mid=001JyApY11tIp6',
+        'url': 'https://y.qq.com/n/yqq/song/001JyApY11tIp6.html',
         'info_dict': {
             'id': '001JyApY11tIp6',
             'ext': 'mp3',
@@ -163,7 +163,8 @@ class QQPlaylistBaseIE(InfoExtractor):
         for item in re.findall(r'class="data"[^<>]*>([^<>]+)</', page):
             song_mid = unescapeHTML(item).split('|')[-5]
             entries.append(cls.url_result(
-                'http://y.qq.com/#type=song&mid=' + song_mid, 'QQMusic',
+                # https://y.qq.com/n/yqq/song/004Dbsoo1yCbNZ.html
+                'https://y.qq.com/n/yqq/song/' + song_mid + ".html", 'QQMusic',
                 song_mid))
 
         return entries
@@ -172,9 +173,9 @@ class QQPlaylistBaseIE(InfoExtractor):
 class QQMusicSingerIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:singer'
     IE_DESC = 'QQ音乐 - 歌手'
-    _VALID_URL = r'https?://y\.qq\.com/#type=singer&mid=(?P<id>[0-9A-Za-z]+)'
+    _VALID_URL = r'https?://y\.qq\.com/n/yqq/singer/(?P<id>[0-9A-Za-z]+)\.html'
     _TEST = {
-        'url': 'http://y.qq.com/#type=singer&mid=001BLpXF2DyJe2',
+        'url': 'https://y.qq.com/n/yqq/singer/001BLpXF2DyJe2.html',
         'info_dict': {
             'id': '001BLpXF2DyJe2',
             'title': '林俊杰',
@@ -217,10 +218,10 @@ class QQMusicSingerIE(QQPlaylistBaseIE):
 class QQMusicAlbumIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:album'
     IE_DESC = 'QQ音乐 - 专辑'
-    _VALID_URL = r'https?://y\.qq\.com/#type=album&mid=(?P<id>[0-9A-Za-z]+)'
+    _VALID_URL = r'https?://y\.qq\.com/n/yqq/album/(?P<id>[0-9A-Za-z]+)\.html'
 
     _TESTS = [{
-        'url': 'http://y.qq.com/#type=album&mid=000gXCTb2AhRR1',
+        'url': 'https://y.qq.com/n/yqq/album/000gXCTb2AhRR1.html',
         'info_dict': {
             'id': '000gXCTb2AhRR1',
             'title': '我们都是这样长大的',
@@ -228,7 +229,7 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
         },
         'playlist_count': 4,
     }, {
-        'url': 'http://y.qq.com/#type=album&mid=002Y5a3b3AlCu3',
+        'url': 'https://y.qq.com/n/yqq/album/002Y5a3b3AlCu3.html',
         'info_dict': {
             'id': '002Y5a3b3AlCu3',
             'title': '그리고...',
@@ -246,7 +247,7 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
 
         entries = [
             self.url_result(
-                'http://y.qq.com/#type=song&mid=' + song['songmid'], 'QQMusic', song['songmid']
+                'https://y.qq.com/n/yqq/song/' + song['songmid'] + ".html", 'QQMusic', song['songmid']
             ) for song in album['list']
         ]
         album_name = album.get('name')
@@ -260,17 +261,17 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
 class QQMusicToplistIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:toplist'
     IE_DESC = 'QQ音乐 - 排行榜'
-    _VALID_URL = r'https?://y\.qq\.com/#type=toplist&p=(?P<id>(top|global)_[0-9]+)'
+    _VALID_URL = r'https?://y\.qq\.com/n/yqq/toplist/(?P<id>(top|global)_[0-9]+)\.html'
 
     _TESTS = [{
-        'url': 'http://y.qq.com/#type=toplist&p=global_123',
+        'url': 'https://y.qq.com/n/yqq/toplist/123.html',
         'info_dict': {
             'id': 'global_123',
             'title': '美国iTunes榜',
         },
         'playlist_count': 10,
     }, {
-        'url': 'http://y.qq.com/#type=toplist&p=top_3',
+        'url': 'https://y.qq.com/n/yqq/toplist/3.html',
         'info_dict': {
             'id': 'top_3',
             'title': '巅峰榜·欧美',
@@ -281,7 +282,7 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
         },
         'playlist_count': 100,
     }, {
-        'url': 'http://y.qq.com/#type=toplist&p=global_106',
+        'url': 'https://y.qq.com/n/yqq/toplist/106.html',
         'info_dict': {
             'id': 'global_106',
             'title': '韩国Mnet榜',
@@ -301,7 +302,7 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
 
         entries = [
             self.url_result(
-                'http://y.qq.com/#type=song&mid=' + song['data']['songmid'], 'QQMusic', song['data']['songmid']
+                'https://y.qq.com/n/yqq/song/' + song['data']['songmid'] + ".html", 'QQMusic', song['data']['songmid']
             ) for song in toplist_json['songlist']
         ]
 
@@ -314,10 +315,10 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
 class QQMusicPlaylistIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:playlist'
     IE_DESC = 'QQ音乐 - 歌单'
-    _VALID_URL = r'https?://y\.qq\.com/#type=taoge&id=(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://y\.qq\.com/n/yqq/playlist/(?P<id>[0-9]+)\.html'
 
     _TESTS = [{
-        'url': 'http://y.qq.com/#type=taoge&id=3462654915',
+        'url': 'http://y.qq.com/n/yqq/playlist/3462654915.html',
         'info_dict': {
             'id': '3462654915',
             'title': '韩国5月新歌精选下旬',
@@ -326,7 +327,7 @@ class QQMusicPlaylistIE(QQPlaylistBaseIE):
         'playlist_count': 40,
         'skip': 'playlist gone',
     }, {
-        'url': 'http://y.qq.com/#type=taoge&id=1374105607',
+        'url': 'https://y.qq.com/n/yqq/playlist/1374105607.html',
         'info_dict': {
             'id': '1374105607',
             'title': '易入人心的华语民谣',
@@ -352,7 +353,7 @@ class QQMusicPlaylistIE(QQPlaylistBaseIE):
         cdlist = list_json['cdlist'][0]
         entries = [
             self.url_result(
-                'http://y.qq.com/#type=song&mid=' + song['songmid'], 'QQMusic', song['songmid']
+                'https://y.qq.com/n/yqq/song/' + song['songmid'] + ".html", 'QQMusic', song['songmid']
             ) for song in cdlist['songlist']
         ]
 

From 5d1bd3b907d22eab7c47b8b408c07a26dbc358ea Mon Sep 17 00:00:00 2001
From: forDream <gam2046@gmail.com>
Date: Wed, 2 Aug 2017 12:20:53 +0800
Subject: [PATCH 1216/1696] [qqmusic]update valid url

---
 youtube_dl/extractor/qqmusic.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 6bff6baa8..7513acba9 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -261,7 +261,7 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
 class QQMusicToplistIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:toplist'
     IE_DESC = 'QQ音乐 - 排行榜'
-    _VALID_URL = r'https?://y\.qq\.com/n/yqq/toplist/(?P<id>(top|global)_[0-9]+)\.html'
+    _VALID_URL = r'https?://y\.qq\.com/n/yqq/toplist/(?P<id>[0-9]+)\.html'
 
     _TESTS = [{
         'url': 'https://y.qq.com/n/yqq/toplist/123.html',
@@ -293,7 +293,9 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
     def _real_extract(self, url):
         list_id = self._match_id(url)
 
-        list_type, num_id = list_id.split("_")
+        # list_type, num_id = list_id.split("_")
+        list_type = "toplist"
+        num_id = list_id
 
         toplist_json = self._download_json(
             'http://i.y.qq.com/v8/fcg-bin/fcg_v8_toplist_cp.fcg?type=%s&topid=%s&format=json'

From 5c037c0d1f155e951050533690d6e990654cfcc9 Mon Sep 17 00:00:00 2001
From: forDream <gam2046@gmail.com>
Date: Wed, 2 Aug 2017 15:08:38 +0800
Subject: [PATCH 1217/1696] [qqmusic]support QQMusicSingerIE

---
 youtube_dl/extractor/qqmusic.py | 52 +++++++++++++++++----------------
 1 file changed, 27 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 7513acba9..42be6bc7b 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -156,16 +156,27 @@ class QQPlaylistBaseIE(InfoExtractor):
     def qq_static_url(category, mid):
         return 'http://y.qq.com/y/static/%s/%s/%s/%s.html' % (category, mid[-2], mid[-1], mid)
 
-    @classmethod
-    def get_entries_from_page(cls, page):
+    def get_singer_all_songs(self, singmid, num):
+        return self._download_webpage(
+            r'https://c.y.qq.com/v8/fcg-bin/fcg_v8_singer_track_cp.fcg?format=json&inCharset=utf8&outCharset=utf-8&platform=yqq&needNewCode=0&singermid=%s&order=listen&begin=0&num=%s&songstatus=1' %
+            (singmid, num), singmid)
+
+    def get_entries_from_page(self, singmid):
         entries = []
 
-        for item in re.findall(r'class="data"[^<>]*>([^<>]+)</', page):
-            song_mid = unescapeHTML(item).split('|')[-5]
-            entries.append(cls.url_result(
-                # https://y.qq.com/n/yqq/song/004Dbsoo1yCbNZ.html
-                'https://y.qq.com/n/yqq/song/' + song_mid + ".html", 'QQMusic',
-                song_mid))
+        default_num = 1
+        json_text = self.get_singer_all_songs(singmid, default_num)
+        json_obj = self._parse_json(json_text, singmid)
+
+        if json_obj['code'] == 0:
+            total = json_obj['data']['total']
+            json_text = self.get_singer_all_songs(singmid, total)
+            json_obj = self._parse_json(json_text, singmid)
+
+        for item in json_obj['data']['list']:
+            if not (item['musicData'].get('songmid') is None):
+                songmid = item['musicData']['songmid']
+                entries.append(self.url_result(r'https://y.qq.com/n/yqq/song/%s.html' % songmid, 'QQMusic', songmid))
 
         return entries
 
@@ -187,26 +198,16 @@ class QQMusicSingerIE(QQPlaylistBaseIE):
     def _real_extract(self, url):
         mid = self._match_id(url)
 
-        singer_page = self._download_webpage(
-            self.qq_static_url('singer', mid), mid, 'Download singer page')
-
-        entries = self.get_entries_from_page(singer_page)
-
-        singer_name = self._html_search_regex(
-            r"singername\s*:\s*'([^']+)'", singer_page, 'singer name',
-            default=None)
-
-        singer_id = self._html_search_regex(
-            r"singerid\s*:\s*'([0-9]+)'", singer_page, 'singer id',
-            default=None)
-
+        entries = self.get_entries_from_page(mid)
+        singer_page = self._download_webpage(url, mid, 'Download singer page')
+        singer_name = self._html_search_regex(r"singername : '(.*?)'", singer_page, 'singer name', default=None)
         singer_desc = None
 
-        if singer_id:
+        if mid:
             req = sanitized_Request(
-                'http://s.plcloud.music.qq.com/fcgi-bin/fcg_get_singer_desc.fcg?utf8=1&outCharset=utf-8&format=xml&singerid=%s' % singer_id)
+                'http://s.plcloud.music.qq.com/fcgi-bin/fcg_get_singer_desc.fcg?utf8=1&outCharset=utf-8&format=xml&singermid=%s' % mid)
             req.add_header(
-                'Referer', 'http://s.plcloud.music.qq.com/xhr_proxy_utf8.html')
+                'Referer', 'https://y.qq.com/n/yqq/singer/')
             singer_desc_page = self._download_xml(
                 req, mid, 'Donwload singer description XML')
 
@@ -304,7 +305,8 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
 
         entries = [
             self.url_result(
-                'https://y.qq.com/n/yqq/song/' + song['data']['songmid'] + ".html", 'QQMusic', song['data']['songmid']
+                'https://y.qq.com/n/yqq/song/' + song['data']['songmid'] + ".html", 'QQMusic',
+                song['data']['songmid']
             ) for song in toplist_json['songlist']
         ]
 

From 134d85a7bda9189dc3b07dc10a69699084529a6a Mon Sep 17 00:00:00 2001
From: forDream <gam2046@gmail.com>
Date: Mon, 14 Aug 2017 08:28:41 +0800
Subject: [PATCH 1218/1696] [qqmusic] review

---
 youtube_dl/extractor/qqmusic.py | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 42be6bc7b..38f4b2cab 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -166,15 +166,15 @@ class QQPlaylistBaseIE(InfoExtractor):
 
         default_num = 1
         json_text = self.get_singer_all_songs(singmid, default_num)
-        json_obj = self._parse_json(json_text, singmid)
+        json_obj_all_songs = self._parse_json(json_text, singmid)
 
-        if json_obj['code'] == 0:
-            total = json_obj['data']['total']
+        if json_obj_all_songs['code'] == 0:
+            total = json_obj_all_songs['data']['total']
             json_text = self.get_singer_all_songs(singmid, total)
-            json_obj = self._parse_json(json_text, singmid)
+            json_obj_all_songs = self._parse_json(json_text, singmid)
 
-        for item in json_obj['data']['list']:
-            if not (item['musicData'].get('songmid') is None):
+        for item in json_obj_all_songs['data']['list']:
+            if item['musicData'].get('songmid') is not None:
                 songmid = item['musicData']['songmid']
                 entries.append(self.url_result(r'https://y.qq.com/n/yqq/song/%s.html' % songmid, 'QQMusic', songmid))
 
@@ -248,7 +248,7 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
 
         entries = [
             self.url_result(
-                'https://y.qq.com/n/yqq/song/' + song['songmid'] + ".html", 'QQMusic', song['songmid']
+                'https://y.qq.com/n/yqq/song/' + song['songmid'] + '.html', 'QQMusic', song['songmid']
             ) for song in album['list']
         ]
         album_name = album.get('name')
@@ -294,8 +294,7 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
     def _real_extract(self, url):
         list_id = self._match_id(url)
 
-        # list_type, num_id = list_id.split("_")
-        list_type = "toplist"
+        list_type = 'toplist'
         num_id = list_id
 
         toplist_json = self._download_json(
@@ -305,7 +304,7 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
 
         entries = [
             self.url_result(
-                'https://y.qq.com/n/yqq/song/' + song['data']['songmid'] + ".html", 'QQMusic',
+                'https://y.qq.com/n/yqq/song/' + song['data']['songmid'] + '.html', 'QQMusic',
                 song['data']['songmid']
             ) for song in toplist_json['songlist']
         ]
@@ -357,7 +356,7 @@ class QQMusicPlaylistIE(QQPlaylistBaseIE):
         cdlist = list_json['cdlist'][0]
         entries = [
             self.url_result(
-                'https://y.qq.com/n/yqq/song/' + song['songmid'] + ".html", 'QQMusic', song['songmid']
+                'https://y.qq.com/n/yqq/song/' + song['songmid'] + '.html', 'QQMusic', song['songmid']
             ) for song in cdlist['songlist']
         ]
 

From 485047854376465f95309daad4966971f56728ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 15 Aug 2017 23:58:00 +0700
Subject: [PATCH 1219/1696] [extractor/common] Add support for float durations
 in _parse_mpd_formats (closes #13919)

---
 test/test_InfoExtractor.py           | 86 ++++++++++++++++++++++++++++
 test/testdata/mpd/float_duration.mpd | 18 ++++++
 youtube_dl/extractor/common.py       |  2 +-
 3 files changed, 105 insertions(+), 1 deletion(-)
 create mode 100644 test/testdata/mpd/float_duration.mpd

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 6f52e11f7..f18a823fc 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -10,6 +10,7 @@ import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from test.helper import FakeYDL, expect_dict, expect_value
+from youtube_dl.compat import compat_etree_fromstring
 from youtube_dl.extractor.common import InfoExtractor
 from youtube_dl.extractor import YoutubeIE, get_info_extractor
 from youtube_dl.utils import encode_data_uri, strip_jsonp, ExtractorError, RegexNotFoundError
@@ -488,6 +489,91 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 self.ie._sort_formats(formats)
                 expect_value(self, formats, expected_formats, None)
 
+    def test_parse_mpd_formats(self):
+        _TEST_CASES = [
+            (
+                # https://github.com/rg3/youtube-dl/issues/13919
+                'float_duration',
+                'http://unknown/manifest.mpd',
+                [{
+                    'manifest_url': 'http://unknown/manifest.mpd',
+                    'ext': 'mp4',
+                    'format_id': '318597',
+                    'format_note': 'DASH video',
+                    'protocol': 'http_dash_segments',
+                    'acodec': 'none',
+                    'vcodec': 'avc1.42001f',
+                    'tbr': 318.597,
+                    'width': 340,
+                    'height': 192,
+                }, {
+                    'manifest_url': 'http://unknown/manifest.mpd',
+                    'ext': 'mp4',
+                    'format_id': '638590',
+                    'format_note': 'DASH video',
+                    'protocol': 'http_dash_segments',
+                    'acodec': 'none',
+                    'vcodec': 'avc1.42001f',
+                    'tbr': 638.59,
+                    'width': 512,
+                    'height': 288,
+                }, {
+                    'manifest_url': 'http://unknown/manifest.mpd',
+                    'ext': 'mp4',
+                    'format_id': '1022565',
+                    'format_note': 'DASH video',
+                    'protocol': 'http_dash_segments',
+                    'acodec': 'none',
+                    'vcodec': 'avc1.4d001f',
+                    'tbr': 1022.565,
+                    'width': 688,
+                    'height': 384,
+                }, {
+                    'manifest_url': 'http://unknown/manifest.mpd',
+                    'ext': 'mp4',
+                    'format_id': '2046506',
+                    'format_note': 'DASH video',
+                    'protocol': 'http_dash_segments',
+                    'acodec': 'none',
+                    'vcodec': 'avc1.4d001f',
+                    'tbr': 2046.506,
+                    'width': 1024,
+                    'height': 576,
+                }, {
+                    'manifest_url': 'http://unknown/manifest.mpd',
+                    'ext': 'mp4',
+                    'format_id': '3998017',
+                    'format_note': 'DASH video',
+                    'protocol': 'http_dash_segments',
+                    'acodec': 'none',
+                    'vcodec': 'avc1.640029',
+                    'tbr': 3998.017,
+                    'width': 1280,
+                    'height': 720,
+                }, {
+                    'manifest_url': 'http://unknown/manifest.mpd',
+                    'ext': 'mp4',
+                    'format_id': '5997485',
+                    'format_note': 'DASH video',
+                    'protocol': 'http_dash_segments',
+                    'acodec': 'none',
+                    'vcodec': 'avc1.640032',
+                    'tbr': 5997.485,
+                    'width': 1920,
+                    'height': 1080,
+                }]
+            ),
+        ]
+
+        for mpd_file, mpd_url, expected_formats in _TEST_CASES:
+            with io.open('./test/testdata/mpd/%s.mpd' % mpd_file,
+                         mode='r', encoding='utf-8') as f:
+                formats = self.ie._parse_mpd_formats(
+                    compat_etree_fromstring(f.read().encode('utf-8')),
+                    mpd_url=mpd_url)
+                self.ie._sort_formats(formats)
+                expect_value(self, formats, expected_formats, None)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/testdata/mpd/float_duration.mpd b/test/testdata/mpd/float_duration.mpd
new file mode 100644
index 000000000..8dc1d2d5e
--- /dev/null
+++ b/test/testdata/mpd/float_duration.mpd
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<MPD xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="urn:mpeg:dash:schema:mpd:2011" type="static" minBufferTime="PT2S" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" mediaPresentationDuration="PT6014S">
+	<Period bitstreamSwitching="true">
+		<AdaptationSet mimeType="audio/mp4" codecs="mp4a.40.2" startWithSAP="1" segmentAlignment="true">
+			<SegmentTemplate timescale="1000000" presentationTimeOffset="0" initialization="ai_$RepresentationID$.mp4d" media="a_$RepresentationID$_$Number$.mp4d" duration="2000000.0" startNumber="0"></SegmentTemplate>
+			<Representation id="318597" bandwidth="61587"></Representation>
+		</AdaptationSet>
+		<AdaptationSet mimeType="video/mp4" startWithSAP="1" segmentAlignment="true">
+			<SegmentTemplate timescale="1000000" presentationTimeOffset="0" initialization="vi_$RepresentationID$.mp4d" media="v_$RepresentationID$_$Number$.mp4d" duration="2000000.0" startNumber="0"></SegmentTemplate>
+			<Representation id="318597" codecs="avc1.42001f" width="340" height="192" bandwidth="318597"></Representation>
+			<Representation id="638590" codecs="avc1.42001f" width="512" height="288" bandwidth="638590"></Representation>
+			<Representation id="1022565" codecs="avc1.4d001f" width="688" height="384" bandwidth="1022565"></Representation>
+			<Representation id="2046506" codecs="avc1.4d001f" width="1024" height="576" bandwidth="2046506"></Representation>
+			<Representation id="3998017" codecs="avc1.640029" width="1280" height="720" bandwidth="3998017"></Representation>
+			<Representation id="5997485" codecs="avc1.640032" width="1920" height="1080" bandwidth="5997485"></Representation>
+		</AdaptationSet>
+	</Period>
+</MPD>
\ No newline at end of file
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 7fe888462..e747258aa 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1786,7 +1786,7 @@ class InfoExtractor(object):
                     ms_info['timescale'] = int(timescale)
                 segment_duration = source.get('duration')
                 if segment_duration:
-                    ms_info['segment_duration'] = int(segment_duration)
+                    ms_info['segment_duration'] = float(segment_duration)
 
             def extract_Initialization(source):
                 initialization = source.find(_add_ns('Initialization'))

From a1aa6596626a98d068780f092367b87398840c47 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Aug 2017 23:03:42 +0700
Subject: [PATCH 1220/1696] [periscope] Renew HLS extraction (closes #13917)

---
 youtube_dl/extractor/periscope.py | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index bfa12edc9..e5e08538c 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -80,18 +80,24 @@ class PeriscopeIE(PeriscopeBaseIE):
         stream = self._call_api(
             'getAccessPublic', {'broadcast_id': token}, token)
 
+        video_urls = set()
         formats = []
-        for format_id in ('replay', 'rtmp', 'hls', 'https_hls'):
+        for format_id in ('replay', 'rtmp', 'hls', 'https_hls', 'lhls', 'lhlsweb'):
             video_url = stream.get(format_id + '_url')
-            if not video_url:
+            if not video_url or video_url in video_urls:
                 continue
-            f = {
+            video_urls.add(video_url)
+            if format_id != 'rtmp':
+                formats.extend(self._extract_m3u8_formats(
+                    video_url, token, 'mp4',
+                    entry_protocol='m3u8_native'
+                    if state in ('ended', 'timed_out') else 'm3u8',
+                    m3u8_id=format_id, fatal=False))
+                continue
+            formats.append({
                 'url': video_url,
                 'ext': 'flv' if format_id == 'rtmp' else 'mp4',
-            }
-            if format_id != 'rtmp':
-                f['protocol'] = 'm3u8_native' if state in ('ended', 'timed_out') else 'm3u8'
-            formats.append(f)
+            })
         self._sort_formats(formats)
 
         return {

From 25a6e769a1af3a79f439369fb683a1d487777cb9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 17 Aug 2017 16:39:57 +0800
Subject: [PATCH 1221/1696] [qqmusic] Fix tests and cleanup

---
 youtube_dl/extractor/qqmusic.py | 89 ++++++++++++++++++---------------
 1 file changed, 48 insertions(+), 41 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 38f4b2cab..62f986050 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -7,11 +7,10 @@ import time
 
 from .common import InfoExtractor
 from ..utils import (
-    sanitized_Request,
-    strip_jsonp,
-    unescapeHTML,
     clean_html,
     ExtractorError,
+    strip_jsonp,
+    unescapeHTML,
 )
 
 
@@ -21,14 +20,14 @@ class QQMusicIE(InfoExtractor):
     _VALID_URL = r'https?://y\.qq\.com/n/yqq/song/(?P<id>[0-9A-Za-z]+)\.html'
     _TESTS = [{
         'url': 'https://y.qq.com/n/yqq/song/004295Et37taLD.html',
-        'md5': '9ce1c1c8445f561506d2e3cfb0255705',
+        'md5': '5f1e6cea39e182857da7ffc5ef5e6bb8',
         'info_dict': {
             'id': '004295Et37taLD',
             'ext': 'mp3',
             'title': '可惜没如果',
             'release_date': '20141227',
             'creator': '林俊杰',
-            'description': 'md5:d327722d0361576fde558f1ac68a7065',
+            'description': 'md5:d85afb3051952ecc50a1ee8a286d1eac',
             'thumbnail': r're:^https?://.*\.jpg$',
         }
     }, {
@@ -53,7 +52,7 @@ class QQMusicIE(InfoExtractor):
             'title': 'Shadows Over Transylvania',
             'release_date': '19970225',
             'creator': 'Dark Funeral',
-            'description': 'md5:ed14d5bd7ecec19609108052c25b2c11',
+            'description': 'md5:c9b20210587cbcd6836a1c597bab4525',
             'thumbnail': r're:^https?://.*\.jpg$',
         },
         'params': {
@@ -105,7 +104,7 @@ class QQMusicIE(InfoExtractor):
             [r'albummid:\'([0-9a-zA-Z]+)\'', r'"albummid":"([0-9a-zA-Z]+)"'],
             detail_info_page, 'album mid', default=None)
         if albummid:
-            thumbnail_url = "http://i.gtimg.cn/music/photo/mid_album_500/%s/%s/%s.jpg" \
+            thumbnail_url = 'http://i.gtimg.cn/music/photo/mid_album_500/%s/%s/%s.jpg' \
                             % (albummid[-2:-1], albummid[-1], albummid)
 
         guid = self.m_r_get_ruin()
@@ -158,8 +157,19 @@ class QQPlaylistBaseIE(InfoExtractor):
 
     def get_singer_all_songs(self, singmid, num):
         return self._download_webpage(
-            r'https://c.y.qq.com/v8/fcg-bin/fcg_v8_singer_track_cp.fcg?format=json&inCharset=utf8&outCharset=utf-8&platform=yqq&needNewCode=0&singermid=%s&order=listen&begin=0&num=%s&songstatus=1' %
-            (singmid, num), singmid)
+            r'https://c.y.qq.com/v8/fcg-bin/fcg_v8_singer_track_cp.fcg', singmid,
+            query={
+                'format': 'json',
+                'inCharset': 'utf8',
+                'outCharset': 'utf-8',
+                'platform': 'yqq',
+                'needNewCode': 0,
+                'singermid': singmid,
+                'order': 'listen',
+                'begin': 0,
+                'num': num,
+                'songstatus': 1,
+            })
 
     def get_entries_from_page(self, singmid):
         entries = []
@@ -176,7 +186,8 @@ class QQPlaylistBaseIE(InfoExtractor):
         for item in json_obj_all_songs['data']['list']:
             if item['musicData'].get('songmid') is not None:
                 songmid = item['musicData']['songmid']
-                entries.append(self.url_result(r'https://y.qq.com/n/yqq/song/%s.html' % songmid, 'QQMusic', songmid))
+                entries.append(self.url_result(
+                    r'https://y.qq.com/n/yqq/song/%s.html' % songmid, 'QQMusic', songmid))
 
         return entries
 
@@ -192,7 +203,7 @@ class QQMusicSingerIE(QQPlaylistBaseIE):
             'title': '林俊杰',
             'description': 'md5:870ec08f7d8547c29c93010899103751',
         },
-        'playlist_count': 12,
+        'playlist_mincount': 12,
     }
 
     def _real_extract(self, url):
@@ -200,16 +211,16 @@ class QQMusicSingerIE(QQPlaylistBaseIE):
 
         entries = self.get_entries_from_page(mid)
         singer_page = self._download_webpage(url, mid, 'Download singer page')
-        singer_name = self._html_search_regex(r"singername : '(.*?)'", singer_page, 'singer name', default=None)
+        singer_name = self._html_search_regex(
+            r"singername\s*:\s*'(.*?)'", singer_page, 'singer name', default=None)
         singer_desc = None
 
         if mid:
-            req = sanitized_Request(
-                'http://s.plcloud.music.qq.com/fcgi-bin/fcg_get_singer_desc.fcg?utf8=1&outCharset=utf-8&format=xml&singermid=%s' % mid)
-            req.add_header(
-                'Referer', 'https://y.qq.com/n/yqq/singer/')
             singer_desc_page = self._download_xml(
-                req, mid, 'Donwload singer description XML')
+                'http://s.plcloud.music.qq.com/fcgi-bin/fcg_get_singer_desc.fcg', mid,
+                'Donwload singer description XML',
+                query={'utf8': 1, 'outCharset': 'utf-8', 'format': 'xml', 'singermid': mid},
+                headers={'Referer': 'https://y.qq.com/n/yqq/singer/'})
 
             singer_desc = singer_desc_page.find('./data/info/desc').text
 
@@ -267,26 +278,25 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
     _TESTS = [{
         'url': 'https://y.qq.com/n/yqq/toplist/123.html',
         'info_dict': {
-            'id': 'global_123',
+            'id': '123',
             'title': '美国iTunes榜',
+            'description': 'md5:89db2335fdbb10678dee2d43fe9aba08',
         },
-        'playlist_count': 10,
+        'playlist_count': 100,
     }, {
         'url': 'https://y.qq.com/n/yqq/toplist/3.html',
         'info_dict': {
-            'id': 'top_3',
+            'id': '3',
             'title': '巅峰榜·欧美',
-            'description': 'QQ音乐巅峰榜·欧美根据用户收听行为自动生成，集结当下最流行的欧美新歌！:更新时间：每周四22点|统'
-                           '计周期：一周（上周四至本周三）|统计对象：三个月内发行的欧美歌曲|统计数量：100首|统计算法：根据'
-                           '歌曲在一周内的有效播放次数，由高到低取前100名（同一歌手最多允许5首歌曲同时上榜）|有效播放次数：'
-                           '登录用户完整播放一首歌曲，记为一次有效播放；同一用户收听同一首歌曲，每天记录为1次有效播放'
+            'description': 'md5:5a600d42c01696b26b71f8c4d43407da',
         },
         'playlist_count': 100,
     }, {
         'url': 'https://y.qq.com/n/yqq/toplist/106.html',
         'info_dict': {
-            'id': 'global_106',
+            'id': '106',
             'title': '韩国Mnet榜',
+            'description': 'md5:cb84b325215e1d21708c615cac82a6e7',
         },
         'playlist_count': 50,
     }]
@@ -298,16 +308,14 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
         num_id = list_id
 
         toplist_json = self._download_json(
-            'http://i.y.qq.com/v8/fcg-bin/fcg_v8_toplist_cp.fcg?type=%s&topid=%s&format=json'
-            % (list_type, num_id),
-            list_id, 'Download toplist page')
+            'http://i.y.qq.com/v8/fcg-bin/fcg_v8_toplist_cp.fcg', list_id,
+            note='Download toplist page',
+            query={'type': 'toplist', 'topid': list_id, 'format': 'json'})
 
-        entries = [
-            self.url_result(
-                'https://y.qq.com/n/yqq/song/' + song['data']['songmid'] + '.html', 'QQMusic',
-                song['data']['songmid']
-            ) for song in toplist_json['songlist']
-        ]
+        entries = [self.url_result(
+            'https://y.qq.com/n/yqq/song/' + song['data']['songmid'] + '.html', 'QQMusic',
+            song['data']['songmid'])
+            for song in toplist_json['songlist']]
 
         topinfo = toplist_json.get('topinfo', {})
         list_name = topinfo.get('ListName')
@@ -343,8 +351,9 @@ class QQMusicPlaylistIE(QQPlaylistBaseIE):
         list_id = self._match_id(url)
 
         list_json = self._download_json(
-            'http://i.y.qq.com/qzone-music/fcg-bin/fcg_ucc_getcdinfo_byids_cp.fcg?type=1&json=1&utf8=1&onlysong=0&disstid=%s'
-            % list_id, list_id, 'Download list page',
+            'http://i.y.qq.com/qzone-music/fcg-bin/fcg_ucc_getcdinfo_byids_cp.fcg',
+            list_id, 'Download list page',
+            query={'type': 1, 'json': 1, 'utf8': 1, 'onlysong': 0, 'disstid': list_id},
             transform_source=strip_jsonp)
         if not len(list_json.get('cdlist', [])):
             if list_json.get('code'):
@@ -354,11 +363,9 @@ class QQMusicPlaylistIE(QQPlaylistBaseIE):
             raise ExtractorError('Unable to get playlist info')
 
         cdlist = list_json['cdlist'][0]
-        entries = [
-            self.url_result(
-                'https://y.qq.com/n/yqq/song/' + song['songmid'] + '.html', 'QQMusic', song['songmid']
-            ) for song in cdlist['songlist']
-        ]
+        entries = [self.url_result(
+            'https://y.qq.com/n/yqq/song/' + song['songmid'] + '.html', 'QQMusic', song['songmid'])
+            for song in cdlist['songlist']]
 
         list_name = cdlist.get('dissname')
         list_description = clean_html(unescapeHTML(cdlist.get('desc')))

From 12f5304556343fafb6a38ad5b4d5ef9fc908f15c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 17 Aug 2017 16:40:56 +0800
Subject: [PATCH 1222/1696] [ChangeLog] Add entry for #13805

---
 ChangeLog | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 6bafb1e8f..5b897735e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,10 @@
+version <unreleased>
+
+Extractors
+
++ [qqmusic] Support new URL schemes (#13805)
+
+
 version 2017.08.13
 
 Core

From bfabd17b33d47f1e973121483623768010880845 Mon Sep 17 00:00:00 2001
From: Genki Sky <genki@genki.is>
Date: Tue, 8 Aug 2017 22:49:57 -0400
Subject: [PATCH 1223/1696] Add new extractor

---
 youtube_dl/extractor/clippit.py    | 74 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  1 +
 2 files changed, 75 insertions(+)
 create mode 100644 youtube_dl/extractor/clippit.py

diff --git a/youtube_dl/extractor/clippit.py b/youtube_dl/extractor/clippit.py
new file mode 100644
index 000000000..a1a7a774c
--- /dev/null
+++ b/youtube_dl/extractor/clippit.py
@@ -0,0 +1,74 @@
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    parse_iso8601,
+    qualities,
+)
+
+import re
+
+
+class ClippitIE(InfoExtractor):
+
+    _VALID_URL = r'https?://(?:www\.)?clippituser\.tv/c/(?P<id>[a-z]+)'
+    _TEST = {
+        'url': 'https://www.clippituser.tv/c/evmgm',
+        'md5': '963ae7a59a2ec4572ab8bf2f2d2c5f09',
+        'info_dict': {
+            'id': 'evmgm',
+            'ext': 'mp4',
+            'title': 'Bye bye Brutus. #BattleBots  - Clippit',
+            'uploader': 'lizllove',
+            'uploader_url': 'https://www.clippituser.tv/p/lizllove',
+            'timestamp': 1472183818,
+            'upload_date': '20160826',
+            'description': 'BattleBots | ABC',
+            'thumbnail': r're:^https?://.*\.jpg$',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(r'<title.*>(.+?)</title>', webpage, 'title')
+
+        FORMATS = ('sd', 'hd')
+        quality = qualities(FORMATS)
+        formats = []
+        for format_id in FORMATS:
+            url = self._html_search_regex(r'data-%s-file="(.+?)"' % format_id,
+                                          webpage, 'url', fatal=False)
+            if not url:
+                continue
+            match = re.search(r'/(?P<height>\d+)\.mp4', url)
+            formats.append({
+                'url': url,
+                'format_id': format_id,
+                'quality': quality(format_id),
+                'height': int(match.group('height')) if match else None,
+            })
+
+        uploader = self._html_search_regex(r'class="username".*>\s+(.+?)\n',
+                                           webpage, 'uploader', fatal=False)
+        uploader_url = ('https://www.clippituser.tv/p/' + uploader
+                        if uploader else None)
+
+        timestamp = self._html_search_regex(r'datetime="(.+?)"',
+                                            webpage, 'date', fatal=False)
+        thumbnail = self._html_search_regex(r'data-image="(.+?)"',
+                                            webpage, 'thumbnail', fatal=False)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'uploader': uploader,
+            'uploader_url': uploader_url,
+            'timestamp': parse_iso8601(timestamp),
+            'description': self._og_search_description(webpage),
+            'thumbnail': thumbnail,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index fb79a1736..ccfa14e7a 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -187,6 +187,7 @@ from .chirbit import (
 from .cinchcast import CinchcastIE
 from .cjsw import CJSWIE
 from .cliphunter import CliphunterIE
+from .clippit import ClippitIE
 from .cliprs import ClipRsIE
 from .clipsyndicate import ClipsyndicateIE
 from .closertotruth import CloserToTruthIE

From 7ddab7742cad2ff04ec087e3e1d19422c931782b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 17 Aug 2017 16:56:37 +0800
Subject: [PATCH 1224/1696] [ChangeLog] Add an entry for Genki Sky's patch

---
 ChangeLog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 5b897735e..cf7d1beb0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,7 @@
 version <unreleased>
 
 Extractors
-
++ [clippit] Add support for clippituser.tv
 + [qqmusic] Support new URL schemes (#13805)
 
 

From 5d28169747e34850fcb53760c77eccb7f3195ef2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 17 Aug 2017 21:21:17 +0800
Subject: [PATCH 1225/1696] Credit Genki Sky for clippit (bfabd17b33d)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 053159cc3..478c7872f 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -223,3 +223,4 @@ Jan Kundrát
 Giuseppe Fabiano
 Örn Guðjónsson
 Parmjit Virk
+Genki Sky

From 93d0583e34b0cd826f081a766b00381bb5fed52d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 17 Aug 2017 22:45:40 +0700
Subject: [PATCH 1226/1696] [pluralsight] Use RPC API for course extraction
 (closes #13937)

---
 youtube_dl/extractor/pluralsight.py | 52 ++++++++++++++++++++---------
 1 file changed, 36 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index d35f54ce8..f6a9131b1 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -18,6 +18,7 @@ from ..utils import (
     parse_duration,
     qualities,
     srt_subtitles_timecode,
+    try_get,
     update_url_query,
     urlencode_postdata,
 )
@@ -26,6 +27,39 @@ from ..utils import (
 class PluralsightBaseIE(InfoExtractor):
     _API_BASE = 'https://app.pluralsight.com'
 
+    def _download_course(self, course_id, url, display_id):
+        try:
+            return self._download_course_rpc(course_id, url, display_id)
+        except ExtractorError:
+            # Old API fallback
+            return self._download_json(
+                'https://app.pluralsight.com/player/user/api/v1/player/payload',
+                display_id, data=urlencode_postdata({'courseId': course_id}),
+                headers={'Referer': url})
+
+    def _download_course_rpc(self, course_id, url, display_id):
+        response = self._download_json(
+            '%s/player/functions/rpc' % self._API_BASE, display_id,
+            'Downloading course JSON',
+            data=json.dumps({
+                'fn': 'bootstrapPlayer',
+                'payload': {
+                    'courseId': course_id,
+                },
+            }).encode('utf-8'),
+            headers={
+                'Content-Type': 'application/json;charset=utf-8',
+                'Referer': url,
+            })
+
+        course = try_get(response, lambda x: x['payload']['course'], dict)
+        if course:
+            return course
+
+        raise ExtractorError(
+            '%s said: %s' % (self.IE_NAME, response['error']['message']),
+            expected=True)
+
 
 class PluralsightIE(PluralsightBaseIE):
     IE_NAME = 'pluralsight'
@@ -162,10 +196,7 @@ class PluralsightIE(PluralsightBaseIE):
 
         display_id = '%s-%s' % (name, clip_id)
 
-        course = self._download_json(
-            'https://app.pluralsight.com/player/user/api/v1/player/payload',
-            display_id, data=urlencode_postdata({'courseId': course_name}),
-            headers={'Referer': url})
+        course = self._download_course(course_name, url, display_id)
 
         collection = course['modules']
 
@@ -331,18 +362,7 @@ class PluralsightCourseIE(PluralsightBaseIE):
 
         # TODO: PSM cookie
 
-        course = self._download_json(
-            '%s/player/functions/rpc' % self._API_BASE, course_id,
-            'Downloading course JSON',
-            data=json.dumps({
-                'fn': 'bootstrapPlayer',
-                'payload': {
-                    'courseId': course_id,
-                }
-            }).encode('utf-8'),
-            headers={
-                'Content-Type': 'application/json;charset=utf-8'
-            })['payload']['course']
+        course = self._download_course(course_id, url, course_id)
 
         title = course['title']
         course_name = course['name']

From 5f5c7b92dda1da6a0f15af7e3999a6ff298a8c92 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 17 Aug 2017 23:14:46 +0700
Subject: [PATCH 1227/1696] [udemy] Fix paid course detection (#13943)

---
 youtube_dl/extractor/udemy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 3b02f43e3..207c4a6a7 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -74,7 +74,7 @@ class UdemyIE(InfoExtractor):
             return compat_urlparse.urljoin(base_url, url) if not url.startswith('http') else url
 
         checkout_url = unescapeHTML(self._search_regex(
-            r'href=(["\'])(?P<url>(?:https?://(?:www\.)?udemy\.com)?/payment/checkout/.+?)\1',
+            r'href=(["\'])(?P<url>(?:https?://(?:www\.)?udemy\.com)?/(?:payment|cart)/checkout/.+?)\1',
             webpage, 'checkout url', group='url', default=None))
         if checkout_url:
             raise ExtractorError(

From 5551d7714d53caaaae32cdedad11a0bdc95efcf1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 17 Aug 2017 23:57:48 +0700
Subject: [PATCH 1228/1696] [generic] Convert redirect URLs to unicode strings
 (closes #13951)

---
 youtube_dl/extractor/generic.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index eff5fbfe8..d2fb2627d 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2015,7 +2015,7 @@ class GenericIE(InfoExtractor):
 
         if head_response is not False:
             # Check for redirect
-            new_url = head_response.geturl()
+            new_url = compat_str(head_response.geturl())
             if url != new_url:
                 self.report_following_redirect(new_url)
                 if force_videoid:
@@ -2116,7 +2116,7 @@ class GenericIE(InfoExtractor):
             elif re.match(r'(?i)^(?:{[^}]+})?MPD$', doc.tag):
                 info_dict['formats'] = self._parse_mpd_formats(
                     doc, video_id,
-                    mpd_base_url=full_response.geturl().rpartition('/')[0],
+                    mpd_base_url=compat_str(full_response.geturl()).rpartition('/')[0],
                     mpd_url=url)
                 self._sort_formats(info_dict['formats'])
                 return info_dict

From a5ac0c475589fd1dcd3ba04802f28828c24be6c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 17 Aug 2017 23:59:12 +0700
Subject: [PATCH 1229/1696] [YoutubeDL] Sanitize byte string format URLs
 (#13951)

---
 youtube_dl/YoutubeDL.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index df7378f83..5f4c93ea3 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1483,12 +1483,14 @@ class YoutubeDL(object):
 
         def is_wellformed(f):
             url = f.get('url')
-            valid_url = url and isinstance(url, compat_str)
-            if not valid_url:
+            if not url:
                 self.report_warning(
                     '"url" field is missing or empty - skipping format, '
                     'there is an error in extractor')
-            return valid_url
+                return False
+            if isinstance(url, bytes):
+                sanitize_string_field(f, 'url')
+            return True
 
         # Filter out malformed formats for better extraction robustness
         formats = list(filter(is_wellformed, formats))

From c0892b2b465cff95d392eaa725e39bd47e4dff58 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Aug 2017 00:58:23 +0700
Subject: [PATCH 1230/1696] [arte] Detect unavailable videos (closes #13945)

---
 youtube_dl/extractor/arte.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 56baef29d..02613cf5d 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -9,12 +9,13 @@ from ..compat import (
     compat_urllib_parse_urlparse,
 )
 from ..utils import (
+    ExtractorError,
     find_xpath_attr,
-    unified_strdate,
     get_element_by_attribute,
     int_or_none,
     NO_DEFAULT,
     qualities,
+    unified_strdate,
 )
 
 # There are different sources of video in arte.tv, the extraction process
@@ -79,6 +80,13 @@ class ArteTVBaseIE(InfoExtractor):
         info = self._download_json(json_url, video_id)
         player_info = info['videoJsonPlayer']
 
+        vsr = player_info['VSR']
+
+        if not vsr and not player_info.get('VRU'):
+            raise ExtractorError(
+                'Video %s is not available' % player_info.get('VID') or video_id,
+                expected=True)
+
         upload_date_str = player_info.get('shootingDate')
         if not upload_date_str:
             upload_date_str = (player_info.get('VRA') or player_info.get('VDA') or '').split(' ')[0]
@@ -107,7 +115,7 @@ class ArteTVBaseIE(InfoExtractor):
         langcode = LANGS.get(lang, lang)
 
         formats = []
-        for format_id, format_dict in player_info['VSR'].items():
+        for format_id, format_dict in vsr.items():
             f = dict(format_dict)
             versionCode = f.get('versionCode')
             l = re.escape(langcode)

From 4a919103651905d4e5954c5d655b45055384e283 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Aug 2017 01:00:07 +0700
Subject: [PATCH 1231/1696] [qqmusic:toplist] PEP 8

---
 youtube_dl/extractor/qqmusic.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 62f986050..084308aeb 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -304,9 +304,6 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
     def _real_extract(self, url):
         list_id = self._match_id(url)
 
-        list_type = 'toplist'
-        num_id = list_id
-
         toplist_json = self._download_json(
             'http://i.y.qq.com/v8/fcg-bin/fcg_v8_toplist_cp.fcg', list_id,
             note='Download toplist page',

From 2738965d98e1883a781a1e9743de0af086c5acd7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Aug 2017 01:03:20 +0700
Subject: [PATCH 1232/1696] [ChangeLog] Actualize

---
 ChangeLog | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index cf7d1beb0..298e0b059 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,8 +1,19 @@
 version <unreleased>
 
+Core
+* [YoutubeDL] Sanitize byte string format URLs (#13951)
++ [extractor/common] Add support for float durations in _parse_mpd_formats
+  (#13919)
+
 Extractors
+* [arte] Detect unavailable videos (#13945)
+* [generic] Convert redirect URLs to unicode strings (#13951)
+* [udemy] Fix paid course detection (#13943)
+* [pluralsight] Use RPC API for course extraction (#13937)
 + [clippit] Add support for clippituser.tv
 + [qqmusic] Support new URL schemes (#13805)
+* [periscope] Renew HLS extraction (#13917)
+* [mixcloud] Extract decrypt key
 
 
 version 2017.08.13

From ea004d34f83fd7dd9a00fc3e2deb5a101aff6ea7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Aug 2017 01:05:27 +0700
Subject: [PATCH 1233/1696] release 2017.08.18

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 1 +
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 3bd61e0a6..66dd4c480 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.08.13*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.08.13**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.08.18*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.08.18**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.08.13
+[debug] youtube-dl version 2017.08.18
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 298e0b059..9a0fad673 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.08.18
 
 Core
 * [YoutubeDL] Sanitize byte string format URLs (#13951)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index cc442742f..1991975cc 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -156,6 +156,7 @@
  - **Cinchcast**
  - **CJSW**
  - **cliphunter**
+ - **Clippit**
  - **ClipRs**
  - **Clipsyndicate**
  - **CloserToTruth**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index da855a602..4358cd3f2 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.08.13'
+__version__ = '2017.08.18'

From d14d9d8903a532e346dffc3b83730045f18f2c28 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Aug 2017 23:31:42 +0700
Subject: [PATCH 1234/1696] [mixcloud] Fix extraction (closes #13958)

---
 youtube_dl/extractor/mixcloud.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index fcf7beeb2..798968ae3 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -91,12 +91,14 @@ class MixcloudIE(InfoExtractor):
             if js_url:
                 js = self._download_webpage(js_url, track_id, fatal=False)
                 if js:
-                    key = self._search_regex(
-                        r'player\s*:\s*{.*?\bvalue\s*:\s*(["\'])(?P<key>(?:(?!\1).)+)\1',
-                        js, 'key', default=None, group='key')
-                    if key and isinstance(key, compat_str):
-                        self._keys.insert(0, key)
-                        self._current_key = key
+                    KEY_RE_TEMPLATE = r'player\s*:\s*{.*?\b%s\s*:\s*(["\'])(?P<key>(?:(?!\1).)+)\1'
+                    for key_name in ('value', 'key_value'):
+                        key = self._search_regex(
+                            KEY_RE_TEMPLATE % key_name, js, 'key',
+                            default=None, group='key')
+                        if key and isinstance(key, compat_str):
+                            self._keys.insert(0, key)
+                            self._current_key = key
 
         message = self._html_search_regex(
             r'(?s)<div[^>]+class="global-message cloudcast-disabled-notice-light"[^>]*>(.+?)<(?:a|/div)',

From f5469da9e6e259c1690c7ef54f1da1c19f65036f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Aug 2017 19:48:20 +0700
Subject: [PATCH 1235/1696] [laola1tv] Add support for tv.ittf.com (closes
 #13965)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/laola1tv.py   | 18 ++++++++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index ccfa14e7a..bda6826f1 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -509,6 +509,7 @@ from .la7 import LA7IE
 from .laola1tv import (
     Laola1TvEmbedIE,
     Laola1TvIE,
+    ITTFIE,
 )
 from .lci import LCIIE
 from .lcp import (
diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py
index 1f91ba017..c7f813370 100644
--- a/youtube_dl/extractor/laola1tv.py
+++ b/youtube_dl/extractor/laola1tv.py
@@ -215,3 +215,21 @@ class Laola1TvIE(Laola1TvEmbedIE):
             'formats': formats,
             'is_live': is_live,
         }
+
+
+class ITTFIE(InfoExtractor):
+    _VALID_URL = r'https?://tv\.ittf\.com/video/[^/]+/(?P<id>\d+)'
+    _TEST = {
+        'url': 'https://tv.ittf.com/video/peng-wang-wei-matsudaira-kenta/951802',
+        'only_matching': True,
+    }
+
+    def _real_extract(self, url):
+        return self.url_result(
+            update_url_query('https://www.laola1.tv/titanplayer.php', {
+                'videoid': self._match_id(url),
+                'type': 'V',
+                'lang': 'en',
+                'portal': 'int',
+                'customer': 1024,
+            }), Laola1TvEmbedIE.ie_key())

From 95f3f7c20a05e7ac490e768b8470b20538ef8581 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 19 Aug 2017 21:40:53 +0800
Subject: [PATCH 1236/1696] [utils] Fix unescapeHTML for misformed string like
 "&a&quot;" (#13935)

---
 ChangeLog           | 6 ++++++
 test/test_utils.py  | 1 +
 youtube_dl/utils.py | 2 +-
 3 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 9a0fad673..9eab4d1e7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Core
+* [utils] Fix unescapeHTML for misformed string like "&a&quot;" (#13935)
+
+
 version 2017.08.18
 
 Core
diff --git a/test/test_utils.py b/test/test_utils.py
index 2aab16b97..e50f3764e 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -279,6 +279,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(unescapeHTML('&#47;'), '/')
         self.assertEqual(unescapeHTML('&eacute;'), 'é')
         self.assertEqual(unescapeHTML('&#2013266066;'), '&#2013266066;')
+        self.assertEqual(unescapeHTML('&a&quot;'), '&a"')
         # HTML5 entities
         self.assertEqual(unescapeHTML('&period;&apos;'), '.\'')
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index c9cbd5842..2554a2abd 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -596,7 +596,7 @@ def unescapeHTML(s):
     assert type(s) == compat_str
 
     return re.sub(
-        r'&([^;]+;)', lambda m: _htmlentity_transform(m.group(1)), s)
+        r'&([^&;]+;)', lambda m: _htmlentity_transform(m.group(1)), s)
 
 
 def get_subprocess_encoding():

From f8f18f332f235bcfa2f8fc161887e0eef283fec0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 19 Aug 2017 21:44:47 +0800
Subject: [PATCH 1237/1696] [cda] Fix extraction (closes #13935)

---
 ChangeLog                   | 1 +
 youtube_dl/extractor/cda.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 9eab4d1e7..6c32747c4 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Core
+* [cda] Fix extraction (closes #13935)
 * [utils] Fix unescapeHTML for misformed string like "&a&quot;" (#13935)
 
 
diff --git a/youtube_dl/extractor/cda.py b/youtube_dl/extractor/cda.py
index 78b7a923c..0c3af23d5 100755
--- a/youtube_dl/extractor/cda.py
+++ b/youtube_dl/extractor/cda.py
@@ -124,7 +124,7 @@ class CDAIE(InfoExtractor):
         }
 
         def extract_format(page, version):
-            json_str = self._search_regex(
+            json_str = self._html_search_regex(
                 r'player_data=(\\?["\'])(?P<player_data>.+?)\1', page,
                 '%s player_json' % version, fatal=False, group='player_data')
             if not json_str:

From 09747ba7663a9c6f89530c7ffbd95cb4776db6bf Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 19 Aug 2017 22:27:53 +0800
Subject: [PATCH 1238/1696] [liveleak] Support another liveleak embedding
 pattern (closes #13336)

---
 ChangeLog                          |  3 +++
 youtube_dl/extractor/extractors.py |  5 ++++-
 youtube_dl/extractor/generic.py    | 23 +++++++++++++++-----
 youtube_dl/extractor/liveleak.py   | 35 +++++++++++++++++++++++++-----
 4 files changed, 55 insertions(+), 11 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 6c32747c4..4c7997b2e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -4,6 +4,9 @@ Core
 * [cda] Fix extraction (closes #13935)
 * [utils] Fix unescapeHTML for misformed string like "&a&quot;" (#13935)
 
+Extractors
++ [liveleak] Support another liveleak embedding pattern (#13336)
+
 
 version 2017.08.18
 
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index bda6826f1..17048fd6e 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -537,7 +537,10 @@ from .limelight import (
     LimelightChannelListIE,
 )
 from .litv import LiTVIE
-from .liveleak import LiveLeakIE
+from .liveleak import (
+    LiveLeakIE,
+    LiveLeakEmbedIE,
+)
 from .livestream import (
     LivestreamIE,
     LivestreamOriginalIE,
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index d2fb2627d..49b00b87e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1519,14 +1519,27 @@ class GenericIE(InfoExtractor):
         # LiveLeak embed
         {
             'url': 'http://www.wykop.pl/link/3088787/',
-            'md5': 'ace83b9ed19b21f68e1b50e844fdf95d',
+            'md5': '7619da8c820e835bef21a1efa2a0fc71',
             'info_dict': {
                 'id': '874_1459135191',
                 'ext': 'mp4',
                 'title': 'Man shows poor quality of new apartment building',
                 'description': 'The wall is like a sand pile.',
                 'uploader': 'Lake8737',
-            }
+            },
+            'add_ie': [LiveLeakIE.ie_key()],
+        },
+        # Another LiveLeak embed pattern (#13336)
+        {
+            'url': 'https://milo.yiannopoulos.net/2017/06/concealed-carry-robbery/',
+            'info_dict': {
+                'id': '2eb_1496309988',
+                'ext': 'mp4',
+                'title': 'Thief robs place where everyone was armed',
+                'description': 'md5:694d73ee79e535953cf2488562288eee',
+                'uploader': 'brazilwtf',
+            },
+            'add_ie': [LiveLeakIE.ie_key()],
         },
         # Duplicated embedded video URLs
         {
@@ -2757,9 +2770,9 @@ class GenericIE(InfoExtractor):
                 self._proto_relative_url(instagram_embed_url), InstagramIE.ie_key())
 
         # Look for LiveLeak embeds
-        liveleak_url = LiveLeakIE._extract_url(webpage)
-        if liveleak_url:
-            return self.url_result(liveleak_url, 'LiveLeak')
+        liveleak_urls = LiveLeakIE._extract_urls(webpage)
+        if liveleak_urls:
+            return self.playlist_from_matches(liveleak_urls, video_id, video_title)
 
         # Look for 3Q SDN embeds
         threeqsdn_url = ThreeQSDNIE._extract_url(webpage)
diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index b2247a84d..d23eaa355 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -75,12 +75,10 @@ class LiveLeakIE(InfoExtractor):
     }]
 
     @staticmethod
-    def _extract_url(webpage):
-        mobj = re.search(
-            r'<iframe[^>]+src="https?://(?:\w+\.)?liveleak\.com/ll_embed\?(?:.*?)i=(?P<id>[\w_]+)(?:.*)',
+    def _extract_urls(webpage):
+        return re.findall(
+            r'<iframe[^>]+src="(https?://(?:\w+\.)?liveleak\.com/ll_embed\?[^"]*[if]=[\w_]+[^"]+)"',
             webpage)
-        if mobj:
-            return 'http://www.liveleak.com/view?i=%s' % mobj.group('id')
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -131,3 +129,30 @@ class LiveLeakIE(InfoExtractor):
         })
 
         return info_dict
+
+
+class LiveLeakEmbedIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?liveleak\.com/ll_embed\?.*?\b(?P<kind>[if])=(?P<id>[\w_]+)'
+
+    # See generic.py for actual test cases
+    _TESTS = [{
+        'url': 'https://www.liveleak.com/ll_embed?i=874_1459135191',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.liveleak.com/ll_embed?f=ab065df993c1',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        kind, video_id = mobj.group('kind', 'id')
+
+        if kind == 'f':
+            webpage = self._download_webpage(url, video_id)
+            liveleak_url = self._search_regex(
+                r'logourl\s*:\s*(?P<q1>[\'"])(?P<url>%s)(?P=q1)' % LiveLeakIE._VALID_URL,
+                webpage, 'LiveLeak URL', group='url')
+        elif kind == 'i':
+            liveleak_url = 'http://www.liveleak.com/view?i=%s' % video_id
+
+        return self.url_result(liveleak_url, ie=LiveLeakIE.ie_key())

From e2481b9b6e621e43fd77e395fd2283ce262b71f3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 19 Aug 2017 22:28:58 +0800
Subject: [PATCH 1239/1696] [ChangeLog] Fix

---
 ChangeLog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 4c7997b2e..320609a4f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,11 +1,11 @@
 version <unreleased>
 
 Core
-* [cda] Fix extraction (closes #13935)
 * [utils] Fix unescapeHTML for misformed string like "&a&quot;" (#13935)
 
 Extractors
 + [liveleak] Support another liveleak embedding pattern (#13336)
+* [cda] Fix extraction (#13935)
 
 
 version 2017.08.18

From 381ad4f30998443fabc4c8633caa548685f49c6b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 19 Aug 2017 22:48:00 +0800
Subject: [PATCH 1240/1696] [liveleak] Support multi-video pages (closes #6542)

---
 ChangeLog                        |  1 +
 youtube_dl/extractor/liveleak.py | 43 ++++++++++++++++++++------------
 2 files changed, 28 insertions(+), 16 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 320609a4f..c07cb9648 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -4,6 +4,7 @@ Core
 * [utils] Fix unescapeHTML for misformed string like "&a&quot;" (#13935)
 
 Extractors
++ [liveleak] Support multi-video pages (#6542)
 + [liveleak] Support another liveleak embedding pattern (#13336)
 * [cda] Fix extraction (#13935)
 
diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index d23eaa355..246aac576 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -72,6 +72,13 @@ class LiveLeakIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        'url': 'https://www.liveleak.com/view?i=677_1439397581',
+        'info_dict': {
+            'id': '677_1439397581',
+            'title': 'Fuel Depot in China Explosion caught on video',
+        },
+        'playlist_count': 3,
     }]
 
     @staticmethod
@@ -109,26 +116,30 @@ class LiveLeakIE(InfoExtractor):
                 'age_limit': age_limit,
             }
 
-        info_dict = entries[0]
+        for idx, info_dict in enumerate(entries):
+            for a_format in info_dict['formats']:
+                if not a_format.get('height'):
+                    a_format['height'] = int_or_none(self._search_regex(
+                        r'([0-9]+)p\.mp4', a_format['url'], 'height label',
+                        default=None))
 
-        for a_format in info_dict['formats']:
-            if not a_format.get('height'):
-                a_format['height'] = int_or_none(self._search_regex(
-                    r'([0-9]+)p\.mp4', a_format['url'], 'height label',
-                    default=None))
+            self._sort_formats(info_dict['formats'])
 
-        self._sort_formats(info_dict['formats'])
+            # Don't append entry ID for one-video pages to keep backward compatibility
+            if len(entries) > 1:
+                info_dict['id'] = '%s_%s' % (video_id, idx + 1)
+            else:
+                info_dict['id'] = video_id
 
-        info_dict.update({
-            'id': video_id,
-            'title': video_title,
-            'description': video_description,
-            'uploader': video_uploader,
-            'age_limit': age_limit,
-            'thumbnail': video_thumbnail,
-        })
+            info_dict.update({
+                'title': video_title,
+                'description': video_description,
+                'uploader': video_uploader,
+                'age_limit': age_limit,
+                'thumbnail': video_thumbnail,
+            })
 
-        return info_dict
+        return self.playlist_result(entries, video_id, video_title)
 
 
 class LiveLeakEmbedIE(InfoExtractor):

From d3d45e0a451bab2cc36181bb50bf3c129a7a5ec4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Aug 2017 23:54:15 +0700
Subject: [PATCH 1241/1696] [bbccouk] Add support for events URLs (closes
 #13893)

---
 youtube_dl/extractor/bbc.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 79ded6ba1..911ae6780 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -37,7 +37,8 @@ class BBCCoUkIE(InfoExtractor):
                             programmes/(?!articles/)|
                             iplayer(?:/[^/]+)?/(?:episode/|playlist/)|
                             music/(?:clips|audiovideo/popular)[/#]|
-                            radio/player/
+                            radio/player/|
+                            events/[^/]+/play/[^/]+/
                         )
                         (?P<id>%s)(?!/(?:episodes|broadcasts|clips))
                     ''' % _ID_REGEX

From 305d99f0bd1effc0e164792199bf93a872da2962 Mon Sep 17 00:00:00 2001
From: "Bernhard M. Wiedemann" <bwiedemann@suse.de>
Date: Mon, 17 Jul 2017 13:49:09 +0200
Subject: [PATCH 1242/1696] [build] Override timestamps in zip file

to make build reproducible.
See https://reproducible-builds.org/ for why this is good

Copying files to not interfere with freshness detection.
---
 Makefile | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 84ccce2b3..41e3a683a 100644
--- a/Makefile
+++ b/Makefile
@@ -46,8 +46,15 @@ tar: youtube-dl.tar.gz
 pypi-files: youtube-dl.bash-completion README.txt youtube-dl.1 youtube-dl.fish
 
 youtube-dl: youtube_dl/*.py youtube_dl/*/*.py
-	zip --quiet youtube-dl youtube_dl/*.py youtube_dl/*/*.py
-	zip --quiet --junk-paths youtube-dl youtube_dl/__main__.py
+	mkdir -p zip
+	for d in youtube_dl youtube_dl/downloader youtube_dl/extractor youtube_dl/postprocessor ; do \
+	  mkdir -p zip/$$d ;\
+	  cp -a $$d/*.py zip/$$d/ ;\
+	done
+	touch -t 200001010101 zip/youtube_dl/*.py zip/youtube_dl/*/*.py
+	mv zip/youtube_dl/__main__.py zip/
+	cd zip ; zip --quiet ../youtube-dl youtube_dl/*.py youtube_dl/*/*.py __main__.py
+	rm -rf zip
 	echo '#!$(PYTHON)' > youtube-dl
 	cat youtube-dl.zip >> youtube-dl
 	rm youtube-dl.zip

From b359e977b9bdff704cd58f6f3b34185ecbe450e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Aug 2017 14:16:58 +0700
Subject: [PATCH 1243/1696] [extractor/common] Make HLS and DASH extraction non
 fatal in _parse_html5_media_entries (closes #13970)

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e747258aa..ceba4ca1c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2123,11 +2123,11 @@ class InfoExtractor(object):
                 formats = self._extract_m3u8_formats(
                     full_url, video_id, ext='mp4',
                     entry_protocol=m3u8_entry_protocol, m3u8_id=m3u8_id,
-                    preference=preference)
+                    preference=preference, fatal=False)
             elif ext == 'mpd':
                 is_plain_url = False
                 formats = self._extract_mpd_formats(
-                    full_url, video_id, mpd_id=mpd_id)
+                    full_url, video_id, mpd_id=mpd_id, fatal=False)
             else:
                 is_plain_url = True
                 formats = [{

From 8239c6791a36813cacc337c5c4a8801d181b8b54 Mon Sep 17 00:00:00 2001
From: Luca Steeb <contact@luca-steeb.com>
Date: Sun, 20 Aug 2017 09:32:33 -0700
Subject: [PATCH 1244/1696] [bandcamp:album] Extract track titles

---
 youtube_dl/extractor/bandcamp.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index 9ddb9af17..be41bd5a2 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -242,7 +242,12 @@ class BandcampAlbumIE(InfoExtractor):
             raise ExtractorError('The page doesn\'t contain any tracks')
         # Only tracks with duration info have songs
         entries = [
-            self.url_result(compat_urlparse.urljoin(url, t_path), ie=BandcampIE.ie_key())
+            self.url_result(
+                compat_urlparse.urljoin(url, t_path),
+                ie=BandcampIE.ie_key(),
+                video_title=self._search_regex(
+                    r'<span\b[^>]+\bitemprop=["\']name["\'][^>]*>([^<]+)',
+                    elem_content, 'track title', fatal=False))
             for elem_content, t_path in track_elements
             if self._html_search_meta('duration', elem_content, default=None)]
 

From 903d4d1625f59b6fb359a898fbb512cb2d6181e9 Mon Sep 17 00:00:00 2001
From: Alan Yee <alyee@ucsd.edu>
Date: Sun, 20 Aug 2017 09:35:39 -0700
Subject: [PATCH 1245/1696] [README.md] Switch to HTTPS URLs

---
 README.md | 50 +++++++++++++++++++++++++-------------------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/README.md b/README.md
index 0067184be..6f5d00df3 100644
--- a/README.md
+++ b/README.md
@@ -25,7 +25,7 @@ If you do not have curl, you can alternatively use a recent wget:
     sudo wget https://yt-dl.org/downloads/latest/youtube-dl -O /usr/local/bin/youtube-dl
     sudo chmod a+rx /usr/local/bin/youtube-dl
 
-Windows users can [download an .exe file](https://yt-dl.org/latest/youtube-dl.exe) and place it in any location on their [PATH](http://en.wikipedia.org/wiki/PATH_%28variable%29) except for `%SYSTEMROOT%\System32` (e.g. **do not** put in `C:\Windows\System32`).
+Windows users can [download an .exe file](https://yt-dl.org/latest/youtube-dl.exe) and place it in any location on their [PATH](https://en.wikipedia.org/wiki/PATH_%28variable%29) except for `%SYSTEMROOT%\System32` (e.g. **do not** put in `C:\Windows\System32`).
 
 You can also use pip:
 
@@ -33,7 +33,7 @@ You can also use pip:
     
 This command will update youtube-dl if you have already installed it. See the [pypi page](https://pypi.python.org/pypi/youtube_dl) for more information.
 
-OS X users can install youtube-dl with [Homebrew](http://brew.sh/):
+OS X users can install youtube-dl with [Homebrew](https://brew.sh/):
 
     brew install youtube-dl
 
@@ -458,7 +458,7 @@ You can also use `--config-location` if you want to use custom configuration fil
 
 ### Authentication with `.netrc` file
 
-You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every youtube-dl execution and prevent tracking plain text passwords in the shell command history. You can achieve this using a [`.netrc` file](http://stackoverflow.com/tags/.netrc/info) on a per extractor basis. For that you will need to create a `.netrc` file in your `$HOME` and restrict permissions to read/write by only you:
+You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every youtube-dl execution and prevent tracking plain text passwords in the shell command history. You can achieve this using a [`.netrc` file](https://stackoverflow.com/tags/.netrc/info) on a per extractor basis. For that you will need to create a `.netrc` file in your `$HOME` and restrict permissions to read/write by only you:
 ```
 touch $HOME/.netrc
 chmod a-rwx,u+rw $HOME/.netrc
@@ -485,7 +485,7 @@ The `-o` option allows users to indicate a template for the output file names.
 
 **tl;dr:** [navigate me to examples](#output-template-examples).
 
-The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "http://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences may be formatted according to [python string formatting operations](https://docs.python.org/2/library/stdtypes.html#string-formatting). For example, `%(NAME)s` or `%(NAME)05d`. To clarify, that is a percent symbol followed by a name in parentheses, followed by a formatting operations. Allowed names along with sequence type are:
+The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "https://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences may be formatted according to [python string formatting operations](https://docs.python.org/2/library/stdtypes.html#string-formatting). For example, `%(NAME)s` or `%(NAME)05d`. To clarify, that is a percent symbol followed by a name in parentheses, followed by a formatting operations. Allowed names along with sequence type are:
 
  - `id` (string): Video identifier
  - `title` (string): Video title
@@ -603,7 +603,7 @@ $ youtube-dl -o '%(uploader)s/%(playlist)s/%(playlist_index)s - %(title)s.%(ext)
 $ youtube-dl -u user -p password -o '~/MyVideos/%(playlist)s/%(chapter_number)s - %(chapter)s/%(title)s.%(ext)s' https://www.udemy.com/java-tutorial/
 
 # Download entire series season keeping each series and each season in separate directory under C:/MyVideos
-$ youtube-dl -o "C:/MyVideos/%(series)s/%(season_number)s - %(season)s/%(episode_number)s - %(episode)s.%(ext)s" http://videomore.ru/kino_v_detalayah/5_sezon/367617
+$ youtube-dl -o "C:/MyVideos/%(series)s/%(season_number)s - %(season)s/%(episode_number)s - %(episode)s.%(ext)s" https://videomore.ru/kino_v_detalayah/5_sezon/367617
 
 # Stream the video being downloaded to stdout
 $ youtube-dl -o - BaW_jenozKc
@@ -716,17 +716,17 @@ $ youtube-dl --dateafter 20000101 --datebefore 20091231
 
 ### How do I update youtube-dl?
 
-If you've followed [our manual installation instructions](http://rg3.github.io/youtube-dl/download.html), you can simply run `youtube-dl -U` (or, on Linux, `sudo youtube-dl -U`).
+If you've followed [our manual installation instructions](https://rg3.github.io/youtube-dl/download.html), you can simply run `youtube-dl -U` (or, on Linux, `sudo youtube-dl -U`).
 
 If you have used pip, a simple `sudo pip install -U youtube-dl` is sufficient to update.
 
-If you have installed youtube-dl using a package manager like *apt-get* or *yum*, use the standard system update mechanism to update. Note that distribution packages are often outdated. As a rule of thumb, youtube-dl releases at least once a month, and often weekly or even daily. Simply go to http://yt-dl.org/ to find out the current version. Unfortunately, there is nothing we youtube-dl developers can do if your distribution serves a really outdated version. You can (and should) complain to your distribution in their bugtracker or support forum.
+If you have installed youtube-dl using a package manager like *apt-get* or *yum*, use the standard system update mechanism to update. Note that distribution packages are often outdated. As a rule of thumb, youtube-dl releases at least once a month, and often weekly or even daily. Simply go to https://yt-dl.org to find out the current version. Unfortunately, there is nothing we youtube-dl developers can do if your distribution serves a really outdated version. You can (and should) complain to your distribution in their bugtracker or support forum.
 
 As a last resort, you can also uninstall the version installed by your package manager and follow our manual installation instructions. For that, remove the distribution's package, with a line like
 
     sudo apt-get remove -y youtube-dl
 
-Afterwards, simply follow [our manual installation instructions](http://rg3.github.io/youtube-dl/download.html):
+Afterwards, simply follow [our manual installation instructions](https://rg3.github.io/youtube-dl/download.html):
 
 ```
 sudo wget https://yt-dl.org/latest/youtube-dl -O /usr/local/bin/youtube-dl
@@ -766,11 +766,11 @@ Apparently YouTube requires you to pass a CAPTCHA test if you download too much.
 
 youtube-dl works fine on its own on most sites. However, if you want to convert video/audio, you'll need [avconv](https://libav.org/) or [ffmpeg](https://www.ffmpeg.org/). On some sites - most notably YouTube - videos can be retrieved in a higher quality format without sound. youtube-dl will detect whether avconv/ffmpeg is present and automatically pick the best option.
 
-Videos or video formats streamed via RTMP protocol can only be downloaded when [rtmpdump](https://rtmpdump.mplayerhq.hu/) is installed. Downloading MMS and RTSP videos requires either [mplayer](http://mplayerhq.hu/) or [mpv](https://mpv.io/) to be installed.
+Videos or video formats streamed via RTMP protocol can only be downloaded when [rtmpdump](https://rtmpdump.mplayerhq.hu/) is installed. Downloading MMS and RTSP videos requires either [mplayer](https://mplayerhq.hu/) or [mpv](https://mpv.io/) to be installed.
 
 ### I have downloaded a video but how can I play it?
 
-Once the video is fully downloaded, use any video player, such as [mpv](https://mpv.io/), [vlc](http://www.videolan.org/) or [mplayer](http://www.mplayerhq.hu/).
+Once the video is fully downloaded, use any video player, such as [mpv](https://mpv.io/), [vlc](https://www.videolan.org/) or [mplayer](https://www.mplayerhq.hu/).
 
 ### I extracted a video URL with `-g`, but it does not play on another machine / in my web browser.
 
@@ -845,10 +845,10 @@ Use the `-o` to specify an [output template](#output-template), for example `-o
 
 ### How do I download a video starting with a `-`?
 
-Either prepend `http://www.youtube.com/watch?v=` or separate the ID from the options with `--`:
+Either prepend `https://www.youtube.com/watch?v=` or separate the ID from the options with `--`:
 
     youtube-dl -- -wNyEUrxzFU
-    youtube-dl "http://www.youtube.com/watch?v=-wNyEUrxzFU"
+    youtube-dl "https://www.youtube.com/watch?v=-wNyEUrxzFU"
 
 ### How do I pass cookies to youtube-dl?
 
@@ -862,9 +862,9 @@ Passing cookies to youtube-dl is a good way to workaround login when a particula
 
 ### How do I stream directly to media player?
 
-You will first need to tell youtube-dl to stream media to stdout with `-o -`, and also tell your media player to read from stdin (it must be capable of this for streaming) and then pipe former to latter. For example, streaming to [vlc](http://www.videolan.org/) can be achieved with:
+You will first need to tell youtube-dl to stream media to stdout with `-o -`, and also tell your media player to read from stdin (it must be capable of this for streaming) and then pipe former to latter. For example, streaming to [vlc](https://www.videolan.org/) can be achieved with:
 
-    youtube-dl -o - "http://www.youtube.com/watch?v=BaW_jenozKcj" | vlc -
+    youtube-dl -o - "https://www.youtube.com/watch?v=BaW_jenozKcj" | vlc -
 
 ### How do I download only new videos from a playlist?
 
@@ -884,7 +884,7 @@ When youtube-dl detects an HLS video, it can download it either with the built-i
 
 When youtube-dl knows that one particular downloader works better for a given website, that downloader will be picked. Otherwise, youtube-dl will pick the best downloader for general compatibility, which at the moment happens to be ffmpeg. This choice may change in future versions of youtube-dl, with improvements of the built-in downloader and/or ffmpeg.
 
-In particular, the generic extractor (used when your website is not in the [list of supported sites by youtube-dl](http://rg3.github.io/youtube-dl/supportedsites.html) cannot mandate one specific downloader.
+In particular, the generic extractor (used when your website is not in the [list of supported sites by youtube-dl](https://rg3.github.io/youtube-dl/supportedsites.html) cannot mandate one specific downloader.
 
 If you put either `--hls-prefer-native` or `--hls-prefer-ffmpeg` into your configuration, a different subset of videos will fail to download correctly. Instead, it is much better to [file an issue](https://yt-dl.org/bug) or a pull request which details why the native or the ffmpeg HLS downloader is a better choice for your use case.
 
@@ -910,7 +910,7 @@ Feel free to bump the issue from time to time by writing a small comment ("Issue
 
 ### How can I detect whether a given URL is supported by youtube-dl?
 
-For one, have a look at the [list of supported sites](docs/supportedsites.md). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/video/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.
+For one, have a look at the [list of supported sites](docs/supportedsites.md). Note that it can sometimes happen that the site changes its URL scheme (say, from https://example.com/video/1234567 to https://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.
 
 It is *not* possible to detect whether a URL is supported or not. That's because youtube-dl contains a generic extractor which matches **all** URLs. You may be tempted to disable, exclude, or remove the generic extractor, but the generic extractor not only allows users to extract videos from lots of websites that embed a video from another service, but may also be used to extract video from a service that it's hosting itself. Therefore, we neither recommend nor support disabling, excluding, or removing the generic extractor.
 
@@ -924,7 +924,7 @@ youtube-dl is an open-source project manned by too few volunteers, so we'd rathe
 
 # DEVELOPER INSTRUCTIONS
 
-Most users do not need to build youtube-dl and can [download the builds](http://rg3.github.io/youtube-dl/download.html) or get them from their distribution.
+Most users do not need to build youtube-dl and can [download the builds](https://rg3.github.io/youtube-dl/download.html) or get them from their distribution.
 
 To run youtube-dl as a developer, you don't need to build anything either. Simply execute
 
@@ -972,7 +972,7 @@ After you have ensured this site is distributing its content legally, you can fo
     class YourExtractorIE(InfoExtractor):
         _VALID_URL = r'https?://(?:www\.)?yourextractor\.com/watch/(?P<id>[0-9]+)'
         _TEST = {
-            'url': 'http://yourextractor.com/watch/42',
+            'url': 'https://yourextractor.com/watch/42',
             'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)',
             'info_dict': {
                 'id': '42',
@@ -1005,8 +1005,8 @@ After you have ensured this site is distributing its content legally, you can fo
 5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
 7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L74-L252). Add tests and code for as many as you want.
-8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://pypi.python.org/pypi/flake8). Also make sure your code works under all [Python](http://www.python.org/) versions claimed supported by youtube-dl, namely 2.6, 2.7, and 3.2+.
-9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
+8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://pypi.python.org/pypi/flake8). Also make sure your code works under all [Python](https://www.python.org/) versions claimed supported by youtube-dl, namely 2.6, 2.7, and 3.2+.
+9. When the tests pass, [add](https://git-scm.com/docs/git-add) the new files and [commit](https://git-scm.com/docs/git-commit) them and [push](https://git-scm.com/docs/git-push) the result, like this:
 
         $ git add youtube_dl/extractor/extractors.py
         $ git add youtube_dl/extractor/yourextractor.py
@@ -1162,7 +1162,7 @@ import youtube_dl
 
 ydl_opts = {}
 with youtube_dl.YoutubeDL(ydl_opts) as ydl:
-    ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
+    ydl.download(['https://www.youtube.com/watch?v=BaW_jenozKc'])
 ```
 
 Most likely, you'll want to use various options. For a list of options available, have a look at [`youtube_dl/YoutubeDL.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L129-L279). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
@@ -1201,19 +1201,19 @@ ydl_opts = {
     'progress_hooks': [my_hook],
 }
 with youtube_dl.YoutubeDL(ydl_opts) as ydl:
-    ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
+    ydl.download(['https://www.youtube.com/watch?v=BaW_jenozKc'])
 ```
 
 # BUGS
 
-Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/issues>. Unless you were prompted to or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the IRC channel [#youtube-dl](irc://chat.freenode.net/#youtube-dl) on freenode ([webchat](http://webchat.freenode.net/?randomnick=1&channels=youtube-dl)).
+Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/issues>. Unless you were prompted to or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the IRC channel [#youtube-dl](irc://chat.freenode.net/#youtube-dl) on freenode ([webchat](https://webchat.freenode.net/?randomnick=1&channels=youtube-dl)).
 
 **Please include the full output of youtube-dl when run with `-v`**, i.e. **add** `-v` flag to **your command line**, copy the **whole** output and post it in the issue body wrapped in \`\`\` for better formatting. It should look similar to this:
 ```
 $ youtube-dl -v <your command line>
 [debug] System config: []
 [debug] User config: []
-[debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
+[debug] Command-line args: [u'-v', u'https://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
 [debug] youtube-dl version 2015.12.06
 [debug] Git HEAD: 135392e
@@ -1244,7 +1244,7 @@ For bug reports, this means that your report should contain the *complete* outpu
 
 If your server has multiple IPs or you suspect censorship, adding `--call-home` may be a good idea to get more diagnostics. If the error is `ERROR: Unable to extract ...` and you cannot reproduce it from multiple countries, add `--dump-pages` (warning: this will yield a rather large output, redirect it to the file `log.txt` by adding `>log.txt 2>&1` to your command-line) or upload the `.dump` files you get when you add `--write-pages` [somewhere](https://gist.github.com/).
 
-**Site support requests must contain an example URL**. An example URL is a URL you might want to download, like `http://www.youtube.com/watch?v=BaW_jenozKc`. There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. `http://www.youtube.com/`) is *not* an example URL.
+**Site support requests must contain an example URL**. An example URL is a URL you might want to download, like `https://www.youtube.com/watch?v=BaW_jenozKc`. There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. `https://www.youtube.com/`) is *not* an example URL.
 
 ###  Are you using the latest version?
 

From 8d9c2a681a1dcf99ab949e79b1b9da17513e11d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Aug 2017 23:06:27 +0700
Subject: [PATCH 1246/1696] [pornhub] Relax uploader regex (closes #13906,
 closes #13975)

---
 youtube_dl/extractor/pornhub.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index f6777cd26..3428458af 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -186,7 +186,7 @@ class PornHubIE(InfoExtractor):
             title, thumbnail, duration = [None] * 3
 
         video_uploader = self._html_search_regex(
-            r'(?s)From:&nbsp;.+?<(?:a href="/users/|a href="/channels/|span class="username)[^>]+>(.+?)<',
+            r'(?s)From:&nbsp;.+?<(?:a\b[^>]+\bhref=["\']/(?:user|channel)s/|span\b[^>]+\bclass=["\']username)[^>]+>(.+?)<',
             webpage, 'uploader', fatal=False)
 
         view_count = self._extract_count(

From 05915e379a2406988f752722dfaa815804fb7fb8 Mon Sep 17 00:00:00 2001
From: Parmjit Virk <pvirk@mts.net>
Date: Tue, 22 Aug 2017 11:48:59 -0500
Subject: [PATCH 1247/1696] [googledrive] Add support for subtitles (fixes
 #13619)

---
 youtube_dl/extractor/googledrive.py | 104 +++++++++++++++++++++++++++-
 1 file changed, 103 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
index c40da85c5..35edc7440 100644
--- a/youtube_dl/extractor/googledrive.py
+++ b/youtube_dl/extractor/googledrive.py
@@ -7,6 +7,8 @@ from ..utils import (
     ExtractorError,
     int_or_none,
     lowercase_escape,
+    error_to_compat_str,
+    update_url_query,
 )
 
 
@@ -24,7 +26,14 @@ class GoogleDriveIE(InfoExtractor):
     }, {
         # video id is longer than 28 characters
         'url': 'https://drive.google.com/file/d/1ENcQ_jeCuj7y19s66_Ou9dRP4GKGsodiDQ/edit',
-        'only_matching': True,
+        'md5': 'c230c67252874fddd8170e3fd1a45886',
+        'info_dict': {
+            'id': '1ENcQ_jeCuj7y19s66_Ou9dRP4GKGsodiDQ',
+            'ext': 'mp4',
+            'title': 'Andreea Banica feat Smiley - Hooky Song (Official Video).mp4',
+            'duration': 189,
+        },
+        'only_matching': True
     }]
     _FORMATS_EXT = {
         '5': 'flv',
@@ -44,6 +53,13 @@ class GoogleDriveIE(InfoExtractor):
         '46': 'webm',
         '59': 'mp4',
     }
+    _BASE_URL_CAPTIONS = 'https://drive.google.com/timedtext'
+    _CAPTIONS_ENTRY_TAG = {
+        'subtitles': 'track',
+        'automatic_captions': 'target',
+    }
+    _caption_formats_ext = []
+    _captions_by_country_xml = None
 
     @staticmethod
     def _extract_url(webpage):
@@ -53,6 +69,81 @@ class GoogleDriveIE(InfoExtractor):
         if mobj:
             return 'https://drive.google.com/file/d/%s' % mobj.group('id')
 
+    def _set_captions_data(self, video_id, video_subtitles_id, hl):
+        try:
+            self._captions_by_country_xml = self._download_xml(self._BASE_URL_CAPTIONS, video_id, query={
+                'id': video_id,
+                'vid': video_subtitles_id,
+                'hl': hl,
+                'v': video_id,
+                'type': 'list',
+                'tlangs': '1',
+                'fmts': '1',
+                'vssids': '1',
+            })
+        except ExtractorError as ee:
+            self.report_warning('unable to download video subtitles: %s' % error_to_compat_str(ee))
+        if self._captions_by_country_xml is not None:
+            caption_available_extensions = self._captions_by_country_xml.findall('format')
+            for caption_extension in caption_available_extensions:
+                if caption_extension.attrib.get('fmt_code') and not caption_extension.attrib.get('default'):
+                    self._caption_formats_ext.append(caption_extension.attrib['fmt_code'])
+
+    def _get_captions_by_type(self, video_id, video_subtitles_id, caption_type, caption_original_lang_code=None):
+        if not video_subtitles_id or not caption_type:
+            return None
+        captions = {}
+        for caption_entry in self._captions_by_country_xml.findall(self._CAPTIONS_ENTRY_TAG[caption_type]):
+            caption_lang_code = caption_entry.attrib.get('lang_code')
+            if not caption_lang_code:
+                continue
+            caption_format_data = []
+            for caption_format in self._caption_formats_ext:
+                query = {
+                    'vid': video_subtitles_id,
+                    'v': video_id,
+                    'fmt': caption_format,
+                    'lang': caption_lang_code if caption_original_lang_code is None else caption_original_lang_code,
+                    'type': 'track',
+                    'name': '',
+                    'kind': '',
+                }
+                if caption_original_lang_code is not None:
+                    query.update({'tlang': caption_lang_code})
+                caption_format_data.append({
+                    'url': update_url_query(self._BASE_URL_CAPTIONS, query),
+                    'ext': caption_format,
+                })
+            captions[caption_lang_code] = caption_format_data
+        if not captions:
+            self.report_warning('video doesn\'t have %s' % caption_type.replace('_', ' '))
+        return captions
+
+    def _get_subtitles(self, video_id, video_subtitles_id, hl):
+        if not video_subtitles_id or not hl:
+            return None
+        if self._captions_by_country_xml is None:
+            self._set_captions_data(video_id, video_subtitles_id, hl)
+            if self._captions_by_country_xml is None:
+                return None
+        return self._get_captions_by_type(video_id, video_subtitles_id, 'subtitles')
+
+    def _get_automatic_captions(self, video_id, video_subtitles_id, hl):
+        if not video_subtitles_id or not hl:
+            return None
+        if self._captions_by_country_xml is None:
+            self._set_captions_data(video_id, video_subtitles_id, hl)
+            if self._captions_by_country_xml is None:
+                return None
+        self.to_screen('%s: Looking for automatic captions' % video_id)
+        subtitle_original_track = self._captions_by_country_xml.find('track')
+        if subtitle_original_track is None:
+            return None
+        subtitle_original_lang_code = subtitle_original_track.attrib.get('lang_code')
+        if not subtitle_original_lang_code:
+            return None
+        return self._get_captions_by_type(video_id, video_subtitles_id, 'automatic_captions', subtitle_original_lang_code)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(
@@ -97,10 +188,21 @@ class GoogleDriveIE(InfoExtractor):
             formats.append(f)
         self._sort_formats(formats)
 
+        hl = self._search_regex(
+            r'"hl"\s*,\s*"([^"]+)', webpage, 'hl', default=None)
+        video_subtitles_id = None
+        ttsurl = self._search_regex(
+            r'"ttsurl"\s*,\s*"([^"]+)', webpage, 'ttsurl', default=None)
+        if ttsurl:
+            # the video Id for subtitles will be the last value in the ttsurl query string
+            video_subtitles_id = ttsurl.encode('utf-8').decode('unicode_escape').split('=')[-1]
+
         return {
             'id': video_id,
             'title': title,
             'thumbnail': self._og_search_thumbnail(webpage, default=None),
             'duration': duration,
             'formats': formats,
+            'subtitles': self.extract_subtitles(video_id, video_subtitles_id, hl),
+            'automatic_captions': self.extract_automatic_captions(video_id, video_subtitles_id, hl),
         }

From e01c3d2ef7264b5d3d6f99e7e0b61340885ed661 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Aug 2017 00:32:41 +0700
Subject: [PATCH 1248/1696] [extractor/common] Introduce _parse_xml

---
 youtube_dl/extractor/common.py | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index ceba4ca1c..1804c4de0 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -27,6 +27,7 @@ from ..compat import (
     compat_urllib_parse_urlencode,
     compat_urllib_request,
     compat_urlparse,
+    compat_xml_parse_error,
 )
 from ..downloader.f4m import remove_encrypted_media
 from ..utils import (
@@ -646,15 +647,29 @@ class InfoExtractor(object):
 
     def _download_xml(self, url_or_request, video_id,
                       note='Downloading XML', errnote='Unable to download XML',
-                      transform_source=None, fatal=True, encoding=None, data=None, headers={}, query={}):
+                      transform_source=None, fatal=True, encoding=None,
+                      data=None, headers={}, query={}):
         """Return the xml as an xml.etree.ElementTree.Element"""
         xml_string = self._download_webpage(
-            url_or_request, video_id, note, errnote, fatal=fatal, encoding=encoding, data=data, headers=headers, query=query)
+            url_or_request, video_id, note, errnote, fatal=fatal,
+            encoding=encoding, data=data, headers=headers, query=query)
         if xml_string is False:
             return xml_string
+        return self._parse_xml(
+            xml_string, video_id, transform_source=transform_source,
+            fatal=fatal)
+
+    def _parse_xml(self, xml_string, video_id, transform_source=None, fatal=True):
         if transform_source:
             xml_string = transform_source(xml_string)
-        return compat_etree_fromstring(xml_string.encode('utf-8'))
+        try:
+            return compat_etree_fromstring(xml_string.encode('utf-8'))
+        except compat_xml_parse_error as ve:
+            errmsg = '%s: Failed to parse XML ' % video_id
+            if fatal:
+                raise ExtractorError(errmsg, cause=ve)
+            else:
+                self.report_warning(errmsg + str(ve))
 
     def _download_json(self, url_or_request, video_id,
                        note='Downloading JSON metadata',

From 37d9af306a928ce2184dcb60883e98ec0dd570ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Aug 2017 00:33:53 +0700
Subject: [PATCH 1249/1696] [googledrive] Simplify and carry long lines
 (#13638)

---
 youtube_dl/extractor/googledrive.py | 115 +++++++++++++++-------------
 1 file changed, 60 insertions(+), 55 deletions(-)

diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
index 35edc7440..97ff28219 100644
--- a/youtube_dl/extractor/googledrive.py
+++ b/youtube_dl/extractor/googledrive.py
@@ -7,7 +7,6 @@ from ..utils import (
     ExtractorError,
     int_or_none,
     lowercase_escape,
-    error_to_compat_str,
     update_url_query,
 )
 
@@ -59,7 +58,7 @@ class GoogleDriveIE(InfoExtractor):
         'automatic_captions': 'target',
     }
     _caption_formats_ext = []
-    _captions_by_country_xml = None
+    _captions_xml = None
 
     @staticmethod
     def _extract_url(webpage):
@@ -69,96 +68,99 @@ class GoogleDriveIE(InfoExtractor):
         if mobj:
             return 'https://drive.google.com/file/d/%s' % mobj.group('id')
 
-    def _set_captions_data(self, video_id, video_subtitles_id, hl):
-        try:
-            self._captions_by_country_xml = self._download_xml(self._BASE_URL_CAPTIONS, video_id, query={
+    def _download_subtitles_xml(self, video_id, subtitles_id, hl):
+        if self._captions_xml:
+            return
+        self._captions_xml = self._download_xml(
+            self._BASE_URL_CAPTIONS, video_id, query={
                 'id': video_id,
-                'vid': video_subtitles_id,
+                'vid': subtitles_id,
                 'hl': hl,
                 'v': video_id,
                 'type': 'list',
                 'tlangs': '1',
                 'fmts': '1',
                 'vssids': '1',
-            })
-        except ExtractorError as ee:
-            self.report_warning('unable to download video subtitles: %s' % error_to_compat_str(ee))
-        if self._captions_by_country_xml is not None:
-            caption_available_extensions = self._captions_by_country_xml.findall('format')
-            for caption_extension in caption_available_extensions:
-                if caption_extension.attrib.get('fmt_code') and not caption_extension.attrib.get('default'):
-                    self._caption_formats_ext.append(caption_extension.attrib['fmt_code'])
+            }, note='Downloading subtitles XML',
+            errnote='Unable to download subtitles XML', fatal=False)
+        if self._captions_xml:
+            for f in self._captions_xml.findall('format'):
+                if f.attrib.get('fmt_code') and not f.attrib.get('default'):
+                    self._caption_formats_ext.append(f.attrib['fmt_code'])
 
-    def _get_captions_by_type(self, video_id, video_subtitles_id, caption_type, caption_original_lang_code=None):
-        if not video_subtitles_id or not caption_type:
-            return None
+    def _get_captions_by_type(self, video_id, subtitles_id, caption_type,
+                              origin_lang_code=None):
+        if not subtitles_id or not caption_type:
+            return
         captions = {}
-        for caption_entry in self._captions_by_country_xml.findall(self._CAPTIONS_ENTRY_TAG[caption_type]):
+        for caption_entry in self._captions_xml.findall(
+                self._CAPTIONS_ENTRY_TAG[caption_type]):
             caption_lang_code = caption_entry.attrib.get('lang_code')
             if not caption_lang_code:
                 continue
             caption_format_data = []
             for caption_format in self._caption_formats_ext:
                 query = {
-                    'vid': video_subtitles_id,
+                    'vid': subtitles_id,
                     'v': video_id,
                     'fmt': caption_format,
-                    'lang': caption_lang_code if caption_original_lang_code is None else caption_original_lang_code,
+                    'lang': (caption_lang_code if origin_lang_code is None
+                             else origin_lang_code),
                     'type': 'track',
                     'name': '',
                     'kind': '',
                 }
-                if caption_original_lang_code is not None:
+                if origin_lang_code is not None:
                     query.update({'tlang': caption_lang_code})
                 caption_format_data.append({
                     'url': update_url_query(self._BASE_URL_CAPTIONS, query),
                     'ext': caption_format,
                 })
             captions[caption_lang_code] = caption_format_data
-        if not captions:
-            self.report_warning('video doesn\'t have %s' % caption_type.replace('_', ' '))
         return captions
 
-    def _get_subtitles(self, video_id, video_subtitles_id, hl):
-        if not video_subtitles_id or not hl:
-            return None
-        if self._captions_by_country_xml is None:
-            self._set_captions_data(video_id, video_subtitles_id, hl)
-            if self._captions_by_country_xml is None:
-                return None
-        return self._get_captions_by_type(video_id, video_subtitles_id, 'subtitles')
+    def _get_subtitles(self, video_id, subtitles_id, hl):
+        if not subtitles_id or not hl:
+            return
+        self._download_subtitles_xml(video_id, subtitles_id, hl)
+        if not self._captions_xml:
+            return
+        return self._get_captions_by_type(video_id, subtitles_id, 'subtitles')
 
-    def _get_automatic_captions(self, video_id, video_subtitles_id, hl):
-        if not video_subtitles_id or not hl:
-            return None
-        if self._captions_by_country_xml is None:
-            self._set_captions_data(video_id, video_subtitles_id, hl)
-            if self._captions_by_country_xml is None:
-                return None
-        self.to_screen('%s: Looking for automatic captions' % video_id)
-        subtitle_original_track = self._captions_by_country_xml.find('track')
-        if subtitle_original_track is None:
-            return None
-        subtitle_original_lang_code = subtitle_original_track.attrib.get('lang_code')
-        if not subtitle_original_lang_code:
-            return None
-        return self._get_captions_by_type(video_id, video_subtitles_id, 'automatic_captions', subtitle_original_lang_code)
+    def _get_automatic_captions(self, video_id, subtitles_id, hl):
+        if not subtitles_id or not hl:
+            return
+        self._download_subtitles_xml(video_id, subtitles_id, hl)
+        if not self._captions_xml:
+            return
+        track = self._captions_xml.find('track')
+        if track is None:
+            return
+        origin_lang_code = track.attrib.get('lang_code')
+        if not origin_lang_code:
+            return
+        return self._get_captions_by_type(
+            video_id, subtitles_id, 'automatic_captions', origin_lang_code)
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(
             'http://docs.google.com/file/d/%s' % video_id, video_id)
 
-        reason = self._search_regex(r'"reason"\s*,\s*"([^"]+)', webpage, 'reason', default=None)
+        reason = self._search_regex(
+            r'"reason"\s*,\s*"([^"]+)', webpage, 'reason', default=None)
         if reason:
             raise ExtractorError(reason)
 
         title = self._search_regex(r'"title"\s*,\s*"([^"]+)', webpage, 'title')
         duration = int_or_none(self._search_regex(
-            r'"length_seconds"\s*,\s*"([^"]+)', webpage, 'length seconds', default=None))
+            r'"length_seconds"\s*,\s*"([^"]+)', webpage, 'length seconds',
+            default=None))
         fmt_stream_map = self._search_regex(
-            r'"fmt_stream_map"\s*,\s*"([^"]+)', webpage, 'fmt stream map').split(',')
-        fmt_list = self._search_regex(r'"fmt_list"\s*,\s*"([^"]+)', webpage, 'fmt_list').split(',')
+            r'"fmt_stream_map"\s*,\s*"([^"]+)', webpage,
+            'fmt stream map').split(',')
+        fmt_list = self._search_regex(
+            r'"fmt_list"\s*,\s*"([^"]+)', webpage, 'fmt_list').split(',')
 
         resolutions = {}
         for fmt in fmt_list:
@@ -190,12 +192,14 @@ class GoogleDriveIE(InfoExtractor):
 
         hl = self._search_regex(
             r'"hl"\s*,\s*"([^"]+)', webpage, 'hl', default=None)
-        video_subtitles_id = None
+        subtitles_id = None
         ttsurl = self._search_regex(
             r'"ttsurl"\s*,\s*"([^"]+)', webpage, 'ttsurl', default=None)
         if ttsurl:
-            # the video Id for subtitles will be the last value in the ttsurl query string
-            video_subtitles_id = ttsurl.encode('utf-8').decode('unicode_escape').split('=')[-1]
+            # the video Id for subtitles will be the last value in the ttsurl
+            # query string
+            subtitles_id = ttsurl.encode('utf-8').decode(
+                'unicode_escape').split('=')[-1]
 
         return {
             'id': video_id,
@@ -203,6 +207,7 @@ class GoogleDriveIE(InfoExtractor):
             'thumbnail': self._og_search_thumbnail(webpage, default=None),
             'duration': duration,
             'formats': formats,
-            'subtitles': self.extract_subtitles(video_id, video_subtitles_id, hl),
-            'automatic_captions': self.extract_automatic_captions(video_id, video_subtitles_id, hl),
+            'subtitles': self.extract_subtitles(video_id, subtitles_id, hl),
+            'automatic_captions': self.extract_automatic_captions(
+                video_id, subtitles_id, hl),
         }

From 8d7a24aff60a57e651bab40f16a81eb7dffb405c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Aug 2017 22:28:09 +0700
Subject: [PATCH 1250/1696] [toutv] Relax DRM check (closes #13994)

---
 youtube_dl/extractor/toutv.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/toutv.py b/youtube_dl/extractor/toutv.py
index 26d770992..071388dcc 100644
--- a/youtube_dl/extractor/toutv.py
+++ b/youtube_dl/extractor/toutv.py
@@ -78,8 +78,10 @@ class TouTvIE(InfoExtractor):
     def _real_extract(self, url):
         path = self._match_id(url)
         metadata = self._download_json('http://ici.tou.tv/presentation/%s' % path, path)
+        # IsDrm does not necessarily mean the video is DRM protected (see
+        # https://github.com/rg3/youtube-dl/issues/13994).
         if metadata.get('IsDrm'):
-            raise ExtractorError('This video is DRM protected.', expected=True)
+            self.report_warning('This video is probably DRM protected.', path)
         video_id = metadata['IdMedia']
         details = metadata['Details']
         title = details['OriginalTitle']

From 0830f3e04842a58eb563962940ceb2bed27aac1f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Aug 2017 22:45:45 +0700
Subject: [PATCH 1251/1696] [cbc:watch] Bypass geo-restriction (closes #13993)

---
 youtube_dl/extractor/cbc.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/cbc.py b/youtube_dl/extractor/cbc.py
index 87ad14e91..9faf40227 100644
--- a/youtube_dl/extractor/cbc.py
+++ b/youtube_dl/extractor/cbc.py
@@ -200,6 +200,7 @@ class CBCWatchBaseIE(InfoExtractor):
         'media': 'http://search.yahoo.com/mrss/',
         'clearleap': 'http://www.clearleap.com/namespace/clearleap/1.0/',
     }
+    _GEO_COUNTRIES = ['CA']
 
     def _call_api(self, path, video_id):
         url = path if path.startswith('http') else self._API_BASE_URL + path
@@ -287,6 +288,11 @@ class CBCWatchBaseIE(InfoExtractor):
 class CBCWatchVideoIE(CBCWatchBaseIE):
     IE_NAME = 'cbc.ca:watch:video'
     _VALID_URL = r'https?://api-cbc\.cloud\.clearleap\.com/cloffice/client/web/play/?\?.*?\bcontentId=(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
+    _TEST = {
+        # geo-restricted to Canada, bypassable
+        'url': 'https://api-cbc.cloud.clearleap.com/cloffice/client/web/play/?contentId=3c84472a-1eea-4dee-9267-2655d5055dcf&categoryId=ebc258f5-ee40-4cca-b66b-ba6bd55b7235',
+        'only_matching': True,
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -323,9 +329,10 @@ class CBCWatchIE(CBCWatchBaseIE):
     IE_NAME = 'cbc.ca:watch'
     _VALID_URL = r'https?://watch\.cbc\.ca/(?:[^/]+/)+(?P<id>[0-9a-f-]+)'
     _TESTS = [{
+        # geo-restricted to Canada, bypassable
         'url': 'http://watch.cbc.ca/doc-zone/season-6/customer-disservice/38e815a-009e3ab12e4',
         'info_dict': {
-            'id': '38e815a-009e3ab12e4',
+            'id': '9673749a-5e77-484c-8b62-a1092a6b5168',
             'ext': 'mp4',
             'title': 'Customer (Dis)Service',
             'description': 'md5:8bdd6913a0fe03d4b2a17ebe169c7c87',
@@ -337,8 +344,8 @@ class CBCWatchIE(CBCWatchBaseIE):
             'skip_download': True,
             'format': 'bestvideo',
         },
-        'skip': 'Geo-restricted to Canada',
     }, {
+        # geo-restricted to Canada, bypassable
         'url': 'http://watch.cbc.ca/arthur/all/1ed4b385-cd84-49cf-95f0-80f004680057',
         'info_dict': {
             'id': '1ed4b385-cd84-49cf-95f0-80f004680057',
@@ -346,7 +353,6 @@ class CBCWatchIE(CBCWatchBaseIE):
             'description': 'Arthur, the sweetest 8-year-old aardvark, and his pals solve all kinds of problems with humour, kindness and teamwork.',
         },
         'playlist_mincount': 30,
-        'skip': 'Geo-restricted to Canada',
     }]
 
     def _real_extract(self, url):

From 5bae33485c223fdf230254fa424f972b3c51e77f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Aug 2017 22:50:00 +0700
Subject: [PATCH 1252/1696] [toutv] PEP 8

---
 youtube_dl/extractor/toutv.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/toutv.py b/youtube_dl/extractor/toutv.py
index 071388dcc..e59ed2661 100644
--- a/youtube_dl/extractor/toutv.py
+++ b/youtube_dl/extractor/toutv.py
@@ -5,7 +5,6 @@ from .common import InfoExtractor
 from ..utils import (
     int_or_none,
     js_to_json,
-    ExtractorError,
     urlencode_postdata,
     extract_attributes,
     smuggle_url,

From c4bdc6811307c002a399b59860d311591145397f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Aug 2017 23:21:19 +0700
Subject: [PATCH 1253/1696] [ChangeLog] Actualize

---
 ChangeLog | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index c07cb9648..4104b6ded 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,12 +1,23 @@
 version <unreleased>
 
 Core
++ [extractor/common] Introduce _parse_xml
+* [extractor/common] Make HLS and DASH extraction in_parse_html5_media_entries
+  non fatal (#13970)
 * [utils] Fix unescapeHTML for misformed string like "&a&quot;" (#13935)
 
 Extractors
+* [cbc:watch] Bypass geo restriction (#13993)
+* [toutv] Relax DRM check (#13994)
++ [googledrive] Add support for subtitles (#13619, #13638)
+* [pornhub] Relax uploader regular expression (#13906, #13975)
+* [bandcamp:album] Extract track titles (#13962)
++ [bbccouk] Add support for events URLs (#13893)
 + [liveleak] Support multi-video pages (#6542)
 + [liveleak] Support another liveleak embedding pattern (#13336)
 * [cda] Fix extraction (#13935)
++ [laola1tv] Add support for tv.ittf.com (#13965)
+* [mixcloud] Fix extraction (#13958, #13974, #13980, #14003)
 
 
 version 2017.08.18
@@ -129,7 +140,7 @@ Extractors
 * [youku:show] Fix playlist extraction (#13248)
 + [dispeak] Recognize sevt subdomain (#13276)
 * [adn] Improve error reporting (#13663)
-* [crunchyroll] Relax series and season regex (#13659)
+* [crunchyroll] Relax series and season regular expression (#13659)
 + [spiegel:article] Add support for nexx iframe embeds (#13029)
 + [nexx:embed] Add support for iframe embeds
 * [nexx] Improve JS embed extraction

From df235dbba8d8ae3b51ad3432f67d0cb661dadd75 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Aug 2017 23:23:13 +0700
Subject: [PATCH 1254/1696] release 2017.08.23

---
 .github/ISSUE_TEMPLATE.md |  6 +++---
 CONTRIBUTING.md           | 12 ++++++------
 ChangeLog                 |  2 +-
 docs/supportedsites.md    |  2 ++
 youtube_dl/version.py     |  2 +-
 5 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 66dd4c480..3e1ff1536 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.08.18*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.08.18**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.08.23*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.08.23**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.08.18
+[debug] youtube-dl version 2017.08.23
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index d606eab0e..a8091e7b5 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -3,7 +3,7 @@
 $ youtube-dl -v <your command line>
 [debug] System config: []
 [debug] User config: []
-[debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
+[debug] Command-line args: [u'-v', u'https://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
 [debug] youtube-dl version 2015.12.06
 [debug] Git HEAD: 135392e
@@ -34,7 +34,7 @@ For bug reports, this means that your report should contain the *complete* outpu
 
 If your server has multiple IPs or you suspect censorship, adding `--call-home` may be a good idea to get more diagnostics. If the error is `ERROR: Unable to extract ...` and you cannot reproduce it from multiple countries, add `--dump-pages` (warning: this will yield a rather large output, redirect it to the file `log.txt` by adding `>log.txt 2>&1` to your command-line) or upload the `.dump` files you get when you add `--write-pages` [somewhere](https://gist.github.com/).
 
-**Site support requests must contain an example URL**. An example URL is a URL you might want to download, like `http://www.youtube.com/watch?v=BaW_jenozKc`. There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. `http://www.youtube.com/`) is *not* an example URL.
+**Site support requests must contain an example URL**. An example URL is a URL you might want to download, like `https://www.youtube.com/watch?v=BaW_jenozKc`. There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. `https://www.youtube.com/`) is *not* an example URL.
 
 ###  Are you using the latest version?
 
@@ -70,7 +70,7 @@ It may sound strange, but some bug reports we receive are completely unrelated t
 
 # DEVELOPER INSTRUCTIONS
 
-Most users do not need to build youtube-dl and can [download the builds](http://rg3.github.io/youtube-dl/download.html) or get them from their distribution.
+Most users do not need to build youtube-dl and can [download the builds](https://rg3.github.io/youtube-dl/download.html) or get them from their distribution.
 
 To run youtube-dl as a developer, you don't need to build anything either. Simply execute
 
@@ -118,7 +118,7 @@ After you have ensured this site is distributing its content legally, you can fo
     class YourExtractorIE(InfoExtractor):
         _VALID_URL = r'https?://(?:www\.)?yourextractor\.com/watch/(?P<id>[0-9]+)'
         _TEST = {
-            'url': 'http://yourextractor.com/watch/42',
+            'url': 'https://yourextractor.com/watch/42',
             'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)',
             'info_dict': {
                 'id': '42',
@@ -151,8 +151,8 @@ After you have ensured this site is distributing its content legally, you can fo
 5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
 7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L74-L252). Add tests and code for as many as you want.
-8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://pypi.python.org/pypi/flake8). Also make sure your code works under all [Python](http://www.python.org/) versions claimed supported by youtube-dl, namely 2.6, 2.7, and 3.2+.
-9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
+8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://pypi.python.org/pypi/flake8). Also make sure your code works under all [Python](https://www.python.org/) versions claimed supported by youtube-dl, namely 2.6, 2.7, and 3.2+.
+9. When the tests pass, [add](https://git-scm.com/docs/git-add) the new files and [commit](https://git-scm.com/docs/git-commit) them and [push](https://git-scm.com/docs/git-push) the result, like this:
 
         $ git add youtube_dl/extractor/extractors.py
         $ git add youtube_dl/extractor/yourextractor.py
diff --git a/ChangeLog b/ChangeLog
index 4104b6ded..a60bd5fc8 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.08.23
 
 Core
 + [extractor/common] Introduce _parse_xml
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 1991975cc..dbec6c8dc 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -363,6 +363,7 @@
  - **IPrima**
  - **iqiyi**: 爱奇艺
  - **Ir90Tv**
+ - **ITTF**
  - **ITV**
  - **ivi**: ivi.ru
  - **ivi:compilation**: ivi.ru compilations
@@ -419,6 +420,7 @@
  - **limelight:channel_list**
  - **LiTV**
  - **LiveLeak**
+ - **LiveLeakEmbed**
  - **livestream**
  - **livestream:original**
  - **LnkGo**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4358cd3f2..94d35a66a 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.08.18'
+__version__ = '2017.08.23'

From 745968bc72f5dcf1271559d58abd3f0d9a2ea01e Mon Sep 17 00:00:00 2001
From: Vijay Singh <sudovijay@users.noreply.github.com>
Date: Thu, 24 Aug 2017 20:58:44 +0530
Subject: [PATCH 1255/1696] [mixcloud] Fix extraction (closes #14015)

---
 youtube_dl/extractor/mixcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 798968ae3..25d3fc4b3 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -92,7 +92,7 @@ class MixcloudIE(InfoExtractor):
                 js = self._download_webpage(js_url, track_id, fatal=False)
                 if js:
                     KEY_RE_TEMPLATE = r'player\s*:\s*{.*?\b%s\s*:\s*(["\'])(?P<key>(?:(?!\1).)+)\1'
-                    for key_name in ('value', 'key_value'):
+                    for key_name in ('value', 'key_value', 'key_value_two'):
                         key = self._search_regex(
                             KEY_RE_TEMPLATE % key_name, js, 'key',
                             default=None, group='key')

From c7121fa7b804d2b9e35dec05f8861e4ebba8afd2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Aug 2017 15:38:38 +0700
Subject: [PATCH 1256/1696] [youtube] Fix controversy videos extraction (closes
 #14027, closes #14029)

---
 youtube_dl/extractor/youtube.py | 36 ++++++++++++++++++++++++++++-----
 1 file changed, 31 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2e71795e7..3d0f5a5c9 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1003,6 +1003,27 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'Skipping DASH manifest',
             ],
         },
+        {
+            # The following content has been identified by the YouTube community
+            # as inappropriate or offensive to some audiences.
+            'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
+            'info_dict': {
+                'id': '6SJNVb0GnPI',
+                'ext': 'mp4',
+                'title': 'Race Differences in Intelligence',
+                'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
+                'duration': 965,
+                'upload_date': '20140124',
+                'uploader': 'New Century Foundation',
+                'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
+                'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
+                'license': 'Standard YouTube License',
+                'view_count': int,
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
         {
             # itag 212
             'url': '1t24XAntNCY',
@@ -1437,9 +1458,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             if dash_mpd and dash_mpd[0] not in dash_mpds:
                 dash_mpds.append(dash_mpd[0])
 
+        is_live = None
+        view_count = None
+
+        def extract_view_count(v_info):
+            return int_or_none(try_get(v_info, lambda x: x['view_count'][0]))
+
         # Get video info
         embed_webpage = None
-        is_live = None
         if re.search(r'player-age-gate-content">', video_webpage) is not None:
             age_gate = True
             # We simulate the access to the video from www.youtube.com/v/{video_id}
@@ -1509,6 +1535,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         continue
                     get_video_info = compat_parse_qs(video_info_webpage)
                     add_dash_mpd(get_video_info)
+                    if view_count is None:
+                        view_count = extract_view_count(get_video_info)
                     if not video_info:
                         video_info = get_video_info
                     if 'token' in get_video_info:
@@ -1592,10 +1620,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 return self.playlist_result(entries, video_id, video_title, video_description)
             self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
 
-        if 'view_count' in video_info:
-            view_count = int(video_info['view_count'][0])
-        else:
-            view_count = None
+        if view_count is None:
+            view_count = extract_view_count(get_video_info)
 
         # Check for "rental" videos
         if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:

From 151978f38a915afd0169e68955b3b7394c54e367 Mon Sep 17 00:00:00 2001
From: Vijay Singh <sudovijay@users.noreply.github.com>
Date: Sat, 26 Aug 2017 18:02:57 +0530
Subject: [PATCH 1257/1696] [mixcloud] Fix extraction (closes #14020)

---
 youtube_dl/extractor/mixcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 25d3fc4b3..f6360cce6 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -92,7 +92,7 @@ class MixcloudIE(InfoExtractor):
                 js = self._download_webpage(js_url, track_id, fatal=False)
                 if js:
                     KEY_RE_TEMPLATE = r'player\s*:\s*{.*?\b%s\s*:\s*(["\'])(?P<key>(?:(?!\1).)+)\1'
-                    for key_name in ('value', 'key_value', 'key_value_two'):
+                    for key_name in ('value', 'key_value', 'key_value.*?', '.*?value.*?'):
                         key = self._search_regex(
                             KEY_RE_TEMPLATE % key_name, js, 'key',
                             default=None, group='key')

From 085d9dd9bebfd1692cfe07e8bcb844780bfe4700 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Aug 2017 22:02:49 +0700
Subject: [PATCH 1258/1696] [rai] Fix audio formats extraction (closes #14024)

---
 youtube_dl/extractor/rai.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index e11bf8f9a..5bf64a56b 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -345,11 +345,11 @@ class RaiIE(RaiBaseIE):
         media_type = media['type']
         if 'Audio' in media_type:
             relinker_info = {
-                'formats': {
+                'formats': [{
                     'format_id': media.get('formatoAudio'),
                     'url': media['audioUrl'],
                     'ext': media.get('formatoAudio'),
-                }
+                }]
             }
         elif 'Video' in media_type:
             relinker_info = self._extract_relinker_info(media['mediaUri'], content_id)

From a3c3a1e12898a57fc2323e4c7cc37ace35482ecf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Aug 2017 23:55:48 +0700
Subject: [PATCH 1259/1696] [http] Rework HTTP downloader * Simplify code and
 split into separate routines to facilitate maintaining * Make retry mechanism
 work on errors during actual download not only during connection
 establishment phase * Retry on ECONNRESET and ETIMEDOUT during reading data
 from network * Retry on content too short and various timeout errors * Show
 error description on retry * Closes #506, closes #809, closes #2849, closes
 #4240, closes #6023, closes #8625, closes #9483

---
 youtube_dl/downloader/common.py |   6 +-
 youtube_dl/downloader/http.py   | 343 ++++++++++++++++++--------------
 2 files changed, 197 insertions(+), 152 deletions(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 77242dacc..75b8166c5 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -304,11 +304,11 @@ class FileDownloader(object):
         """Report attempt to resume at given byte."""
         self.to_screen('[download] Resuming download at byte %s' % resume_len)
 
-    def report_retry(self, count, retries):
+    def report_retry(self, err, count, retries):
         """Report retry in case of HTTP error 5xx"""
         self.to_screen(
-            '[download] Got server HTTP error. Retrying (attempt %d of %s)...'
-            % (count, self.format_retries(retries)))
+            '[download] Got server HTTP error: %s. Retrying (attempt %d of %s)...'
+            % (error_to_compat_str(err), count, self.format_retries(retries)))
 
     def report_file_already_downloaded(self, file_name):
         """Report file has already been fully downloaded."""
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index af405b950..8a6638cc2 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -22,8 +22,16 @@ from ..utils import (
 class HttpFD(FileDownloader):
     def real_download(self, filename, info_dict):
         url = info_dict['url']
-        tmpfilename = self.temp_name(filename)
-        stream = None
+
+        class DownloadContext(dict):
+            __getattr__ = dict.get
+            __setattr__ = dict.__setitem__
+            __delattr__ = dict.__delitem__
+
+        ctx = DownloadContext()
+        ctx.filename = filename
+        ctx.tmpfilename = self.temp_name(filename)
+        ctx.stream = None
 
         # Do not include the Accept-Encoding header
         headers = {'Youtubedl-no-compression': 'True'}
@@ -38,46 +46,51 @@ class HttpFD(FileDownloader):
         if is_test:
             request.add_header('Range', 'bytes=0-%s' % str(self._TEST_FILE_SIZE - 1))
 
-        # Establish possible resume length
-        if os.path.isfile(encodeFilename(tmpfilename)):
-            resume_len = os.path.getsize(encodeFilename(tmpfilename))
-        else:
-            resume_len = 0
+        ctx.open_mode = 'wb'
+        ctx.resume_len = 0
 
-        open_mode = 'wb'
-        if resume_len != 0:
-            if self.params.get('continuedl', True):
-                self.report_resuming_byte(resume_len)
-                request.add_header('Range', 'bytes=%d-' % resume_len)
-                open_mode = 'ab'
-            else:
-                resume_len = 0
+        if self.params.get('continuedl', True):
+            # Establish possible resume length
+            if os.path.isfile(encodeFilename(ctx.tmpfilename)):
+                ctx.resume_len = os.path.getsize(encodeFilename(ctx.tmpfilename))
 
         count = 0
         retries = self.params.get('retries', 0)
-        while count <= retries:
+
+        class SucceedDownload(Exception):
+            pass
+
+        class RetryDownload(Exception):
+            def __init__(self, source_error):
+                self.source_error = source_error
+
+        def establish_connection():
+            if ctx.resume_len != 0:
+                self.report_resuming_byte(ctx.resume_len)
+                request.add_header('Range', 'bytes=%d-' % ctx.resume_len)
+                ctx.open_mode = 'ab'
             # Establish connection
             try:
-                data = self.ydl.urlopen(request)
+                ctx.data = self.ydl.urlopen(request)
                 # When trying to resume, Content-Range HTTP header of response has to be checked
                 # to match the value of requested Range HTTP header. This is due to a webservers
                 # that don't support resuming and serve a whole file with no Content-Range
                 # set in response despite of requested Range (see
                 # https://github.com/rg3/youtube-dl/issues/6057#issuecomment-126129799)
-                if resume_len > 0:
-                    content_range = data.headers.get('Content-Range')
+                if ctx.resume_len > 0:
+                    content_range = ctx.data.headers.get('Content-Range')
                     if content_range:
                         content_range_m = re.search(r'bytes (\d+)-', content_range)
                         # Content-Range is present and matches requested Range, resume is possible
-                        if content_range_m and resume_len == int(content_range_m.group(1)):
-                            break
+                        if content_range_m and ctx.resume_len == int(content_range_m.group(1)):
+                            return
                     # Content-Range is either not present or invalid. Assuming remote webserver is
                     # trying to send the whole file, resume is not possible, so wiping the local file
                     # and performing entire redownload
                     self.report_unable_to_resume()
-                    resume_len = 0
-                    open_mode = 'wb'
-                break
+                    ctx.resume_len = 0
+                    ctx.open_mode = 'wb'
+                return
             except (compat_urllib_error.HTTPError, ) as err:
                 if (err.code < 500 or err.code >= 600) and err.code != 416:
                     # Unexpected HTTP error
@@ -86,15 +99,15 @@ class HttpFD(FileDownloader):
                     # Unable to resume (requested range not satisfiable)
                     try:
                         # Open the connection again without the range header
-                        data = self.ydl.urlopen(basic_request)
-                        content_length = data.info()['Content-Length']
+                        ctx.data = self.ydl.urlopen(basic_request)
+                        content_length = ctx.data.info()['Content-Length']
                     except (compat_urllib_error.HTTPError, ) as err:
                         if err.code < 500 or err.code >= 600:
                             raise
                     else:
                         # Examine the reported length
                         if (content_length is not None and
-                                (resume_len - 100 < int(content_length) < resume_len + 100)):
+                                (ctx.resume_len - 100 < int(content_length) < ctx.resume_len + 100)):
                             # The file had already been fully downloaded.
                             # Explanation to the above condition: in issue #175 it was revealed that
                             # YouTube sometimes adds or removes a few bytes from the end of the file,
@@ -102,152 +115,184 @@ class HttpFD(FileDownloader):
                             # I decided to implement a suggested change and consider the file
                             # completely downloaded if the file size differs less than 100 bytes from
                             # the one in the hard drive.
-                            self.report_file_already_downloaded(filename)
-                            self.try_rename(tmpfilename, filename)
+                            self.report_file_already_downloaded(ctx.filename)
+                            self.try_rename(ctx.tmpfilename, ctx.filename)
                             self._hook_progress({
-                                'filename': filename,
+                                'filename': ctx.filename,
                                 'status': 'finished',
-                                'downloaded_bytes': resume_len,
-                                'total_bytes': resume_len,
+                                'downloaded_bytes': ctx.resume_len,
+                                'total_bytes': ctx.resume_len,
                             })
-                            return True
+                            raise SucceedDownload()
                         else:
                             # The length does not match, we start the download over
                             self.report_unable_to_resume()
-                            resume_len = 0
-                            open_mode = 'wb'
-                            break
-            except socket.error as e:
-                if e.errno != errno.ECONNRESET:
+                            ctx.resume_len = 0
+                            ctx.open_mode = 'wb'
+                            return
+                raise RetryDownload(err)
+            except socket.error as err:
+                if err.errno != errno.ECONNRESET:
                     # Connection reset is no problem, just retry
                     raise
+                raise RetryDownload(err)
 
-            # Retry
-            count += 1
-            if count <= retries:
-                self.report_retry(count, retries)
+        def download():
+            data_len = ctx.data.info().get('Content-length', None)
 
-        if count > retries:
-            self.report_error('giving up after %s retries' % retries)
-            return False
+            # Range HTTP header may be ignored/unsupported by a webserver
+            # (e.g. extractor/scivee.py, extractor/bambuser.py).
+            # However, for a test we still would like to download just a piece of a file.
+            # To achieve this we limit data_len to _TEST_FILE_SIZE and manually control
+            # block size when downloading a file.
+            if is_test and (data_len is None or int(data_len) > self._TEST_FILE_SIZE):
+                data_len = self._TEST_FILE_SIZE
 
-        data_len = data.info().get('Content-length', None)
-
-        # Range HTTP header may be ignored/unsupported by a webserver
-        # (e.g. extractor/scivee.py, extractor/bambuser.py).
-        # However, for a test we still would like to download just a piece of a file.
-        # To achieve this we limit data_len to _TEST_FILE_SIZE and manually control
-        # block size when downloading a file.
-        if is_test and (data_len is None or int(data_len) > self._TEST_FILE_SIZE):
-            data_len = self._TEST_FILE_SIZE
-
-        if data_len is not None:
-            data_len = int(data_len) + resume_len
-            min_data_len = self.params.get('min_filesize')
-            max_data_len = self.params.get('max_filesize')
-            if min_data_len is not None and data_len < min_data_len:
-                self.to_screen('\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len))
-                return False
-            if max_data_len is not None and data_len > max_data_len:
-                self.to_screen('\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len))
-                return False
-
-        byte_counter = 0 + resume_len
-        block_size = self.params.get('buffersize', 1024)
-        start = time.time()
-
-        # measure time over whole while-loop, so slow_down() and best_block_size() work together properly
-        now = None  # needed for slow_down() in the first loop run
-        before = start  # start measuring
-        while True:
-
-            # Download and write
-            data_block = data.read(block_size if not is_test else min(block_size, data_len - byte_counter))
-            byte_counter += len(data_block)
-
-            # exit loop when download is finished
-            if len(data_block) == 0:
-                break
-
-            # Open destination file just in time
-            if stream is None:
-                try:
-                    (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
-                    assert stream is not None
-                    filename = self.undo_temp_name(tmpfilename)
-                    self.report_destination(filename)
-                except (OSError, IOError) as err:
-                    self.report_error('unable to open for writing: %s' % str(err))
+            if data_len is not None:
+                data_len = int(data_len) + ctx.resume_len
+                min_data_len = self.params.get('min_filesize')
+                max_data_len = self.params.get('max_filesize')
+                if min_data_len is not None and data_len < min_data_len:
+                    self.to_screen('\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len))
+                    return False
+                if max_data_len is not None and data_len > max_data_len:
+                    self.to_screen('\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len))
                     return False
 
-                if self.params.get('xattr_set_filesize', False) and data_len is not None:
+            byte_counter = 0 + ctx.resume_len
+            block_size = self.params.get('buffersize', 1024)
+            start = time.time()
+
+            # measure time over whole while-loop, so slow_down() and best_block_size() work together properly
+            now = None  # needed for slow_down() in the first loop run
+            before = start  # start measuring
+
+            def retry(e):
+                if ctx.tmpfilename != '-':
+                    ctx.stream.close()
+                ctx.stream = None
+                ctx.resume_len = os.path.getsize(encodeFilename(ctx.tmpfilename))
+                raise RetryDownload(e)
+
+            while True:
+                try:
+                    # Download and write
+                    data_block = ctx.data.read(block_size if not is_test else min(block_size, data_len - byte_counter))
+                # socket.timeout is a subclass of socket.error but may not have
+                # errno set
+                except socket.timeout as e:
+                    retry(e)
+                except socket.error as e:
+                    if e.errno not in (errno.ECONNRESET, errno.ETIMEDOUT):
+                        raise
+                    retry(e)
+
+                byte_counter += len(data_block)
+
+                # exit loop when download is finished
+                if len(data_block) == 0:
+                    break
+
+                # Open destination file just in time
+                if ctx.stream is None:
                     try:
-                        write_xattr(tmpfilename, 'user.ytdl.filesize', str(data_len).encode('utf-8'))
-                    except (XAttrUnavailableError, XAttrMetadataError) as err:
-                        self.report_error('unable to set filesize xattr: %s' % str(err))
+                        ctx.stream, ctx.tmpfilename = sanitize_open(
+                            ctx.tmpfilename, ctx.open_mode)
+                        assert ctx.stream is not None
+                        ctx.filename = self.undo_temp_name(ctx.tmpfilename)
+                        self.report_destination(ctx.filename)
+                    except (OSError, IOError) as err:
+                        self.report_error('unable to open for writing: %s' % str(err))
+                        return False
 
-            try:
-                stream.write(data_block)
-            except (IOError, OSError) as err:
+                    if self.params.get('xattr_set_filesize', False) and data_len is not None:
+                        try:
+                            write_xattr(ctx.tmpfilename, 'user.ytdl.filesize', str(data_len).encode('utf-8'))
+                        except (XAttrUnavailableError, XAttrMetadataError) as err:
+                            self.report_error('unable to set filesize xattr: %s' % str(err))
+
+                try:
+                    ctx.stream.write(data_block)
+                except (IOError, OSError) as err:
+                    self.to_stderr('\n')
+                    self.report_error('unable to write data: %s' % str(err))
+                    return False
+
+                # Apply rate limit
+                self.slow_down(start, now, byte_counter - ctx.resume_len)
+
+                # end measuring of one loop run
+                now = time.time()
+                after = now
+
+                # Adjust block size
+                if not self.params.get('noresizebuffer', False):
+                    block_size = self.best_block_size(after - before, len(data_block))
+
+                before = after
+
+                # Progress message
+                speed = self.calc_speed(start, now, byte_counter - ctx.resume_len)
+                if data_len is None:
+                    eta = None
+                else:
+                    eta = self.calc_eta(start, time.time(), data_len - ctx.resume_len, byte_counter - ctx.resume_len)
+
+                self._hook_progress({
+                    'status': 'downloading',
+                    'downloaded_bytes': byte_counter,
+                    'total_bytes': data_len,
+                    'tmpfilename': ctx.tmpfilename,
+                    'filename': ctx.filename,
+                    'eta': eta,
+                    'speed': speed,
+                    'elapsed': now - start,
+                })
+
+                if is_test and byte_counter == data_len:
+                    break
+
+            if ctx.stream is None:
                 self.to_stderr('\n')
-                self.report_error('unable to write data: %s' % str(err))
+                self.report_error('Did not get any data blocks')
                 return False
+            if ctx.tmpfilename != '-':
+                ctx.stream.close()
 
-            # Apply rate limit
-            self.slow_down(start, now, byte_counter - resume_len)
+            if data_len is not None and byte_counter != data_len:
+                err = ContentTooShortError(byte_counter, int(data_len))
+                if count <= retries:
+                    retry(err)
+                raise err
 
-            # end measuring of one loop run
-            now = time.time()
-            after = now
+            self.try_rename(ctx.tmpfilename, ctx.filename)
 
-            # Adjust block size
-            if not self.params.get('noresizebuffer', False):
-                block_size = self.best_block_size(after - before, len(data_block))
-
-            before = after
-
-            # Progress message
-            speed = self.calc_speed(start, now, byte_counter - resume_len)
-            if data_len is None:
-                eta = None
-            else:
-                eta = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
+            # Update file modification time
+            if self.params.get('updatetime', True):
+                info_dict['filetime'] = self.try_utime(ctx.filename, ctx.data.info().get('last-modified', None))
 
             self._hook_progress({
-                'status': 'downloading',
                 'downloaded_bytes': byte_counter,
-                'total_bytes': data_len,
-                'tmpfilename': tmpfilename,
-                'filename': filename,
-                'eta': eta,
-                'speed': speed,
-                'elapsed': now - start,
+                'total_bytes': byte_counter,
+                'filename': ctx.filename,
+                'status': 'finished',
+                'elapsed': time.time() - start,
             })
 
-            if is_test and byte_counter == data_len:
-                break
+            return True
 
-        if stream is None:
-            self.to_stderr('\n')
-            self.report_error('Did not get any data blocks')
-            return False
-        if tmpfilename != '-':
-            stream.close()
+        while count <= retries:
+            try:
+                establish_connection()
+                download()
+                return True
+            except RetryDownload as e:
+                count += 1
+                if count <= retries:
+                    self.report_retry(e.source_error, count, retries)
+                continue
+            except SucceedDownload:
+                return True
 
-        if data_len is not None and byte_counter != data_len:
-            raise ContentTooShortError(byte_counter, int(data_len))
-        self.try_rename(tmpfilename, filename)
-
-        # Update file modification time
-        if self.params.get('updatetime', True):
-            info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None))
-
-        self._hook_progress({
-            'downloaded_bytes': byte_counter,
-            'total_bytes': byte_counter,
-            'filename': filename,
-            'status': 'finished',
-            'elapsed': time.time() - start,
-        })
-
-        return True
+        self.report_error('giving up after %s retries' % retries)
+        return False

From dd121cc1cab2f077cbf68ee432e83d4d094f0f9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Aug 2017 03:12:56 +0700
Subject: [PATCH 1260/1696] [extractor/common] Extract height from res
 attribute of source tag for HTML5 videos (closes #14034)

---
 youtube_dl/extractor/common.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 1804c4de0..b4af3f987 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2184,6 +2184,9 @@ class InfoExtractor(object):
                     f = parse_content_type(source_attributes.get('type'))
                     is_plain_url, formats = _media_formats(src, media_type, f)
                     if is_plain_url:
+                        # res attribute is not standard but seen several times
+                        # in the wild
+                        f['height'] = int_or_none(source_attributes.get('res'))
                         f.update(formats[0])
                         media_info['formats'].append(f)
                     else:

From 1ed4549942c34cce52b9c641cf9f532c38866149 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Aug 2017 03:27:05 +0700
Subject: [PATCH 1261/1696] [extractor/common] Extract format id from label
 attribute of source tag for HTML5 videos (#14034)

---
 youtube_dl/extractor/common.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b4af3f987..74d30ec50 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2186,7 +2186,10 @@ class InfoExtractor(object):
                     if is_plain_url:
                         # res attribute is not standard but seen several times
                         # in the wild
-                        f['height'] = int_or_none(source_attributes.get('res'))
+                        f.update({
+                            'height': int_or_none(source_attributes.get('res')),
+                            'format_id': source_attributes.get('label'),
+                        })
                         f.update(formats[0])
                         media_info['formats'].append(f)
                     else:

From ff17be3ac921910e5cab6f54a579f379dacae068 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Aug 2017 03:27:20 +0700
Subject: [PATCH 1262/1696] [extractor/generic] Extract from LD-JSON last of
 all Previous sources may contain several formats, e.g.
 http://tamasha.com/v/PgGZ

---
 youtube_dl/extractor/generic.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 49b00b87e..c81efdc00 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2871,12 +2871,6 @@ class GenericIE(InfoExtractor):
                     merged[k] = v
             return merged
 
-        # Looking for http://schema.org/VideoObject
-        json_ld = self._search_json_ld(
-            webpage, video_id, default={}, expected_type='VideoObject')
-        if json_ld.get('url'):
-            return merge_dicts(json_ld, info_dict)
-
         # Look for HTML5 media
         entries = self._parse_html5_media_entries(url, webpage, video_id, m3u8_id='hls')
         if entries:
@@ -2895,6 +2889,12 @@ class GenericIE(InfoExtractor):
                 jwplayer_data, video_id, require_title=False, base_url=url)
             return merge_dicts(info, info_dict)
 
+        # Looking for http://schema.org/VideoObject
+        json_ld = self._search_json_ld(
+            webpage, video_id, default={}, expected_type='VideoObject')
+        if json_ld.get('url'):
+            return merge_dicts(json_ld, info_dict)
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True

From 62c06c593d19c1314c84639128bb3354665399bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Aug 2017 04:24:41 +0700
Subject: [PATCH 1263/1696] [ChangeLog] Actualize

---
 ChangeLog | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index a60bd5fc8..46d76ff3d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,23 @@
+version <unreleased>
+
+Core
++ [extractor/common] Extract height and format id for HTML5 videos (#14034)
+* [downloader/http] Rework HTTP downloader (#506, #809, #2849, #4240, #6023,
+  #8625, #9483)
+    * Simplify code and split into separate routines to facilitate maintaining
+    * Make retry mechanism work on errors during actual download not only
+      during connection establishment phase
+    * Retry on ECONNRESET and ETIMEDOUT during reading data from network
+    * Retry on content too short
+    * Show error description on retry
+
+Extractors
+* [generic] Lower preference for extraction from LD-JSON
+* [rai] Fix audio formats extraction (#14024)
+* [youtube] Fix controversy videos extraction (#14027, #14029)
+* [mixcloud] Fix extraction (#14015, #14020)
+
+
 version 2017.08.23
 
 Core

From f031b760650f57672deecacb3f1991bd6fcd778e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Aug 2017 04:28:04 +0700
Subject: [PATCH 1264/1696] release 2017.08.27

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 3e1ff1536..777932f93 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.08.23*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.08.23**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.08.27*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.08.27**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.08.23
+[debug] youtube-dl version 2017.08.27
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 46d76ff3d..a997759b0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.08.27
 
 Core
 + [extractor/common] Extract height and format id for HTML5 videos (#14034)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 94d35a66a..6a24b70d7 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.08.23'
+__version__ = '2017.08.27'

From 1c9c8de29e83bbfc38027dabcd9642f1c41a64b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Aug 2017 05:59:08 +0700
Subject: [PATCH 1265/1696] [youtube] Fix extraction with
 --youtube-skip-dash-manifest enabled (closes #14037)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 3d0f5a5c9..5a6b735a0 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1621,7 +1621,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
 
         if view_count is None:
-            view_count = extract_view_count(get_video_info)
+            view_count = extract_view_count(video_info)
 
         # Check for "rental" videos
         if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:

From cc0412ef915047233391c978fa1f8d4d1479ddfe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Aug 2017 06:06:49 +0700
Subject: [PATCH 1266/1696] [ChangeLog] Actualize

---
 ChangeLog | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index a997759b0..c38a83670 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,10 @@
+version <unreleased>
+
+Extractors
+
+* [youtube] Fix extraction with --youtube-skip-dash-manifest enabled (#14037)
+
+
 version 2017.08.27
 
 Core

From 2cfa7cbdd09353c8f8b135ba1c0b3a325bb03a16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Aug 2017 06:09:29 +0700
Subject: [PATCH 1267/1696] release 2017.08.27.1

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 777932f93..7959d910b 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.08.27*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.08.27**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.08.27.1*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.08.27.1**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.08.27
+[debug] youtube-dl version 2017.08.27.1
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index c38a83670..ef9ac4660 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.08.27.1
 
 Extractors
 
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6a24b70d7..7504ef0f0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.08.27'
+__version__ = '2017.08.27.1'

From 3902cdd0e3309364809407b895e4060199b89d11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Aug 2017 22:36:57 +0700
Subject: [PATCH 1268/1696] [pornhd] Fix extraction (closes #14005)

---
 youtube_dl/extractor/pornhd.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/pornhd.py b/youtube_dl/extractor/pornhd.py
index 36761788d..b52879c7a 100644
--- a/youtube_dl/extractor/pornhd.py
+++ b/youtube_dl/extractor/pornhd.py
@@ -54,7 +54,7 @@ class PornHdIE(InfoExtractor):
              r'<title>(.+?) - .*?[Pp]ornHD.*?</title>'], webpage, 'title')
 
         sources = self._parse_json(js_to_json(self._search_regex(
-            r"(?s)sources'?\s*:\s*(\{.+?\})\s*\}[;,)]",
+            r"(?s)sources'?\s*[:=]\s*(\{.+?\})",
             webpage, 'sources', default='{}')), video_id)
 
         if not sources:
@@ -82,7 +82,8 @@ class PornHdIE(InfoExtractor):
         view_count = int_or_none(self._html_search_regex(
             r'(\d+) views\s*<', webpage, 'view count', fatal=False))
         thumbnail = self._search_regex(
-            r"'poster'\s*:\s*'([^']+)'", webpage, 'thumbnail', fatal=False)
+            r"poster'?\s*:\s*([\"'])(?P<url>(?:(?!\1).)+)\1", webpage,
+            'thumbnail', fatal=False, group='url')
 
         return {
             'id': video_id,

From fea82c1780cc751267fd2b9b4145996bfc0c1994 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 28 Aug 2017 00:39:22 +0700
Subject: [PATCH 1269/1696] [googledrive] Add support for source format (closes
 #14046)

---
 youtube_dl/extractor/googledrive.py | 121 +++++++++++++++++++---------
 1 file changed, 84 insertions(+), 37 deletions(-)

diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
index 97ff28219..37d373901 100644
--- a/youtube_dl/extractor/googledrive.py
+++ b/youtube_dl/extractor/googledrive.py
@@ -4,6 +4,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    determine_ext,
     ExtractorError,
     int_or_none,
     lowercase_escape,
@@ -15,17 +16,26 @@ class GoogleDriveIE(InfoExtractor):
     _VALID_URL = r'https?://(?:(?:docs|drive)\.google\.com/(?:uc\?.*?id=|file/d/)|video\.google\.com/get_player\?.*?docid=)(?P<id>[a-zA-Z0-9_-]{28,})'
     _TESTS = [{
         'url': 'https://drive.google.com/file/d/0ByeS4oOUV-49Zzh4R1J6R09zazQ/edit?pli=1',
-        'md5': 'd109872761f7e7ecf353fa108c0dbe1e',
+        'md5': '5c602afbbf2c1db91831f5d82f678554',
         'info_dict': {
             'id': '0ByeS4oOUV-49Zzh4R1J6R09zazQ',
             'ext': 'mp4',
             'title': 'Big Buck Bunny.mp4',
             'duration': 45,
         }
+    }, {
+        # video can't be watched anonymously due to view count limit reached,
+        # but can be downloaded (see https://github.com/rg3/youtube-dl/issues/14046)
+        'url': 'https://drive.google.com/file/d/0B-vUyvmDLdWDcEt4WjBqcmI2XzQ/view',
+        'md5': 'bfbd670d03a470bb1e6d4a257adec12e',
+        'info_dict': {
+            'id': '0B-vUyvmDLdWDcEt4WjBqcmI2XzQ',
+            'ext': 'mp4',
+            'title': 'Annabelle Creation (2017)- Z.V1 [TH].MP4',
+        }
     }, {
         # video id is longer than 28 characters
         'url': 'https://drive.google.com/file/d/1ENcQ_jeCuj7y19s66_Ou9dRP4GKGsodiDQ/edit',
-        'md5': 'c230c67252874fddd8170e3fd1a45886',
         'info_dict': {
             'id': '1ENcQ_jeCuj7y19s66_Ou9dRP4GKGsodiDQ',
             'ext': 'mp4',
@@ -147,47 +157,84 @@ class GoogleDriveIE(InfoExtractor):
         webpage = self._download_webpage(
             'http://docs.google.com/file/d/%s' % video_id, video_id)
 
-        reason = self._search_regex(
-            r'"reason"\s*,\s*"([^"]+)', webpage, 'reason', default=None)
-        if reason:
-            raise ExtractorError(reason)
-
-        title = self._search_regex(r'"title"\s*,\s*"([^"]+)', webpage, 'title')
+        title = self._search_regex(
+            r'"title"\s*,\s*"([^"]+)', webpage, 'title',
+            default=None) or self._og_search_title(webpage)
         duration = int_or_none(self._search_regex(
             r'"length_seconds"\s*,\s*"([^"]+)', webpage, 'length seconds',
             default=None))
-        fmt_stream_map = self._search_regex(
-            r'"fmt_stream_map"\s*,\s*"([^"]+)', webpage,
-            'fmt stream map').split(',')
-        fmt_list = self._search_regex(
-            r'"fmt_list"\s*,\s*"([^"]+)', webpage, 'fmt_list').split(',')
-
-        resolutions = {}
-        for fmt in fmt_list:
-            mobj = re.search(
-                r'^(?P<format_id>\d+)/(?P<width>\d+)[xX](?P<height>\d+)', fmt)
-            if mobj:
-                resolutions[mobj.group('format_id')] = (
-                    int(mobj.group('width')), int(mobj.group('height')))
 
         formats = []
-        for fmt_stream in fmt_stream_map:
-            fmt_stream_split = fmt_stream.split('|')
-            if len(fmt_stream_split) < 2:
-                continue
-            format_id, format_url = fmt_stream_split[:2]
-            f = {
-                'url': lowercase_escape(format_url),
-                'format_id': format_id,
-                'ext': self._FORMATS_EXT[format_id],
-            }
-            resolution = resolutions.get(format_id)
-            if resolution:
-                f.update({
-                    'width': resolution[0],
-                    'height': resolution[1],
+        fmt_stream_map = self._search_regex(
+            r'"fmt_stream_map"\s*,\s*"([^"]+)', webpage,
+            'fmt stream map', default='').split(',')
+        fmt_list = self._search_regex(
+            r'"fmt_list"\s*,\s*"([^"]+)', webpage,
+            'fmt_list', default='').split(',')
+        if fmt_stream_map and fmt_list:
+            resolutions = {}
+            for fmt in fmt_list:
+                mobj = re.search(
+                    r'^(?P<format_id>\d+)/(?P<width>\d+)[xX](?P<height>\d+)', fmt)
+                if mobj:
+                    resolutions[mobj.group('format_id')] = (
+                        int(mobj.group('width')), int(mobj.group('height')))
+
+            for fmt_stream in fmt_stream_map:
+                fmt_stream_split = fmt_stream.split('|')
+                if len(fmt_stream_split) < 2:
+                    continue
+                format_id, format_url = fmt_stream_split[:2]
+                f = {
+                    'url': lowercase_escape(format_url),
+                    'format_id': format_id,
+                    'ext': self._FORMATS_EXT[format_id],
+                }
+                resolution = resolutions.get(format_id)
+                if resolution:
+                    f.update({
+                        'width': resolution[0],
+                        'height': resolution[1],
+                    })
+                formats.append(f)
+
+        source_url = update_url_query(
+            'https://drive.google.com/uc', {
+                'id': video_id,
+                'export': 'download',
+            })
+        urlh = self._request_webpage(
+            source_url, video_id, note='Requesting source file',
+            errnote='Unable to request source file', fatal=False)
+        if urlh:
+            def add_source_format(src_url):
+                formats.append({
+                    'url': src_url,
+                    'ext': determine_ext(title, 'mp4').lower(),
+                    'format_id': 'source',
+                    'quality': 1,
                 })
-            formats.append(f)
+            if urlh.headers.get('Content-Disposition'):
+                add_source_format(source_url)
+            else:
+                confirmation_webpage = self._webpage_read_content(
+                    urlh, url, video_id, note='Downloading confirmation page',
+                    errnote='Unable to confirm download', fatal=False)
+                if confirmation_webpage:
+                    confirm = self._search_regex(
+                        r'confirm=([^&"\']+)', confirmation_webpage,
+                        'confirmation code', fatal=False)
+                    if confirm:
+                        add_source_format(update_url_query(source_url, {
+                            'confirm': confirm,
+                        }))
+
+        if not formats:
+            reason = self._search_regex(
+                r'"reason"\s*,\s*"([^"]+)', webpage, 'reason', default=None)
+            if reason:
+                raise ExtractorError(reason, expected=True)
+
         self._sort_formats(formats)
 
         hl = self._search_regex(

From 1b41da488df64bd41463a79691330e555f79ac2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 28 Aug 2017 00:50:41 +0700
Subject: [PATCH 1270/1696] [googledrive] Extend _VALID_URL (closes #9785)

---
 youtube_dl/extractor/googledrive.py | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
index 37d373901..3bf462d63 100644
--- a/youtube_dl/extractor/googledrive.py
+++ b/youtube_dl/extractor/googledrive.py
@@ -13,7 +13,18 @@ from ..utils import (
 
 
 class GoogleDriveIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:docs|drive)\.google\.com/(?:uc\?.*?id=|file/d/)|video\.google\.com/get_player\?.*?docid=)(?P<id>[a-zA-Z0-9_-]{28,})'
+    _VALID_URL = r'''(?x)
+                        https?://
+                            (?:
+                                (?:docs|drive)\.google\.com/
+                                (?:
+                                    (?:uc|open)\?.*?id=|
+                                    file/d/
+                                )|
+                                video\.google\.com/get_player\?.*?docid=
+                            )
+                            (?P<id>[a-zA-Z0-9_-]{28,})
+                    '''
     _TESTS = [{
         'url': 'https://drive.google.com/file/d/0ByeS4oOUV-49Zzh4R1J6R09zazQ/edit?pli=1',
         'md5': '5c602afbbf2c1db91831f5d82f678554',
@@ -42,7 +53,13 @@ class GoogleDriveIE(InfoExtractor):
             'title': 'Andreea Banica feat Smiley - Hooky Song (Official Video).mp4',
             'duration': 189,
         },
-        'only_matching': True
+        'only_matching': True,
+    }, {
+        'url': 'https://drive.google.com/open?id=0B2fjwgkl1A_CX083Tkowdmt6d28',
+        'only_matching': True,
+    }, {
+        'url': 'https://drive.google.com/uc?id=0B2fjwgkl1A_CX083Tkowdmt6d28',
+        'only_matching': True,
     }]
     _FORMATS_EXT = {
         '5': 'flv',

From c75c384fb6d64f8d44c10d798cb64c8c00c61175 Mon Sep 17 00:00:00 2001
From: Ryan Schmidt <ryandesign@macports.org>
Date: Sun, 27 Aug 2017 18:07:09 -0500
Subject: [PATCH 1271/1696] Fix build failures with old cp and zip

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 41e3a683a..c74eea792 100644
--- a/Makefile
+++ b/Makefile
@@ -49,11 +49,11 @@ youtube-dl: youtube_dl/*.py youtube_dl/*/*.py
 	mkdir -p zip
 	for d in youtube_dl youtube_dl/downloader youtube_dl/extractor youtube_dl/postprocessor ; do \
 	  mkdir -p zip/$$d ;\
-	  cp -a $$d/*.py zip/$$d/ ;\
+	  cp -pPR $$d/*.py zip/$$d/ ;\
 	done
 	touch -t 200001010101 zip/youtube_dl/*.py zip/youtube_dl/*/*.py
 	mv zip/youtube_dl/__main__.py zip/
-	cd zip ; zip --quiet ../youtube-dl youtube_dl/*.py youtube_dl/*/*.py __main__.py
+	cd zip ; zip -q ../youtube-dl youtube_dl/*.py youtube_dl/*/*.py __main__.py
 	rm -rf zip
 	echo '#!$(PYTHON)' > youtube-dl
 	cat youtube-dl.zip >> youtube-dl

From 53647dfd0ae5f3d369e01f04acd5eff9713cc91d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Aug 2017 05:27:56 +0700
Subject: [PATCH 1272/1696] [bbccouk] Add support for w-prefixed ids (closes
 #14056)

---
 youtube_dl/extractor/bbc.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 911ae6780..8b20c03d6 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -29,7 +29,7 @@ from ..compat import (
 class BBCCoUkIE(InfoExtractor):
     IE_NAME = 'bbc.co.uk'
     IE_DESC = 'BBC iPlayer'
-    _ID_REGEX = r'[pb][\da-z]{7}'
+    _ID_REGEX = r'[pbw][\da-z]{7}'
     _VALID_URL = r'''(?x)
                     https?://
                         (?:www\.)?bbc\.co\.uk/
@@ -233,6 +233,9 @@ class BBCCoUkIE(InfoExtractor):
         }, {
             'url': 'https://www.bbc.co.uk/music/audiovideo/popular#p055bc55',
             'only_matching': True,
+        }, {
+            'url': 'http://www.bbc.co.uk/programmes/w3csv1y9',
+            'only_matching': True,
         }]
 
     _USP_RE = r'/([^/]+?)\.ism(?:\.hlsv2\.ism)?/[^/]+\.m3u8'

From 5b4bfbfc3bba2174234834a08a97e261e492a2f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Aug 2017 23:50:33 +0700
Subject: [PATCH 1273/1696] [charlierose] Add support for episodes (closes
 #14062)

---
 youtube_dl/extractor/charlierose.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/charlierose.py b/youtube_dl/extractor/charlierose.py
index 2d517f231..42c9af263 100644
--- a/youtube_dl/extractor/charlierose.py
+++ b/youtube_dl/extractor/charlierose.py
@@ -5,7 +5,7 @@ from ..utils import remove_end
 
 
 class CharlieRoseIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?charlierose\.com/video(?:s|/player)/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?charlierose\.com/(?:video|episode)(?:s|/player)/(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://charlierose.com/videos/27996',
         'md5': 'fda41d49e67d4ce7c2411fd2c4702e09',
@@ -24,6 +24,9 @@ class CharlieRoseIE(InfoExtractor):
     }, {
         'url': 'https://charlierose.com/videos/27996',
         'only_matching': True,
+    }, {
+        'url': 'https://charlierose.com/episodes/30887?autoplay=true',
+        'only_matching': True,
     }]
 
     _PLAYER_BASE = 'https://charlierose.com/video/player/%s'

From 7998520933db70625ff3859c464fa4bbf2ff7fac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 31 Aug 2017 00:47:58 +0700
Subject: [PATCH 1274/1696] [youtube] Fix upload date extraction (closes
 #14065)

---
 youtube_dl/extractor/youtube.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 5a6b735a0..563cf6274 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1665,10 +1665,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         if not upload_date:
             upload_date = self._search_regex(
                 [r'(?s)id="eow-date.*?>(.*?)</span>',
-                 r'id="watch-uploader-info".*?>.*?(?:Published|Uploaded|Streamed live|Started) on (.+?)</strong>'],
+                 r'(?:id="watch-uploader-info".*?>.*?|["\']simpleText["\']\s*:\s*["\'])(?:Published|Uploaded|Streamed live|Started) on (.+?)[<"\']'],
                 video_webpage, 'upload date', default=None)
-            if upload_date:
-                upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
         upload_date = unified_strdate(upload_date)
 
         video_license = self._html_search_regex(

From 8d81f3e36d3caaeabcdff99d3340d4075d30741e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Sep 2017 00:57:14 +0700
Subject: [PATCH 1275/1696] [youtube] Force old layout for each webpage (closes
 #14083)

---
 youtube_dl/extractor/youtube.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 563cf6274..953e38227 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -16,6 +16,7 @@ from ..jsinterp import JSInterpreter
 from ..swfinterp import SWFInterpreter
 from ..compat import (
     compat_chr,
+    compat_kwargs,
     compat_parse_qs,
     compat_urllib_parse_unquote,
     compat_urllib_parse_unquote_plus,
@@ -245,6 +246,11 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
 
         return True
 
+    def _download_webpage(self, *args, **kwargs):
+        kwargs.setdefault('query', {})['disable_polymer'] = 'true'
+        return super(YoutubeBaseInfoExtractor, self)._download_webpage(
+            *args, **compat_kwargs(kwargs))
+
     def _real_initialize(self):
         if self._downloader is None:
             return
@@ -2052,7 +2058,7 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
                      |
                         (%(playlist_id)s)
                      )""" % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
-    _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s&disable_polymer=true'
+    _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s'
     _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})&amp;[^"]*?index=(?P<index>\d+)(?:[^>]+>(?P<title>[^<]+))?'
     IE_NAME = 'youtube:playlist'
     _TESTS = [{

From 8681ed7fc80a6b4b99e3a57adf5ec8d4177a601c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Sep 2017 01:04:22 +0700
Subject: [PATCH 1276/1696] [ChangeLog] Actualize

---
 ChangeLog | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index ef9ac4660..d89fb3d18 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,16 @@
+version <unreleased>
+
+Extractors
+* [youtube] Force old layout for each webpage (#14068, #14072, #14074, #14076,
+  #14077, #14079, #14082, #14083, #14094, #14095, #14096)
+* [youtube] Fix upload date extraction (#14065)
++ [charlierose] Add support for episodes (#14062)
++ [bbccouk] Add support for w-prefixed ids (#14056)
+* [googledrive] Extend URL regular expression (#9785)
++ [googledrive] Add support for source format (#14046)
+* [pornhd] Fix extraction (#14005)
+
+
 version 2017.08.27.1
 
 Extractors

From a2022b0c406286aa3d5101f1c85e7d11453e89d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Sep 2017 01:08:32 +0700
Subject: [PATCH 1277/1696] release 2017.09.02

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 7959d910b..bd9e21983 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.08.27.1*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.08.27.1**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.09.02*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.09.02**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.08.27.1
+[debug] youtube-dl version 2017.09.02
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index d89fb3d18..c439c8ef9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.09.02
 
 Extractors
 * [youtube] Force old layout for each webpage (#14068, #14072, #14074, #14076,
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 7504ef0f0..60ed35de9 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.08.27.1'
+__version__ = '2017.09.02'

From a3431e12249530aa7a6962e54bcdbfce190c4c4d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Sep 2017 15:33:54 +0700
Subject: [PATCH 1278/1696] [radiocanada] Skip unsupported platforms (closes
 #14100)

---
 youtube_dl/extractor/radiocanada.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/radiocanada.py b/youtube_dl/extractor/radiocanada.py
index 3b40002a8..6bbc2781c 100644
--- a/youtube_dl/extractor/radiocanada.py
+++ b/youtube_dl/extractor/radiocanada.py
@@ -59,6 +59,7 @@ class RadioCanadaIE(InfoExtractor):
             device_types.append('android')
 
         formats = []
+        error = None
         # TODO: extract f4m formats
         # f4m formats can be extracted using flashhd device_type but they produce unplayable file
         for device_type in device_types:
@@ -84,8 +85,8 @@ class RadioCanadaIE(InfoExtractor):
             if not v_url:
                 continue
             if v_url == 'null':
-                raise ExtractorError('%s said: %s' % (
-                    self.IE_NAME, xpath_text(v_data, 'message')), expected=True)
+                error = xpath_text(v_data, 'message')
+                continue
             ext = determine_ext(v_url)
             if ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
@@ -129,6 +130,9 @@ class RadioCanadaIE(InfoExtractor):
                             formats.extend(self._extract_f4m_formats(
                                 base_url + '/manifest.f4m', video_id,
                                 f4m_id='hds', fatal=False))
+        if not formats and error:
+            raise ExtractorError(
+                '%s said: %s' % (self.IE_NAME, error), expected=True)
         self._sort_formats(formats)
 
         subtitles = {}

From 64f0e30b93db804323a6db382d4ccdf1dac4e38b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Sep 2017 15:44:49 +0700
Subject: [PATCH 1279/1696] [viidea] Capture and output lecture error message
 (#14099)

---
 youtube_dl/extractor/viidea.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/viidea.py b/youtube_dl/extractor/viidea.py
index 4adcd1830..a0abbae60 100644
--- a/youtube_dl/extractor/viidea.py
+++ b/youtube_dl/extractor/viidea.py
@@ -4,12 +4,14 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urlparse,
+    compat_HTTPError,
     compat_str,
+    compat_urlparse,
 )
 from ..utils import (
-    parse_duration,
+    ExtractorError,
     js_to_json,
+    parse_duration,
     parse_iso8601,
 )
 
@@ -128,9 +130,16 @@ class ViideaIE(InfoExtractor):
 
         base_url = self._proto_relative_url(cfg['livepipe'], 'http:')
 
-        lecture_data = self._download_json(
-            '%s/site/api/lecture/%s?format=json' % (base_url, lecture_id),
-            lecture_id)['lecture'][0]
+        try:
+            lecture_data = self._download_json(
+                '%s/site/api/lecture/%s?format=json' % (base_url, lecture_id),
+                lecture_id)['lecture'][0]
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+                msg = self._parse_json(
+                    e.cause.read().decode('utf-8'), lecture_id)
+                raise ExtractorError(msg['detail'], expected=True)
+            raise
 
         lecture_info = {
             'id': lecture_id,

From 503115540d8f135dc944ae48e40ba78f36238867 Mon Sep 17 00:00:00 2001
From: dubber0 <rexa.mose@gmail.com>
Date: Sat, 22 Jul 2017 21:32:51 +0200
Subject: [PATCH 1280/1696] [aliexpress:live] Add extractor

---
 youtube_dl/extractor/aliexpress.py | 40 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  1 +
 2 files changed, 41 insertions(+)
 create mode 100644 youtube_dl/extractor/aliexpress.py

diff --git a/youtube_dl/extractor/aliexpress.py b/youtube_dl/extractor/aliexpress.py
new file mode 100644
index 000000000..3997213f8
--- /dev/null
+++ b/youtube_dl/extractor/aliexpress.py
@@ -0,0 +1,40 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+
+import re
+
+from .common import InfoExtractor
+from ..utils import try_get, float_or_none
+from ..compat import compat_str
+
+
+class AliExpressLiveIE(InfoExtractor):
+
+    _VALID_URL = r'https?://live\.aliexpress\.com/live/(?P<id>[0-9]{16})'
+    _TEST = {
+        'url': 'https://live.aliexpress.com/live/2800002704436634',
+        'md5': '7ac2bc46afdd18f0b45a0a340fc47ffe',
+        'info_dict': {
+            'id': '2800002704436634',
+            'ext': 'm3u8',
+            'title': 'CASIMA7.22',
+            'uploader': 'CASIMA Official Store',
+            'upload_date': '20170714',
+            'timestamp': 1500027138,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        page = self._download_webpage(url, video_id)
+        run_params_json = self._search_regex(r'runParams = (.+)[\s+]var myCtl', page, 'runParams', flags=re.DOTALL)
+        run_params = self._parse_json(run_params_json, video_id)
+
+        return {
+            'id': video_id,
+            'title': run_params['title'],
+            'url': run_params['replyStreamUrl'],
+            'uploader': try_get(run_params, lambda x: x['followBar']['name'], compat_str),
+            'timestamp': float_or_none(try_get(run_params, lambda x: x['followBar']['createTime']) / 1000),
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 17048fd6e..d335f9fff 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -45,6 +45,7 @@ from .anvato import AnvatoIE
 from .anysex import AnySexIE
 from .aol import AolIE
 from .allocine import AllocineIE
+from .aliexpress import AliExpressLiveIE
 from .aparat import AparatIE
 from .appleconnect import AppleConnectIE
 from .appletrailers import (

From 23b2df82c70a832e485aaf52befa26e27a904995 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Sep 2017 16:04:36 +0700
Subject: [PATCH 1281/1696] [aliexpress:live] Fix issues (closes #13698, closes
 #13707)

---
 youtube_dl/extractor/aliexpress.py | 47 +++++++++++++++++++-----------
 1 file changed, 30 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/aliexpress.py b/youtube_dl/extractor/aliexpress.py
index 3997213f8..6f241e683 100644
--- a/youtube_dl/extractor/aliexpress.py
+++ b/youtube_dl/extractor/aliexpress.py
@@ -1,40 +1,53 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-
-import re
-
 from .common import InfoExtractor
-from ..utils import try_get, float_or_none
 from ..compat import compat_str
+from ..utils import (
+    float_or_none,
+    try_get,
+)
 
 
 class AliExpressLiveIE(InfoExtractor):
-
-    _VALID_URL = r'https?://live\.aliexpress\.com/live/(?P<id>[0-9]{16})'
+    _VALID_URL = r'https?://live\.aliexpress\.com/live/(?P<id>\d+)'
     _TEST = {
         'url': 'https://live.aliexpress.com/live/2800002704436634',
-        'md5': '7ac2bc46afdd18f0b45a0a340fc47ffe',
+        'md5': 'e729e25d47c5e557f2630eaf99b740a5',
         'info_dict': {
             'id': '2800002704436634',
-            'ext': 'm3u8',
+            'ext': 'mp4',
             'title': 'CASIMA7.22',
+            'thumbnail': r're:http://.*\.jpg',
             'uploader': 'CASIMA Official Store',
-            'upload_date': '20170714',
-            'timestamp': 1500027138,
+            'timestamp': 1500717600,
+            'upload_date': '20170722',
         },
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        page = self._download_webpage(url, video_id)
-        run_params_json = self._search_regex(r'runParams = (.+)[\s+]var myCtl', page, 'runParams', flags=re.DOTALL)
-        run_params = self._parse_json(run_params_json, video_id)
+
+        webpage = self._download_webpage(url, video_id)
+
+        data = self._parse_json(
+            self._search_regex(
+                r'(?s)runParams\s*=\s*({.+?})\s*;?\s*var',
+                webpage, 'runParams'),
+            video_id)
+
+        title = data['title']
+
+        formats = self._extract_m3u8_formats(
+            data['replyStreamUrl'], video_id, 'mp4',
+            entry_protocol='m3u8_native', m3u8_id='hls')
 
         return {
             'id': video_id,
-            'title': run_params['title'],
-            'url': run_params['replyStreamUrl'],
-            'uploader': try_get(run_params, lambda x: x['followBar']['name'], compat_str),
-            'timestamp': float_or_none(try_get(run_params, lambda x: x['followBar']['createTime']) / 1000),
+            'title': title,
+            'thumbnail': data.get('coverUrl'),
+            'uploader': try_get(
+                data, lambda x: x['followBar']['name'], compat_str),
+            'timestamp': float_or_none(data.get('startTimeLong'), scale=1000),
+            'formats': formats,
         }

From 73602bcd0c254b735cc93ce5ffeca9e98228190e Mon Sep 17 00:00:00 2001
From: Tatsuyuki Ishi <ishitatsuyuki@gmail.com>
Date: Fri, 1 Sep 2017 17:08:24 +0900
Subject: [PATCH 1282/1696] [soundcloud] Fix download URL with private tracks

---
 youtube_dl/extractor/soundcloud.py | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 2e52e092b..23dcac803 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -1,8 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
 import itertools
+import re
 
 from .common import (
     InfoExtractor,
@@ -17,7 +17,7 @@ from ..utils import (
     ExtractorError,
     int_or_none,
     unified_strdate,
-)
+    update_url_query)
 
 
 class SoundcloudIE(InfoExtractor):
@@ -160,11 +160,13 @@ class SoundcloudIE(InfoExtractor):
             'license': info.get('license'),
         }
         formats = []
+        query = {'client_id': self._CLIENT_ID}
+        if secret_token is not None:
+            query['secret_token'] = secret_token
         if info.get('downloadable', False):
             # We can build a direct link to the song
-            format_url = (
-                'https://api.soundcloud.com/tracks/{0}/download?client_id={1}'.format(
-                    track_id, self._CLIENT_ID))
+            format_url = update_url_query(
+                'https://api.soundcloud.com/tracks/{0}/download'.format(track_id), query)
             formats.append({
                 'format_id': 'download',
                 'ext': info.get('original_format', 'mp3'),
@@ -176,10 +178,7 @@ class SoundcloudIE(InfoExtractor):
         # We have to retrieve the url
         format_dict = self._download_json(
             'https://api.soundcloud.com/i1/tracks/%s/streams' % track_id,
-            track_id, 'Downloading track url', query={
-                'client_id': self._CLIENT_ID,
-                'secret_token': secret_token,
-            })
+            track_id, 'Downloading track url', query=query)
 
         for key, stream_url in format_dict.items():
             abr = int_or_none(self._search_regex(
@@ -216,7 +215,7 @@ class SoundcloudIE(InfoExtractor):
             # cannot be always used, sometimes it can give an HTTP 404 error
             formats.append({
                 'format_id': 'fallback',
-                'url': info['stream_url'] + '?client_id=' + self._CLIENT_ID,
+                'url': update_url_query(info['stream_url'], query),
                 'ext': ext,
             })
 

From d7c7100e3d920512a11bf7c6fee21e26da7ffa73 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 3 Sep 2017 16:18:24 +0700
Subject: [PATCH 1283/1696] [soundcloud] Simplify and add test (closes #14093)

---
 youtube_dl/extractor/soundcloud.py | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 23dcac803..1c6799d57 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -17,7 +17,8 @@ from ..utils import (
     ExtractorError,
     int_or_none,
     unified_strdate,
-    update_url_query)
+    update_url_query,
+)
 
 
 class SoundcloudIE(InfoExtractor):
@@ -120,6 +121,21 @@ class SoundcloudIE(InfoExtractor):
                 'license': 'cc-by-sa',
             },
         },
+        # private link, downloadable format
+        {
+            'url': 'https://soundcloud.com/oriuplift/uponly-238-no-talking-wav/s-AyZUd',
+            'md5': '64a60b16e617d41d0bef032b7f55441e',
+            'info_dict': {
+                'id': '340344461',
+                'ext': 'wav',
+                'title': 'Uplifting Only 238 [No Talking] (incl. Alex Feed Guestmix) (Aug 31, 2017) [wav]',
+                'description': 'md5:fa20ee0fca76a3d6df8c7e57f3715366',
+                'uploader': 'Ori Uplift Music',
+                'upload_date': '20170831',
+                'duration': 7449,
+                'license': 'all-rights-reserved',
+            },
+        },
     ]
 
     _CLIENT_ID = 'JlZIsxg2hY5WnBgtn3jfS0UYCl0K8DOg'
@@ -166,7 +182,7 @@ class SoundcloudIE(InfoExtractor):
         if info.get('downloadable', False):
             # We can build a direct link to the song
             format_url = update_url_query(
-                'https://api.soundcloud.com/tracks/{0}/download'.format(track_id), query)
+                'https://api.soundcloud.com/tracks/%s/download' % track_id, query)
             formats.append({
                 'format_id': 'download',
                 'ext': info.get('original_format', 'mp3'),

From 0cbb841ba94c8d813ff81e817154c5491a796f20 Mon Sep 17 00:00:00 2001
From: Timendum <timedum@gmail.com>
Date: Thu, 31 Aug 2017 12:56:37 +0200
Subject: [PATCH 1284/1696] [bpb] Fix extraction (closes #14043)

---
 youtube_dl/extractor/bpb.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bpb.py b/youtube_dl/extractor/bpb.py
index 9661ade4f..14bc0f77d 100644
--- a/youtube_dl/extractor/bpb.py
+++ b/youtube_dl/extractor/bpb.py
@@ -33,13 +33,13 @@ class BpbIE(InfoExtractor):
         title = self._html_search_regex(
             r'<h2 class="white">(.*?)</h2>', webpage, 'title')
         video_info_dicts = re.findall(
-            r"({\s*src:\s*'http://film\.bpb\.de/[^}]+})", webpage)
+            r"({\s*src\s*:\s*'https://film\.bpb\.de/[^}]+})", webpage)
 
         formats = []
         for video_info in video_info_dicts:
             video_info = self._parse_json(video_info, video_id, transform_source=js_to_json)
-            quality = video_info['quality']
             video_url = video_info['src']
+            quality = 'high' if re.search(r'_high\.', video_url) else 'low'
             formats.append({
                 'url': video_url,
                 'preference': 10 if quality == 'high' else 0,

From c1c1585b316995ca47b59e8dc1e3b463beb1c54a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Sep 2017 16:38:43 +0700
Subject: [PATCH 1285/1696] [bpb] Improve (closes #14086)

---
 youtube_dl/extractor/bpb.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/bpb.py b/youtube_dl/extractor/bpb.py
index 14bc0f77d..07833532e 100644
--- a/youtube_dl/extractor/bpb.py
+++ b/youtube_dl/extractor/bpb.py
@@ -33,13 +33,18 @@ class BpbIE(InfoExtractor):
         title = self._html_search_regex(
             r'<h2 class="white">(.*?)</h2>', webpage, 'title')
         video_info_dicts = re.findall(
-            r"({\s*src\s*:\s*'https://film\.bpb\.de/[^}]+})", webpage)
+            r"({\s*src\s*:\s*'https?://film\.bpb\.de/[^}]+})", webpage)
 
         formats = []
         for video_info in video_info_dicts:
-            video_info = self._parse_json(video_info, video_id, transform_source=js_to_json)
-            video_url = video_info['src']
-            quality = 'high' if re.search(r'_high\.', video_url) else 'low'
+            video_info = self._parse_json(
+                video_info, video_id, transform_source=js_to_json, fatal=False)
+            if not video_info:
+                continue
+            video_url = video_info.get('src')
+            if not video_url:
+                continue
+            quality = 'high' if '_high' in video_url else 'low'
             formats.append({
                 'url': video_url,
                 'preference': 10 if quality == 'high' else 0,

From 0b4a8eb3ac823c26b037eb368c114ce6d976c5c3 Mon Sep 17 00:00:00 2001
From: theychx <boomingzooming@gmail.com>
Date: Mon, 28 Aug 2017 21:35:57 +0200
Subject: [PATCH 1286/1696] [vidme:user] Relax _VALID_URLs

---
 youtube_dl/extractor/vidme.py | 34 ++++++++++++++++++++++++----------
 1 file changed, 24 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index a7971d72e..39b65ed2f 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -263,29 +263,43 @@ class VidmeListBaseIE(InfoExtractor):
 
 class VidmeUserIE(VidmeListBaseIE):
     IE_NAME = 'vidme:user'
-    _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]{6,})(?!/likes)(?:[^\da-zA-Z]|$)'
+    _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z_-]{6,})(?!/likes)(?:[^\da-zA-Z_-]|$)'
     _API_ITEM = 'list'
     _TITLE = 'Videos'
-    _TEST = {
-        'url': 'https://vid.me/EFARCHIVE',
+    _TESTS = [{
+        'url': 'https://vid.me/MasakoX',
         'info_dict': {
-            'id': '3834632',
-            'title': 'EFARCHIVE - %s' % _TITLE,
+            'id': '16112341',
+            'title': 'MasakoX - %s' % _TITLE,
         },
-        'playlist_mincount': 238,
-    }
+        'playlist_mincount': 191,
+    }, {
+        'url': 'https://vid.me/unsQuare_netWork',
+        'info_dict': {
+            'id': '16148757',
+            'title': 'unsQuare_netWork - %s' % _TITLE,
+        },
+        'playlist_mincount': 73,
+    }]
 
 
 class VidmeUserLikesIE(VidmeListBaseIE):
     IE_NAME = 'vidme:user:likes'
-    _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]{6,})/likes'
+    _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z_-]{6,})/likes'
     _API_ITEM = 'likes'
     _TITLE = 'Likes'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://vid.me/ErinAlexis/likes',
         'info_dict': {
             'id': '6483530',
             'title': 'ErinAlexis - %s' % _TITLE,
         },
         'playlist_mincount': 415,
-    }
+    }, {
+        'url': 'https://vid.me/Kaleidoscope-Ish/likes',
+        'info_dict': {
+            'id': '16908594',
+            'title': 'Kaleidoscope-Ish - %s' % _TITLE,
+        },
+        'playlist_mincount': 43,
+    }]

From bc35f075370ed1e67fe71c544e6243a2fc4fa430 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Sep 2017 17:02:11 +0700
Subject: [PATCH 1287/1696] [vidme:user] Make tests only matching (closes
 #14054)

---
 youtube_dl/extractor/vidme.py | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 39b65ed2f..59adb2377 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -275,11 +275,7 @@ class VidmeUserIE(VidmeListBaseIE):
         'playlist_mincount': 191,
     }, {
         'url': 'https://vid.me/unsQuare_netWork',
-        'info_dict': {
-            'id': '16148757',
-            'title': 'unsQuare_netWork - %s' % _TITLE,
-        },
-        'playlist_mincount': 73,
+        'only_matching': True,
     }]
 
 
@@ -297,9 +293,5 @@ class VidmeUserLikesIE(VidmeListBaseIE):
         'playlist_mincount': 415,
     }, {
         'url': 'https://vid.me/Kaleidoscope-Ish/likes',
-        'info_dict': {
-            'id': '16908594',
-            'title': 'Kaleidoscope-Ish - %s' % _TITLE,
-        },
-        'playlist_mincount': 43,
+        'only_matching': True,
     }]

From e9b865267aaa90e3b9e1b0468d20a4df31e13393 Mon Sep 17 00:00:00 2001
From: John D <jdong1992@gmail.com>
Date: Wed, 30 Aug 2017 00:14:43 -0700
Subject: [PATCH 1288/1696] [manyvids] Add support for preview videos (closes
 #14053)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/manyvids.py   | 36 ++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+)
 create mode 100644 youtube_dl/extractor/manyvids.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d335f9fff..46a11f3ef 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -564,6 +564,7 @@ from .mangomolo import (
     MangomoloVideoIE,
     MangomoloLiveIE,
 )
+from .manyvids import ManyVidsIE
 from .matchtv import MatchTVIE
 from .mdr import MDRIE
 from .mediaset import MediasetIE
diff --git a/youtube_dl/extractor/manyvids.py b/youtube_dl/extractor/manyvids.py
new file mode 100644
index 000000000..ea739ce3f
--- /dev/null
+++ b/youtube_dl/extractor/manyvids.py
@@ -0,0 +1,36 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse_unquote
+
+
+class ManyVidsIE(InfoExtractor):
+    _VALID_URL = r'https?://www.manyvids\.com/Video/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'https://www.manyvids.com/Video/133957/everthing-about-me/',
+        'md5': '03f11bb21c52dd12a05be21a5c7dcc97',
+        'info_dict': {
+            'id': '133957',
+            'ext': 'mp4',
+            'title': 'everthing about me',
+
+        }
+    }
+
+    def _real_extract(self, url):
+        formats = []
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        video_url = compat_urllib_parse_unquote(self._search_regex(
+            r'data-video-filepath=\"(.+?)\"', webpage, 'video URL', default=''))
+
+        title = self._html_search_regex(r'<h2[^>]+class="m-a-0"[^>]*>([^<]+)', webpage, 'title')
+        formats.append({
+            'url': video_url
+        })
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+        }

From efc57145c10bdf22da9d8571c35ccd0404e3b7c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Sep 2017 17:30:02 +0700
Subject: [PATCH 1289/1696] [manyvids] Improve (closes #14059)

---
 youtube_dl/extractor/manyvids.py | 40 +++++++++++++++++++++-----------
 1 file changed, 26 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/manyvids.py b/youtube_dl/extractor/manyvids.py
index ea739ce3f..b94b3c2ab 100644
--- a/youtube_dl/extractor/manyvids.py
+++ b/youtube_dl/extractor/manyvids.py
@@ -2,35 +2,47 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_unquote
+from ..utils import int_or_none
 
 
 class ManyVidsIE(InfoExtractor):
-    _VALID_URL = r'https?://www.manyvids\.com/Video/(?P<id>[0-9]+)'
+    _VALID_URL = r'(?i)https?://(?:www\.)?manyvids\.com/video/(?P<id>\d+)'
     _TEST = {
         'url': 'https://www.manyvids.com/Video/133957/everthing-about-me/',
         'md5': '03f11bb21c52dd12a05be21a5c7dcc97',
         'info_dict': {
             'id': '133957',
             'ext': 'mp4',
-            'title': 'everthing about me',
-
-        }
+            'title': 'everthing about me (Preview)',
+            'view_count': int,
+            'like_count': int,
+        },
     }
 
     def _real_extract(self, url):
-        formats = []
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        video_url = compat_urllib_parse_unquote(self._search_regex(
-            r'data-video-filepath=\"(.+?)\"', webpage, 'video URL', default=''))
 
-        title = self._html_search_regex(r'<h2[^>]+class="m-a-0"[^>]*>([^<]+)', webpage, 'title')
-        formats.append({
-            'url': video_url
-        })
+        webpage = self._download_webpage(url, video_id)
+
+        video_url = self._search_regex(
+            r'data-(?:video-filepath|meta-video)\s*=s*(["\'])(?P<url>(?:(?!\1).)+)\1',
+            webpage, 'video URL', group='url')
+
+        title = '%s (Preview)' % self._html_search_regex(
+            r'<h2[^>]+class="m-a-0"[^>]*>([^<]+)', webpage, 'title')
+
+        like_count = int_or_none(self._search_regex(
+            r'data-likes=["\'](\d+)', webpage, 'like count', default=None))
+        view_count = int_or_none(self._html_search_regex(
+            r'(?s)<span[^>]+class="views-wrapper"[^>]*>(.+?)</span', webpage,
+            'view count', default=None))
+
         return {
             'id': video_id,
             'title': title,
-            'formats': formats,
+            'view_count': view_count,
+            'like_count': like_count,
+            'formats': [{
+                'url': video_url,
+            }],
         }

From 6348671c4a4e9f45cebc107ff4c148ef4970bb39 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 Sep 2017 23:08:07 +0700
Subject: [PATCH 1290/1696] [arte] Relax unavailability check (closes #14112)

---
 youtube_dl/extractor/arte.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 02613cf5d..5cde90c5b 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -82,7 +82,7 @@ class ArteTVBaseIE(InfoExtractor):
 
         vsr = player_info['VSR']
 
-        if not vsr and not player_info.get('VRU'):
+        if not vsr:
             raise ExtractorError(
                 'Video %s is not available' % player_info.get('VID') or video_id,
                 expected=True)

From 880fa66f4ffa9afcfce91b5ce39f05909050da67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Sep 2017 22:45:07 +0700
Subject: [PATCH 1291/1696] [redtube] Fix formats extraction (closes #14122)

---
 youtube_dl/extractor/redtube.py | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/redtube.py b/youtube_dl/extractor/redtube.py
index c367a6ae7..f70a75256 100644
--- a/youtube_dl/extractor/redtube.py
+++ b/youtube_dl/extractor/redtube.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -62,7 +63,23 @@ class RedTubeIE(InfoExtractor):
                         'format_id': format_id,
                         'height': int_or_none(format_id),
                     })
-        else:
+        medias = self._parse_json(
+            self._search_regex(
+                r'mediaDefinition\s*:\s*(\[.+?\])', webpage,
+                'media definitions', default='{}'),
+            video_id, fatal=False)
+        if medias and isinstance(medias, list):
+            for media in medias:
+                format_url = media.get('videoUrl')
+                if not format_url or not isinstance(format_url, compat_str):
+                    continue
+                format_id = media.get('quality')
+                formats.append({
+                    'url': format_url,
+                    'format_id': format_id,
+                    'height': int_or_none(format_id),
+                })
+        if not formats:
             video_url = self._html_search_regex(
                 r'<source src="(.+?)" type="video/mp4">', webpage, 'video URL')
             formats.append({'url': video_url})
@@ -73,7 +90,7 @@ class RedTubeIE(InfoExtractor):
             r'<span[^>]+class="added-time"[^>]*>ADDED ([^<]+)<',
             webpage, 'upload date', fatal=False))
         duration = int_or_none(self._search_regex(
-            r'videoDuration\s*:\s*(\d+)', webpage, 'duration', fatal=False))
+            r'videoDuration\s*:\s*(\d+)', webpage, 'duration', default=None))
         view_count = str_to_int(self._search_regex(
             r'<span[^>]*>VIEWS</span></td>\s*<td>([\d,.]+)',
             webpage, 'view count', fatal=False))

From c5c9bf0c120d2c481124a0c3913b981cf061fb95 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Sep 2017 23:31:34 +0700
Subject: [PATCH 1292/1696] [YoutubeDL] Ensure dir existence for each requested
 format (closes #14116)

---
 youtube_dl/YoutubeDL.py | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 5f4c93ea3..4f208f1e1 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1710,12 +1710,17 @@ class YoutubeDL(object):
         if filename is None:
             return
 
-        try:
-            dn = os.path.dirname(sanitize_path(encodeFilename(filename)))
-            if dn and not os.path.exists(dn):
-                os.makedirs(dn)
-        except (OSError, IOError) as err:
-            self.report_error('unable to create directory ' + error_to_compat_str(err))
+        def ensure_dir_exists(path):
+            try:
+                dn = os.path.dirname(path)
+                if dn and not os.path.exists(dn):
+                    os.makedirs(dn)
+                return True
+            except (OSError, IOError) as err:
+                self.report_error('unable to create directory ' + error_to_compat_str(err))
+                return False
+
+        if not ensure_dir_exists(sanitize_path(encodeFilename(filename))):
             return
 
         if self.params.get('writedescription', False):
@@ -1853,8 +1858,11 @@ class YoutubeDL(object):
                         for f in requested_formats:
                             new_info = dict(info_dict)
                             new_info.update(f)
-                            fname = self.prepare_filename(new_info)
-                            fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext'])
+                            fname = prepend_extension(
+                                self.prepare_filename(new_info),
+                                'f%s' % f['format_id'], new_info['ext'])
+                            if not ensure_dir_exists(fname):
+                                return
                             downloaded.append(fname)
                             partial_success = dl(fname, new_info)
                             success = success and partial_success

From 66c9fa36c10860b380806b9de48f38d628289e03 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 6 Sep 2017 00:48:37 +0700
Subject: [PATCH 1293/1696] [youtube] Separate methods for embeds extraction

---
 youtube_dl/extractor/generic.py | 33 ++++-------------------------
 youtube_dl/extractor/youtube.py | 37 +++++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c81efdc00..b83c18380 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2243,36 +2243,11 @@ class GenericIE(InfoExtractor):
         if vid_me_embed_url is not None:
             return self.url_result(vid_me_embed_url, 'Vidme')
 
-        # Look for embedded YouTube player
-        matches = re.findall(r'''(?x)
-            (?:
-                <iframe[^>]+?src=|
-                data-video-url=|
-                <embed[^>]+?src=|
-                embedSWF\(?:\s*|
-                <object[^>]+data=|
-                new\s+SWFObject\(
-            )
-            (["\'])
-                (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
-                (?:embed|v|p)/.+?)
-            \1''', webpage)
-        if matches:
+        # Look for YouTube embeds
+        youtube_urls = YoutubeIE._extract_urls(webpage)
+        if youtube_urls:
             return self.playlist_from_matches(
-                matches, video_id, video_title, lambda m: unescapeHTML(m[1]))
-
-        # Look for lazyYT YouTube embed
-        matches = re.findall(
-            r'class="lazyYT" data-youtube-id="([^"]+)"', webpage)
-        if matches:
-            return self.playlist_from_matches(matches, video_id, video_title, lambda m: unescapeHTML(m))
-
-        # Look for Wordpress "YouTube Video Importer" plugin
-        matches = re.findall(r'''(?x)<div[^>]+
-            class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
-            data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
-        if matches:
-            return self.playlist_from_matches(matches, video_id, video_title, lambda m: m[-1])
+                youtube_urls, video_id, video_title, ie=YoutubeIE.ie_key())
 
         matches = DailymotionIE._extract_urls(webpage)
         if matches:
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 953e38227..ad2e933ee 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1374,6 +1374,43 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             playback_url, video_id, 'Marking watched',
             'Unable to mark watched', fatal=False)
 
+    @staticmethod
+    def _extract_urls(webpage):
+        # Embedded YouTube player
+        entries = [
+            unescapeHTML(mobj.group('url'))
+            for mobj in re.finditer(r'''(?x)
+            (?:
+                <iframe[^>]+?src=|
+                data-video-url=|
+                <embed[^>]+?src=|
+                embedSWF\(?:\s*|
+                <object[^>]+data=|
+                new\s+SWFObject\(
+            )
+            (["\'])
+                (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
+                (?:embed|v|p)/.+?)
+            \1''', webpage)]
+
+        # lazyYT YouTube embed
+        entries.extend(list(map(
+            unescapeHTML,
+            re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage))))
+
+        # Wordpress "YouTube Video Importer" plugin
+        matches = re.findall(r'''(?x)<div[^>]+
+            class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
+            data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
+        entries.extend(m[-1] for m in matches)
+
+        return entries
+
+    @staticmethod
+    def _extract_url(webpage):
+        urls = YoutubeIE._extract_urls(webpage)
+        return urls[0] if urls else None
+
     @classmethod
     def extract_id(cls, url):
         mobj = re.match(cls._VALID_URL, url, re.VERBOSE)

From 5113b6912467619bd463c5ebefe759d07078bea1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 6 Sep 2017 00:50:25 +0700
Subject: [PATCH 1294/1696] [abcnews,chilloutsoze,cracked,vice,vk] Use
 dedicated YouTube embeds extraction routines

---
 youtube_dl/extractor/abcnews.py      | 7 +++----
 youtube_dl/extractor/chilloutzone.py | 9 ++++-----
 youtube_dl/extractor/cracked.py      | 7 +++----
 youtube_dl/extractor/vice.py         | 7 +++----
 youtube_dl/extractor/vk.py           | 7 +++----
 5 files changed, 16 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/abcnews.py b/youtube_dl/extractor/abcnews.py
index 74d54560c..f770fe901 100644
--- a/youtube_dl/extractor/abcnews.py
+++ b/youtube_dl/extractor/abcnews.py
@@ -7,6 +7,7 @@ import time
 
 from .amp import AMPIE
 from .common import InfoExtractor
+from .youtube import YoutubeIE
 from ..compat import compat_urlparse
 
 
@@ -108,9 +109,7 @@ class AbcNewsIE(InfoExtractor):
             r'window\.abcnvideo\.url\s*=\s*"([^"]+)"', webpage, 'video URL')
         full_video_url = compat_urlparse.urljoin(url, video_url)
 
-        youtube_url = self._html_search_regex(
-            r'<iframe[^>]+src="(https://www\.youtube\.com/embed/[^"]+)"',
-            webpage, 'YouTube URL', default=None)
+        youtube_url = YoutubeIE._extract_url(webpage)
 
         timestamp = None
         date_str = self._html_search_regex(
@@ -140,7 +139,7 @@ class AbcNewsIE(InfoExtractor):
         }
 
         if youtube_url:
-            entries = [entry, self.url_result(youtube_url, 'Youtube')]
+            entries = [entry, self.url_result(youtube_url, ie=YoutubeIE.ie_key())]
             return self.playlist_result(entries)
 
         return entry
diff --git a/youtube_dl/extractor/chilloutzone.py b/youtube_dl/extractor/chilloutzone.py
index 0206d96db..d4769da75 100644
--- a/youtube_dl/extractor/chilloutzone.py
+++ b/youtube_dl/extractor/chilloutzone.py
@@ -5,6 +5,7 @@ import base64
 import json
 
 from .common import InfoExtractor
+from .youtube import YoutubeIE
 from ..utils import (
     clean_html,
     ExtractorError
@@ -70,11 +71,9 @@ class ChilloutzoneIE(InfoExtractor):
 
         # If nativePlatform is None a fallback mechanism is used (i.e. youtube embed)
         if native_platform is None:
-            youtube_url = self._html_search_regex(
-                r'<iframe.* src="((?:https?:)?//(?:[^.]+\.)?youtube\.com/.+?)"',
-                webpage, 'fallback video URL', default=None)
-            if youtube_url is not None:
-                return self.url_result(youtube_url, ie='Youtube')
+            youtube_url = YoutubeIE._extract_url(webpage)
+            if youtube_url:
+                return self.url_result(youtube_url, ie=YoutubeIE.ie_key())
 
         # Non Fallback: Decide to use native source (e.g. youtube or vimeo) or
         # the own CDN
diff --git a/youtube_dl/extractor/cracked.py b/youtube_dl/extractor/cracked.py
index 94d03ce2a..f77a68ece 100644
--- a/youtube_dl/extractor/cracked.py
+++ b/youtube_dl/extractor/cracked.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from .youtube import YoutubeIE
 from ..utils import (
     parse_iso8601,
     str_to_int,
@@ -41,11 +42,9 @@ class CrackedIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        youtube_url = self._search_regex(
-            r'<iframe[^>]+src="((?:https?:)?//www\.youtube\.com/embed/[^"]+)"',
-            webpage, 'youtube url', default=None)
+        youtube_url = YoutubeIE._extract_url(webpage)
         if youtube_url:
-            return self.url_result(youtube_url, 'Youtube')
+            return self.url_result(youtube_url, ie=YoutubeIE.ie_key())
 
         video_url = self._html_search_regex(
             [r'var\s+CK_vidSrc\s*=\s*"([^"]+)"', r'<video\s+src="([^"]+)"'],
diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index 54e207b39..b8b8bf979 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -7,6 +7,7 @@ import hashlib
 import json
 
 from .adobepass import AdobePassIE
+from .youtube import YoutubeIE
 from .common import InfoExtractor
 from ..compat import compat_HTTPError
 from ..utils import (
@@ -261,11 +262,9 @@ class ViceArticleIE(InfoExtractor):
         if embed_code:
             return _url_res('ooyala:%s' % embed_code, 'Ooyala')
 
-        youtube_url = self._html_search_regex(
-            r'<iframe[^>]+src="(.*youtube\.com/.*)"',
-            body, 'YouTube URL', default=None)
+        youtube_url = YoutubeIE._extract_url(body)
         if youtube_url:
-            return _url_res(youtube_url, 'Youtube')
+            return _url_res(youtube_url, YoutubeIE.ie_key())
 
         video_url = self._html_search_regex(
             r'data-video-url="([^"]+)"',
diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index dc2719cf9..105e172d5 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -25,6 +25,7 @@ from ..utils import (
 from .dailymotion import DailymotionIE
 from .pladform import PladformIE
 from .vimeo import VimeoIE
+from .youtube import YoutubeIE
 
 
 class VKBaseIE(InfoExtractor):
@@ -345,11 +346,9 @@ class VKIE(VKBaseIE):
             if re.search(error_re, info_page):
                 raise ExtractorError(error_msg % video_id, expected=True)
 
-        youtube_url = self._search_regex(
-            r'<iframe[^>]+src="((?:https?:)?//www.youtube.com/embed/[^"]+)"',
-            info_page, 'youtube iframe', default=None)
+        youtube_url = YoutubeIE._extract_url(info_page)
         if youtube_url:
-            return self.url_result(youtube_url, 'Youtube')
+            return self.url_result(youtube_url, ie=YoutubeIE.ie_key())
 
         vimeo_url = VimeoIE._extract_url(url, info_page)
         if vimeo_url is not None:

From 931edb2ada89db2bf3596ce1ad5c4d808914c7ab Mon Sep 17 00:00:00 2001
From: Olivier Bilodeau <olivier@bottomlesspit.org>
Date: Fri, 8 Sep 2017 10:53:24 -0400
Subject: [PATCH 1295/1696] [radiocanada] Add fallback for title extraction

---
 youtube_dl/extractor/radiocanada.py | 45 ++++++++++++++++++++---------
 1 file changed, 31 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/radiocanada.py b/youtube_dl/extractor/radiocanada.py
index 6bbc2781c..b952e59b4 100644
--- a/youtube_dl/extractor/radiocanada.py
+++ b/youtube_dl/extractor/radiocanada.py
@@ -20,20 +20,37 @@ from ..utils import (
 class RadioCanadaIE(InfoExtractor):
     IE_NAME = 'radiocanada'
     _VALID_URL = r'(?:radiocanada:|https?://ici\.radio-canada\.ca/widgets/mediaconsole/)(?P<app_code>[^:/]+)[:/](?P<id>[0-9]+)'
-    _TEST = {
-        'url': 'http://ici.radio-canada.ca/widgets/mediaconsole/medianet/7184272',
-        'info_dict': {
-            'id': '7184272',
-            'ext': 'mp4',
-            'title': 'Le parcours du tireur capté sur vidéo',
-            'description': 'Images des caméras de surveillance fournies par la GRC montrant le parcours du tireur d\'Ottawa',
-            'upload_date': '20141023',
+    _TESTS = [
+        {
+            'url': 'http://ici.radio-canada.ca/widgets/mediaconsole/medianet/7184272',
+            'info_dict': {
+                'id': '7184272',
+                'ext': 'mp4',
+                'title': 'Le parcours du tireur capté sur vidéo',
+                'description': 'Images des caméras de surveillance fournies par la GRC montrant le parcours du tireur d\'Ottawa',
+                'upload_date': '20141023',
+            },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            }
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }
+        {
+            # empty Title
+            'url': 'http://ici.radio-canada.ca/widgets/mediaconsole/medianet/7754998/',
+            'info_dict': {
+                'id': '7754998',
+                'ext': 'mp4',
+                'title': 'letelejournal22h',
+                'description': 'INTEGRALE WEB 22H-TJ',
+                'upload_date': '20170720',
+            },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
+        }
+    ]
 
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})
@@ -145,7 +162,7 @@ class RadioCanadaIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'title': get_meta('Title'),
+            'title': get_meta('Title') or get_meta('AV-nomEmission'),
             'description': get_meta('Description') or get_meta('ShortDescription'),
             'thumbnail': get_meta('imageHR') or get_meta('imageMR') or get_meta('imageBR'),
             'duration': int_or_none(get_meta('length')),

From 51aee72d16eb844377a44c12e50dbb95cd4ced27 Mon Sep 17 00:00:00 2001
From: kayb94 <30302445+kayb94@users.noreply.github.com>
Date: Fri, 8 Sep 2017 15:13:17 +0000
Subject: [PATCH 1296/1696] [README.md] Clarify how to run extractor specific
 test cases

---
 README.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 6f5d00df3..28ee63f40 100644
--- a/README.md
+++ b/README.md
@@ -936,6 +936,8 @@ To run the test, simply invoke your favorite test runner, or execute a test file
     python test/test_download.py
     nosetests
 
+See item 6 of [new extractor tutorial](#adding-support-for-a-new-site) for how to run extractor specific test cases.
+
 If you want to create a build of youtube-dl yourself, you'll need
 
 * python
@@ -1003,7 +1005,7 @@ After you have ensured this site is distributing its content legally, you can fo
             }
     ```
 5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
-6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
+6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in.
 7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L74-L252). Add tests and code for as many as you want.
 8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://pypi.python.org/pypi/flake8). Also make sure your code works under all [Python](https://www.python.org/) versions claimed supported by youtube-dl, namely 2.6, 2.7, and 3.2+.
 9. When the tests pass, [add](https://git-scm.com/docs/git-add) the new files and [commit](https://git-scm.com/docs/git-commit) them and [push](https://git-scm.com/docs/git-push) the result, like this:

From debed8d759e74507371758d2344ce5afe5e237c2 Mon Sep 17 00:00:00 2001
From: luceatnobis <wehrmeyer.martin@web.de>
Date: Tue, 4 Jul 2017 11:26:02 +0200
Subject: [PATCH 1297/1696] [rutube:playlist] Add extractor (closes #13534)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/rutube.py     | 84 ++++++++++++++++++++++++++++++
 2 files changed, 85 insertions(+)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 46a11f3ef..aefadc56f 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -899,6 +899,7 @@ from .rutube import (
     RutubeEmbedIE,
     RutubeMovieIE,
     RutubePersonIE,
+    RutubePlaylistIE,
 )
 from .rutv import RUTVIE
 from .ruutu import RuutuIE
diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index 889fa7628..a6b17c0ef 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -7,10 +7,14 @@ import itertools
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
+    compat_parse_qs,
+    compat_urllib_parse_urlparse,
 )
 from ..utils import (
     determine_ext,
     unified_strdate,
+    try_get,
+    int_or_none,
 )
 
 
@@ -42,8 +46,24 @@ class RutubeIE(InfoExtractor):
     }, {
         'url': 'http://rutube.ru/embed/a10e53b86e8f349080f718582ce4c661',
         'only_matching': True,
+    }, {
+        'url': 'http://rutube.ru/video/3eac3b4561676c17df9132a9a1e62e3e/?pl_id=4252',
+        'only_matching': True,
     }]
 
+    @classmethod
+    def suitable(cls, url):
+        parts = compat_urllib_parse_urlparse(url)
+        params = compat_parse_qs(parts.query)
+
+        # see if URL without parameters is OK
+        res = super(RutubeIE, cls).suitable(url)
+
+        if params:  # we only allow pl_id parameter in the url
+            res = res and 'pl_id' in params and len(params) == 1
+
+        return res
+
     @staticmethod
     def _extract_urls(webpage):
         return [mobj.group('url') for mobj in re.finditer(
@@ -193,3 +213,67 @@ class RutubePersonIE(RutubeChannelIE):
     }]
 
     _PAGE_TEMPLATE = 'http://rutube.ru/api/video/person/%s/?page=%s&format=json'
+
+
+class RutubePlaylistIE(InfoExtractor):
+    IE_NAME = 'rutube:playlist'
+    IE_DESC = 'Rutube playlists'
+    _TESTS = [{
+        'url': 'https://rutube.ru/video/10b3a03fc01d5bbcc632a2f3514e8aab/?pl_id=4252&pl_type=source',
+        'info_dict': {
+            'id': '4252',
+        },
+        'playlist_count': 25,
+    }]
+
+    _VALID_URL = r'https?://rutube\.ru/(?:video|(?:play/)?embed)/[\da-z]{32}/\?(?:.+)?pl_id=(?P<id>\d+)'
+    _PAGE_TEMPLATE = 'http://rutube.ru/api/playlist/source/%s/?page=%s'
+
+    @staticmethod
+    def suitable(url):
+        params = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
+        return params.get('pl_id') and int_or_none(params['pl_id'][0]) \
+            and params.get('pl_type')
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        return self._extract_playlist(playlist_id)
+
+    def _extract_playlist(self, playlist_id):
+        entries = []
+        for pagenum in itertools.count(1):
+            page_url = self._PAGE_TEMPLATE % (playlist_id, pagenum)
+
+            # download_json will sent an accept: application/xml header
+            page = self._download_json(page_url, playlist_id,
+                                       "Downloading metadata for page %s" % pagenum,
+                                       headers={'Accept': 'application/json'})
+
+            if not page['results']:
+                break
+
+            results = page['results']
+            for result in results:
+                entry = self.url_result(result.get('video_url'), 'Rutube')
+                category = try_get(result, lambda x: x['category']['name'])
+                entry.update({
+                    'id': result.get('id'),
+                    'uploader': try_get(result, lambda x: x['author']['name']),
+                    'uploader_id': try_get(result, lambda x: x['author']['id']),
+                    'upload_date': unified_strdate(result.get('created_ts')),
+                    'title': result.get('title'),
+                    'description': result.get('description'),
+                    'thumbnail': result.get('thumbnail_url'),
+                    'duration': int_or_none(result.get('duration')),
+                    'category': [category] if category else None,
+                    'age_limit': 18 if result.get('is_adult') else 0,
+                    'view_count': int_or_none(result.get('hits')),
+                    'is_live': result.get('is_livestream'),
+                    'webpage_url': result.get('video_url'),
+                })
+                entries.append(entry)
+
+            if page['has_next'] is False:
+                break
+
+        return self.playlist_result(entries, playlist_id, page['name'])

From 48b813748d91acc7e9efc15075079a03faea18ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Sep 2017 18:39:13 +0700
Subject: [PATCH 1298/1696] [rutube] Rework and generalize playlist extractors
 (closes #13565)

---
 youtube_dl/extractor/rutube.py | 216 ++++++++++++++++-----------------
 1 file changed, 108 insertions(+), 108 deletions(-)

diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index a6b17c0ef..5a184879f 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -12,34 +12,60 @@ from ..compat import (
 )
 from ..utils import (
     determine_ext,
-    unified_strdate,
+    unified_timestamp,
     try_get,
     int_or_none,
 )
 
 
-class RutubeIE(InfoExtractor):
+class RutubeBaseIE(InfoExtractor):
+    def _extract_video(self, video, video_id=None, require_title=True):
+        title = video['title'] if require_title else video.get('title')
+
+        age_limit = video.get('is_adult')
+        if age_limit is not None:
+            age_limit = 18 if age_limit is True else 0
+
+        uploader_id = try_get(video, lambda x: x['author']['id'])
+        category = try_get(video, lambda x: x['category']['name'])
+
+        return {
+            'id': video.get('id') or video_id,
+            'title': title,
+            'description': video.get('description'),
+            'thumbnail': video.get('thumbnail_url'),
+            'duration': int_or_none(video.get('duration')),
+            'uploader': try_get(video, lambda x: x['author']['name']),
+            'uploader_id': compat_str(uploader_id) if uploader_id else None,
+            'timestamp': unified_timestamp(video.get('created_ts')),
+            'category': [category] if category else None,
+            'age_limit': age_limit,
+            'view_count': int_or_none(video.get('hits')),
+            'comment_count': int_or_none(video.get('comments_count')),
+            'is_live': video.get('is_livestream'),
+        }
+
+
+class RutubeIE(RutubeBaseIE):
     IE_NAME = 'rutube'
     IE_DESC = 'Rutube videos'
     _VALID_URL = r'https?://rutube\.ru/(?:video|(?:play/)?embed)/(?P<id>[\da-z]{32})'
 
     _TESTS = [{
         'url': 'http://rutube.ru/video/3eac3b4561676c17df9132a9a1e62e3e/',
+        'md5': '79938ade01294ef7e27574890d0d3769',
         'info_dict': {
             'id': '3eac3b4561676c17df9132a9a1e62e3e',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Раненный кенгуру забежал в аптеку',
             'description': 'http://www.ntdtv.ru ',
             'duration': 80,
             'uploader': 'NTDRussian',
             'uploader_id': '29790',
+            'timestamp': 1381943602,
             'upload_date': '20131016',
             'age_limit': 0,
         },
-        'params': {
-            # It requires ffmpeg (m3u8 download)
-            'skip_download': True,
-        },
     }, {
         'url': 'http://rutube.ru/play/embed/a10e53b86e8f349080f718582ce4c661',
         'only_matching': True,
@@ -49,20 +75,14 @@ class RutubeIE(InfoExtractor):
     }, {
         'url': 'http://rutube.ru/video/3eac3b4561676c17df9132a9a1e62e3e/?pl_id=4252',
         'only_matching': True,
+    }, {
+        'url': 'https://rutube.ru/video/10b3a03fc01d5bbcc632a2f3514e8aab/?pl_type=source',
+        'only_matching': True,
     }]
 
     @classmethod
     def suitable(cls, url):
-        parts = compat_urllib_parse_urlparse(url)
-        params = compat_parse_qs(parts.query)
-
-        # see if URL without parameters is OK
-        res = super(RutubeIE, cls).suitable(url)
-
-        if params:  # we only allow pl_id parameter in the url
-            res = res and 'pl_id' in params and len(params) == 1
-
-        return res
+        return False if RutubePlaylistIE.suitable(url) else super(RutubeIE, cls).suitable(url)
 
     @staticmethod
     def _extract_urls(webpage):
@@ -72,12 +92,12 @@ class RutubeIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         video = self._download_json(
             'http://rutube.ru/api/video/%s/?format=json' % video_id,
             video_id, 'Downloading video JSON')
 
-        # Some videos don't have the author field
-        author = video.get('author') or {}
+        info = self._extract_video(video, video_id)
 
         options = self._download_json(
             'http://rutube.ru/api/play/options/%s/?format=json' % video_id,
@@ -99,19 +119,8 @@ class RutubeIE(InfoExtractor):
                 })
         self._sort_formats(formats)
 
-        return {
-            'id': video['id'],
-            'title': video['title'],
-            'description': video['description'],
-            'duration': video['duration'],
-            'view_count': video['hits'],
-            'formats': formats,
-            'thumbnail': video['thumbnail_url'],
-            'uploader': author.get('name'),
-            'uploader_id': compat_str(author['id']) if author else None,
-            'upload_date': unified_strdate(video['created_ts']),
-            'age_limit': 18 if video['is_adult'] else 0,
-        }
+        info['formats'] = formats
+        return info
 
 
 class RutubeEmbedIE(InfoExtractor):
@@ -123,7 +132,8 @@ class RutubeEmbedIE(InfoExtractor):
         'url': 'http://rutube.ru/video/embed/6722881?vk_puid37=&vk_puid38=',
         'info_dict': {
             'id': 'a10e53b86e8f349080f718582ce4c661',
-            'ext': 'mp4',
+            'ext': 'flv',
+            'timestamp': 1387830582,
             'upload_date': '20131223',
             'uploader_id': '297833',
             'description': 'Видео группы ★http://vk.com/foxkidsreset★ музей Fox Kids и Jetix<br/><br/> восстановлено и сделано в шикоформате subziro89 http://vk.com/subziro89',
@@ -131,7 +141,7 @@ class RutubeEmbedIE(InfoExtractor):
             'title': 'Мистический городок Эйри в Индиан 5 серия озвучка subziro89',
         },
         'params': {
-            'skip_download': 'Requires ffmpeg',
+            'skip_download': True,
         },
     }, {
         'url': 'http://rutube.ru/play/embed/8083783',
@@ -145,10 +155,51 @@ class RutubeEmbedIE(InfoExtractor):
         canonical_url = self._html_search_regex(
             r'<link\s+rel="canonical"\s+href="([^"]+?)"', webpage,
             'Canonical URL')
-        return self.url_result(canonical_url, 'Rutube')
+        return self.url_result(canonical_url, RutubeIE.ie_key())
 
 
-class RutubeChannelIE(InfoExtractor):
+class RutubePlaylistBaseIE(RutubeBaseIE):
+    def _next_page_url(self, page_num, playlist_id, *args, **kwargs):
+        return self._PAGE_TEMPLATE % (playlist_id, page_num)
+
+    def _entries(self, playlist_id, *args, **kwargs):
+        next_page_url = None
+        for pagenum in itertools.count(1):
+            page = self._download_json(
+                next_page_url or self._next_page_url(
+                    pagenum, playlist_id, *args, **kwargs),
+                playlist_id, 'Downloading page %s' % pagenum)
+
+            results = page.get('results')
+            if not results or not isinstance(results, list):
+                break
+
+            for result in results:
+                video_url = result.get('video_url')
+                if not video_url or not isinstance(video_url, compat_str):
+                    continue
+                entry = self._extract_video(result, require_title=False)
+                entry.update({
+                    '_type': 'url',
+                    'url': video_url,
+                    'ie_key': RutubeIE.ie_key(),
+                })
+                yield entry
+
+            next_page_url = page.get('next')
+            if not next_page_url or not page.get('has_next'):
+                break
+
+    def _extract_playlist(self, playlist_id, *args, **kwargs):
+        return self.playlist_result(
+            self._entries(playlist_id, *args, **kwargs),
+            playlist_id, kwargs.get('playlist_name'))
+
+    def _real_extract(self, url):
+        return self._extract_playlist(self._match_id(url))
+
+
+class RutubeChannelIE(RutubePlaylistBaseIE):
     IE_NAME = 'rutube:channel'
     IE_DESC = 'Rutube channels'
     _VALID_URL = r'https?://rutube\.ru/tags/video/(?P<id>\d+)'
@@ -162,27 +213,8 @@ class RutubeChannelIE(InfoExtractor):
 
     _PAGE_TEMPLATE = 'http://rutube.ru/api/tags/video/%s/?page=%s&format=json'
 
-    def _extract_videos(self, channel_id, channel_title=None):
-        entries = []
-        for pagenum in itertools.count(1):
-            page = self._download_json(
-                self._PAGE_TEMPLATE % (channel_id, pagenum),
-                channel_id, 'Downloading page %s' % pagenum)
-            results = page['results']
-            if not results:
-                break
-            entries.extend(self.url_result(result['video_url'], 'Rutube') for result in results)
-            if not page['has_next']:
-                break
-        return self.playlist_result(entries, channel_id, channel_title)
 
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        channel_id = mobj.group('id')
-        return self._extract_videos(channel_id)
-
-
-class RutubeMovieIE(RutubeChannelIE):
+class RutubeMovieIE(RutubePlaylistBaseIE):
     IE_NAME = 'rutube:movie'
     IE_DESC = 'Rutube movies'
     _VALID_URL = r'https?://rutube\.ru/metainfo/tv/(?P<id>\d+)'
@@ -196,11 +228,11 @@ class RutubeMovieIE(RutubeChannelIE):
         movie = self._download_json(
             self._MOVIE_TEMPLATE % movie_id, movie_id,
             'Downloading movie JSON')
-        movie_name = movie['name']
-        return self._extract_videos(movie_id, movie_name)
+        return self._extract_playlist(
+            movie_id, playlist_name=movie.get('name'))
 
 
-class RutubePersonIE(RutubeChannelIE):
+class RutubePersonIE(RutubePlaylistBaseIE):
     IE_NAME = 'rutube:person'
     IE_DESC = 'Rutube person videos'
     _VALID_URL = r'https?://rutube\.ru/video/person/(?P<id>\d+)'
@@ -215,65 +247,33 @@ class RutubePersonIE(RutubeChannelIE):
     _PAGE_TEMPLATE = 'http://rutube.ru/api/video/person/%s/?page=%s&format=json'
 
 
-class RutubePlaylistIE(InfoExtractor):
+class RutubePlaylistIE(RutubePlaylistBaseIE):
     IE_NAME = 'rutube:playlist'
     IE_DESC = 'Rutube playlists'
+    _VALID_URL = r'https?://rutube\.ru/(?:video|(?:play/)?embed)/[\da-z]{32}/\?.*?\bpl_id=(?P<id>\d+)'
     _TESTS = [{
-        'url': 'https://rutube.ru/video/10b3a03fc01d5bbcc632a2f3514e8aab/?pl_id=4252&pl_type=source',
+        'url': 'https://rutube.ru/video/cecd58ed7d531fc0f3d795d51cee9026/?pl_id=3097&pl_type=tag',
         'info_dict': {
-            'id': '4252',
+            'id': '3097',
         },
-        'playlist_count': 25,
+        'playlist_count': 27,
+    }, {
+        'url': 'https://rutube.ru/video/10b3a03fc01d5bbcc632a2f3514e8aab/?pl_id=4252&pl_type=source',
+        'only_matching': True,
     }]
 
-    _VALID_URL = r'https?://rutube\.ru/(?:video|(?:play/)?embed)/[\da-z]{32}/\?(?:.+)?pl_id=(?P<id>\d+)'
-    _PAGE_TEMPLATE = 'http://rutube.ru/api/playlist/source/%s/?page=%s'
+    _PAGE_TEMPLATE = 'http://rutube.ru/api/playlist/%s/%s/?page=%s&format=json'
 
     @staticmethod
     def suitable(url):
         params = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
-        return params.get('pl_id') and int_or_none(params['pl_id'][0]) \
-            and params.get('pl_type')
+        return params.get('pl_type', [None])[0] and int_or_none(params.get('pl_id', [None])[0])
+
+    def _next_page_url(self, page_num, playlist_id, item_kind):
+        return self._PAGE_TEMPLATE % (item_kind, playlist_id, page_num)
 
     def _real_extract(self, url):
-        playlist_id = self._match_id(url)
-        return self._extract_playlist(playlist_id)
-
-    def _extract_playlist(self, playlist_id):
-        entries = []
-        for pagenum in itertools.count(1):
-            page_url = self._PAGE_TEMPLATE % (playlist_id, pagenum)
-
-            # download_json will sent an accept: application/xml header
-            page = self._download_json(page_url, playlist_id,
-                                       "Downloading metadata for page %s" % pagenum,
-                                       headers={'Accept': 'application/json'})
-
-            if not page['results']:
-                break
-
-            results = page['results']
-            for result in results:
-                entry = self.url_result(result.get('video_url'), 'Rutube')
-                category = try_get(result, lambda x: x['category']['name'])
-                entry.update({
-                    'id': result.get('id'),
-                    'uploader': try_get(result, lambda x: x['author']['name']),
-                    'uploader_id': try_get(result, lambda x: x['author']['id']),
-                    'upload_date': unified_strdate(result.get('created_ts')),
-                    'title': result.get('title'),
-                    'description': result.get('description'),
-                    'thumbnail': result.get('thumbnail_url'),
-                    'duration': int_or_none(result.get('duration')),
-                    'category': [category] if category else None,
-                    'age_limit': 18 if result.get('is_adult') else 0,
-                    'view_count': int_or_none(result.get('hits')),
-                    'is_live': result.get('is_livestream'),
-                    'webpage_url': result.get('video_url'),
-                })
-                entries.append(entry)
-
-            if page['has_next'] is False:
-                break
-
-        return self.playlist_result(entries, playlist_id, page['name'])
+        qs = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
+        playlist_kind = qs['pl_type'][0]
+        playlist_id = qs['pl_id'][0]
+        return self._extract_playlist(playlist_id, item_kind=playlist_kind)

From c7e327c4d46a9b72f3f707710194dccf6eee50d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Sep 2017 19:08:39 +0700
Subject: [PATCH 1299/1696] [utils] Introduce bool_or_none

---
 youtube_dl/utils.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 2554a2abd..c42dd4c3a 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1815,6 +1815,10 @@ def float_or_none(v, scale=1, invscale=1, default=None):
         return default
 
 
+def bool_or_none(v, default=None):
+    return v if isinstance(v, bool) else default
+
+
 def strip_or_none(v):
     return None if v is None else v.strip()
 

From c3dd44e08577c2ae0d08951037db5d1db7a321c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Sep 2017 19:09:27 +0700
Subject: [PATCH 1300/1696] [rutube] Use bool_or_none

---
 youtube_dl/extractor/rutube.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index 5a184879f..828c03b48 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -12,9 +12,10 @@ from ..compat import (
 )
 from ..utils import (
     determine_ext,
-    unified_timestamp,
-    try_get,
+    bool_or_none,
     int_or_none,
+    try_get,
+    unified_timestamp,
 )
 
 
@@ -42,7 +43,7 @@ class RutubeBaseIE(InfoExtractor):
             'age_limit': age_limit,
             'view_count': int_or_none(video.get('hits')),
             'comment_count': int_or_none(video.get('comments_count')),
-            'is_live': video.get('is_livestream'),
+            'is_live': bool_or_none(video.get('is_livestream')),
         }
 
 

From bf6ec2fea9087235c14df2a079620fcc2c17b5eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Sep 2017 22:08:32 +0700
Subject: [PATCH 1301/1696] [fox] Fix extraction (#14147)

---
 youtube_dl/extractor/fox.py | 121 ++++++++++++++++++++++++------------
 1 file changed, 82 insertions(+), 39 deletions(-)

diff --git a/youtube_dl/extractor/fox.py b/youtube_dl/extractor/fox.py
index 159fdf9c4..facc665f6 100644
--- a/youtube_dl/extractor/fox.py
+++ b/youtube_dl/extractor/fox.py
@@ -3,56 +3,99 @@ from __future__ import unicode_literals
 
 from .adobepass import AdobePassIE
 from ..utils import (
-    smuggle_url,
-    update_url_query,
+    int_or_none,
+    parse_age_limit,
+    parse_duration,
+    try_get,
+    unified_timestamp,
 )
 
 
 class FOXIE(AdobePassIE):
-    _VALID_URL = r'https?://(?:www\.)?fox\.com/watch/(?P<id>[0-9]+)'
-    _TEST = {
-        'url': 'http://www.fox.com/watch/255180355939/7684182528',
+    _VALID_URL = r'https?://(?:www\.)?fox\.com/watch/(?P<id>[\da-fA-F]+)'
+    _TESTS = [{
+        # clip
+        'url': 'https://www.fox.com/watch/4b765a60490325103ea69888fb2bd4e8/',
         'md5': 'ebd296fcc41dd4b19f8115d8461a3165',
         'info_dict': {
-            'id': '255180355939',
+            'id': '4b765a60490325103ea69888fb2bd4e8',
             'ext': 'mp4',
-            'title': 'Official Trailer: Gotham',
-            'description': 'Tracing the rise of the great DC Comics Super-Villains and vigilantes, Gotham reveals an entirely new chapter that has never been told.',
-            'duration': 129,
-            'timestamp': 1400020798,
-            'upload_date': '20140513',
-            'uploader': 'NEWA-FNG-FOXCOM',
+            'title': 'Aftermath: Bruce Wayne Develops Into The Dark Knight',
+            'description': 'md5:549cd9c70d413adb32ce2a779b53b486',
+            'duration': 102,
+            'timestamp': 1504291893,
+            'upload_date': '20170901',
+            'creator': 'FOX',
+            'series': 'Gotham',
         },
-        'add_ie': ['ThePlatform'],
-    }
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        # episode, geo-restricted
+        'url': 'https://www.fox.com/watch/087036ca7f33c8eb79b08152b4dd75c1/',
+        'only_matching': True,
+    }, {
+        # episode, geo-restricted, tv provided required
+        'url': 'https://www.fox.com/watch/30056b295fb57f7452aeeb4920bc3024/',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
 
-        settings = self._parse_json(self._search_regex(
-            r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);',
-            webpage, 'drupal settings'), video_id)
-        fox_pdk_player = settings['fox_pdk_player']
-        release_url = fox_pdk_player['release_url']
-        query = {
-            'mbr': 'true',
-            'switch': 'http'
-        }
-        if fox_pdk_player.get('access') == 'locked':
-            ap_p = settings['foxAdobePassProvider']
-            rating = ap_p.get('videoRating')
-            if rating == 'n/a':
-                rating = None
-            resource = self._get_mvpd_resource('fbc-fox', None, ap_p['videoGUID'], rating)
-            query['auth'] = self._extract_mvpd_auth(url, video_id, 'fbc-fox', resource)
+        video = self._download_json(
+            'https://api.fox.com/fbc-content/v1_4/video/%s' % video_id,
+            video_id, headers={
+                'apikey': 'abdcbed02c124d393b39e818a4312055',
+                'Content-Type': 'application/json',
+                'Referer': url,
+            })
 
-        info = self._search_json_ld(webpage, video_id, fatal=False)
-        info.update({
-            '_type': 'url_transparent',
-            'ie_key': 'ThePlatform',
-            'url': smuggle_url(update_url_query(release_url, query), {'force_smil_url': True}),
+        title = video['name']
+
+        m3u8_url = self._download_json(
+            video['videoRelease']['url'], video_id)['playURL']
+
+        formats = self._extract_m3u8_formats(
+            m3u8_url, video_id, 'mp4',
+            entry_protocol='m3u8_native', m3u8_id='hls')
+        self._sort_formats(formats)
+
+        description = video.get('description')
+        duration = int_or_none(video.get('durationInSeconds')) or int_or_none(
+            video.get('duration')) or parse_duration(video.get('duration'))
+        timestamp = unified_timestamp(video.get('datePublished'))
+        age_limit = parse_age_limit(video.get('contentRating'))
+
+        data = try_get(
+            video, lambda x: x['trackingData']['properties'], dict) or {}
+
+        creator = data.get('brand') or data.get('network') or video.get('network')
+
+        series = video.get('seriesName') or data.get(
+            'seriesName') or data.get('show')
+        season_number = int_or_none(video.get('seasonNumber'))
+        episode = video.get('name')
+        episode_number = int_or_none(video.get('episodeNumber'))
+        release_year = int_or_none(video.get('releaseYear'))
+
+        if data.get('authRequired'):
+            # TODO: AP
+            pass
+
+        return {
             'id': video_id,
-        })
-
-        return info
+            'title': title,
+            'description': description,
+            'duration': duration,
+            'timestamp': timestamp,
+            'age_limit': age_limit,
+            'creator': creator,
+            'series': series,
+            'season_number': season_number,
+            'episode': episode,
+            'episode_number': episode_number,
+            'release_year': release_year,
+            'formats': formats,
+        }

From b98339b54b1de517def970a955cbbdda3e1d4874 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Sep 2017 22:15:55 +0700
Subject: [PATCH 1302/1696] [ChangeLog] Actualize

---
 ChangeLog | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index c439c8ef9..86b36c37e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,32 @@
+version <unreleased>
+
+Core
++ [utils] Introduce bool_or_none
+* [YoutubeDL] Ensure dir existence for each requested format (#14116)
+
+Extractors
+* [fox] Fix extraction (#14147)
+* [rutube] Use bool_or_none
+* [rutube] Rework and generalize playlist extractors (#13565)
++ [rutube:playlist] Add support for playlists (#13534, #13565)
++ [radiocanada] Add fallback for title extraction (#14145)
+* [vk] Use dedicated YouTube embeds extraction routine
+* [vice] Use dedicated YouTube embeds extraction routine
+* [cracked] Use dedicated YouTube embeds extraction routine
+* [chilloutzone] Use dedicated YouTube embeds extraction routine
+* [abcnews] Use dedicated YouTube embeds extraction routine
+* [youtube] Separate methods for embeds extraction
+* [redtube] Fix formats extraction (#14122)
+* [arte] Relax unavailability check (#14112)
++ [manyvids] Add support for preview videos from manyvids.com (#14053, #14059)
+* [vidme:user] Relax URL regular expression (#14054)
+* [bpb] Fix extraction (#14043, #14086)
+* [soundcloud] Fix download URL with private tracks (#14093)
+* [aliexpress:live] Add support for live.aliexpress.com (#13698, #13707)
+* [viidea] Capture and output lecture error message (#14099)
+* [radiocanada] Skip unsupported platforms (#14100)
+
+
 version 2017.09.02
 
 Extractors

From 806498cf2f35cc98cf0e6c5b46f58ca357a842de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Sep 2017 22:16:55 +0700
Subject: [PATCH 1303/1696] release 2017.09.10

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 CONTRIBUTING.md           | 4 +++-
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 3 +++
 youtube_dl/version.py     | 2 +-
 5 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index bd9e21983..fb934d4ba 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.09.02*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.09.02**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.09.10*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.09.10**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.09.02
+[debug] youtube-dl version 2017.09.10
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index a8091e7b5..333acee80 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -82,6 +82,8 @@ To run the test, simply invoke your favorite test runner, or execute a test file
     python test/test_download.py
     nosetests
 
+See item 6 of [new extractor tutorial](#adding-support-for-a-new-site) for how to run extractor specific test cases.
+
 If you want to create a build of youtube-dl yourself, you'll need
 
 * python
@@ -149,7 +151,7 @@ After you have ensured this site is distributing its content legally, you can fo
             }
     ```
 5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
-6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
+6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in.
 7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L74-L252). Add tests and code for as many as you want.
 8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://pypi.python.org/pypi/flake8). Also make sure your code works under all [Python](https://www.python.org/) versions claimed supported by youtube-dl, namely 2.6, 2.7, and 3.2+.
 9. When the tests pass, [add](https://git-scm.com/docs/git-add) the new files and [commit](https://git-scm.com/docs/git-commit) them and [push](https://git-scm.com/docs/git-push) the result, like this:
diff --git a/ChangeLog b/ChangeLog
index 86b36c37e..99667e5e2 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.09.10
 
 Core
 + [utils] Introduce bool_or_none
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index dbec6c8dc..798a81d3c 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -38,6 +38,7 @@
  - **afreecatv**: afreecatv.com
  - **afreecatv:global**: afreecatv.com
  - **AirMozilla**
+ - **AliExpressLive**
  - **AlJazeera**
  - **Allocine**
  - **AlphaPorno**
@@ -437,6 +438,7 @@
  - **MakerTV**
  - **mangomolo:live**
  - **mangomolo:video**
+ - **ManyVids**
  - **MatchTV**
  - **MDR**: MDR.DE and KiKA
  - **media.ccc.de**
@@ -701,6 +703,7 @@
  - **rutube:embed**: Rutube embedded videos
  - **rutube:movie**: Rutube movies
  - **rutube:person**: Rutube person videos
+ - **rutube:playlist**: Rutube playlists
  - **RUTV**: RUTV.RU
  - **Ruutu**
  - **Ruv**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 60ed35de9..736f753df 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.09.02'
+__version__ = '2017.09.10'

From f12a6e88b2c2632b10c156eb94d91675327485f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 Sep 2017 03:22:27 +0700
Subject: [PATCH 1304/1696] [rutube:playlist] Fix suitable (closes #14166)

---
 youtube_dl/extractor/rutube.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index 828c03b48..89d89b65a 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -265,8 +265,10 @@ class RutubePlaylistIE(RutubePlaylistBaseIE):
 
     _PAGE_TEMPLATE = 'http://rutube.ru/api/playlist/%s/%s/?page=%s&format=json'
 
-    @staticmethod
-    def suitable(url):
+    @classmethod
+    def suitable(cls, url):
+        if not super(RutubePlaylistIE, cls).suitable(url):
+            return False
         params = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
         return params.get('pl_type', [None])[0] and int_or_none(params.get('pl_id', [None])[0])
 

From 43df248f10548e3c43f0f02584a360136f1129d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 Sep 2017 03:27:43 +0700
Subject: [PATCH 1305/1696] [ChangeLog] Actualize

---
 ChangeLog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 99667e5e2..189276408 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [rutube:playlist] Fix suitable (#14166)
+
+
 version 2017.09.10
 
 Core

From 7dacceae75d3c513f442cfd20d778a31bb35d3d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 Sep 2017 03:30:33 +0700
Subject: [PATCH 1306/1696] release 2017.09.11

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index fb934d4ba..f40cb2c4e 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.09.10*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.09.10**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.09.11*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.09.11**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.09.10
+[debug] youtube-dl version 2017.09.11
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 189276408..c286da6c6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.09.11
 
 Extractors
 * [rutube:playlist] Fix suitable (#14166)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 736f753df..cdcb32e06 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.09.10'
+__version__ = '2017.09.11'

From 2709d9fa28155f7abc84d3b57ce4491391d185ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 Sep 2017 04:14:54 +0700
Subject: [PATCH 1307/1696] [animeondemand] Add support for flash videos
 (closes #9944)

---
 youtube_dl/extractor/animeondemand.py | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index 9e28f2579..c22530778 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -46,6 +46,10 @@ class AnimeOnDemandIE(InfoExtractor):
         # Full length film, non-series, ger/jap, Dub/OmU, account required
         'url': 'https://www.anime-on-demand.de/anime/185',
         'only_matching': True,
+    }, {
+        # Flash videos
+        'url': 'https://www.anime-on-demand.de/anime/12',
+        'only_matching': True,
     }]
 
     def _login(self):
@@ -120,10 +124,11 @@ class AnimeOnDemandIE(InfoExtractor):
             formats = []
 
             for input_ in re.findall(
-                    r'<input[^>]+class=["\'].*?streamstarter_html5[^>]+>', html):
+                    r'<input[^>]+class=["\'].*?streamstarter[^>]+>', html):
                 attributes = extract_attributes(input_)
+                title = attributes.get('data-dialog-header')
                 playlist_urls = []
-                for playlist_key in ('data-playlist', 'data-otherplaylist'):
+                for playlist_key in ('data-playlist', 'data-otherplaylist', 'data-stream'):
                     playlist_url = attributes.get(playlist_key)
                     if isinstance(playlist_url, compat_str) and re.match(
                             r'/?[\da-zA-Z]+', playlist_url):
@@ -160,6 +165,23 @@ class AnimeOnDemandIE(InfoExtractor):
                         fatal=False)
                     if not playlist:
                         continue
+                    stream_url = playlist.get('streamurl')
+                    if stream_url:
+                        rtmp = re.search(
+                            r'^(?P<url>rtmpe?://(?P<host>[^/]+)/(?P<app>.+/))(?P<playpath>mp[34]:.+)',
+                            stream_url)
+                        if rtmp:
+                            formats.append({
+                                'url': rtmp.group('url'),
+                                'app': rtmp.group('app'),
+                                'play_path': rtmp.group('playpath'),
+                                'page_url': url,
+                                'player_url': 'https://www.anime-on-demand.de/assets/jwplayer.flash-55abfb34080700304d49125ce9ffb4a6.swf',
+                                'rtmp_real_time': True,
+                                'format_id': 'rtmp',
+                                'ext': 'flv',
+                            })
+                            continue
                     start_video = playlist.get('startvideo', 0)
                     playlist = playlist.get('playlist')
                     if not playlist or not isinstance(playlist, list):

From 018cc61549f417cf1e88af46ff68a17b75e62630 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 Sep 2017 04:22:55 +0700
Subject: [PATCH 1308/1696] [animeondemand] Bypass geo restriction

---
 youtube_dl/extractor/animeondemand.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index c22530778..2a1cd6520 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -21,6 +21,8 @@ class AnimeOnDemandIE(InfoExtractor):
     _LOGIN_URL = 'https://www.anime-on-demand.de/users/sign_in'
     _APPLY_HTML5_URL = 'https://www.anime-on-demand.de/html5apply'
     _NETRC_MACHINE = 'animeondemand'
+    # German-speaking countries of Europe
+    _GEO_COUNTRIES = ['AT', 'CH', 'DE', 'LI', 'LU']
     _TESTS = [{
         # jap, OmU
         'url': 'https://www.anime-on-demand.de/anime/161',

From 2f483758bc6a6661f1215c38161ee626d90ab655 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 Sep 2017 04:32:35 +0700
Subject: [PATCH 1309/1696] [animeondemand] Improve and modernize

---
 youtube_dl/extractor/animeondemand.py | 34 +++++++++++++--------------
 1 file changed, 16 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index 2a1cd6520..69d363311 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -3,16 +3,13 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urlparse,
-    compat_str,
-)
+from ..compat import compat_str
 from ..utils import (
     determine_ext,
     extract_attributes,
     ExtractorError,
-    sanitized_Request,
     urlencode_postdata,
+    urljoin,
 )
 
 
@@ -78,14 +75,13 @@ class AnimeOnDemandIE(InfoExtractor):
             'post url', default=self._LOGIN_URL, group='url')
 
         if not post_url.startswith('http'):
-            post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
-
-        request = sanitized_Request(
-            post_url, urlencode_postdata(login_form))
-        request.add_header('Referer', self._LOGIN_URL)
+            post_url = urljoin(self._LOGIN_URL, post_url)
 
         response = self._download_webpage(
-            request, None, 'Logging in as %s' % username)
+            post_url, None, 'Logging in as %s' % username,
+            data=urlencode_postdata(login_form), headers={
+                'Referer': self._LOGIN_URL,
+            })
 
         if all(p not in response for p in ('>Logout<', 'href="/users/sign_out"')):
             error = self._search_regex(
@@ -154,17 +150,19 @@ class AnimeOnDemandIE(InfoExtractor):
                         format_id_list.append(compat_str(num))
                     format_id = '-'.join(format_id_list)
                     format_note = ', '.join(filter(None, (kind, lang_note)))
-                    request = sanitized_Request(
-                        compat_urlparse.urljoin(url, playlist_url),
+                    item_id_list = []
+                    if format_id:
+                        item_id_list.append(format_id)
+                    item_id_list.append('videomaterial')
+                    playlist = self._download_json(
+                        urljoin(url, playlist_url), video_id,
+                        'Downloading %s JSON' % ' '.join(item_id_list),
                         headers={
                             'X-Requested-With': 'XMLHttpRequest',
                             'X-CSRF-Token': csrf_token,
                             'Referer': url,
                             'Accept': 'application/json, text/javascript, */*; q=0.01',
-                        })
-                    playlist = self._download_json(
-                        request, video_id, 'Downloading %s playlist JSON' % format_id,
-                        fatal=False)
+                        }, fatal=False)
                     if not playlist:
                         continue
                     stream_url = playlist.get('streamurl')
@@ -246,7 +244,7 @@ class AnimeOnDemandIE(InfoExtractor):
                     f.update({
                         'id': '%s-%s' % (f['id'], m.group('kind').lower()),
                         'title': m.group('title'),
-                        'url': compat_urlparse.urljoin(url, m.group('href')),
+                        'url': urljoin(url, m.group('href')),
                     })
                     entries.append(f)
 

From e7c3e33456e155106ad08347d8ab9a2ecd0c8a11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Sep 2017 23:19:53 +0700
Subject: [PATCH 1310/1696] [downloader/fragment] Restart inconsistent
 incomplete fragment downloads (#13731)

---
 youtube_dl/downloader/fragment.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index bccc8ecc1..6f6fb4a77 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -151,10 +151,15 @@ class FragmentFD(FileDownloader):
         if self.__do_ytdl_file(ctx):
             if os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename']))):
                 self._read_ytdl_file(ctx)
+                if ctx['fragment_index'] > 0 and resume_len == 0:
+                    self.report_error(
+                        'Inconsistent state of incomplete fragment download. '
+                        'Restarting from the beginning...')
+                    ctx['fragment_index'] = resume_len = 0
+                    self._write_ytdl_file(ctx)
             else:
                 self._write_ytdl_file(ctx)
-            if ctx['fragment_index'] > 0:
-                assert resume_len > 0
+                assert ctx['fragment_index'] == 0
 
         dest_stream, tmpfilename = sanitize_open(tmpfilename, open_mode)
 

From 319fc70676fea19b71437aab4078d4d72cc1ba5f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Sep 2017 23:50:19 +0700
Subject: [PATCH 1311/1696] [tv4] Relax _VALID_URL (closes #14206)

---
 youtube_dl/extractor/tv4.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tv4.py b/youtube_dl/extractor/tv4.py
index 7aeb2c620..6487039e5 100644
--- a/youtube_dl/extractor/tv4.py
+++ b/youtube_dl/extractor/tv4.py
@@ -18,7 +18,7 @@ class TV4IE(InfoExtractor):
             tv4\.se/(?:[^/]+)/klipp/(?:.*)-|
             tv4play\.se/
             (?:
-                (?:program|barn)/(?:[^\?]+)\?video_id=|
+                (?:program|barn)/(?:[^/]+/|(?:[^\?]+)\?video_id=)|
                 iframe/video/|
                 film/|
                 sport/|
@@ -63,6 +63,10 @@ class TV4IE(InfoExtractor):
             'url': 'http://www.tv4play.se/barn/looney-tunes?video_id=3062412',
             'only_matching': True,
         },
+        {
+            'url': ' http://www.tv4play.se/program/farang/3922081',
+            'only_matching': True,
+        }
     ]
 
     def _real_extract(self, url):

From 0732a90579091ad60b124fd693bdda8ee526e305 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 14 Sep 2017 20:37:46 +0200
Subject: [PATCH 1312/1696] [orf] Add new extractor for f4m stories

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/orf.py        | 116 +++++++++++++++++++++++++++--
 2 files changed, 112 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index aefadc56f..a3a97e940 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -768,6 +768,7 @@ from .ora import OraTVIE
 from .orf import (
     ORFTVthekIE,
     ORFFM4IE,
+    ORFFM4StoryIE,
     ORFOE1IE,
     ORFIPTVIE,
 )
diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index cc296eabd..74fe8017e 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -6,14 +6,15 @@ import re
 from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
-    HEADRequest,
-    unified_strdate,
-    strip_jsonp,
-    int_or_none,
-    float_or_none,
     determine_ext,
+    float_or_none,
+    HEADRequest,
+    int_or_none,
+    orderedSet,
     remove_end,
+    strip_jsonp,
     unescapeHTML,
+    unified_strdate,
 )
 
 
@@ -307,3 +308,108 @@ class ORFIPTVIE(InfoExtractor):
             'upload_date': upload_date,
             'formats': formats,
         }
+
+
+class ORFFM4StoryIE(InfoExtractor):
+    IE_NAME = 'orf:fm4:story'
+    IE_DESC = 'fm4.orf.at stories'
+    _VALID_URL = r'https?://fm4\.orf\.at/stories/(?P<id>\d+)'
+
+    _TEST = {
+        'url': 'http://fm4.orf.at/stories/2865738/',
+        'playlist': [{
+            'md5': 'e1c2c706c45c7b34cf478bbf409907ca',
+            'info_dict': {
+                'id': '547792',
+                'ext': 'flv',
+                'title': 'Manu Delago und Inner Tongue live',
+                'description': 'Manu Delago und Inner Tongue haben bei der FM4 Soundpark Session live alles gegeben. Hier gibt es Fotos und die gesamte Session als Video.',
+                'duration': 1748.52,
+                'thumbnail': r're:^https?://.*\.jpg$',
+                'upload_date': '20170913',
+            },
+        }, {
+            'md5': 'c6dd2179731f86f4f55a7b49899d515f',
+            'info_dict': {
+                'id': '547798',
+                'ext': 'flv',
+                'title': 'Manu Delago und Inner Tongue live (2)',
+                'duration': 1504.08,
+                'thumbnail': r're:^https?://.*\.jpg$',
+                'upload_date': '20170913',
+                'description': 'Manu Delago und Inner Tongue haben bei der FM4 Soundpark Session live alles gegeben. Hier gibt es Fotos und die gesamte Session als Video.',
+            },
+        }],
+    }
+
+    def _real_extract(self, url):
+        story_id = self._match_id(url)
+        webpage = self._download_webpage(url, story_id)
+
+        entries = []
+        all_ids = orderedSet(re.findall(r'data-video(?:id)?="(\d+)"', webpage))
+        for idx, video_id in enumerate(all_ids):
+            data = self._download_json(
+                'http://bits.orf.at/filehandler/static-api/json/current/data.json?file=%s' % video_id,
+                video_id)[0]
+
+            duration = float_or_none(data['duration'], 1000)
+
+            video = data['sources']['q8c']
+            load_balancer_url = video['loadBalancerUrl']
+            abr = int_or_none(video.get('audioBitrate'))
+            vbr = int_or_none(video.get('bitrate'))
+            fps = int_or_none(video.get('videoFps'))
+            width = int_or_none(video.get('videoWidth'))
+            height = int_or_none(video.get('videoHeight'))
+            thumbnail = video.get('preview')
+
+            rendition = self._download_json(
+                load_balancer_url, video_id, transform_source=strip_jsonp)
+
+            f = {
+                'abr': abr,
+                'vbr': vbr,
+                'fps': fps,
+                'width': width,
+                'height': height,
+            }
+
+            formats = []
+            for format_id, format_url in rendition['redirect'].items():
+                if format_id == 'rtmp':
+                    ff = f.copy()
+                    ff.update({
+                        'url': format_url,
+                        'format_id': format_id,
+                    })
+                    formats.append(ff)
+                elif determine_ext(format_url) == 'f4m':
+                    formats.extend(self._extract_f4m_formats(
+                        format_url, video_id, f4m_id=format_id))
+                elif determine_ext(format_url) == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        format_url, video_id, 'mp4', m3u8_id=format_id))
+                else:
+                    continue
+            self._sort_formats(formats)
+
+            title = remove_end(self._og_search_title(webpage), ' - fm4.ORF.at')
+            if idx >= 1:
+                # Titles are duplicates, make them unique
+                title += ' (' + str(idx + 1) + ')'
+            description = self._og_search_description(webpage)
+            upload_date = unified_strdate(self._html_search_meta(
+                'dc.date', webpage, 'upload date'))
+
+            entries.append({
+                'id': video_id,
+                'title': title,
+                'description': description,
+                'duration': duration,
+                'thumbnail': thumbnail,
+                'upload_date': upload_date,
+                'formats': formats,
+            })
+
+        return self.playlist_result(entries)

From fad9fc537d8ab5141f402f6ccdf60161a8d0302a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 14 Sep 2017 20:47:23 +0200
Subject: [PATCH 1313/1696] [tv4] fix a test URL

---
 youtube_dl/extractor/tv4.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tv4.py b/youtube_dl/extractor/tv4.py
index 6487039e5..cfcce020a 100644
--- a/youtube_dl/extractor/tv4.py
+++ b/youtube_dl/extractor/tv4.py
@@ -64,7 +64,7 @@ class TV4IE(InfoExtractor):
             'only_matching': True,
         },
         {
-            'url': ' http://www.tv4play.se/program/farang/3922081',
+            'url': 'http://www.tv4play.se/program/farang/3922081',
             'only_matching': True,
         }
     ]

From c46680fb2a0ee61faa25863d3c10b7b098cdbe67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Sep 2017 01:59:47 +0700
Subject: [PATCH 1314/1696] [condenast] Fix extraction (closes #14196, closes
 #14207)

---
 youtube_dl/extractor/condenast.py | 54 ++++++++++++++++++-------------
 1 file changed, 32 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/condenast.py b/youtube_dl/extractor/condenast.py
index 0c3f0c0e4..ed278fefc 100644
--- a/youtube_dl/extractor/condenast.py
+++ b/youtube_dl/extractor/condenast.py
@@ -116,16 +116,16 @@ class CondeNastIE(InfoExtractor):
         entries = [self.url_result(build_url(path), 'CondeNast') for path in paths]
         return self.playlist_result(entries, playlist_title=title)
 
-    def _extract_video_params(self, webpage):
-        query = {}
-        params = self._search_regex(
-            r'(?s)var params = {(.+?)}[;,]', webpage, 'player params', default=None)
-        if params:
-            query.update({
-                'videoId': self._search_regex(r'videoId: [\'"](.+?)[\'"]', params, 'video id'),
-                'playerId': self._search_regex(r'playerId: [\'"](.+?)[\'"]', params, 'player id'),
-                'target': self._search_regex(r'target: [\'"](.+?)[\'"]', params, 'target'),
-            })
+    def _extract_video_params(self, webpage, display_id):
+        query = self._parse_json(
+            self._search_regex(
+                r'(?s)var\s+params\s*=\s*({.+?})[;,]', webpage, 'player params',
+                default='{}'),
+            display_id, transform_source=js_to_json, fatal=False)
+        if query:
+            query['videoId'] = self._search_regex(
+                r'(?:data-video-id=|currentVideoId\s*=\s*)["\']([\da-f]+)',
+                webpage, 'video id', default=None)
         else:
             params = extract_attributes(self._search_regex(
                 r'(<[^>]+data-js="video-player"[^>]+>)',
@@ -141,17 +141,27 @@ class CondeNastIE(InfoExtractor):
         video_id = params['videoId']
 
         video_info = None
-        if params.get('playerId'):
-            info_page = self._download_json(
-                'http://player.cnevids.com/player/video.js',
-                video_id, 'Downloading video info', fatal=False, query=params)
-            if info_page:
-                video_info = info_page.get('video')
-            if not video_info:
-                info_page = self._download_webpage(
-                    'http://player.cnevids.com/player/loader.js',
-                    video_id, 'Downloading loader info', query=params)
-        else:
+
+        # New API path
+        query = params.copy()
+        query['embedType'] = 'inline'
+        info_page = self._download_json(
+            'http://player.cnevids.com/embed-api.json', video_id,
+            'Downloading embed info', fatal=False, query=query)
+
+        # Old fallbacks
+        if not info_page:
+            if params.get('playerId'):
+                info_page = self._download_json(
+                    'http://player.cnevids.com/player/video.js', video_id,
+                    'Downloading video info', fatal=False, query=params)
+        if info_page:
+            video_info = info_page.get('video')
+        if not video_info:
+            info_page = self._download_webpage(
+                'http://player.cnevids.com/player/loader.js',
+                video_id, 'Downloading loader info', query=params)
+        if not video_info:
             info_page = self._download_webpage(
                 'https://player.cnevids.com/inline/video/%s.js' % video_id,
                 video_id, 'Downloading inline info', query={
@@ -215,7 +225,7 @@ class CondeNastIE(InfoExtractor):
         if url_type == 'series':
             return self._extract_series(url, webpage)
         else:
-            params = self._extract_video_params(webpage)
+            params = self._extract_video_params(webpage, display_id)
             info = self._search_json_ld(
                 webpage, display_id, fatal=False)
             info.update(self._extract_video(params))

From 86e55e317cb70f07792cfe543186ad520cbe3230 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Sep 2017 21:45:18 +0700
Subject: [PATCH 1315/1696] [ChangeLog] Actualize

---
 ChangeLog | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index c286da6c6..38d511362 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,18 @@
+version <unreleased>
+
+Core
+* [downloader/fragment] Restart inconsistent incomplete fragment downloads
+  (#13731)
+* [YoutubeDL] Download raw subtitles files (#12909, #14191)
+
+Extractors
+* [condenast] Fix extraction (#14196, #14207)
++ [orf] Add support for f4m stories
+* [tv4] Relax URL regular expression (#14206)
+* [animeondemand] Bypass geo restriction
++ [animeondemand] Add support for flash videos (#9944)
+
+
 version 2017.09.11
 
 Extractors

From 159d304a9fa2b91d91a60fe3bdf2211a59bcf346 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Sep 2017 21:48:06 +0700
Subject: [PATCH 1316/1696] release 2017.09.15

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 1 +
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index f40cb2c4e..98ab5b6ca 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.09.11*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.09.11**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.09.15*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.09.15**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.09.11
+[debug] youtube-dl version 2017.09.15
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 38d511362..041dfd7b9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.09.15
 
 Core
 * [downloader/fragment] Restart inconsistent incomplete fragment downloads
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 798a81d3c..6b01dc910 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -593,6 +593,7 @@
  - **Openload**
  - **OraTV**
  - **orf:fm4**: radio FM4
+ - **orf:fm4:story**: fm4.orf.at stories
  - **orf:iptv**: iptv.ORF.at
  - **orf:oe1**: Radio Österreich 1
  - **orf:tvthek**: ORF TVthek
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index cdcb32e06..8399c04fe 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.09.11'
+__version__ = '2017.09.15'

From cbf85239bbb835162725cd4c8758831ca1003445 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Sep 2017 22:13:30 +0700
Subject: [PATCH 1317/1696] [vgtv] Relax _VALID_URL (closes #14223)

---
 youtube_dl/extractor/vgtv.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 0f8c156a7..c21a09c01 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -42,7 +42,7 @@ class VGTVIE(XstreamIE):
                     )
                     /?
                     (?:
-                        \#!/(?:video|live)/|
+                        (?:\#!/)?(?:video|live)/|
                         embed?.*id=|
                         articles/
                     )|
@@ -146,7 +146,11 @@ class VGTVIE(XstreamIE):
         {
             'url': 'abtv:140026',
             'only_matching': True,
-        }
+        },
+        {
+            'url': 'http://www.vgtv.no/video/84196/hevnen-er-soet-episode-10-abu',
+            'only_matching': True,
+        },
     ]
 
     def _real_extract(self, url):

From b763e1d68c6becc414a802a452f5aa819c5de920 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Sep 2017 22:18:38 +0700
Subject: [PATCH 1318/1696] [twitch] Add support for go.twitch.tv URLs (closes
 #14215)

---
 youtube_dl/extractor/twitch.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 2daf9dfac..c926c99a9 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -28,7 +28,7 @@ from ..utils import (
 
 
 class TwitchBaseIE(InfoExtractor):
-    _VALID_URL_BASE = r'https?://(?:www\.)?twitch\.tv'
+    _VALID_URL_BASE = r'https?://(?:(?:www|go)\.)?twitch\.tv'
 
     _API_BASE = 'https://api.twitch.tv'
     _USHER_BASE = 'https://usher.ttvnw.net'
@@ -217,7 +217,7 @@ class TwitchVodIE(TwitchItemBaseIE):
     _VALID_URL = r'''(?x)
                     https?://
                         (?:
-                            (?:www\.)?twitch\.tv/(?:[^/]+/v|videos)/|
+                            (?:(?:www|go)\.)?twitch\.tv/(?:[^/]+/v|videos)/|
                             player\.twitch\.tv/\?.*?\bvideo=v
                         )
                         (?P<id>\d+)
@@ -458,7 +458,7 @@ class TwitchStreamIE(TwitchBaseIE):
     _VALID_URL = r'''(?x)
                     https?://
                         (?:
-                            (?:www\.)?twitch\.tv/|
+                            (?:(?:www|go)\.)?twitch\.tv/|
                             player\.twitch\.tv/\?.*?\bchannel=
                         )
                         (?P<id>[^/#?]+)
@@ -489,6 +489,9 @@ class TwitchStreamIE(TwitchBaseIE):
     }, {
         'url': 'https://player.twitch.tv/?channel=lotsofs',
         'only_matching': True,
+    }, {
+        'url': 'https://go.twitch.tv/food',
+        'only_matching': True,
     }]
 
     @classmethod

From 6be44a50edfe2e75e31553e7a128ce1849301958 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Sep 2017 22:25:38 +0700
Subject: [PATCH 1319/1696] [dailymotion:playlist] Relax _VALID_URL (closes
 #14219)

---
 youtube_dl/extractor/dailymotion.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 74e991331..e9d0dd19c 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -325,7 +325,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
 
 class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
     IE_NAME = 'dailymotion:playlist'
-    _VALID_URL = r'(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/playlist/(?P<id>.+?)/'
+    _VALID_URL = r'(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/playlist/(?P<id>[^/?#&]+)'
     _MORE_PAGES_INDICATOR = r'(?s)<div class="pages[^"]*">.*?<a\s+class="[^"]*?icon-arrow_right[^"]*?"'
     _PAGE_TEMPLATE = 'https://www.dailymotion.com/playlist/%s/%s'
     _TESTS = [{

From a4245acef85ac2414e77cf2cda4cb39adb617241 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Sep 2017 23:12:19 +0700
Subject: [PATCH 1320/1696] [noovo] Fix extraction (closes #14214)

---
 youtube_dl/extractor/noovo.py | 61 ++++++++++++++++++-----------------
 1 file changed, 32 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/noovo.py b/youtube_dl/extractor/noovo.py
index f7fa098a5..974de3c3e 100644
--- a/youtube_dl/extractor/noovo.py
+++ b/youtube_dl/extractor/noovo.py
@@ -6,6 +6,7 @@ from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
     int_or_none,
+    js_to_json,
     smuggle_url,
     try_get,
 )
@@ -24,8 +25,6 @@ class NoovoIE(InfoExtractor):
             'timestamp': 1491399228,
             'upload_date': '20170405',
             'uploader_id': '618566855001',
-            'creator': 'vtele',
-            'view_count': int,
             'series': 'RPM+',
         },
         'params': {
@@ -37,13 +36,11 @@ class NoovoIE(InfoExtractor):
         'info_dict': {
             'id': '5395865725001',
             'title': 'Épisode 13 : Les retrouvailles',
-            'description': 'md5:336d5ebc5436534e61d16e63ddfca327',
+            'description': 'md5:888c3330f0c1b4476c5bc99a1c040473',
             'ext': 'mp4',
             'timestamp': 1492019320,
             'upload_date': '20170412',
             'uploader_id': '618566855001',
-            'creator': 'vtele',
-            'view_count': int,
             'series': "L'amour est dans le pré",
             'season_number': 5,
             'episode': 'Épisode 13',
@@ -58,40 +55,46 @@ class NoovoIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        data = self._download_json(
-            'http://api.noovo.ca/api/v1/pages/single-episode/%s' % video_id,
-            video_id)['data']
+        webpage = self._download_webpage(url, video_id)
 
-        content = try_get(data, lambda x: x['contents'][0])
+        bc_url = BrightcoveNewIE._extract_url(self, webpage)
 
-        brightcove_id = data.get('brightcoveId') or content['brightcoveId']
+        data = self._parse_json(
+            self._search_regex(
+                r'(?s)dataLayer\.push\(\s*({.+?})\s*\);', webpage, 'data',
+                default='{}'),
+            video_id, transform_source=js_to_json, fatal=False)
+
+        title = try_get(
+            data, lambda x: x['video']['nom'],
+            compat_str) or self._html_search_meta(
+            'dcterms.Title', webpage, 'title', fatal=True)
+
+        description = self._html_search_meta(
+            ('dcterms.Description', 'description'), webpage, 'description')
 
         series = try_get(
-            data, (
-                lambda x: x['show']['title'],
-                lambda x: x['season']['show']['title']),
-            compat_str)
+            data, lambda x: x['emission']['nom']) or self._search_regex(
+            r'<div[^>]+class="banner-card__subtitle h4"[^>]*>([^<]+)',
+            webpage, 'series', default=None)
 
-        episode = None
-        og = data.get('og')
-        if isinstance(og, dict) and og.get('type') == 'video.episode':
-            episode = og.get('title')
+        season_el = try_get(data, lambda x: x['emission']['saison'], dict) or {}
+        season = try_get(season_el, lambda x: x['nom'], compat_str)
+        season_number = int_or_none(try_get(season_el, lambda x: x['numero']))
 
-        video = content or data
+        episode_el = try_get(season_el, lambda x: x['episode'], dict) or {}
+        episode = try_get(episode_el, lambda x: x['nom'], compat_str)
+        episode_number = int_or_none(try_get(episode_el, lambda x: x['numero']))
 
         return {
             '_type': 'url_transparent',
             'ie_key': BrightcoveNewIE.ie_key(),
-            'url': smuggle_url(
-                self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id,
-                {'geo_countries': ['CA']}),
-            'id': brightcove_id,
-            'title': video.get('title'),
-            'creator': video.get('source'),
-            'view_count': int_or_none(video.get('viewsCount')),
+            'url': smuggle_url(bc_url, {'geo_countries': ['CA']}),
+            'title': title,
+            'description': description,
             'series': series,
-            'season_number': int_or_none(try_get(
-                data, lambda x: x['season']['seasonNumber'])),
+            'season': season,
+            'season_number': season_number,
             'episode': episode,
-            'episode_number': int_or_none(data.get('episodeNumber')),
+            'episode_number': episode_number,
         }

From 68d43a61b552007a718894967b869c0f1d8ff00f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Sep 2017 12:14:48 +0800
Subject: [PATCH 1321/1696] Ignore TTML subtitles

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index a5b585f43..fbf7cecb2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -22,6 +22,7 @@ cover/
 updates_key.pem
 *.egg-info
 *.srt
+*.ttml
 *.sbv
 *.vtt
 *.flv

From 3869028ffb6be6ab719e5cf1004276dfdfd1216d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Sep 2017 12:18:38 +0800
Subject: [PATCH 1322/1696] [utils] Use bytes-like objects in dfxp2srt

This fixes handling of non-UTF8 TTML subtitles

Closes #14191
---
 ChangeLog                          |  6 ++++++
 test/test_utils.py                 | 26 +++++++++++++++++++++++---
 youtube_dl/postprocessor/ffmpeg.py |  2 +-
 youtube_dl/utils.py                | 18 +++++++++++-------
 4 files changed, 41 insertions(+), 11 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 041dfd7b9..ba9260e3e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Core
+* [utils] Fix handling raw TTML subtitles (#14191)
+
+
 version 2017.09.15
 
 Core
diff --git a/test/test_utils.py b/test/test_utils.py
index e50f3764e..efa73d0f4 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -1064,7 +1064,7 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
                     <p begin="3" dur="-1">Ignored, three</p>
                 </div>
             </body>
-            </tt>'''
+            </tt>'''.encode('utf-8')
         srt_data = '''1
 00:00:00,000 --> 00:00:01,000
 The following line contains Chinese characters and special symbols
@@ -1089,7 +1089,7 @@ Line
                     <p begin="0" end="1">The first line</p>
                 </div>
             </body>
-            </tt>'''
+            </tt>'''.encode('utf-8')
         srt_data = '''1
 00:00:00,000 --> 00:00:01,000
 The first line
@@ -1115,7 +1115,7 @@ The first line
       <p style="s1" tts:textDecoration="underline" begin="00:00:09.56" id="p2" end="00:00:12.36"><span style="s2" tts:color="lime">inner<br /> </span>style</p>
     </div>
   </body>
-</tt>'''
+</tt>'''.encode('utf-8')
         srt_data = '''1
 00:00:02,080 --> 00:00:05,839
 <font color="white" face="sansSerif" size="16">default style<font color="red">custom style</font></font>
@@ -1138,6 +1138,26 @@ part 3</font></u>
 '''
         self.assertEqual(dfxp2srt(dfxp_data_with_style), srt_data)
 
+        dfxp_data_non_utf8 = '''<?xml version="1.0" encoding="UTF-16"?>
+            <tt xmlns="http://www.w3.org/ns/ttml" xml:lang="en" xmlns:tts="http://www.w3.org/ns/ttml#parameter">
+            <body>
+                <div xml:lang="en">
+                    <p begin="0" end="1">Line 1</p>
+                    <p begin="1" end="2">第二行</p>
+                </div>
+            </body>
+            </tt>'''.encode('utf-16')
+        srt_data = '''1
+00:00:00,000 --> 00:00:01,000
+Line 1
+
+2
+00:00:01,000 --> 00:00:02,000
+第二行
+
+'''
+        self.assertEqual(dfxp2srt(dfxp_data_non_utf8), srt_data)
+
     def test_cli_option(self):
         self.assertEqual(cli_option({'proxy': '127.0.0.1:3128'}, '--proxy', 'proxy'), ['--proxy', '127.0.0.1:3128'])
         self.assertEqual(cli_option({'proxy': None}, '--proxy', 'proxy'), [])
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 51256a3fb..f71d413b5 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -585,7 +585,7 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
                 dfxp_file = old_file
                 srt_file = subtitles_filename(filename, lang, 'srt')
 
-                with io.open(dfxp_file, 'rt', encoding='utf-8') as f:
+                with open(dfxp_file, 'rb') as f:
                     srt_data = dfxp2srt(f.read())
 
                 with io.open(srt_file, 'wt', encoding='utf-8') as f:
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 9e4492d40..b724e0b70 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2572,14 +2572,18 @@ def srt_subtitles_timecode(seconds):
 
 
 def dfxp2srt(dfxp_data):
+    '''
+    @param dfxp_data A bytes-like object containing DFXP data
+    @returns A unicode object containing converted SRT data
+    '''
     LEGACY_NAMESPACES = (
-        ('http://www.w3.org/ns/ttml', [
-            'http://www.w3.org/2004/11/ttaf1',
-            'http://www.w3.org/2006/04/ttaf1',
-            'http://www.w3.org/2006/10/ttaf1',
+        (b'http://www.w3.org/ns/ttml', [
+            b'http://www.w3.org/2004/11/ttaf1',
+            b'http://www.w3.org/2006/04/ttaf1',
+            b'http://www.w3.org/2006/10/ttaf1',
         ]),
-        ('http://www.w3.org/ns/ttml#styling', [
-            'http://www.w3.org/ns/ttml#style',
+        (b'http://www.w3.org/ns/ttml#styling', [
+            b'http://www.w3.org/ns/ttml#style',
         ]),
     )
 
@@ -2674,7 +2678,7 @@ def dfxp2srt(dfxp_data):
         for ns in v:
             dfxp_data = dfxp_data.replace(ns, k)
 
-    dfxp = compat_etree_fromstring(dfxp_data.encode('utf-8'))
+    dfxp = compat_etree_fromstring(dfxp_data)
     out = []
     paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall('.//p')
 

From 790d379e4df2f85ece7cab02e805643234bb5c16 Mon Sep 17 00:00:00 2001
From: Windom <windom@users.noreply.github.com>
Date: Sat, 16 Sep 2017 18:39:46 +0300
Subject: [PATCH 1323/1696] [morningstar] Relax _VALID_URL

---
 youtube_dl/extractor/morningstar.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/morningstar.py b/youtube_dl/extractor/morningstar.py
index 320d27bdd..0093bcd6c 100644
--- a/youtube_dl/extractor/morningstar.py
+++ b/youtube_dl/extractor/morningstar.py
@@ -8,8 +8,8 @@ from .common import InfoExtractor
 
 class MorningstarIE(InfoExtractor):
     IE_DESC = 'morningstar.com'
-    _VALID_URL = r'https?://(?:www\.)?morningstar\.com/[cC]over/video[cC]enter\.aspx\?id=(?P<id>[0-9]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:(?:www|news)\.)morningstar\.com/[cC]over/video[cC]enter\.aspx\?id=(?P<id>[0-9]+)'
+    _TESTS = [{
         'url': 'http://www.morningstar.com/cover/videocenter.aspx?id=615869',
         'md5': '6c0acface7a787aadc8391e4bbf7b0f5',
         'info_dict': {
@@ -19,7 +19,10 @@ class MorningstarIE(InfoExtractor):
             'description': "Vanguard's Joel Dickson on managing higher tax rates for high-income earners and fund capital-gain distributions in 2013.",
             'thumbnail': r're:^https?://.*m(?:orning)?star\.com/.+thumb\.jpg$'
         }
-    }
+    }, {
+        'url': 'http://news.morningstar.com/cover/videocenter.aspx?id=825556',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 8251af63a12cd73cf2578c81dbb869232da2592c Mon Sep 17 00:00:00 2001
From: Vijay Singh <sudovijay@users.noreply.github.com>
Date: Sat, 16 Sep 2017 21:15:23 +0530
Subject: [PATCH 1324/1696] [viki] Update app data (closes #14181)

---
 youtube_dl/extractor/viki.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index e9c8bf824..853e5c75f 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -23,9 +23,9 @@ class VikiBaseIE(InfoExtractor):
     _API_QUERY_TEMPLATE = '/v4/%sapp=%s&t=%s&site=www.viki.com'
     _API_URL_TEMPLATE = 'http://api.viki.io%s&sig=%s'
 
-    _APP = '65535a'
+    _APP = '100005a'
     _APP_VERSION = '2.2.5.1428709186'
-    _APP_SECRET = '-$iJ}@p7!G@SyU/je1bEyWg}upLu-6V6-Lg9VD(]siH,r.,m-r|ulZ,U4LC/SeR)'
+    _APP_SECRET = 'MM_d*yP@`&1@]@!AVrXf_o-HVEnoTnm$O-ti4[G~$JDI/Dc-&piU&z&5.;:}95=Iad'
 
     _GEO_BYPASS = False
     _NETRC_MACHINE = 'viki'

From 4ed2d7b7d1f67e499a46e507d957616e364565ca Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 17 Sep 2017 13:53:04 +0800
Subject: [PATCH 1325/1696] Fix flake8 issues after #14225

---
 youtube_dl/extractor/common.py |  2 +-
 youtube_dl/utils.py            | 33 +++++++++++++++++----------------
 2 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 317a9a76f..2bbbf8f4d 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2452,7 +2452,7 @@ class InfoExtractor(object):
     def _set_cookie(self, domain, name, value, expire_time=None, port=None,
                     path='/', secure=False, discard=False, rest={}, **kwargs):
         cookie = compat_cookiejar.Cookie(
-            0, name, value, port, not port is None, domain, True,
+            0, name, value, port, port is not None, domain, True,
             domain.startswith('.'), path, True, secure, expire_time,
             discard, None, None, rest)
         self._downloader.cookiejar.set_cookie(cookie)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index b724e0b70..acc4f987b 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3830,23 +3830,23 @@ def cookie_to_dict(cookie):
     cookie_dict = {
         'name': cookie.name,
         'value': cookie.value,
-    };
+    }
     if cookie.port_specified:
         cookie_dict['port'] = cookie.port
     if cookie.domain_specified:
         cookie_dict['domain'] = cookie.domain
     if cookie.path_specified:
         cookie_dict['path'] = cookie.path
-    if not cookie.expires is None:
+    if cookie.expires is not None:
         cookie_dict['expires'] = cookie.expires
-    if not cookie.secure is None:
+    if cookie.secure is not None:
         cookie_dict['secure'] = cookie.secure
-    if not cookie.discard is None:
+    if cookie.discard is not None:
         cookie_dict['discard'] = cookie.discard
     try:
         if (cookie.has_nonstandard_attr('httpOnly') or
-            cookie.has_nonstandard_attr('httponly') or
-            cookie.has_nonstandard_attr('HttpOnly')):
+                cookie.has_nonstandard_attr('httponly') or
+                cookie.has_nonstandard_attr('HttpOnly')):
             cookie_dict['httponly'] = True
     except TypeError:
         pass
@@ -3957,7 +3957,7 @@ class PhantomJSwrapper(object):
             cookies = json.loads(f.read().decode('utf-8'))
         for cookie in cookies:
             if cookie['httponly'] is True:
-                cookie['rest'] = { 'httpOnly': None }
+                cookie['rest'] = {'httpOnly': None}
             if 'expiry' in cookie:
                 cookie['expire_time'] = cookie['expiry']
             self.extractor._set_cookie(**cookie)
@@ -3965,7 +3965,7 @@ class PhantomJSwrapper(object):
     def get(self, url, html=None, video_id=None, note=None, note2='Executing JS on webpage', headers={}, jscode='saveAndExit();'):
         """
         Downloads webpage (if needed) and executes JS
-        
+
         Params:
             url: website url
             html: optional, html code of website
@@ -3974,11 +3974,11 @@ class PhantomJSwrapper(object):
             note2: optional, displayed when executing JS
             headers: custom http headers
             jscode: code to be executed when page is loaded
-        
+
         Returns tuple with:
             * downloaded website (after JS execution)
             * anything you print with `console.log` (but not inside `page.execute`!)
-        
+
         In most cases you don't need to add any `jscode`.
         It is executed in `page.onLoadFinished`.
         `saveAndExit();` is mandatory, use it instead of `phantom.exit()`
@@ -3992,7 +3992,7 @@ class PhantomJSwrapper(object):
               else
                 window.setTimeout(check, 500);
             }
-            
+
             page.evaluate(function(){
               document.querySelector('#a').click();
             });
@@ -4024,13 +4024,14 @@ class PhantomJSwrapper(object):
         else:
             self.extractor.to_screen('%s: %s' % (video_id, note2))
 
-        p = subprocess.Popen([self.exe, '--ssl-protocol=any',
-            self._TMP_FILES['script'].name], stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE)
+        p = subprocess.Popen([
+            self.exe, '--ssl-protocol=any',
+            self._TMP_FILES['script'].name
+        ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         out, err = p.communicate()
         if p.returncode != 0:
-            raise ExtractorError('Executing JS failed\n:'
-                                 + encodeArgument(err))
+            raise ExtractorError(
+                'Executing JS failed\n:' + encodeArgument(err))
         with open(self._TMP_FILES['html'].name, 'rb') as f:
             html = f.read().decode('utf-8')
 

From 9c2a17f2ce7b2b9dc45b603be413a943f6637498 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Sep 2017 22:19:57 +0700
Subject: [PATCH 1326/1696] [popcorntv] Add extractor (closes #5914, closes
 #14211)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/popcorntv.py  | 78 ++++++++++++++++++++++++++++++
 2 files changed, 79 insertions(+)
 create mode 100644 youtube_dl/extractor/popcorntv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index a3a97e940..ab95c8575 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -808,6 +808,7 @@ from .polskieradio import (
     PolskieRadioIE,
     PolskieRadioCategoryIE,
 )
+from .popcorntv import PopcornTVIE
 from .porn91 import Porn91IE
 from .porncom import PornComIE
 from .pornflip import PornFlipIE
diff --git a/youtube_dl/extractor/popcorntv.py b/youtube_dl/extractor/popcorntv.py
new file mode 100644
index 000000000..ac901f426
--- /dev/null
+++ b/youtube_dl/extractor/popcorntv.py
@@ -0,0 +1,78 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    extract_attributes,
+    int_or_none,
+    unified_timestamp,
+)
+
+
+class PopcornTVIE(InfoExtractor):
+    _VALID_URL = r'https?://[^/]+\.popcorntv\.it/guarda/(?P<display_id>[^/]+)/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://animemanga.popcorntv.it/guarda/food-wars-battaglie-culinarie-episodio-01/9183',
+        'md5': '47d65a48d147caf692ab8562fe630b45',
+        'info_dict': {
+            'id': '9183',
+            'display_id': 'food-wars-battaglie-culinarie-episodio-01',
+            'ext': 'mp4',
+            'title': 'Food Wars, Battaglie Culinarie | Episodio 01',
+            'description': 'md5:b8bea378faae4651d3b34c6e112463d0',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'timestamp': 1497610857,
+            'upload_date': '20170616',
+            'duration': 1440,
+            'view_count': int,
+        },
+    }, {
+        'url': 'https://cinema.popcorntv.it/guarda/smash-cut/10433',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        display_id, video_id = mobj.group('display_id', 'id')
+
+        webpage = self._download_webpage(url, display_id)
+
+        m3u8_url = extract_attributes(
+            self._search_regex(
+                r'(<link[^>]+itemprop=["\'](?:content|embed)Url[^>]*>)',
+                webpage, 'content'
+            ))['href']
+
+        formats = self._extract_m3u8_formats(
+            m3u8_url, display_id, 'mp4', entry_protocol='m3u8_native',
+            m3u8_id='hls')
+
+        title = self._search_regex(
+            r'<h1[^>]+itemprop=["\']name[^>]*>([^<]+)', webpage,
+            'title', default=None) or self._og_search_title(webpage)
+
+        description = self._html_search_regex(
+            r'(?s)<article[^>]+itemprop=["\']description[^>]*>(.+?)</article>',
+            webpage, 'description', fatal=False)
+        thumbnail = self._og_search_thumbnail(webpage)
+        timestamp = unified_timestamp(self._html_search_meta(
+            'uploadDate', webpage, 'timestamp'))
+        print(self._html_search_meta(
+            'duration', webpage))
+        duration = int_or_none(self._html_search_meta(
+            'duration', webpage), invscale=60)
+        view_count = int_or_none(self._html_search_meta(
+            'interactionCount', webpage, 'view count'))
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'duration': duration,
+            'view_count': view_count,
+            'formats': formats,
+        }

From 4d8c4b46d5668387cc685123f80fb64bbc7c5aff Mon Sep 17 00:00:00 2001
From: kayb94 <30302445+kayb94@users.noreply.github.com>
Date: Sun, 17 Sep 2017 15:46:52 +0000
Subject: [PATCH 1327/1696] [heise] Add support for YouTube embeds

---
 youtube_dl/extractor/heise.py | 33 +++++++++++++++++++++++++++------
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/heise.py b/youtube_dl/extractor/heise.py
index 382f32771..495ffb7dc 100644
--- a/youtube_dl/extractor/heise.py
+++ b/youtube_dl/extractor/heise.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from .youtube import YoutubeIE
 from ..utils import (
     determine_ext,
     int_or_none,
@@ -25,6 +26,22 @@ class HeiseIE(InfoExtractor):
             'description': 'md5:c934cbfb326c669c2bcabcbe3d3fcd20',
             'thumbnail': r're:^https?://.*/gallery/$',
         }
+    }, {
+        # YouTube embed
+        'url': 'http://www.heise.de/newsticker/meldung/Netflix-In-20-Jahren-vom-Videoverleih-zum-TV-Revolutionaer-3814130.html',
+        'md5': 'e403d2b43fea8e405e88e3f8623909f1',
+        'info_dict': {
+            'id': '6kmWbXleKW4',
+            'ext': 'mp4',
+            'title': 'NEU IM SEPTEMBER | Netflix',
+            'description': 'md5:2131f3c7525e540d5fd841de938bd452',
+            'upload_date': '20170830',
+            'uploader': 'Netflix Deutschland, Österreich und Schweiz',
+            'uploader_id': 'netflixdach',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'http://www.heise.de/ct/artikel/c-t-uplink-3-3-Owncloud-Tastaturen-Peilsender-Smartphone-2403911.html',
         'only_matching': True,
@@ -40,6 +57,16 @@ class HeiseIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
+        title = self._html_search_meta('fulltitle', webpage, default=None)
+        if not title or title == "c't":
+            title = self._search_regex(
+                    r'<div[^>]+class="videoplayerjw"[^>]+data-title="([^"]+)"',
+                    webpage, 'title')
+
+        yt_urls = YoutubeIE._extract_urls(webpage)
+        if yt_urls:
+            return self.playlist_from_matches(yt_urls, video_id, title, ie=YoutubeIE.ie_key())
+
         container_id = self._search_regex(
             r'<div class="videoplayerjw"[^>]+data-container="([0-9]+)"',
             webpage, 'container ID')
@@ -47,12 +74,6 @@ class HeiseIE(InfoExtractor):
             r'<div class="videoplayerjw"[^>]+data-sequenz="([0-9]+)"',
             webpage, 'sequenz ID')
 
-        title = self._html_search_meta('fulltitle', webpage, default=None)
-        if not title or title == "c't":
-            title = self._search_regex(
-                r'<div[^>]+class="videoplayerjw"[^>]+data-title="([^"]+)"',
-                webpage, 'title')
-
         doc = self._download_xml(
             'http://www.heise.de/videout/feed', video_id, query={
                 'container': container_id,

From 8a1a60d17397721620e75d83f2aad3a353286f15 Mon Sep 17 00:00:00 2001
From: Kareem Moussa <itstehkman@gmail.com>
Date: Tue, 19 Sep 2017 08:51:20 -0700
Subject: [PATCH 1328/1696] [devscripts/check-porn] Fix gettestcases import

---
 devscripts/check-porn.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/devscripts/check-porn.py b/devscripts/check-porn.py
index 7a219ebe9..72b2ee422 100644
--- a/devscripts/check-porn.py
+++ b/devscripts/check-porn.py
@@ -14,7 +14,7 @@ import os
 import sys
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from test.helper import get_testcases
+from test.helper import gettestcases
 from youtube_dl.utils import compat_urllib_parse_urlparse
 from youtube_dl.utils import compat_urllib_request
 
@@ -24,7 +24,7 @@ if len(sys.argv) > 1:
 else:
     METHOD = 'EURISTIC'
 
-for test in get_testcases():
+for test in gettestcases():
     if METHOD == 'EURISTIC':
         try:
             webpage = compat_urllib_request.urlopen(test['url'], timeout=10).read()

From dc76eef092dc10f7e3f599fa7d85a04de8d84b8f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 Sep 2017 23:59:36 +0700
Subject: [PATCH 1329/1696] [tvplay] Bypass geo restriction

---
 youtube_dl/extractor/tvplay.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index 99ff82a5d..46132eda1 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -15,7 +15,9 @@ from ..utils import (
     int_or_none,
     parse_iso8601,
     qualities,
+    smuggle_url,
     try_get,
+    unsmuggle_url,
     update_url_query,
 )
 
@@ -224,6 +226,9 @@ class TVPlayIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
+        self._initialize_geo_bypass(smuggled_data.get('geo_countries'))
+
         video_id = self._match_id(url)
         geo_country = self._search_regex(
             r'https?://[^/]+\.([a-z]{2})', url,
@@ -426,4 +431,9 @@ class ViafreeIE(InfoExtractor):
                 r'currentVideo["\']\s*:\s*.+?["\']id["\']\s*:\s*["\'](\d{6,})',
                 webpage, 'video id')
 
-        return self.url_result('mtg:%s' % video_id, TVPlayIE.ie_key())
+        return self.url_result(
+            smuggle_url(
+                'mtg:%s' % video_id,
+                {'geo_countries': [
+                    compat_urlparse.urlparse(url).netloc.rsplit('.', 1)[-1]]}),
+            ie=TVPlayIE.ie_key(), video_id=video_id)

From 3b65a6fbf31256030ff35210a7be2b50369a6c4f Mon Sep 17 00:00:00 2001
From: capital-G <dennis.scheiba@gmx.de>
Date: Tue, 19 Sep 2017 22:58:06 +0200
Subject: [PATCH 1330/1696] [twitter] Fix duration extraction

---
 youtube_dl/extractor/twitter.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 6eaf360a6..7399cf538 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -229,7 +229,7 @@ class TwitterCardIE(TwitterBaseIE):
 
         title = self._search_regex(r'<title>([^<]+)</title>', webpage, 'title')
         thumbnail = config.get('posterImageUrl') or config.get('image_src')
-        duration = float_or_none(config.get('duration')) or duration
+        duration = float_or_none(config.get('duration'), scale=1000) or duration
 
         return {
             'id': video_id,
@@ -255,6 +255,7 @@ class TwitterIE(InfoExtractor):
             'description': 'FREE THE NIPPLE on Twitter: "FTN supporters on Hollywood Blvd today! http://t.co/c7jHH749xJ"',
             'uploader': 'FREE THE NIPPLE',
             'uploader_id': 'freethenipple',
+            'duration': 12.922,
         },
         'params': {
             'skip_download': True,  # requires ffmpeg
@@ -305,11 +306,12 @@ class TwitterIE(InfoExtractor):
         'info_dict': {
             'id': '700207533655363584',
             'ext': 'mp4',
-            'title': 'Donte - BEAT PROD: @suhmeduh #Damndaniel',
-            'description': 'Donte on Twitter: "BEAT PROD: @suhmeduh  https://t.co/HBrQ4AfpvZ #Damndaniel https://t.co/byBooq2ejZ"',
+            'title': 'あかさ - BEAT PROD: @suhmeduh #Damndaniel',
+            'description': 'あかさ on Twitter: "BEAT PROD: @suhmeduh  https://t.co/HBrQ4AfpvZ #Damndaniel https://t.co/byBooq2ejZ"',
             'thumbnail': r're:^https?://.*\.jpg',
-            'uploader': 'Donte',
+            'uploader': 'あかさ',
             'uploader_id': 'jaydingeer',
+            'duration': 30.0,
         },
         'params': {
             'skip_download': True,  # requires ffmpeg
@@ -337,6 +339,7 @@ class TwitterIE(InfoExtractor):
             'description': 'Captain America on Twitter: "@King0fNerd Are you sure you made the right choice? Find out in theaters. https://t.co/GpgYi9xMJI"',
             'uploader_id': 'captainamerica',
             'uploader': 'Captain America',
+            'duration': 3.17,
         },
         'params': {
             'skip_download': True,  # requires ffmpeg
@@ -364,6 +367,7 @@ class TwitterIE(InfoExtractor):
             'description': 'عالم الأخبار on Twitter: "كلمة تاريخية بجلسة الجناسي التاريخية.. النائب خالد مؤنس العتيبي للمعارضين : اتقوا الله .. الظلم ظلمات يوم القيامة   https://t.co/xg6OhpyKfN"',
             'uploader': 'عالم الأخبار',
             'uploader_id': 'news_al3alm',
+            'duration': 277.4,
         },
         'params': {
             'format': 'best[format_id^=http-]',

From 12ea5c79fb0bfa878d62d130cf67057fc230dfa7 Mon Sep 17 00:00:00 2001
From: Parmjit Virk <pvirk@mts.net>
Date: Wed, 20 Sep 2017 14:53:06 -0500
Subject: [PATCH 1331/1696] [nbcsports:vplayer] Correct theplatform URL (closes
 #13873)

---
 youtube_dl/extractor/nbc.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 62db70b43..836a41f06 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -109,10 +109,10 @@ class NBCSportsVPlayerIE(InfoExtractor):
     _VALID_URL = r'https?://vplayer\.nbcsports\.com/(?:[^/]+/)+(?P<id>[0-9a-zA-Z_]+)'
 
     _TESTS = [{
-        'url': 'https://vplayer.nbcsports.com/p/BxmELC/nbcsports_share/select/9CsDKds0kvHI',
+        'url': 'https://vplayer.nbcsports.com/p/BxmELC/nbcsports_embed/select/9CsDKds0kvHI',
         'info_dict': {
             'id': '9CsDKds0kvHI',
-            'ext': 'flv',
+            'ext': 'mp4',
             'description': 'md5:df390f70a9ba7c95ff1daace988f0d8d',
             'title': 'Tyler Kalinoski hits buzzer-beater to lift Davidson',
             'timestamp': 1426270238,
@@ -120,7 +120,7 @@ class NBCSportsVPlayerIE(InfoExtractor):
             'uploader': 'NBCU-SPORTS',
         }
     }, {
-        'url': 'http://vplayer.nbcsports.com/p/BxmELC/nbc_embedshare/select/_hqLjQ95yx8Z',
+        'url': 'https://vplayer.nbcsports.com/p/BxmELC/nbcsports_embed/select/media/_hqLjQ95yx8Z',
         'only_matching': True,
     }]
 
@@ -134,7 +134,8 @@ class NBCSportsVPlayerIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        theplatform_url = self._og_search_video_url(webpage)
+        theplatform_url = self._og_search_video_url(webpage).replace(
+            'vplayer.nbcsports.com', 'player.theplatform.com')
         return self.url_result(theplatform_url, 'ThePlatform')
 
 

From f6ff52b473c9ed969fadb3e3d50852c4a27ba17e Mon Sep 17 00:00:00 2001
From: Giuseppe Fabiano <gfabiano40@gmail.com>
Date: Wed, 20 Sep 2017 23:05:33 +0200
Subject: [PATCH 1332/1696] [beeg] Fix extraction (closes #14275)

---
 youtube_dl/extractor/beeg.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index d5c5822f2..bbeae4bac 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -9,6 +9,7 @@ from ..compat import (
 from ..utils import (
     int_or_none,
     parse_iso8601,
+    urljoin,
 )
 
 
@@ -36,9 +37,11 @@ class BeegIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         cpl_url = self._search_regex(
-            r'<script[^>]+src=(["\'])(?P<url>(?:https?:)?//static\.beeg\.com/cpl/\d+\.js.*?)\1',
+            r'<script[^>]+src=(["\'])(?P<url>(?:/static|(?:https?:)?//static\.beeg\.com)/cpl/\d+\.js.*?)\1',
             webpage, 'cpl', default=None, group='url')
 
+        cpl_url = urljoin(url, cpl_url)
+
         beeg_version, beeg_salt = [None] * 2
 
         if cpl_url:
@@ -54,7 +57,7 @@ class BeegIE(InfoExtractor):
                     r'beeg_salt\s*=\s*(["\'])(?P<beeg_salt>.+?)\1', cpl, 'beeg salt',
                     default=None, group='beeg_salt')
 
-        beeg_version = beeg_version or '2000'
+        beeg_version = beeg_version or '2185'
         beeg_salt = beeg_salt or 'pmweAkq8lAYKdfWcFCUj0yoVgoPlinamH5UE1CB3H'
 
         video = self._download_json(

From 8c6919e4331e1cd44f50e700e8fc4e630d913a2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 Sep 2017 23:00:35 +0700
Subject: [PATCH 1333/1696] [lynda] Add support for educourse.ga (closes
 #14286)

---
 youtube_dl/extractor/lynda.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index d2f75296a..1b6f5091d 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -94,7 +94,7 @@ class LyndaBaseIE(InfoExtractor):
 class LyndaIE(LyndaBaseIE):
     IE_NAME = 'lynda'
     IE_DESC = 'lynda.com videos'
-    _VALID_URL = r'https?://(?:www\.)?lynda\.com/(?:[^/]+/[^/]+/(?P<course_id>\d+)|player/embed)/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:lynda\.com|educourse\.ga)/(?:[^/]+/[^/]+/(?P<course_id>\d+)|player/embed)/(?P<id>\d+)'
 
     _TIMECODE_REGEX = r'\[(?P<timecode>\d+:\d+:\d+[\.,]\d+)\]'
 
@@ -110,6 +110,9 @@ class LyndaIE(LyndaBaseIE):
     }, {
         'url': 'https://www.lynda.com/player/embed/133770?tr=foo=1;bar=g;fizz=rt&fs=0',
         'only_matching': True,
+    }, {
+        'url': 'https://educourse.ga/Bootstrap-tutorials/Using-exercise-files/110885/114408-4.html',
+        'only_matching': True,
     }]
 
     def _raise_unavailable(self, video_id):
@@ -253,7 +256,7 @@ class LyndaCourseIE(LyndaBaseIE):
 
     # Course link equals to welcome/introduction video link of same course
     # We will recognize it as course link
-    _VALID_URL = r'https?://(?:www|m)\.lynda\.com/(?P<coursepath>[^/]+/[^/]+/(?P<courseid>\d+))-\d\.html'
+    _VALID_URL = r'https?://(?:www|m)\.(?:lynda\.com|educourse\.ga)/(?P<coursepath>[^/]+/[^/]+/(?P<courseid>\d+))-\d\.html'
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 8c2895305dc09920055611c8120f5a65fcd2614f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 23 Sep 2017 02:30:03 +0800
Subject: [PATCH 1334/1696] [options] Accept lrc as a subtitle conversion
 target format (closes #14292)

---
 ChangeLog              | 1 +
 youtube_dl/__init__.py | 2 +-
 youtube_dl/options.py  | 2 +-
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index ba9260e3e..42ba879b1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Core
++ [options] Accept lrc as a subtitle conversion target format (#14292)
 * [utils] Fix handling raw TTML subtitles (#14191)
 
 
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index c4589411e..ba684a075 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -206,7 +206,7 @@ def _real_main(argv=None):
         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv', 'avi']:
             parser.error('invalid video recode format specified')
     if opts.convertsubtitles is not None:
-        if opts.convertsubtitles not in ['srt', 'vtt', 'ass']:
+        if opts.convertsubtitles not in ['srt', 'vtt', 'ass', 'lrc']:
             parser.error('invalid subtitle format specified')
 
     if opts.date is not None:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 38439c971..4c0455044 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -847,7 +847,7 @@ def parseOpts(overrideArguments=None):
     postproc.add_option(
         '--convert-subs', '--convert-subtitles',
         metavar='FORMAT', dest='convertsubtitles', default=None,
-        help='Convert the subtitles to other format (currently supported: srt|ass|vtt)')
+        help='Convert the subtitles to other format (currently supported: srt|ass|vtt|lrc)')
 
     parser.add_option_group(general)
     parser.add_option_group(network)

From 2384f5a64e501d7abb844e8d31fe340b34d8d4e7 Mon Sep 17 00:00:00 2001
From: Tatsuyuki Ishi <ishitatsuyuki@gmail.com>
Date: Wed, 6 Sep 2017 11:24:34 +0900
Subject: [PATCH 1335/1696] [mixcloud] Fix extraction (closes #14088)

---
 youtube_dl/compat.py             |  10 +-
 youtube_dl/extractor/mixcloud.py | 157 ++++++++++++++++++++-----------
 2 files changed, 109 insertions(+), 58 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 9e4e13bcf..2a62248ef 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -6,6 +6,7 @@ import collections
 import email
 import getpass
 import io
+import itertools
 import optparse
 import os
 import re
@@ -15,7 +16,6 @@ import socket
 import struct
 import subprocess
 import sys
-import itertools
 import xml.etree.ElementTree
 
 
@@ -2898,6 +2898,13 @@ else:
     compat_struct_pack = struct.pack
     compat_struct_unpack = struct.unpack
 
+try:
+    from future_builtins import zip as compat_zip
+except ImportError:  # not 2.6+ or is 3.x
+    try:
+        from itertools import izip as compat_zip  # < 2.5 or 3.x
+    except ImportError:
+        compat_zip = zip
 
 __all__ = [
     'compat_HTMLParseError',
@@ -2948,5 +2955,6 @@ __all__ = [
     'compat_urlretrieve',
     'compat_xml_parse_error',
     'compat_xpath',
+    'compat_zip',
     'workaround_optparse_bug9161',
 ]
diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index f6360cce6..481182380 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -9,16 +9,16 @@ from .common import InfoExtractor
 from ..compat import (
     compat_chr,
     compat_ord,
-    compat_str,
     compat_urllib_parse_unquote,
     compat_urlparse,
+    compat_zip
 )
 from ..utils import (
     clean_html,
     ExtractorError,
     OnDemandPagedList,
     str_to_int,
-)
+    try_get)
 
 
 class MixcloudIE(InfoExtractor):
@@ -54,27 +54,19 @@ class MixcloudIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    _keys = [
-        'return { requestAnimationFrame: function(callback) { callback(); }, innerHeight: 500 };',
-        'pleasedontdownloadourmusictheartistswontgetpaid',
-        'window.addEventListener = window.addEventListener || function() {};',
-        '(function() { return new Date().toLocaleDateString(); })()'
-    ]
-    _current_key = None
+    @staticmethod
+    def _decrypt_xor_cipher(key, ciphertext):
+        """Encrypt/Decrypt XOR cipher. Both ways are possible because it's XOR."""
+        return ''.join([
+            compat_chr(compat_ord(ch) ^ compat_ord(k))
+            for ch, k in compat_zip(ciphertext, itertools.cycle(key))])
 
-    # See https://www.mixcloud.com/media/js2/www_js_2.9e23256562c080482435196ca3975ab5.js
-    def _decrypt_play_info(self, play_info, video_id):
-        play_info = base64.b64decode(play_info.encode('ascii'))
-        for num, key in enumerate(self._keys, start=1):
-            try:
-                return self._parse_json(
-                    ''.join([
-                        compat_chr(compat_ord(ch) ^ compat_ord(key[idx % len(key)]))
-                        for idx, ch in enumerate(play_info)]),
-                    video_id)
-            except ExtractorError:
-                if num == len(self._keys):
-                    raise
+    @staticmethod
+    def _decrypt_and_extend(stream_info, url_key, getter, key, formats):
+        maybe_url = stream_info.get(url_key)
+        if maybe_url is not None:
+            decrypted = MixcloudIE._decrypt_xor_cipher(key, base64.b64decode(maybe_url))
+            formats.extend(getter(decrypted))
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -84,54 +76,105 @@ class MixcloudIE(InfoExtractor):
 
         webpage = self._download_webpage(url, track_id)
 
-        if not self._current_key:
-            js_url = self._search_regex(
-                r'<script[^>]+\bsrc=["\"](https://(?:www\.)?mixcloud\.com/media/js2/www_js_4\.[^>]+\.js)',
-                webpage, 'js url', default=None)
-            if js_url:
-                js = self._download_webpage(js_url, track_id, fatal=False)
-                if js:
-                    KEY_RE_TEMPLATE = r'player\s*:\s*{.*?\b%s\s*:\s*(["\'])(?P<key>(?:(?!\1).)+)\1'
-                    for key_name in ('value', 'key_value', 'key_value.*?', '.*?value.*?'):
-                        key = self._search_regex(
-                            KEY_RE_TEMPLATE % key_name, js, 'key',
-                            default=None, group='key')
-                        if key and isinstance(key, compat_str):
-                            self._keys.insert(0, key)
-                            self._current_key = key
+        # Legacy path
+        encrypted_play_info = self._search_regex(
+            r'm-play-info="([^"]+)"', webpage, 'play info', default=None)
+
+        if encrypted_play_info is not None:
+            # Decode
+            encrypted_play_info = base64.b64decode(encrypted_play_info)
+        else:
+            # New path
+            full_info_json = self._parse_json(self._html_search_regex(
+                r'<script id="relay-data" type="text/x-mixcloud">([^<]+)</script>', webpage, 'play info'), 'play info')
+            for item in full_info_json:
+                item_data = try_get(item, lambda x: x['cloudcast']['data']['cloudcastLookup'])
+                if try_get(item_data, lambda x: x['streamInfo']['url']):
+                    info_json = item_data
+                    break
+            else:
+                raise ExtractorError('Failed to extract matching stream info')
 
         message = self._html_search_regex(
             r'(?s)<div[^>]+class="global-message cloudcast-disabled-notice-light"[^>]*>(.+?)<(?:a|/div)',
             webpage, 'error message', default=None)
 
-        encrypted_play_info = self._search_regex(
-            r'm-play-info="([^"]+)"', webpage, 'play info')
+        js_url = self._search_regex(
+            r'<script[^>]+\bsrc=["\"](https://(?:www\.)?mixcloud\.com/media/js2/www_js_4\.[^>]+\.js)',
+            webpage, 'js url', default=None)
+        if js_url is None:
+            js_url = self._search_regex(
+                r'<script[^>]+\bsrc=["\"](https://(?:www\.)?mixcloud\.com/media/js/www\.[^>]+\.js)',
+                webpage, 'js url')
+        js = self._download_webpage(js_url, track_id)
+        # Known plaintext attack
+        if encrypted_play_info:
+            kps = ['{"stream_url":']
+            kpa_target = encrypted_play_info
+        else:
+            kps = ['https://', 'http://']
+            kpa_target = base64.b64decode(info_json['streamInfo']['url'])
+        for kp in kps:
+            partial_key = self._decrypt_xor_cipher(kpa_target, kp)
+            for quote in ["'", '"']:
+                key = self._search_regex(r'{0}({1}[^{0}]*){0}'.format(quote, re.escape(partial_key)), js,
+                                         "encryption key", default=None)
+                if key is not None:
+                    break
+            else:
+                continue
+            break
+        else:
+            raise ExtractorError('Failed to extract encryption key')
 
-        play_info = self._decrypt_play_info(encrypted_play_info, track_id)
+        if encrypted_play_info is not None:
+            play_info = self._parse_json(self._decrypt_xor_cipher(key, encrypted_play_info), 'play info')
+            if message and 'stream_url' not in play_info:
+                raise ExtractorError('%s said: %s' % (self.IE_NAME, message), expected=True)
+            song_url = play_info['stream_url']
+            formats = [{
+                'format_id': 'normal',
+                'url': song_url
+            }]
 
-        if message and 'stream_url' not in play_info:
-            raise ExtractorError('%s said: %s' % (self.IE_NAME, message), expected=True)
+            title = self._html_search_regex(r'm-title="([^"]+)"', webpage, 'title')
+            thumbnail = self._proto_relative_url(self._html_search_regex(
+                r'm-thumbnail-url="([^"]+)"', webpage, 'thumbnail', fatal=False))
+            uploader = self._html_search_regex(
+                r'm-owner-name="([^"]+)"', webpage, 'uploader', fatal=False)
+            uploader_id = self._search_regex(
+                r'\s+"profile": "([^"]+)",', webpage, 'uploader id', fatal=False)
+            description = self._og_search_description(webpage)
+            view_count = str_to_int(self._search_regex(
+                [r'<meta itemprop="interactionCount" content="UserPlays:([0-9]+)"',
+                 r'/listeners/?">([0-9,.]+)</a>',
+                 r'(?:m|data)-tooltip=["\']([\d,.]+) plays'],
+                webpage, 'play count', default=None))
 
-        song_url = play_info['stream_url']
+        else:
+            title = info_json['name']
+            thumbnail = try_get(info_json,
+                                lambda x: 'https://thumbnailer.mixcloud.com/unsafe/600x600/' + x['picture']['urlRoot'])
+            uploader = try_get(info_json, lambda x: x['owner']['displayName'])
+            uploader_id = try_get(info_json, lambda x: x['owner']['username'])
+            description = try_get(info_json, lambda x: x['description'])
+            view_count = try_get(info_json, lambda x: x['plays'])
 
-        title = self._html_search_regex(r'm-title="([^"]+)"', webpage, 'title')
-        thumbnail = self._proto_relative_url(self._html_search_regex(
-            r'm-thumbnail-url="([^"]+)"', webpage, 'thumbnail', fatal=False))
-        uploader = self._html_search_regex(
-            r'm-owner-name="([^"]+)"', webpage, 'uploader', fatal=False)
-        uploader_id = self._search_regex(
-            r'\s+"profile": "([^"]+)",', webpage, 'uploader id', fatal=False)
-        description = self._og_search_description(webpage)
-        view_count = str_to_int(self._search_regex(
-            [r'<meta itemprop="interactionCount" content="UserPlays:([0-9]+)"',
-             r'/listeners/?">([0-9,.]+)</a>',
-             r'(?:m|data)-tooltip=["\']([\d,.]+) plays'],
-            webpage, 'play count', default=None))
+            stream_info = info_json['streamInfo']
+            formats = []
+            self._decrypt_and_extend(stream_info, 'url', lambda x: [{
+                'format_id': 'normal',
+                'url': x
+            }], key, formats)
+            self._decrypt_and_extend(stream_info, 'hlsUrl', lambda x: self._extract_m3u8_formats(x, title), key,
+                                     formats)
+            self._decrypt_and_extend(stream_info, 'dashUrl', lambda x: self._extract_mpd_formats(x, title), key,
+                                     formats)
 
         return {
             'id': track_id,
             'title': title,
-            'url': song_url,
+            'formats': formats,
             'description': description,
             'thumbnail': thumbnail,
             'uploader': uploader,

From 095774e59130c999ed8ce132f80a7164c5ee39a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Sep 2017 05:35:55 +0700
Subject: [PATCH 1336/1696] [mixcloud] Improve and simplify (closes #14132)

---
 youtube_dl/extractor/mixcloud.py | 71 ++++++++++++++++++--------------
 1 file changed, 41 insertions(+), 30 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 481182380..f331db890 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -9,6 +9,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_chr,
     compat_ord,
+    compat_str,
     compat_urllib_parse_unquote,
     compat_urlparse,
     compat_zip
@@ -16,9 +17,12 @@ from ..compat import (
 from ..utils import (
     clean_html,
     ExtractorError,
+    int_or_none,
     OnDemandPagedList,
     str_to_int,
-    try_get)
+    try_get,
+    urljoin,
+)
 
 
 class MixcloudIE(InfoExtractor):
@@ -61,13 +65,6 @@ class MixcloudIE(InfoExtractor):
             compat_chr(compat_ord(ch) ^ compat_ord(k))
             for ch, k in compat_zip(ciphertext, itertools.cycle(key))])
 
-    @staticmethod
-    def _decrypt_and_extend(stream_info, url_key, getter, key, formats):
-        maybe_url = stream_info.get(url_key)
-        if maybe_url is not None:
-            decrypted = MixcloudIE._decrypt_xor_cipher(key, base64.b64decode(maybe_url))
-            formats.extend(getter(decrypted))
-
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         uploader = mobj.group(1)
@@ -86,9 +83,12 @@ class MixcloudIE(InfoExtractor):
         else:
             # New path
             full_info_json = self._parse_json(self._html_search_regex(
-                r'<script id="relay-data" type="text/x-mixcloud">([^<]+)</script>', webpage, 'play info'), 'play info')
+                r'<script id="relay-data" type="text/x-mixcloud">([^<]+)</script>',
+                webpage, 'play info'), 'play info')
             for item in full_info_json:
-                item_data = try_get(item, lambda x: x['cloudcast']['data']['cloudcastLookup'])
+                item_data = try_get(
+                    item, lambda x: x['cloudcast']['data']['cloudcastLookup'],
+                    dict)
                 if try_get(item_data, lambda x: x['streamInfo']['url']):
                     info_json = item_data
                     break
@@ -100,13 +100,9 @@ class MixcloudIE(InfoExtractor):
             webpage, 'error message', default=None)
 
         js_url = self._search_regex(
-            r'<script[^>]+\bsrc=["\"](https://(?:www\.)?mixcloud\.com/media/js2/www_js_4\.[^>]+\.js)',
-            webpage, 'js url', default=None)
-        if js_url is None:
-            js_url = self._search_regex(
-                r'<script[^>]+\bsrc=["\"](https://(?:www\.)?mixcloud\.com/media/js/www\.[^>]+\.js)',
-                webpage, 'js url')
-        js = self._download_webpage(js_url, track_id)
+            r'<script[^>]+\bsrc=["\"](https://(?:www\.)?mixcloud\.com/media/(?:js2/www_js_4|js/www)\.[^>]+\.js)',
+            webpage, 'js url')
+        js = self._download_webpage(js_url, track_id, 'Downloading JS')
         # Known plaintext attack
         if encrypted_play_info:
             kps = ['{"stream_url":']
@@ -117,8 +113,9 @@ class MixcloudIE(InfoExtractor):
         for kp in kps:
             partial_key = self._decrypt_xor_cipher(kpa_target, kp)
             for quote in ["'", '"']:
-                key = self._search_regex(r'{0}({1}[^{0}]*){0}'.format(quote, re.escape(partial_key)), js,
-                                         "encryption key", default=None)
+                key = self._search_regex(
+                    r'{0}({1}[^{0}]*){0}'.format(quote, re.escape(partial_key)),
+                    js, 'encryption key', default=None)
                 if key is not None:
                     break
             else:
@@ -153,23 +150,37 @@ class MixcloudIE(InfoExtractor):
 
         else:
             title = info_json['name']
-            thumbnail = try_get(info_json,
-                                lambda x: 'https://thumbnailer.mixcloud.com/unsafe/600x600/' + x['picture']['urlRoot'])
+            thumbnail = urljoin(
+                'https://thumbnailer.mixcloud.com/unsafe/600x600/',
+                try_get(info_json, lambda x: x['picture']['urlRoot'], compat_str))
             uploader = try_get(info_json, lambda x: x['owner']['displayName'])
             uploader_id = try_get(info_json, lambda x: x['owner']['username'])
             description = try_get(info_json, lambda x: x['description'])
-            view_count = try_get(info_json, lambda x: x['plays'])
+            view_count = int_or_none(try_get(info_json, lambda x: x['plays']))
 
             stream_info = info_json['streamInfo']
             formats = []
-            self._decrypt_and_extend(stream_info, 'url', lambda x: [{
-                'format_id': 'normal',
-                'url': x
-            }], key, formats)
-            self._decrypt_and_extend(stream_info, 'hlsUrl', lambda x: self._extract_m3u8_formats(x, title), key,
-                                     formats)
-            self._decrypt_and_extend(stream_info, 'dashUrl', lambda x: self._extract_mpd_formats(x, title), key,
-                                     formats)
+
+            for url_key in ('url', 'hlsUrl', 'dashUrl'):
+                format_url = stream_info.get(url_key)
+                if not format_url:
+                    continue
+                decrypted = self._decrypt_xor_cipher(key, base64.b64decode(format_url))
+                if not decrypted:
+                    continue
+                if url_key == 'hlsUrl':
+                    formats.extend(self._extract_m3u8_formats(
+                        decrypted, track_id, 'mp4', entry_protocol='m3u8_native',
+                        m3u8_id='hls', fatal=False))
+                elif url_key == 'dashUrl':
+                    formats.extend(self._extract_mpd_formats(
+                        decrypted, track_id, mpd_id='dash', fatal=False))
+                else:
+                    formats.append({
+                        'format_id': 'http',
+                        'url': decrypted,
+                    })
+            self._sort_formats(formats)
 
         return {
             'id': track_id,

From 9ce1ac404648142139f8b231c674d434ad4f9ffe Mon Sep 17 00:00:00 2001
From: kayb94 <30302445+kayb94@users.noreply.github.com>
Date: Fri, 22 Sep 2017 22:49:48 +0000
Subject: [PATCH 1337/1696] [generic] Fix support for multiple HTML5 videos on
 one page (closes #14080)

---
 youtube_dl/extractor/generic.py | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index b83c18380..7d0edf09c 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1879,6 +1879,15 @@ class GenericIE(InfoExtractor):
                 'title': 'Building A Business Online: Principal Chairs Q & A',
             },
         },
+        {
+            # multiple HTML5 videos on one page
+            'url': 'https://www.paragon-software.com/home/rk-free/keyscenarios.html',
+            'info_dict': {
+                'id': 'keyscenarios',
+                'title': 'Rescue Kit 14 Free Edition - Getting started',
+            },
+            'playlist_count': 4,
+        }
         # {
         #     # TODO: find another test
         #     # http://schema.org/VideoObject
@@ -2849,13 +2858,20 @@ class GenericIE(InfoExtractor):
         # Look for HTML5 media
         entries = self._parse_html5_media_entries(url, webpage, video_id, m3u8_id='hls')
         if entries:
-            for entry in entries:
-                entry.update({
+            if len(entries) == 1:
+                entries[0].update({
                     'id': video_id,
                     'title': video_title,
                 })
+            else:
+                for num, entry in enumerate(entries, start=1):
+                    entry.update({
+                        'id': '%s-%s' % (video_id, num),
+                        'title': '%s (%d)' % (video_title, num),
+                    })
+            for entry in entries:
                 self._sort_formats(entry['formats'])
-            return self.playlist_result(entries)
+            return self.playlist_result(entries, video_id, video_title)
 
         jwplayer_data = self._find_jwplayer_data(
             webpage, video_id, transform_source=js_to_json)

From 13de91c9e92bd831fee38fddbdabce7f6e82ef91 Mon Sep 17 00:00:00 2001
From: Dan Weber <dan.b.weber@me.com>
Date: Tue, 12 Sep 2017 22:52:54 -0400
Subject: [PATCH 1338/1696] [americastestkitchen] Add extractor (closes #10764)

---
 youtube_dl/extractor/americastestkitchen.py | 85 +++++++++++++++++++++
 youtube_dl/extractor/extractors.py          |  1 +
 2 files changed, 86 insertions(+)
 create mode 100755 youtube_dl/extractor/americastestkitchen.py

diff --git a/youtube_dl/extractor/americastestkitchen.py b/youtube_dl/extractor/americastestkitchen.py
new file mode 100755
index 000000000..f231e7f6e
--- /dev/null
+++ b/youtube_dl/extractor/americastestkitchen.py
@@ -0,0 +1,85 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class AmericasTestKitchenIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?americastestkitchen\.com/episode/(?P<id>\d+)'
+    _TESTS = [{
+        'url':
+        'https://www.americastestkitchen.com/episode/548-summer-dinner-party',
+        'md5': 'b861c3e365ac38ad319cfd509c30577f',
+        'info_dict': {
+            'id': '1_5g5zua6e',
+            'title': 'atk_s17_e24.mp4',
+            'ext': 'mp4',
+            'description': '<p>Host Julia Collin Davison goes into the test kitchen with test cook Dan Souza to learn how to make the ultimate Grill-Roasted Beef Tenderloin. Next, equipment expert Adam Ried reviews gas grills in the Equipment Corner. Then, gadget guru Lisa McManus uncovers the best quirky gadgets. Finally, test cook Erin McMurrer shows host Bridget Lancaster how to make an elegant Pear-Walnut Upside-Down Cake.</p>',
+            'timestamp': 1497285541,
+            'upload_date': '20170612',
+            'uploader_id': 'roger.metcalf@americastestkitchen.com',
+            'release_date': '2017-06-17',
+            'thumbnail': 'http://d3cizcpymoenau.cloudfront.net/images/35973/e24-tenderloin-16.jpg',
+            'episode_number': 24,
+            'episode': 'Summer Dinner Party',
+            'episode_id': '548-summer-dinner-party',
+            'season_number': 17
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        'url':
+        'https://www.americastestkitchen.com/episode/546-a-spanish-affair',
+        'only_matching':
+        True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        partner_id = self._search_regex(
+            r'partner_id/(?P<partner_id>\d+)',
+            webpage,
+            'partner_id',
+            group='partner_id')
+
+        video_data = self._parse_json(
+            self._search_regex(
+                r'window\.__INITIAL_STATE__\s*=\s*({.+?});\s*</script>',
+                webpage, 'initial context'),
+            video_id)
+
+        episode_data = video_data['episodeDetail']['content']['data']
+        episode_content_meta = episode_data['full_video']
+        external_id = episode_content_meta['external_id']
+
+        # photo data
+        photo_data = episode_content_meta.get('photo')
+        thumbnail = photo_data.get('image_url') if photo_data else None
+
+        # meta
+        release_date = episode_data.get('aired_at')
+        description = episode_content_meta.get('description')
+        episode_number = int(episode_content_meta.get('episode_number'))
+        episode = episode_content_meta.get('title')
+        episode_id = episode_content_meta.get('episode_slug')
+        season_number = int(episode_content_meta.get('season_number'))
+
+        return {
+            '_type': 'url_transparent',
+            'url': 'kaltura:%s:%s' % (partner_id, external_id),
+            'ie_key': 'Kaltura',
+            'id': video_id,
+            'release_date': release_date,
+            'thumbnail': thumbnail,
+            'description': description,
+            'episode_number': episode_number,
+            'episode': episode,
+            'episode_id': episode_id,
+            'season_number': season_number
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index ab95c8575..585300500 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -39,6 +39,7 @@ from .airmozilla import AirMozillaIE
 from .aljazeera import AlJazeeraIE
 from .alphaporno import AlphaPornoIE
 from .amcnetworks import AMCNetworksIE
+from .americastestkitchen import AmericasTestKitchenIE
 from .animeondemand import AnimeOnDemandIE
 from .anitube import AnitubeIE
 from .anvato import AnvatoIE

From 4bb58fa118a8c75b2ecf05f7b29a0ae27eef6239 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Sep 2017 06:28:46 +0700
Subject: [PATCH 1339/1696] [americastestkitchen] Improve (closes #13996)

---
 youtube_dl/extractor/americastestkitchen.py | 82 ++++++++++-----------
 1 file changed, 41 insertions(+), 41 deletions(-)

diff --git a/youtube_dl/extractor/americastestkitchen.py b/youtube_dl/extractor/americastestkitchen.py
index f231e7f6e..01736872d 100755
--- a/youtube_dl/extractor/americastestkitchen.py
+++ b/youtube_dl/extractor/americastestkitchen.py
@@ -1,85 +1,85 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    int_or_none,
+    try_get,
+    unified_strdate,
+)
 
 
 class AmericasTestKitchenIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?americastestkitchen\.com/episode/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?americastestkitchen\.com/(?:episode|videos)/(?P<id>\d+)'
     _TESTS = [{
-        'url':
-        'https://www.americastestkitchen.com/episode/548-summer-dinner-party',
+        'url': 'https://www.americastestkitchen.com/episode/548-summer-dinner-party',
         'md5': 'b861c3e365ac38ad319cfd509c30577f',
         'info_dict': {
             'id': '1_5g5zua6e',
-            'title': 'atk_s17_e24.mp4',
+            'title': 'Summer Dinner Party',
             'ext': 'mp4',
-            'description': '<p>Host Julia Collin Davison goes into the test kitchen with test cook Dan Souza to learn how to make the ultimate Grill-Roasted Beef Tenderloin. Next, equipment expert Adam Ried reviews gas grills in the Equipment Corner. Then, gadget guru Lisa McManus uncovers the best quirky gadgets. Finally, test cook Erin McMurrer shows host Bridget Lancaster how to make an elegant Pear-Walnut Upside-Down Cake.</p>',
+            'description': 'md5:858d986e73a4826979b6a5d9f8f6a1ec',
+            'thumbnail': r're:^https?://.*\.jpg',
             'timestamp': 1497285541,
             'upload_date': '20170612',
             'uploader_id': 'roger.metcalf@americastestkitchen.com',
-            'release_date': '2017-06-17',
-            'thumbnail': 'http://d3cizcpymoenau.cloudfront.net/images/35973/e24-tenderloin-16.jpg',
-            'episode_number': 24,
+            'release_date': '20170617',
+            'series': "America's Test Kitchen",
+            'season_number': 17,
             'episode': 'Summer Dinner Party',
-            'episode_id': '548-summer-dinner-party',
-            'season_number': 17
+            'episode_number': 24,
         },
         'params': {
-            # m3u8 download
             'skip_download': True,
         },
     }, {
-        'url':
-        'https://www.americastestkitchen.com/episode/546-a-spanish-affair',
-        'only_matching':
-        True,
+        'url': 'https://www.americastestkitchen.com/videos/3420-pan-seared-salmon',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
 
         partner_id = self._search_regex(
-            r'partner_id/(?P<partner_id>\d+)',
-            webpage,
-            'partner_id',
-            group='partner_id')
+            r'src=["\'](?:https?:)?//(?:[^/]+\.)kaltura\.com/(?:[^/]+/)*(?:p|partner_id)/(\d+)',
+            webpage, 'kaltura partner id')
 
         video_data = self._parse_json(
             self._search_regex(
-                r'window\.__INITIAL_STATE__\s*=\s*({.+?});\s*</script>',
+                r'window\.__INITIAL_STATE__\s*=\s*({.+?})\s*;\s*</script>',
                 webpage, 'initial context'),
             video_id)
 
-        episode_data = video_data['episodeDetail']['content']['data']
-        episode_content_meta = episode_data['full_video']
-        external_id = episode_content_meta['external_id']
+        ep_data = try_get(
+            video_data,
+            (lambda x: x['episodeDetail']['content']['data'],
+             lambda x: x['videoDetail']['content']['data']), dict)
+        ep_meta = ep_data.get('full_video', {})
+        external_id = ep_data.get('external_id') or ep_meta['external_id']
 
-        # photo data
-        photo_data = episode_content_meta.get('photo')
-        thumbnail = photo_data.get('image_url') if photo_data else None
+        title = ep_data.get('title') or ep_meta.get('title')
+        description = clean_html(ep_meta.get('episode_description') or ep_data.get(
+            'description') or ep_meta.get('description'))
+        thumbnail = try_get(ep_meta, lambda x: x['photo']['image_url'])
+        release_date = unified_strdate(ep_data.get('aired_at'))
 
-        # meta
-        release_date = episode_data.get('aired_at')
-        description = episode_content_meta.get('description')
-        episode_number = int(episode_content_meta.get('episode_number'))
-        episode = episode_content_meta.get('title')
-        episode_id = episode_content_meta.get('episode_slug')
-        season_number = int(episode_content_meta.get('season_number'))
+        season_number = int_or_none(ep_meta.get('season_number'))
+        episode = ep_meta.get('title')
+        episode_number = int_or_none(ep_meta.get('episode_number'))
 
         return {
             '_type': 'url_transparent',
             'url': 'kaltura:%s:%s' % (partner_id, external_id),
             'ie_key': 'Kaltura',
-            'id': video_id,
-            'release_date': release_date,
-            'thumbnail': thumbnail,
+            'title': title,
             'description': description,
-            'episode_number': episode_number,
+            'thumbnail': thumbnail,
+            'release_date': release_date,
+            'series': "America's Test Kitchen",
+            'season_number': season_number,
             'episode': episode,
-            'episode_id': episode_id,
-            'season_number': season_number
+            'episode_number': episode_number,
         }

From 5c1452e8f1e744db14be1baef840e9f531e8f144 Mon Sep 17 00:00:00 2001
From: Giuseppe Fabiano <gfabiano40@gmail.com>
Date: Sat, 23 Sep 2017 01:38:09 +0200
Subject: [PATCH 1340/1696] [twitter] Add support for user_id-less URLs (closes
 #14270)

---
 youtube_dl/extractor/twitter.py | 30 +++++++++++++++++++++++++-----
 1 file changed, 25 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 7399cf538..0df3ad7c7 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -242,8 +242,9 @@ class TwitterCardIE(TwitterBaseIE):
 
 class TwitterIE(InfoExtractor):
     IE_NAME = 'twitter'
-    _VALID_URL = r'https?://(?:www\.|m\.|mobile\.)?twitter\.com/(?P<user_id>[^/]+)/status/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.|m\.|mobile\.)?twitter\.com/(?:i/web|(?P<user_id>[^/]+))/status/(?P<id>\d+)'
     _TEMPLATE_URL = 'https://twitter.com/%s/status/%s'
+    _TEMPLATE_STATUSES_URL = 'https://twitter.com/statuses/%s'
 
     _TESTS = [{
         'url': 'https://twitter.com/freethenipple/status/643211948184596480',
@@ -322,9 +323,9 @@ class TwitterIE(InfoExtractor):
         'info_dict': {
             'id': 'MIOxnrUteUd',
             'ext': 'mp4',
-            'title': 'FilmDrunk - Vine of the day',
-            'description': 'FilmDrunk on Twitter: "Vine of the day https://t.co/xmTvRdqxWf"',
-            'uploader': 'FilmDrunk',
+            'title': 'Vince Mancini - Vine of the day',
+            'description': 'Vince Mancini on Twitter: "Vine of the day https://t.co/xmTvRdqxWf"',
+            'uploader': 'Vince Mancini',
             'uploader_id': 'Filmdrunk',
             'timestamp': 1402826626,
             'upload_date': '20140615',
@@ -372,6 +373,21 @@ class TwitterIE(InfoExtractor):
         'params': {
             'format': 'best[format_id^=http-]',
         },
+    }, {
+        'url': 'https://twitter.com/i/web/status/910031516746514432',
+        'info_dict': {
+            'id': '910031516746514432',
+            'ext': 'mp4',
+            'title': 'Préfet de Guadeloupe - [Direct] #Maria Le centre se trouve actuellement au sud de Basse-Terre. Restez confinés. Réfugiez-vous dans la pièce la + sûre.',
+            'thumbnail': r're:^https?://.*\.jpg',
+            'description': 'Préfet de Guadeloupe on Twitter: "[Direct] #Maria Le centre se trouve actuellement au sud de Basse-Terre. Restez confinés. Réfugiez-vous dans la pièce la + sûre. https://t.co/mwx01Rs4lo"',
+            'uploader': 'Préfet de Guadeloupe',
+            'uploader_id': 'Prefet971',
+            'duration': 47.48,
+        },
+        'params': {
+            'skip_download': True,  # requires ffmpeg
+        },
     }]
 
     def _real_extract(self, url):
@@ -380,11 +396,15 @@ class TwitterIE(InfoExtractor):
         twid = mobj.group('id')
 
         webpage, urlh = self._download_webpage_handle(
-            self._TEMPLATE_URL % (user_id, twid), twid)
+            self._TEMPLATE_STATUSES_URL % twid, twid)
 
         if 'twitter.com/account/suspended' in urlh.geturl():
             raise ExtractorError('Account suspended by Twitter.', expected=True)
 
+        if user_id is None:
+            mobj = re.match(self._VALID_URL, urlh.geturl())
+            user_id = mobj.group('user_id')
+
         username = remove_end(self._og_search_title(webpage), ' on Twitter')
 
         title = description = self._og_search_description(webpage).strip('').replace('\n', ' ').strip('“”')

From 1c22d7a7f30917abfd2b7495f7bd02d51cb8528a Mon Sep 17 00:00:00 2001
From: Namnamseo <0201ssw+github@gmail.com>
Date: Thu, 24 Aug 2017 11:32:24 +0900
Subject: [PATCH 1341/1696] [kakao] Add extractor (closes #12298)

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/kakao.py      | 140 +++++++++++++++++++++++++++++
 2 files changed, 141 insertions(+)
 create mode 100644 youtube_dl/extractor/kakao.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 585300500..4232a4fef 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -483,6 +483,7 @@ from .jove import JoveIE
 from .joj import JojIE
 from .jwplatform import JWPlatformIE
 from .jpopsukitv import JpopsukiIE
+from .kakao import KakaoIE
 from .kaltura import KalturaIE
 from .kamcord import KamcordIE
 from .kanalplay import KanalPlayIE
diff --git a/youtube_dl/extractor/kakao.py b/youtube_dl/extractor/kakao.py
new file mode 100644
index 000000000..0caa41e9e
--- /dev/null
+++ b/youtube_dl/extractor/kakao.py
@@ -0,0 +1,140 @@
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    compat_str,
+    unified_timestamp,
+)
+
+
+class KakaoIE(InfoExtractor):
+    _VALID_URL = r'https?://tv.kakao.com/channel/(?P<channel>\d+)/cliplink/(?P<id>\d+)'
+    IE_NAME = 'kakao.com'
+
+    _TESTS = [{
+        'url': 'http://tv.kakao.com/channel/2671005/cliplink/301965083',
+        'md5': '702b2fbdeb51ad82f5c904e8c0766340',
+        'info_dict': {
+            'id': '301965083',
+            'ext': 'mp4',
+            'title': '乃木坂46 バナナマン 「3期生紹介コーナーが始動！顔高低差GPも！」 『乃木坂工事中』',
+            'uploader_id': 2671005,
+            'uploader': '그랑그랑이',
+            'timestamp': 1488160199,
+            'upload_date': '20170227',
+        }
+    }, {
+        'url': 'http://tv.kakao.com/channel/2653210/cliplink/300103180',
+        'md5': 'a8917742069a4dd442516b86e7d66529',
+        'info_dict': {
+            'id': '300103180',
+            'ext': 'mp4',
+            'description': '러블리즈 - Destiny (나의 지구) (Lovelyz - Destiny)\r\n\r\n[쇼! 음악중심] 20160611, 507회',
+            'title': '러블리즈 - Destiny (나의 지구) (Lovelyz - Destiny)',
+            'uploader_id': 2653210,
+            'uploader': '쇼 음악중심',
+            'timestamp': 1485684628,
+            'upload_date': '20170129',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        player_url = 'http://tv.kakao.com/embed/player/cliplink/' + video_id + \
+            '?service=kakao_tv&autoplay=1&profile=HIGH&wmode=transparent'
+        player_header = {'Referer': player_url}
+
+        impress = self._download_json(
+            'http://tv.kakao.com/api/v1/ft/cliplinks/%s/impress' % video_id,
+            video_id, 'Downloading video info',
+            query={
+                'player': 'monet_html5',
+                'referer': url,
+                'uuid': '',
+                'service': 'kakao_tv',
+                'section': '',
+                'dteType': 'PC',
+                'fields': 'clipLink,clip,channel,hasPlusFriend,-service,-tagList'
+            }, headers=player_header)
+
+        clipLink = impress['clipLink']
+        clip = clipLink['clip']
+
+        video_info = {
+            'id': video_id,
+            'title': clip['title'],
+            'description': clip.get('description'),
+            'uploader': clipLink.get('channel', {}).get('name'),
+            'uploader_id': clipLink.get('channelId'),
+            'duration': int_or_none(clip.get('duration')),
+            'view_count': int_or_none(clip.get('playCount')),
+            'like_count': int_or_none(clip.get('likeCount')),
+            'comment_count': int_or_none(clip.get('commentCount')),
+        }
+
+        tid = impress.get('tid', '')
+        raw = self._download_json(
+            'http://tv.kakao.com/api/v1/ft/cliplinks/%s/raw' % video_id,
+            video_id, 'Downloading video formats info',
+            query={
+                'player': 'monet_html5',
+                'referer': url,
+                'uuid': '',
+                'service': 'kakao_tv',
+                'section': '',
+                'tid': tid,
+                'profile': 'HIGH',
+                'dteType': 'PC',
+            }, headers=player_header, fatal=False)
+
+        formats = []
+        for fmt in raw.get('outputList', []):
+            try:
+                profile_name = fmt['profile']
+                fmt_url_json = self._download_json(
+                    'http://tv.kakao.com/api/v1/ft/cliplinks/%s/raw/videolocation' % video_id,
+                    video_id, 'Downloading video URL for profile %s' % profile_name,
+                    query={
+                        'service': 'kakao_tv',
+                        'section': '',
+                        'tid': tid,
+                        'profile': profile_name
+                    }, headers=player_header, fatal=False)
+
+                if fmt_url_json is None:
+                    continue
+
+                fmt_url = fmt_url_json['url']
+                formats.append({
+                    'url': fmt_url,
+                    'format_id': profile_name,
+                    'width': int_or_none(fmt.get('width')),
+                    'height': int_or_none(fmt.get('height')),
+                    'format_note': fmt.get('label'),
+                    'filesize': int_or_none(fmt.get('filesize'))
+                })
+            except KeyError:
+                pass
+
+        self._sort_formats(formats)
+        video_info['formats'] = formats
+
+        top_thumbnail = clip.get('thumbnailUrl')
+        thumbs = []
+        for thumb in clip.get('clipChapterThumbnailList', []):
+            thumbs.append({
+                'url': thumb.get('thumbnailUrl'),
+                'id': compat_str(thumb.get('timeInSec')),
+                'preference': -1 if thumb.get('isDefault') else 0
+            })
+        video_info['thumbnail'] = top_thumbnail
+        video_info['thumbnails'] = thumbs
+
+        upload_date = unified_timestamp(clipLink.get('createTime'))
+        video_info['timestamp'] = upload_date
+
+        return video_info

From f70ddd4aebbfb0bdf2f63c1eba5b5614d2cfb70f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Sep 2017 07:25:15 +0700
Subject: [PATCH 1342/1696] [kakao] Improve (closes #14007)

---
 youtube_dl/extractor/kakao.py | 117 ++++++++++++++++++----------------
 1 file changed, 63 insertions(+), 54 deletions(-)

diff --git a/youtube_dl/extractor/kakao.py b/youtube_dl/extractor/kakao.py
index 0caa41e9e..c9b438efc 100644
--- a/youtube_dl/extractor/kakao.py
+++ b/youtube_dl/extractor/kakao.py
@@ -3,16 +3,17 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     int_or_none,
-    compat_str,
     unified_timestamp,
+    update_url_query,
 )
 
 
 class KakaoIE(InfoExtractor):
     _VALID_URL = r'https?://tv.kakao.com/channel/(?P<channel>\d+)/cliplink/(?P<id>\d+)'
-    IE_NAME = 'kakao.com'
+    _API_BASE = 'http://tv.kakao.com/api/v1/ft/cliplinks'
 
     _TESTS = [{
         'url': 'http://tv.kakao.com/channel/2671005/cliplink/301965083',
@@ -44,60 +45,57 @@ class KakaoIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        player_url = 'http://tv.kakao.com/embed/player/cliplink/' + video_id + \
-            '?service=kakao_tv&autoplay=1&profile=HIGH&wmode=transparent'
-        player_header = {'Referer': player_url}
-
-        impress = self._download_json(
-            'http://tv.kakao.com/api/v1/ft/cliplinks/%s/impress' % video_id,
-            video_id, 'Downloading video info',
-            query={
-                'player': 'monet_html5',
-                'referer': url,
-                'uuid': '',
-                'service': 'kakao_tv',
-                'section': '',
-                'dteType': 'PC',
-                'fields': 'clipLink,clip,channel,hasPlusFriend,-service,-tagList'
-            }, headers=player_header)
-
-        clipLink = impress['clipLink']
-        clip = clipLink['clip']
-
-        video_info = {
-            'id': video_id,
-            'title': clip['title'],
-            'description': clip.get('description'),
-            'uploader': clipLink.get('channel', {}).get('name'),
-            'uploader_id': clipLink.get('channelId'),
-            'duration': int_or_none(clip.get('duration')),
-            'view_count': int_or_none(clip.get('playCount')),
-            'like_count': int_or_none(clip.get('likeCount')),
-            'comment_count': int_or_none(clip.get('commentCount')),
+        player_header = {
+            'Referer': update_url_query(
+                'http://tv.kakao.com/embed/player/cliplink/%s' % video_id, {
+                    'service': 'kakao_tv',
+                    'autoplay': '1',
+                    'profile': 'HIGH',
+                    'wmode': 'transparent',
+                })
         }
 
+        QUERY_COMMON = {
+            'player': 'monet_html5',
+            'referer': url,
+            'uuid': '',
+            'service': 'kakao_tv',
+            'section': '',
+            'dteType': 'PC',
+        }
+
+        query = QUERY_COMMON.copy()
+        query['fields'] = 'clipLink,clip,channel,hasPlusFriend,-service,-tagList'
+        impress = self._download_json(
+            '%s/%s/impress' % (self._API_BASE, video_id),
+            video_id, 'Downloading video info',
+            query=query, headers=player_header)
+
+        clip_link = impress['clipLink']
+        clip = clip_link['clip']
+
+        title = clip.get('title') or clip_link.get('displayTitle')
+
         tid = impress.get('tid', '')
+
+        query = QUERY_COMMON.copy()
+        query.update({
+            'tid': tid,
+            'profile': 'HIGH',
+        })
         raw = self._download_json(
-            'http://tv.kakao.com/api/v1/ft/cliplinks/%s/raw' % video_id,
+            '%s/%s/raw' % (self._API_BASE, video_id),
             video_id, 'Downloading video formats info',
-            query={
-                'player': 'monet_html5',
-                'referer': url,
-                'uuid': '',
-                'service': 'kakao_tv',
-                'section': '',
-                'tid': tid,
-                'profile': 'HIGH',
-                'dteType': 'PC',
-            }, headers=player_header, fatal=False)
+            query=query, headers=player_header)
 
         formats = []
         for fmt in raw.get('outputList', []):
             try:
                 profile_name = fmt['profile']
                 fmt_url_json = self._download_json(
-                    'http://tv.kakao.com/api/v1/ft/cliplinks/%s/raw/videolocation' % video_id,
-                    video_id, 'Downloading video URL for profile %s' % profile_name,
+                    '%s/%s/raw/videolocation' % (self._API_BASE, video_id),
+                    video_id,
+                    'Downloading video URL for profile %s' % profile_name,
                     query={
                         'service': 'kakao_tv',
                         'section': '',
@@ -119,11 +117,8 @@ class KakaoIE(InfoExtractor):
                 })
             except KeyError:
                 pass
-
         self._sort_formats(formats)
-        video_info['formats'] = formats
 
-        top_thumbnail = clip.get('thumbnailUrl')
         thumbs = []
         for thumb in clip.get('clipChapterThumbnailList', []):
             thumbs.append({
@@ -131,10 +126,24 @@ class KakaoIE(InfoExtractor):
                 'id': compat_str(thumb.get('timeInSec')),
                 'preference': -1 if thumb.get('isDefault') else 0
             })
-        video_info['thumbnail'] = top_thumbnail
-        video_info['thumbnails'] = thumbs
+        top_thumbnail = clip.get('thumbnailUrl')
+        if top_thumbnail:
+            thumbs.append({
+                'url': top_thumbnail,
+                'preference': 10,
+            })
 
-        upload_date = unified_timestamp(clipLink.get('createTime'))
-        video_info['timestamp'] = upload_date
-
-        return video_info
+        return {
+            'id': video_id,
+            'title': title,
+            'description': clip.get('description'),
+            'uploader': clip_link.get('channel', {}).get('name'),
+            'uploader_id': clip_link.get('channelId'),
+            'thumbnails': thumbs,
+            'timestamp': unified_timestamp(clip_link.get('createTime')),
+            'duration': int_or_none(clip.get('duration')),
+            'view_count': int_or_none(clip.get('playCount')),
+            'like_count': int_or_none(clip.get('likeCount')),
+            'comment_count': int_or_none(clip.get('commentCount')),
+            'formats': formats,
+        }

From 7f4921b38d10c17fe354bab20b741b362c5ae0aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Sep 2017 07:26:40 +0700
Subject: [PATCH 1343/1696] [heise] PEP 8

---
 youtube_dl/extractor/heise.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/heise.py b/youtube_dl/extractor/heise.py
index 495ffb7dc..82e11a7d8 100644
--- a/youtube_dl/extractor/heise.py
+++ b/youtube_dl/extractor/heise.py
@@ -60,8 +60,8 @@ class HeiseIE(InfoExtractor):
         title = self._html_search_meta('fulltitle', webpage, default=None)
         if not title or title == "c't":
             title = self._search_regex(
-                    r'<div[^>]+class="videoplayerjw"[^>]+data-title="([^"]+)"',
-                    webpage, 'title')
+                r'<div[^>]+class="videoplayerjw"[^>]+data-title="([^"]+)"',
+                webpage, 'title')
 
         yt_urls = YoutubeIE._extract_urls(webpage)
         if yt_urls:

From 136507b39a2b48cb775249e9724eeeedb56baed2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Sep 2017 07:41:22 +0700
Subject: [PATCH 1344/1696] [24video] Add support for 24video.adult (closes
 #14295)

---
 youtube_dl/extractor/twentyfourvideo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twentyfourvideo.py b/youtube_dl/extractor/twentyfourvideo.py
index 7af11659f..cc51ca0ae 100644
--- a/youtube_dl/extractor/twentyfourvideo.py
+++ b/youtube_dl/extractor/twentyfourvideo.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 class TwentyFourVideoIE(InfoExtractor):
     IE_NAME = '24video'
-    _VALID_URL = r'https?://(?P<host>(?:www\.)?24video\.(?:net|me|xxx|sex|tube))/(?:video/(?:view|xml)/|player/new24_play\.swf\?id=)(?P<id>\d+)'
+    _VALID_URL = r'https?://(?P<host>(?:www\.)?24video\.(?:net|me|xxx|sex|tube|adult))/(?:video/(?:view|xml)/|player/new24_play\.swf\?id=)(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.24video.net/video/view/1044982',

From e3440d824a7326d0ba609d8f0896203208ecc558 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Sep 2017 07:42:17 +0700
Subject: [PATCH 1345/1696] [24video] Fix timestamp extraction and make non
 fatal (#14295)

---
 youtube_dl/extractor/twentyfourvideo.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/twentyfourvideo.py b/youtube_dl/extractor/twentyfourvideo.py
index cc51ca0ae..96e0b96e3 100644
--- a/youtube_dl/extractor/twentyfourvideo.py
+++ b/youtube_dl/extractor/twentyfourvideo.py
@@ -60,8 +60,8 @@ class TwentyFourVideoIE(InfoExtractor):
         duration = int_or_none(self._og_search_property(
             'duration', webpage, 'duration', fatal=False))
         timestamp = parse_iso8601(self._search_regex(
-            r'<time id="video-timeago" datetime="([^"]+)" itemprop="uploadDate">',
-            webpage, 'upload date'))
+            r'<time[^>]+\bdatetime="([^"]+)"[^>]+itemprop="uploadDate"',
+            webpage, 'upload date', fatal=False))
 
         uploader = self._html_search_regex(
             r'class="video-uploaded"[^>]*>\s*<a href="/jsecUser/movies/[^"]+"[^>]*>([^<]+)</a>',
@@ -72,7 +72,7 @@ class TwentyFourVideoIE(InfoExtractor):
             webpage, 'view count', fatal=False))
         comment_count = int_or_none(self._html_search_regex(
             r'<a[^>]+href="#tab-comments"[^>]*>(\d+) комментари',
-            webpage, 'comment count', fatal=False))
+            webpage, 'comment count', default=None))
 
         # Sets some cookies
         self._download_xml(

From 47b5dfb047ce78dfb4ca7091efb4f7d0a1af3545 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Sep 2017 23:14:41 +0700
Subject: [PATCH 1346/1696] Credit @luboss for joj (#13268)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 478c7872f..c2dd5067c 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -224,3 +224,4 @@ Giuseppe Fabiano
 Örn Guðjónsson
 Parmjit Virk
 Genki Sky
+Ľuboš Katrinec

From 07d1344c85781c911d0c1b1f6aa1804cd69f37d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Sep 2017 23:16:27 +0700
Subject: [PATCH 1347/1696] Credit @coreynicholson for vlive:playlist (#13613)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index c2dd5067c..cff6600c3 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -225,3 +225,4 @@ Giuseppe Fabiano
 Parmjit Virk
 Genki Sky
 Ľuboš Katrinec
+Corey Nicholson

From 16f54d0751553c09a512a69f56920235a186d4bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Sep 2017 23:20:20 +0700
Subject: [PATCH 1348/1696] Credit @codeasashu for voot (#11814)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index cff6600c3..295b784f9 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -226,3 +226,4 @@ Parmjit Virk
 Genki Sky
 Ľuboš Katrinec
 Corey Nicholson
+Ashutosh Chaudhary

From aab20aabfc8098a667b0f1bd820fe85ab2653b8e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Sep 2017 23:23:27 +0700
Subject: [PATCH 1349/1696] Credit @jdong92 for voot (#14059)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 295b784f9..147c3e9b4 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -227,3 +227,4 @@ Genki Sky
 Ľuboš Katrinec
 Corey Nicholson
 Ashutosh Chaudhary
+John Dong

From 2f3933aa1e33d7ec24a8701f102a309029a31768 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Sep 2017 23:26:35 +0700
Subject: [PATCH 1350/1696] Credit @ishitatsuyuki for mixcloud fix (#14132)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 147c3e9b4..d1f3bbad3 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -228,3 +228,4 @@ Genki Sky
 Corey Nicholson
 Ashutosh Chaudhary
 John Dong
+Tatsuyuki Ishi

From a87d7b4953d46e75e62fe25a5cc5e689297127ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Sep 2017 23:27:28 +0700
Subject: [PATCH 1351/1696] Credit @nbppp2 for americastestkitchen (#13996)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index d1f3bbad3..1d40c86be 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -229,3 +229,4 @@ Corey Nicholson
 Ashutosh Chaudhary
 John Dong
 Tatsuyuki Ishi
+Daniel Weber

From 4c54b89e0310abd6be643604dc0f1ec3a608b68d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Sep 2017 00:08:27 +0700
Subject: [PATCH 1352/1696] Hide experimental phantomjs wrapper

---
 youtube_dl/YoutubeDL.py          |   2 +-
 youtube_dl/extractor/openload.py | 228 ++++++++++++++++++++++++++++++-
 youtube_dl/utils.py              | 214 -----------------------------
 3 files changed, 228 insertions(+), 216 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index bfb4ff225..0a7f36c98 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -89,10 +89,10 @@ from .utils import (
     write_string,
     YoutubeDLCookieProcessor,
     YoutubeDLHandler,
-    PhantomJSwrapper,
 )
 from .cache import Cache
 from .extractor import get_info_extractor, gen_extractor_classes, _LAZY_LOADER
+from .extractor.openload import PhantomJSwrapper
 from .downloader import get_suitable_downloader
 from .downloader.rtmp import rtmpdump_version
 from .postprocessor import (
diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 292476ef8..da1ef02e5 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -1,17 +1,243 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import json
+import os
 import re
+import subprocess
+import tempfile
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
+    check_executable,
     determine_ext,
+    encodeArgument,
     ExtractorError,
     get_element_by_id,
-    PhantomJSwrapper,
+    get_exe_version,
+    is_outdated_version,
+    std_headers,
 )
 
 
+def cookie_to_dict(cookie):
+    cookie_dict = {
+        'name': cookie.name,
+        'value': cookie.value,
+    }
+    if cookie.port_specified:
+        cookie_dict['port'] = cookie.port
+    if cookie.domain_specified:
+        cookie_dict['domain'] = cookie.domain
+    if cookie.path_specified:
+        cookie_dict['path'] = cookie.path
+    if cookie.expires is not None:
+        cookie_dict['expires'] = cookie.expires
+    if cookie.secure is not None:
+        cookie_dict['secure'] = cookie.secure
+    if cookie.discard is not None:
+        cookie_dict['discard'] = cookie.discard
+    try:
+        if (cookie.has_nonstandard_attr('httpOnly') or
+                cookie.has_nonstandard_attr('httponly') or
+                cookie.has_nonstandard_attr('HttpOnly')):
+            cookie_dict['httponly'] = True
+    except TypeError:
+        pass
+    return cookie_dict
+
+
+def cookie_jar_to_list(cookie_jar):
+    return [cookie_to_dict(cookie) for cookie in cookie_jar]
+
+
+class PhantomJSwrapper(object):
+    """PhantomJS wrapper class
+
+    This class is experimental.
+    """
+
+    _TEMPLATE = r'''
+        phantom.onError = function(msg, trace) {{
+          var msgStack = ['PHANTOM ERROR: ' + msg];
+          if(trace && trace.length) {{
+            msgStack.push('TRACE:');
+            trace.forEach(function(t) {{
+              msgStack.push(' -> ' + (t.file || t.sourceURL) + ': ' + t.line
+                + (t.function ? ' (in function ' + t.function +')' : ''));
+            }});
+          }}
+          console.error(msgStack.join('\n'));
+          phantom.exit(1);
+        }};
+        var page = require('webpage').create();
+        var fs = require('fs');
+        var read = {{ mode: 'r', charset: 'utf-8' }};
+        var write = {{ mode: 'w', charset: 'utf-8' }};
+        JSON.parse(fs.read("{cookies}", read)).forEach(function(x) {{
+          phantom.addCookie(x);
+        }});
+        page.settings.resourceTimeout = {timeout};
+        page.settings.userAgent = "{ua}";
+        page.onLoadStarted = function() {{
+          page.evaluate(function() {{
+            delete window._phantom;
+            delete window.callPhantom;
+          }});
+        }};
+        var saveAndExit = function() {{
+          fs.write("{html}", page.content, write);
+          fs.write("{cookies}", JSON.stringify(phantom.cookies), write);
+          phantom.exit();
+        }};
+        page.onLoadFinished = function(status) {{
+          if(page.url === "") {{
+            page.setContent(fs.read("{html}", read), "{url}");
+          }}
+          else {{
+            {jscode}
+          }}
+        }};
+        page.open("");
+    '''
+
+    _TMP_FILE_NAMES = ['script', 'html', 'cookies']
+
+    @staticmethod
+    def _version():
+        return get_exe_version('phantomjs', version_re=r'([0-9.]+)')
+
+    def __init__(self, extractor, required_version=None, timeout=10000):
+        self.exe = check_executable('phantomjs', ['-v'])
+        if not self.exe:
+            raise ExtractorError('PhantomJS executable not found in PATH, '
+                                 'download it from http://phantomjs.org',
+                                 expected=True)
+
+        self.extractor = extractor
+
+        if required_version:
+            version = self._version()
+            if is_outdated_version(version, required_version):
+                self.extractor._downloader.report_warning(
+                    'Your copy of PhantomJS is outdated, update it to version '
+                    '%s or newer if you encounter any errors.' % required_version)
+
+        self.options = {
+            'timeout': timeout,
+        }
+        self._TMP_FILES = {}
+        for name in self._TMP_FILE_NAMES:
+            tmp = tempfile.NamedTemporaryFile(delete=False)
+            tmp.close()
+            self._TMP_FILES[name] = tmp
+
+    def __del__(self):
+        for name in self._TMP_FILE_NAMES:
+            try:
+                os.remove(self._TMP_FILES[name].name)
+            except:
+                pass
+
+    def _save_cookies(self, url):
+        cookies = cookie_jar_to_list(self.extractor._downloader.cookiejar)
+        for cookie in cookies:
+            if 'path' not in cookie:
+                cookie['path'] = '/'
+            if 'domain' not in cookie:
+                cookie['domain'] = compat_urlparse.urlparse(url).netloc
+        with open(self._TMP_FILES['cookies'].name, 'wb') as f:
+            f.write(json.dumps(cookies).encode('utf-8'))
+
+    def _load_cookies(self):
+        with open(self._TMP_FILES['cookies'].name, 'rb') as f:
+            cookies = json.loads(f.read().decode('utf-8'))
+        for cookie in cookies:
+            if cookie['httponly'] is True:
+                cookie['rest'] = {'httpOnly': None}
+            if 'expiry' in cookie:
+                cookie['expire_time'] = cookie['expiry']
+            self.extractor._set_cookie(**cookie)
+
+    def get(self, url, html=None, video_id=None, note=None, note2='Executing JS on webpage', headers={}, jscode='saveAndExit();'):
+        """
+        Downloads webpage (if needed) and executes JS
+
+        Params:
+            url: website url
+            html: optional, html code of website
+            video_id: video id
+            note: optional, displayed when downloading webpage
+            note2: optional, displayed when executing JS
+            headers: custom http headers
+            jscode: code to be executed when page is loaded
+
+        Returns tuple with:
+            * downloaded website (after JS execution)
+            * anything you print with `console.log` (but not inside `page.execute`!)
+
+        In most cases you don't need to add any `jscode`.
+        It is executed in `page.onLoadFinished`.
+        `saveAndExit();` is mandatory, use it instead of `phantom.exit()`
+        It is possible to wait for some element on the webpage, for example:
+            var check = function() {
+              var elementFound = page.evaluate(function() {
+                return document.querySelector('#b.done') !== null;
+              });
+              if(elementFound)
+                saveAndExit();
+              else
+                window.setTimeout(check, 500);
+            }
+
+            page.evaluate(function(){
+              document.querySelector('#a').click();
+            });
+            check();
+        """
+        if 'saveAndExit();' not in jscode:
+            raise ExtractorError('`saveAndExit();` not found in `jscode`')
+        if not html:
+            html = self.extractor._download_webpage(url, video_id, note=note, headers=headers)
+        with open(self._TMP_FILES['html'].name, 'wb') as f:
+            f.write(html.encode('utf-8'))
+
+        self._save_cookies(url)
+
+        replaces = self.options
+        replaces['url'] = url
+        user_agent = headers.get('User-Agent') or std_headers['User-Agent']
+        replaces['ua'] = user_agent.replace('"', '\\"')
+        replaces['jscode'] = jscode
+
+        for x in self._TMP_FILE_NAMES:
+            replaces[x] = self._TMP_FILES[x].name.replace('\\', '\\\\').replace('"', '\\"')
+
+        with open(self._TMP_FILES['script'].name, 'wb') as f:
+            f.write(self._TEMPLATE.format(**replaces).encode('utf-8'))
+
+        if video_id is None:
+            self.extractor.to_screen('%s' % (note2,))
+        else:
+            self.extractor.to_screen('%s: %s' % (video_id, note2))
+
+        p = subprocess.Popen([
+            self.exe, '--ssl-protocol=any',
+            self._TMP_FILES['script'].name
+        ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        out, err = p.communicate()
+        if p.returncode != 0:
+            raise ExtractorError(
+                'Executing JS failed\n:' + encodeArgument(err))
+        with open(self._TMP_FILES['html'].name, 'rb') as f:
+            html = f.read().decode('utf-8')
+
+        self._load_cookies()
+
+        return (html, encodeArgument(out))
+
+
 class OpenloadIE(InfoExtractor):
     _VALID_URL = r'https?://(?:openload\.(?:co|io)|oload\.tv)/(?:f|embed)/(?P<id>[a-zA-Z0-9-_]+)'
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index acc4f987b..92b22e639 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3826,220 +3826,6 @@ def write_xattr(path, key, value):
                         "or the 'xattr' binary.")
 
 
-def cookie_to_dict(cookie):
-    cookie_dict = {
-        'name': cookie.name,
-        'value': cookie.value,
-    }
-    if cookie.port_specified:
-        cookie_dict['port'] = cookie.port
-    if cookie.domain_specified:
-        cookie_dict['domain'] = cookie.domain
-    if cookie.path_specified:
-        cookie_dict['path'] = cookie.path
-    if cookie.expires is not None:
-        cookie_dict['expires'] = cookie.expires
-    if cookie.secure is not None:
-        cookie_dict['secure'] = cookie.secure
-    if cookie.discard is not None:
-        cookie_dict['discard'] = cookie.discard
-    try:
-        if (cookie.has_nonstandard_attr('httpOnly') or
-                cookie.has_nonstandard_attr('httponly') or
-                cookie.has_nonstandard_attr('HttpOnly')):
-            cookie_dict['httponly'] = True
-    except TypeError:
-        pass
-    return cookie_dict
-
-
-def cookie_jar_to_list(cookie_jar):
-    return [cookie_to_dict(cookie) for cookie in cookie_jar]
-
-
-class PhantomJSwrapper(object):
-    """PhantomJS wrapper class"""
-
-    _TEMPLATE = r'''
-        phantom.onError = function(msg, trace) {{
-          var msgStack = ['PHANTOM ERROR: ' + msg];
-          if(trace && trace.length) {{
-            msgStack.push('TRACE:');
-            trace.forEach(function(t) {{
-              msgStack.push(' -> ' + (t.file || t.sourceURL) + ': ' + t.line
-                + (t.function ? ' (in function ' + t.function +')' : ''));
-            }});
-          }}
-          console.error(msgStack.join('\n'));
-          phantom.exit(1);
-        }};
-        var page = require('webpage').create();
-        var fs = require('fs');
-        var read = {{ mode: 'r', charset: 'utf-8' }};
-        var write = {{ mode: 'w', charset: 'utf-8' }};
-        JSON.parse(fs.read("{cookies}", read)).forEach(function(x) {{
-          phantom.addCookie(x);
-        }});
-        page.settings.resourceTimeout = {timeout};
-        page.settings.userAgent = "{ua}";
-        page.onLoadStarted = function() {{
-          page.evaluate(function() {{
-            delete window._phantom;
-            delete window.callPhantom;
-          }});
-        }};
-        var saveAndExit = function() {{
-          fs.write("{html}", page.content, write);
-          fs.write("{cookies}", JSON.stringify(phantom.cookies), write);
-          phantom.exit();
-        }};
-        page.onLoadFinished = function(status) {{
-          if(page.url === "") {{
-            page.setContent(fs.read("{html}", read), "{url}");
-          }}
-          else {{
-            {jscode}
-          }}
-        }};
-        page.open("");
-    '''
-
-    _TMP_FILE_NAMES = ['script', 'html', 'cookies']
-
-    @staticmethod
-    def _version():
-        return get_exe_version('phantomjs', version_re=r'([0-9.]+)')
-
-    def __init__(self, extractor, required_version=None, timeout=10000):
-        self.exe = check_executable('phantomjs', ['-v'])
-        if not self.exe:
-            raise ExtractorError('PhantomJS executable not found in PATH, '
-                                 'download it from http://phantomjs.org',
-                                 expected=True)
-
-        self.extractor = extractor
-
-        if required_version:
-            version = self._version()
-            if is_outdated_version(version, required_version):
-                self.extractor._downloader.report_warning(
-                    'Your copy of PhantomJS is outdated, update it to version '
-                    '%s or newer if you encounter any errors.' % required_version)
-
-        self.options = {
-            'timeout': timeout,
-        }
-        self._TMP_FILES = {}
-        for name in self._TMP_FILE_NAMES:
-            tmp = tempfile.NamedTemporaryFile(delete=False)
-            tmp.close()
-            self._TMP_FILES[name] = tmp
-
-    def __del__(self):
-        for name in self._TMP_FILE_NAMES:
-            try:
-                os.remove(self._TMP_FILES[name].name)
-            except:
-                pass
-
-    def _save_cookies(self, url):
-        cookies = cookie_jar_to_list(self.extractor._downloader.cookiejar)
-        for cookie in cookies:
-            if 'path' not in cookie:
-                cookie['path'] = '/'
-            if 'domain' not in cookie:
-                cookie['domain'] = compat_urlparse.urlparse(url).netloc
-        with open(self._TMP_FILES['cookies'].name, 'wb') as f:
-            f.write(json.dumps(cookies).encode('utf-8'))
-
-    def _load_cookies(self):
-        with open(self._TMP_FILES['cookies'].name, 'rb') as f:
-            cookies = json.loads(f.read().decode('utf-8'))
-        for cookie in cookies:
-            if cookie['httponly'] is True:
-                cookie['rest'] = {'httpOnly': None}
-            if 'expiry' in cookie:
-                cookie['expire_time'] = cookie['expiry']
-            self.extractor._set_cookie(**cookie)
-
-    def get(self, url, html=None, video_id=None, note=None, note2='Executing JS on webpage', headers={}, jscode='saveAndExit();'):
-        """
-        Downloads webpage (if needed) and executes JS
-
-        Params:
-            url: website url
-            html: optional, html code of website
-            video_id: video id
-            note: optional, displayed when downloading webpage
-            note2: optional, displayed when executing JS
-            headers: custom http headers
-            jscode: code to be executed when page is loaded
-
-        Returns tuple with:
-            * downloaded website (after JS execution)
-            * anything you print with `console.log` (but not inside `page.execute`!)
-
-        In most cases you don't need to add any `jscode`.
-        It is executed in `page.onLoadFinished`.
-        `saveAndExit();` is mandatory, use it instead of `phantom.exit()`
-        It is possible to wait for some element on the webpage, for example:
-            var check = function() {
-              var elementFound = page.evaluate(function() {
-                return document.querySelector('#b.done') !== null;
-              });
-              if(elementFound)
-                saveAndExit();
-              else
-                window.setTimeout(check, 500);
-            }
-
-            page.evaluate(function(){
-              document.querySelector('#a').click();
-            });
-            check();
-        """
-        if 'saveAndExit();' not in jscode:
-            raise ExtractorError('`saveAndExit();` not found in `jscode`')
-        if not html:
-            html = self.extractor._download_webpage(url, video_id, note=note, headers=headers)
-        with open(self._TMP_FILES['html'].name, 'wb') as f:
-            f.write(html.encode('utf-8'))
-
-        self._save_cookies(url)
-
-        replaces = self.options
-        replaces['url'] = url
-        user_agent = headers.get('User-Agent') or std_headers['User-Agent']
-        replaces['ua'] = user_agent.replace('"', '\\"')
-        replaces['jscode'] = jscode
-
-        for x in self._TMP_FILE_NAMES:
-            replaces[x] = self._TMP_FILES[x].name.replace('\\', '\\\\').replace('"', '\\"')
-
-        with open(self._TMP_FILES['script'].name, 'wb') as f:
-            f.write(self._TEMPLATE.format(**replaces).encode('utf-8'))
-
-        if video_id is None:
-            self.extractor.to_screen('%s' % (note2,))
-        else:
-            self.extractor.to_screen('%s: %s' % (video_id, note2))
-
-        p = subprocess.Popen([
-            self.exe, '--ssl-protocol=any',
-            self._TMP_FILES['script'].name
-        ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        out, err = p.communicate()
-        if p.returncode != 0:
-            raise ExtractorError(
-                'Executing JS failed\n:' + encodeArgument(err))
-        with open(self._TMP_FILES['html'].name, 'rb') as f:
-            html = f.read().decode('utf-8')
-
-        self._load_cookies()
-
-        return (html, encodeArgument(out))
-
-
 def random_birthday(year_field, month_field, day_field):
     return {
         year_field: str(random.randint(1950, 1995)),

From 011da618bdb907cbe02aacefba421dad4cf49a4d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Sep 2017 00:12:40 +0700
Subject: [PATCH 1353/1696] [openload] Fix _load_cookies for python 2.6

---
 youtube_dl/extractor/openload.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index da1ef02e5..b50d6c77b 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -8,7 +8,10 @@ import subprocess
 import tempfile
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
+from ..compat import (
+    compat_urlparse,
+    compat_kwargs,
+)
 from ..utils import (
     check_executable,
     determine_ext,
@@ -158,7 +161,7 @@ class PhantomJSwrapper(object):
                 cookie['rest'] = {'httpOnly': None}
             if 'expiry' in cookie:
                 cookie['expire_time'] = cookie['expiry']
-            self.extractor._set_cookie(**cookie)
+            self.extractor._set_cookie(**compat_kwargs(cookie))
 
     def get(self, url, html=None, video_id=None, note=None, note2='Executing JS on webpage', headers={}, jscode='saveAndExit();'):
         """

From 4d182955a2bbe95eb4b931f82204f402ffd5552d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Sep 2017 00:19:27 +0700
Subject: [PATCH 1354/1696] [kakao] Fix _VALID_URL

---
 youtube_dl/extractor/kakao.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kakao.py b/youtube_dl/extractor/kakao.py
index c9b438efc..7fa140b0c 100644
--- a/youtube_dl/extractor/kakao.py
+++ b/youtube_dl/extractor/kakao.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class KakaoIE(InfoExtractor):
-    _VALID_URL = r'https?://tv.kakao.com/channel/(?P<channel>\d+)/cliplink/(?P<id>\d+)'
+    _VALID_URL = r'https?://tv\.kakao\.com/channel/(?P<channel>\d+)/cliplink/(?P<id>\d+)'
     _API_BASE = 'http://tv.kakao.com/api/v1/ft/cliplinks'
 
     _TESTS = [{

From 10cab6613fc52f64f0a5a81281fec2f8d2d92f16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Sep 2017 00:21:34 +0700
Subject: [PATCH 1355/1696] [ChangeLog] Actualize

---
 ChangeLog | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 42ba879b1..87ceabe6c 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -4,6 +4,30 @@ Core
 + [options] Accept lrc as a subtitle conversion target format (#14292)
 * [utils] Fix handling raw TTML subtitles (#14191)
 
+Extractors
+* [24video] Fix timestamp extraction and make non fatal (#14295)
++ [24video] Add support for 24video.adult (#14295)
++ [kakao] Add support for tv.kakao.com (#12298, #14007)
++ [twitter] Add support for URLs without user id (#14270)
++ [americastestkitchen] Add support for americastestkitchen.com (#10764,
+  #13996)
+* [generic] Fix support for multiple HTML5 videos on one page (#14080)
+* [mixcloud] Fix extraction (#14088, #14132)
++ [lynda] Add support for educourse.ga (#14286)
+* [beeg] Fix extraction (#14275)
+* [nbcsports:vplayer] Correct theplatform URL (#13873)
+* [twitter] Fix duration extraction (#14141)
+* [tvplay] Bypass geo restriction
++ [heise] Add support for YouTube embeds (#14109)
++ [popcorntv] Add support for popcorntv.it (#5914, #14211)
+* [viki] Update app data (#14181)
+* [morningstar] Relax URL regular expression (#14222)
+* [openload] Fix extraction (#14225, #14257)
+* [noovo] Fix extraction (#14214)
+* [dailymotion:playlist] Relax URL regular expression (#14219)
++ [twitch] Add support for go.twitch.tv URLs (#14215)
+* [vgtv] Relax URL regular expression (#14223)
+
 
 version 2017.09.15
 

From 9fc41bcb6b94ddb64885b1eb20282c69dc787929 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Sep 2017 00:22:50 +0700
Subject: [PATCH 1356/1696] release 2017.09.24

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 README.md                 | 2 +-
 docs/supportedsites.md    | 3 +++
 youtube_dl/version.py     | 2 +-
 5 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 98ab5b6ca..1954528e3 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.09.15*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.09.15**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.09.24*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.09.24**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.09.15
+[debug] youtube-dl version 2017.09.24
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 87ceabe6c..da60c1b36 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.09.24
 
 Core
 + [options] Accept lrc as a subtitle conversion target format (#14292)
diff --git a/README.md b/README.md
index 28ee63f40..7818e58df 100644
--- a/README.md
+++ b/README.md
@@ -427,7 +427,7 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
                                      syntax. Example: --exec 'adb push {}
                                      /sdcard/Music/ && rm {}'
     --convert-subs FORMAT            Convert the subtitles to other format
-                                     (currently supported: srt|ass|vtt)
+                                     (currently supported: srt|ass|vtt|lrc)
 
 # CONFIGURATION
 
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 6b01dc910..d36a07cf6 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -43,6 +43,7 @@
  - **Allocine**
  - **AlphaPorno**
  - **AMCNetworks**
+ - **AmericasTestKitchen**
  - **anderetijden**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
  - **AnimeOnDemand**
  - **anitube.se**
@@ -378,6 +379,7 @@
  - **Jove**
  - **jpopsuki.tv**
  - **JWPlatform**
+ - **Kakao**
  - **Kaltura**
  - **Kamcord**
  - **KanalPlay**: Kanal 5/9/11 Play
@@ -627,6 +629,7 @@
  - **Pokemon**
  - **PolskieRadio**
  - **PolskieRadioCategory**
+ - **PopcornTV**
  - **PornCom**
  - **PornerBros**
  - **PornFlip**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 8399c04fe..0e2e4dc90 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.09.15'
+__version__ = '2017.09.24'

From 8b389f7e3ce4466824abc7ce2aaf657abf34a682 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Sep 2017 18:21:38 +0700
Subject: [PATCH 1357/1696] Credit the author of multiple generic HTML5 embeds
 fix

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 1d40c86be..7e012247c 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -230,3 +230,4 @@ Ashutosh Chaudhary
 John Dong
 Tatsuyuki Ishi
 Daniel Weber
+Kay Bouché

From db96252831e24ae040de92f4c6668d3eaef82165 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Sep 2017 19:23:08 +0700
Subject: [PATCH 1358/1696] [xhamsterembed] Fix extraction (closes #14308)

---
 youtube_dl/extractor/xhamster.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index c42b59e51..be3624ef2 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -221,7 +221,7 @@ class XHamsterEmbedIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         video_url = self._search_regex(
-            r'href="(https?://xhamster\.com/movies/%s/[^"]*\.html[^"]*)"' % video_id,
+            r'href="(https?://xhamster\.com/(?:movies/{0}/[^"]*\.html|videos/[^/]*-{0})[^"]*)"'.format(video_id),
             webpage, 'xhamster url', default=None)
 
         if not video_url:

From 02d01e15f1a52a23cc4991c39219f98aed542136 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 26 Sep 2017 21:47:18 +0700
Subject: [PATCH 1359/1696] [generic] Fix infinite recursion for twitter:player
 URLs (closes #14339)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 7d0edf09c..096e2aab4 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2973,7 +2973,7 @@ class GenericIE(InfoExtractor):
             # be supported by youtube-dl thus this is checked the very last (see
             # https://dev.twitter.com/cards/types/player#On_twitter.com_via_desktop_browser)
             embed_url = self._html_search_meta('twitter:player', webpage, default=None)
-            if embed_url:
+            if embed_url and embed_url != url:
                 return self.url_result(embed_url)
 
         if not found:

From b14b2283a0b9f66f54e862641b5b1cd22c07bd62 Mon Sep 17 00:00:00 2001
From: Timendum <timedum@gmail.com>
Date: Wed, 27 Sep 2017 17:48:47 +0200
Subject: [PATCH 1360/1696] [gfycat] Add support for /gifs/detail URLs (closes
 #14322)

---
 youtube_dl/extractor/gfycat.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gfycat.py b/youtube_dl/extractor/gfycat.py
index 45ccc11c1..a0670b645 100644
--- a/youtube_dl/extractor/gfycat.py
+++ b/youtube_dl/extractor/gfycat.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class GfycatIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?gfycat\.com/(?:ifr/)?(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?gfycat\.com/(?:ifr/|gifs/detail/)?(?P<id>[^/?#]+)'
     _TESTS = [{
         'url': 'http://gfycat.com/DeadlyDecisiveGermanpinscher',
         'info_dict': {
@@ -44,6 +44,9 @@ class GfycatIE(InfoExtractor):
             'categories': list,
             'age_limit': 0,
         }
+    }, {
+        'url': 'https://gfycat.com/gifs/detail/UnconsciousLankyIvorygull',
+        'only_matching': True
     }]
 
     def _real_extract(self, url):

From 63d990d2859d0e981da2e416097655798334431b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Sep 2017 00:29:42 +0700
Subject: [PATCH 1361/1696] [generic] Add support for Video.js embeds

---
 youtube_dl/extractor/generic.py | 56 +++++++++++++++++++++++++++++++++
 1 file changed, 56 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 096e2aab4..a3d09a036 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -22,6 +22,8 @@ from ..utils import (
     HEADRequest,
     is_html,
     js_to_json,
+    KNOWN_EXTENSIONS,
+    mimetype2ext,
     orderedSet,
     sanitized_Request,
     smuggle_url,
@@ -1130,6 +1132,22 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             }
         },
+        {
+            # Video.js embed
+            'url': 'http://ortcam.com/solidworks-урок-6-настройка-чертежа_33f9b7351.html',
+            'info_dict': {
+                'id': 'yygqldloqIk',
+                'ext': 'mp4',
+                'title': 'SolidWorks. Урок 6 Настройка чертежа',
+                'description': 'md5:baf95267792646afdbf030e4d06b2ab3',
+                'upload_date': '20130314',
+                'uploader': 'PROстое3D',
+                'uploader_id': 'PROstoe3D',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
         # rtl.nl embed
         {
             'url': 'http://www.rtlnieuws.nl/nieuws/buitenland/aanslagen-kopenhagen',
@@ -2880,6 +2898,44 @@ class GenericIE(InfoExtractor):
                 jwplayer_data, video_id, require_title=False, base_url=url)
             return merge_dicts(info, info_dict)
 
+        # Video.js embed
+        mobj = re.search(
+            r'(?s)\bvideojs\s*\(.+?\bplayer\.src\s*\(\s*(\[.+?\])\s*\)\s*;',
+            webpage)
+        if mobj is not None:
+            sources = self._parse_json(
+                mobj.group(1), video_id, transform_source=js_to_json,
+                fatal=False) or []
+            formats = []
+            for source in sources:
+                src = source.get('src')
+                if not src or not isinstance(src, compat_str):
+                    continue
+                src = compat_urlparse.urljoin(url, src)
+                src_type = source.get('type')
+                if isinstance(src_type, compat_str):
+                    src_type = src_type.lower()
+                ext = determine_ext(src).lower()
+                if src_type == 'video/youtube':
+                    return self.url_result(src, YoutubeIE.ie_key())
+                if src_type == 'application/dash+xml' or ext == 'mpd':
+                    formats.extend(self._extract_mpd_formats(
+                        src, video_id, mpd_id='dash', fatal=False))
+                elif src_type == 'application/x-mpegurl' or ext == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        src, video_id, 'mp4', entry_protocol='m3u8_native',
+                        m3u8_id='hls', fatal=False))
+                else:
+                    formats.append({
+                        'url': src,
+                        'ext': (mimetype2ext(src_type) or
+                                ext if ext in KNOWN_EXTENSIONS else 'mp4'),
+                    })
+            if formats:
+                self._sort_formats(formats)
+                info_dict['formats'] = formats
+                return info_dict
+
         # Looking for http://schema.org/VideoObject
         json_ld = self._search_json_ld(
             webpage, video_id, default={}, expected_type='VideoObject')

From 3600fd591dc1ee4963f6940d19ef39d464d3a689 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Sep 2017 00:46:48 +0700
Subject: [PATCH 1362/1696] [YoutubeDL] Document youtube_include_dash_manifest

---
 youtube_dl/YoutubeDL.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 0a7f36c98..20aa1e49e 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -304,6 +304,12 @@ class YoutubeDL(object):
                        otherwise prefer avconv.
     postprocessor_args: A list of additional command-line arguments for the
                         postprocessor.
+                        
+    The following options are used by the Youtube extractor:
+    youtube_include_dash_manifest: If True (default), DASH manifests and related
+                        data will be downloaded and processed by extractor.
+                        You can reduce network I/O by disabling it if you don't
+                        care about DASH.
     """
 
     _NUMERIC_FIELDS = set((

From eb9a15be6091d09f46d4306516b58cf2fe91e647 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Sep 2017 22:47:03 +0700
Subject: [PATCH 1363/1696] [yahoo] Add support for custom brigthcove embeds
 (closes #14210)

---
 youtube_dl/extractor/yahoo.py | 29 ++++++++++++++++++++++++++---
 1 file changed, 26 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 38f82bf44..e92337969 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -28,7 +28,7 @@ from .nbc import NBCSportsVPlayerIE
 
 class YahooIE(InfoExtractor):
     IE_DESC = 'Yahoo screen and movies'
-    _VALID_URL = r'(?P<url>(?P<host>https?://(?:[a-zA-Z]{2}\.)?[\da-zA-Z_-]+\.yahoo\.com)/(?:[^/]+/)*(?P<display_id>.+)?-(?P<id>[0-9]+)(?:-[a-z]+)?(?:\.html)?)'
+    _VALID_URL = r'(?P<host>https?://(?:[a-zA-Z]{2}\.)?[\da-zA-Z_-]+\.yahoo\.com)/(?:[^/]+/)*(?:(?P<display_id>.+)?-)?(?P<id>[0-9]+)(?:-[a-z]+)?(?:\.html)?'
     _TESTS = [
         {
             'url': 'http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html',
@@ -227,13 +227,28 @@ class YahooIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            # custom brightcove
+            'url': 'https://au.tv.yahoo.com/plus7/sunrise/-/watch/37083565/clown-entertainers-say-it-is-hurting-their-business/',
+            'info_dict': {
+                'id': '5575377707001',
+                'ext': 'mp4',
+                'title': "Clown entertainers say 'It' is hurting their business",
+                'description': 'Stephen King s horror film has much to answer for. Jelby and Mr Loopy the Clowns join us.',
+                'timestamp': 1505341164,
+                'upload_date': '20170913',
+                'uploader_id': '2376984109001',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        }
     ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('display_id') or self._match_id(url)
         page_id = mobj.group('id')
-        url = mobj.group('url')
+        display_id = mobj.group('display_id') or page_id
         host = mobj.group('host')
         webpage, urlh = self._download_webpage_handle(url, display_id)
         if 'err=404' in urlh.geturl():
@@ -262,6 +277,14 @@ class YahooIE(InfoExtractor):
         if bc_url:
             return self.url_result(bc_url, BrightcoveNewIE.ie_key())
 
+        brightcove_id = self._search_regex(
+            r'<iframe[^>]+data-video-id=["\'](\d+)', webpage, 'brightcove id',
+            default=None)
+        if brightcove_id:
+            return self.url_result(
+                'http://players.brightcove.net/2376984109001/default_default/index.html?videoId=%s' % brightcove_id,
+                BrightcoveNewIE.ie_key())
+
         # Query result is often embedded in webpage as JSON. Sometimes explicit requests
         # to video API results in a failure with geo restriction reason therefore using
         # embedded query result when present sounds reasonable.

From fa3fdeb41fdd637283baeb6d681450a982043a1c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Sep 2017 22:54:22 +0700
Subject: [PATCH 1364/1696] [yahoo] Fix some tests

---
 youtube_dl/extractor/yahoo.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index e92337969..a3bb1b0b1 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -50,6 +50,7 @@ class YahooIE(InfoExtractor):
                 'description': 'md5:66b627ab0a282b26352136ca96ce73c1',
                 'duration': 151,
             },
+            'skip': 'HTTP Error 404',
         },
         {
             'url': 'https://screen.yahoo.com/community/community-sizzle-reel-203225340.html?format=embed',
@@ -142,7 +143,7 @@ class YahooIE(InfoExtractor):
             'skip': 'Domain name in.lifestyle.yahoo.com gone',
         }, {
             'url': 'https://www.yahoo.com/movies/v/true-story-trailer-173000497.html',
-            'md5': '2a9752f74cb898af5d1083ea9f661b58',
+            'md5': '989396ae73d20c6f057746fb226aa215',
             'info_dict': {
                 'id': '071c4013-ce30-3a93-a5b2-e0413cd4a9d1',
                 'ext': 'mp4',

From 3836b02ce8df654ea70276f1d196b347ea9bc257 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Sep 2017 22:56:40 +0700
Subject: [PATCH 1365/1696] [YoutubeDL] PEP 8

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 20aa1e49e..474d6c95f 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -304,7 +304,7 @@ class YoutubeDL(object):
                        otherwise prefer avconv.
     postprocessor_args: A list of additional command-line arguments for the
                         postprocessor.
-                        
+
     The following options are used by the Youtube extractor:
     youtube_include_dash_manifest: If True (default), DASH manifests and related
                         data will be downloaded and processed by extractor.

From 2c53bd51c6b0bef836d4e84d9a8f9127c3b67660 Mon Sep 17 00:00:00 2001
From: Giuseppe Fabiano <gfabiano40@gmail.com>
Date: Sat, 30 Sep 2017 22:21:17 +0200
Subject: [PATCH 1366/1696] [rtve:alacarta] Fix extraction (closes #14290)

---
 youtube_dl/extractor/rtve.py | 36 +++++++++++++++++++++++++++++++++---
 1 file changed, 33 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 746677a24..d9edf9da2 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -10,6 +10,7 @@ from ..compat import (
     compat_struct_unpack,
 )
 from ..utils import (
+    determine_ext,
     ExtractorError,
     float_or_none,
     remove_end,
@@ -84,6 +85,18 @@ class RTVEALaCartaIE(InfoExtractor):
             'title': 'TODO',
         },
         'skip': 'The f4m manifest can\'t be used yet',
+    }, {
+        'url': 'http://www.rtve.es/alacarta/videos/servir-y-proteger/servir-proteger-capitulo-104/4236788/',
+        'md5': 'e55e162379ad587e9640eda4f7353c0f',
+        'info_dict': {
+            'id': '4236788',
+            'ext': 'mp4',
+            'title': 'Servir y proteger - Capítulo 104 ',
+            'duration': 3222.0,
+        },
+        'params': {
+            'skip_download': True,  # requires ffmpeg
+        },
     }, {
         'url': 'http://www.rtve.es/m/alacarta/videos/cuentame-como-paso/cuentame-como-paso-t16-ultimo-minuto-nuestra-vida-capitulo-276/2969138/?media=tve',
         'only_matching': True,
@@ -107,24 +120,41 @@ class RTVEALaCartaIE(InfoExtractor):
             video_id)['page']['items'][0]
         if info['state'] == 'DESPU':
             raise ExtractorError('The video is no longer available', expected=True)
+        title = info['title']
         png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/%s/videos/%s.png' % (self._manager, video_id)
         png_request = sanitized_Request(png_url)
         png_request.add_header('Referer', url)
         png = self._download_webpage(png_request, video_id, 'Downloading url information')
         video_url = _decrypt_url(png)
-        if not video_url.endswith('.f4m'):
+        ext = determine_ext(video_url)
+
+        formats = []
+        if not video_url.endswith('.f4m') and ext != 'm3u8':
             if '?' not in video_url:
                 video_url = video_url.replace('resources/', 'auth/resources/')
             video_url = video_url.replace('.net.rtve', '.multimedia.cdn.rtve')
 
+        if ext == 'm3u8':
+            formats.extend(self._extract_m3u8_formats(
+                video_url, video_id, ext='mp4', entry_protocol='m3u8_native',
+                m3u8_id='hls', fatal=False))
+        elif ext == 'f4m':
+            formats.extend(self._extract_f4m_formats(
+                video_url, video_id, f4m_id='hds', fatal=False))
+        else:
+            formats.append({
+                'url': video_url,
+            })
+        self._sort_formats(formats)
+
         subtitles = None
         if info.get('sbtFile') is not None:
             subtitles = self.extract_subtitles(video_id, info['sbtFile'])
 
         return {
             'id': video_id,
-            'title': info['title'],
-            'url': video_url,
+            'title': title,
+            'formats': formats,
             'thumbnail': info.get('image'),
             'page_url': url,
             'subtitles': subtitles,

From b69ca0ccfcd3bc7b64aa227339e92576baf5fc9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Oct 2017 04:37:42 +0700
Subject: [PATCH 1367/1696] [yahoo] Use extracted brightcove account id (closes
 #14210)

---
 youtube_dl/extractor/yahoo.py | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index a3bb1b0b1..87615cd47 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -12,11 +12,12 @@ from ..compat import (
 )
 from ..utils import (
     clean_html,
-    unescapeHTML,
+    determine_ext,
     ExtractorError,
+    extract_attributes,
     int_or_none,
     mimetype2ext,
-    determine_ext,
+    unescapeHTML,
 )
 
 from .brightcove import (
@@ -278,13 +279,21 @@ class YahooIE(InfoExtractor):
         if bc_url:
             return self.url_result(bc_url, BrightcoveNewIE.ie_key())
 
-        brightcove_id = self._search_regex(
-            r'<iframe[^>]+data-video-id=["\'](\d+)', webpage, 'brightcove id',
-            default=None)
-        if brightcove_id:
-            return self.url_result(
-                'http://players.brightcove.net/2376984109001/default_default/index.html?videoId=%s' % brightcove_id,
-                BrightcoveNewIE.ie_key())
+        brightcove_iframe = self._search_regex(
+            r'(<iframe[^>]+data-video-id=["\']\d+[^>]+>)', webpage,
+            'brightcove iframe', default=None)
+        if brightcove_iframe:
+            attr = extract_attributes(brightcove_iframe)
+            src = attr.get('src')
+            if src:
+                parsed_src = compat_urlparse.urlparse(src)
+                qs = compat_urlparse.parse_qs(parsed_src.query)
+                account_id = qs.get('accountId', ['2376984109001'])[0]
+                brightcove_id = attr.get('data-video-id') or qs.get('videoId', [None])[0]
+                if account_id and brightcove_id:
+                    return self.url_result(
+                        'http://players.brightcove.net/%s/default_default/index.html?videoId=%s' % (account_id, brightcove_id),
+                        BrightcoveNewIE.ie_key())
 
         # Query result is often embedded in webpage as JSON. Sometimes explicit requests
         # to video API results in a failure with geo restriction reason therefore using

From c8da40d834697c343f64609b2cfbb3ff759c18a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Oct 2017 04:49:27 +0700
Subject: [PATCH 1368/1696] [yahoo] Bypass geo restriction for brightcove
 (#14210)

---
 youtube_dl/extractor/yahoo.py | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 87615cd47..552013a74 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -17,6 +17,7 @@ from ..utils import (
     extract_attributes,
     int_or_none,
     mimetype2ext,
+    smuggle_url,
     unescapeHTML,
 )
 
@@ -29,7 +30,7 @@ from .nbc import NBCSportsVPlayerIE
 
 class YahooIE(InfoExtractor):
     IE_DESC = 'Yahoo screen and movies'
-    _VALID_URL = r'(?P<host>https?://(?:[a-zA-Z]{2}\.)?[\da-zA-Z_-]+\.yahoo\.com)/(?:[^/]+/)*(?:(?P<display_id>.+)?-)?(?P<id>[0-9]+)(?:-[a-z]+)?(?:\.html)?'
+    _VALID_URL = r'(?P<host>https?://(?:(?P<country>[a-zA-Z]{2})\.)?[\da-zA-Z_-]+\.yahoo\.com)/(?:[^/]+/)*(?:(?P<display_id>.+)?-)?(?P<id>[0-9]+)(?:-[a-z]+)?(?:\.html)?'
     _TESTS = [
         {
             'url': 'http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html',
@@ -244,6 +245,11 @@ class YahooIE(InfoExtractor):
             'params': {
                 'skip_download': True,
             },
+        },
+        {
+            # custom brightcove, geo-restricted to Australia, bypassable
+            'url': 'https://au.tv.yahoo.com/plus7/sunrise/-/watch/37263964/sunrise-episode-wed-27-sep/',
+            'only_matching': True,
         }
     ]
 
@@ -274,10 +280,15 @@ class YahooIE(InfoExtractor):
         if bc_url:
             return self.url_result(bc_url, BrightcoveLegacyIE.ie_key())
 
+        def brightcove_url_result(bc_url):
+            return self.url_result(
+                smuggle_url(bc_url, {'geo_countries': [mobj.group('country')]}),
+                BrightcoveNewIE.ie_key())
+
         # Look for Brightcove New Studio embeds
         bc_url = BrightcoveNewIE._extract_url(self, webpage)
         if bc_url:
-            return self.url_result(bc_url, BrightcoveNewIE.ie_key())
+            return brightcove_url_result(bc_url)
 
         brightcove_iframe = self._search_regex(
             r'(<iframe[^>]+data-video-id=["\']\d+[^>]+>)', webpage,
@@ -291,9 +302,9 @@ class YahooIE(InfoExtractor):
                 account_id = qs.get('accountId', ['2376984109001'])[0]
                 brightcove_id = attr.get('data-video-id') or qs.get('videoId', [None])[0]
                 if account_id and brightcove_id:
-                    return self.url_result(
-                        'http://players.brightcove.net/%s/default_default/index.html?videoId=%s' % (account_id, brightcove_id),
-                        BrightcoveNewIE.ie_key())
+                    return brightcove_url_result(
+                        'http://players.brightcove.net/%s/default_default/index.html?videoId=%s'
+                        % (account_id, brightcove_id))
 
         # Query result is often embedded in webpage as JSON. Sometimes explicit requests
         # to video API results in a failure with geo restriction reason therefore using

From c5b7014a9c936ab8cd0c3b8514a278575c414aa1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Oct 2017 07:01:21 +0700
Subject: [PATCH 1369/1696] [generic] Add support for single format Video.js
 embeds (closes #14371)

---
 youtube_dl/extractor/generic.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a3d09a036..1721a3dbd 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1133,7 +1133,7 @@ class GenericIE(InfoExtractor):
             }
         },
         {
-            # Video.js embed
+            # Video.js embed, multiple formats
             'url': 'http://ortcam.com/solidworks-урок-6-настройка-чертежа_33f9b7351.html',
             'info_dict': {
                 'id': 'yygqldloqIk',
@@ -1148,6 +1148,19 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            # Video.js embed, single format
+            'url': 'https://www.vooplayer.com/v3/watch/watch.php?v=NzgwNTg=',
+            'info_dict': {
+                'id': 'watch',
+                'ext': 'mp4',
+                'title': 'Step 1 -  Good Foundation',
+                'description': 'md5:d1e7ff33a29fc3eb1673d6c270d344f4',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
         # rtl.nl embed
         {
             'url': 'http://www.rtlnieuws.nl/nieuws/buitenland/aanslagen-kopenhagen',
@@ -2900,12 +2913,14 @@ class GenericIE(InfoExtractor):
 
         # Video.js embed
         mobj = re.search(
-            r'(?s)\bvideojs\s*\(.+?\bplayer\.src\s*\(\s*(\[.+?\])\s*\)\s*;',
+            r'(?s)\bvideojs\s*\(.+?\.src\s*\(\s*((?:\[.+?\]|{.+?}))\s*\)\s*;',
             webpage)
         if mobj is not None:
             sources = self._parse_json(
                 mobj.group(1), video_id, transform_source=js_to_json,
                 fatal=False) or []
+            if not isinstance(sources, list):
+                sources = [sources]
             formats = []
             for source in sources:
                 src = source.get('src')

From 4e599194d6cf3dbe214dceb3c66ba5f718c6579a Mon Sep 17 00:00:00 2001
From: Rafal Borczuch <rafalq.b+github@gmail.com>
Date: Sun, 1 Oct 2017 12:59:00 +0100
Subject: [PATCH 1370/1696] [tvp] Add support for new URL schema (closes
 #14368)

---
 youtube_dl/extractor/tvp.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index c5b3288ad..3954f0b93 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -15,16 +15,16 @@ from ..utils import (
 class TVPIE(InfoExtractor):
     IE_NAME = 'tvp'
     IE_DESC = 'Telewizja Polska'
-    _VALID_URL = r'https?://[^/]+\.tvp\.(?:pl|info)/(?:(?!\d+/)[^/]+/)*(?P<id>\d+)'
+    _VALID_URL = r'https?://[^/]+\.tvp\.(?:pl|info)/(?:video/(?:[^,\s]*,)*|(?:(?!\d+/)[^/]+/)*)(?P<id>\d+)'
 
     _TESTS = [{
-        'url': 'http://vod.tvp.pl/194536/i-seria-odc-13',
+        'url': 'https://vod.tvp.pl/video/czas-honoru,i-seria-odc-13,194536',
         'md5': '8aa518c15e5cc32dfe8db400dc921fbb',
         'info_dict': {
             'id': '194536',
             'ext': 'mp4',
             'title': 'Czas honoru, I seria – odc. 13',
-            'description': 'md5:76649d2014f65c99477be17f23a4dead',
+            'description': 'md5:381afa5bca72655fe94b05cfe82bf53d',
         },
     }, {
         'url': 'http://www.tvp.pl/there-can-be-anything-so-i-shortened-it/17916176',
@@ -37,12 +37,13 @@ class TVPIE(InfoExtractor):
         },
     }, {
         # page id is not the same as video id(#7799)
-        'url': 'http://vod.tvp.pl/22704887/08122015-1500',
-        'md5': 'cf6a4705dfd1489aef8deb168d6ba742',
+        'url': 'https://wiadomosci.tvp.pl/33908820/28092017-1930',
+        'md5': '84cd3c8aec4840046e5ab712416b73d0',
         'info_dict': {
-            'id': '22680786',
+            'id': '33908820',
             'ext': 'mp4',
-            'title': 'Wiadomości, 08.12.2015, 15:00',
+            'title': 'Wiadomości, 28.09.2017, 19:30',
+            'description': 'Wydanie główne codziennego serwisu informacyjnego.'
         },
     }, {
         'url': 'http://vod.tvp.pl/seriale/obyczajowe/na-sygnale/sezon-2-27-/odc-39/17834272',

From 1dd126180eb7c3266af5480a95a3e5c4eb81ed1d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Oct 2017 21:45:56 +0700
Subject: [PATCH 1371/1696] [ChangeLog] Actualize

---
 ChangeLog | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index da60c1b36..66bb79042 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,21 @@
+version <unreleased>
+
+Core
+* [YoutubeDL] Document youtube_include_dash_manifest
+
+Extractors
++ [tvp] Add support for new URL schema (#14368)
++ [generic] Add support for single format Video.js embeds (#14371)
+* [yahoo] Bypass geo restriction for brightcove (#14210)
+* [yahoo] Use extracted brightcove account id (#14210)
+* [rtve:alacarta] Fix extraction (#14290)
++ [yahoo] Add support for custom brigthcove embeds (#14210)
++ [generic] Add support for Video.js embeds
++ [gfycat] Add support for /gifs/detail URLs (#14322)
+* [generic] Fix infinite recursion for twitter:player URLs (#14339)
+* [xhamsterembed] Fix extraction (#14308)
+
+
 version 2017.09.24
 
 Core

From fcdd37d053c97c1bc363635c46634c778ce3eee5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Oct 2017 21:54:11 +0700
Subject: [PATCH 1372/1696] release 2017.10.01

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 1954528e3..3be306203 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.09.24*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.09.24**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.10.01*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.10.01**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.09.24
+[debug] youtube-dl version 2017.10.01
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 66bb79042..80299d522 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.10.01
 
 Core
 * [YoutubeDL] Document youtube_include_dash_manifest
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0e2e4dc90..49fa02d43 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.09.24'
+__version__ = '2017.10.01'

From 839728f5bfad3a6166be3839009b13963f00dfac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Oct 2017 03:28:25 +0700
Subject: [PATCH 1373/1696] [afreecatv] Add support for adult videos (closes
 #14376)

---
 youtube_dl/extractor/afreecatv.py | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py
index c8cb91dcb..2c58f4617 100644
--- a/youtube_dl/extractor/afreecatv.py
+++ b/youtube_dl/extractor/afreecatv.py
@@ -138,6 +138,23 @@ class AfreecaTVIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        # adult video
+        'url': 'http://vod.afreecatv.com/PLAYER/STATION/26542731',
+        'info_dict': {
+            'id': '20171001_F1AE1711_196617479_1',
+            'ext': 'mp4',
+            'title': '[생]서아 초심 찾기 방송 (part 1)',
+            'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
+            'uploader': 'BJ서아',
+            'uploader_id': 'bjdyrksu',
+            'upload_date': '20171001',
+            'duration': 3600,
+            'age_limit': 18,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'http://www.afreecatv.com/player/Player.swf?szType=szBjId=djleegoon&nStationNo=11273158&nBbsNo=13161095&nTitleNo=36327652',
         'only_matching': True,
@@ -160,7 +177,15 @@ class AfreecaTVIE(InfoExtractor):
 
         video_xml = self._download_xml(
             'http://afbbs.afreecatv.com:8080/api/video/get_video_info.php',
-            video_id, query={'nTitleNo': video_id})
+            video_id, query={
+                'nTitleNo': video_id,
+                'partialView': 'SKIP_ADULT',
+            })
+
+        flag = xpath_text(video_xml, './track/flag', 'flag', default=None)
+        if flag and flag != 'SUCCEED':
+            raise ExtractorError(
+                '%s said: %s' % (self.IE_NAME, flag), expected=True)
 
         video_element = video_xml.findall(compat_xpath('./track/video'))[1]
         if video_element is None or video_element.text is None:

From 117589dfa2ac32566dc705ee7e712139105e6dd8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Oct 2017 04:14:36 +0700
Subject: [PATCH 1374/1696] [canvas] Generalize mediazone.vrt.be extractor and
 rework canvas and een

---
 youtube_dl/extractor/canvas.py     | 142 +++++++++++++++++++----------
 youtube_dl/extractor/extractors.py |   5 +-
 2 files changed, 99 insertions(+), 48 deletions(-)

diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py
index aada02917..6899f8431 100644
--- a/youtube_dl/extractor/canvas.py
+++ b/youtube_dl/extractor/canvas.py
@@ -3,24 +3,104 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import float_or_none
+from ..utils import (
+    float_or_none,
+    strip_or_none,
+)
 
 
 class CanvasIE(InfoExtractor):
+    _VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?P<site_id>canvas|een|ketnet)/assets/(?P<id>m[dz]-ast-[^/?#&]+)'
+    _TESTS = [{
+        'url': 'https://mediazone.vrt.be/api/v1/ketnet/assets/md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
+        'md5': '90139b746a0a9bd7bb631283f6e2a64e',
+        'info_dict': {
+            'id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
+            'display_id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
+            'ext': 'flv',
+            'title': 'Nachtwacht: De Greystook',
+            'description': 'md5:1db3f5dc4c7109c821261e7512975be7',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 1468.03,
+        },
+        'expected_warnings': ['is not a supported codec', 'Unknown MIME type'],
+    }, {
+        'url': 'https://mediazone.vrt.be/api/v1/canvas/assets/mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        site_id, video_id = mobj.group('site_id'), mobj.group('id')
+
+        data = self._download_json(
+            'https://mediazone.vrt.be/api/v1/%s/assets/%s'
+            % (site_id, video_id), video_id)
+
+        title = data['title']
+        description = data.get('description')
+
+        formats = []
+        for target in data['targetUrls']:
+            format_url, format_type = target.get('url'), target.get('type')
+            if not format_url or not format_type:
+                continue
+            if format_type == 'HLS':
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id=format_type, fatal=False))
+            elif format_type == 'HDS':
+                formats.extend(self._extract_f4m_formats(
+                    format_url, video_id, f4m_id=format_type, fatal=False))
+            elif format_type == 'MPEG_DASH':
+                formats.extend(self._extract_mpd_formats(
+                    format_url, video_id, mpd_id=format_type, fatal=False))
+            elif format_type == 'HSS':
+                formats.extend(self._extract_ism_formats(
+                    format_url, video_id, ism_id='mss', fatal=False))
+            else:
+                formats.append({
+                    'format_id': format_type,
+                    'url': format_url,
+                })
+        self._sort_formats(formats)
+
+        subtitles = {}
+        subtitle_urls = data.get('subtitleUrls')
+        if isinstance(subtitle_urls, list):
+            for subtitle in subtitle_urls:
+                subtitle_url = subtitle.get('url')
+                if subtitle_url and subtitle.get('type') == 'CLOSED':
+                    subtitles.setdefault('nl', []).append({'url': subtitle_url})
+
+        return {
+            'id': video_id,
+            'display_id': video_id,
+            'title': title,
+            'description': description,
+            'formats': formats,
+            'duration': float_or_none(data.get('duration'), 1000),
+            'thumbnail': data.get('posterImageUrl'),
+            'subtitles': subtitles,
+        }
+
+
+class CanvasEenIE(InfoExtractor):
     IE_DESC = 'canvas.be and een.be'
     _VALID_URL = r'https?://(?:www\.)?(?P<site_id>canvas|een)\.be/(?:[^/]+/)*(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://www.canvas.be/video/de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week',
-        'md5': 'ea838375a547ac787d4064d8c7860a6c',
+        'md5': 'ed66976748d12350b118455979cca293',
         'info_dict': {
             'id': 'mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
             'display_id': 'de-afspraak-veilt-voor-de-warmste-week',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'De afspraak veilt voor de Warmste Week',
             'description': 'md5:24cb860c320dc2be7358e0e5aa317ba6',
             'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 49.02,
-        }
+        },
+        'expected_warnings': ['is not a supported codec'],
     }, {
         # with subtitles
         'url': 'http://www.canvas.be/video/panorama/2016/pieter-0167',
@@ -40,7 +120,8 @@ class CanvasIE(InfoExtractor):
         },
         'params': {
             'skip_download': True,
-        }
+        },
+        'skip': 'Pagina niet gevonden',
     }, {
         'url': 'https://www.een.be/sorry-voor-alles/herbekijk-sorry-voor-alles',
         'info_dict': {
@@ -54,7 +135,8 @@ class CanvasIE(InfoExtractor):
         },
         'params': {
             'skip_download': True,
-        }
+        },
+        'skip': 'Episode no longer available',
     }, {
         'url': 'https://www.canvas.be/check-point/najaar-2016/de-politie-uw-vriend',
         'only_matching': True,
@@ -66,55 +148,21 @@ class CanvasIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
-        title = (self._search_regex(
+        title = strip_or_none(self._search_regex(
             r'<h1[^>]+class="video__body__header__title"[^>]*>(.+?)</h1>',
             webpage, 'title', default=None) or self._og_search_title(
-            webpage)).strip()
+            webpage, default=None))
 
         video_id = self._html_search_regex(
-            r'data-video=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video id', group='id')
-
-        data = self._download_json(
-            'https://mediazone.vrt.be/api/v1/%s/assets/%s'
-            % (site_id, video_id), display_id)
-
-        formats = []
-        for target in data['targetUrls']:
-            format_url, format_type = target.get('url'), target.get('type')
-            if not format_url or not format_type:
-                continue
-            if format_type == 'HLS':
-                formats.extend(self._extract_m3u8_formats(
-                    format_url, display_id, entry_protocol='m3u8_native',
-                    ext='mp4', preference=0, fatal=False, m3u8_id=format_type))
-            elif format_type == 'HDS':
-                formats.extend(self._extract_f4m_formats(
-                    format_url, display_id, f4m_id=format_type, fatal=False))
-            elif format_type == 'MPEG_DASH':
-                formats.extend(self._extract_mpd_formats(
-                    format_url, display_id, mpd_id=format_type, fatal=False))
-            else:
-                formats.append({
-                    'format_id': format_type,
-                    'url': format_url,
-                })
-        self._sort_formats(formats)
-
-        subtitles = {}
-        subtitle_urls = data.get('subtitleUrls')
-        if isinstance(subtitle_urls, list):
-            for subtitle in subtitle_urls:
-                subtitle_url = subtitle.get('url')
-                if subtitle_url and subtitle.get('type') == 'CLOSED':
-                    subtitles.setdefault('nl', []).append({'url': subtitle_url})
+            r'data-video=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video id',
+            group='id')
 
         return {
+            '_type': 'url_transparent',
+            'url': 'https://mediazone.vrt.be/api/v1/%s/assets/%s' % (site_id, video_id),
+            'ie_key': CanvasIE.ie_key(),
             'id': video_id,
             'display_id': display_id,
             'title': title,
             'description': self._og_search_description(webpage),
-            'formats': formats,
-            'duration': float_or_none(data.get('duration'), 1000),
-            'thumbnail': data.get('posterImageUrl'),
-            'subtitles': subtitles,
         }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 4232a4fef..24e9acda6 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -150,7 +150,10 @@ from .camdemy import (
 from .camwithher import CamWithHerIE
 from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
-from .canvas import CanvasIE
+from .canvas import (
+    CanvasIE,
+    CanvasEenIE,
+)
 from .carambatv import (
     CarambaTVIE,
     CarambaTVPageIE,

From 544ffb7790ea1ac7bbc081d2e486101c0382b900 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Oct 2017 04:15:12 +0700
Subject: [PATCH 1375/1696] [ketnet] Add support for videos without direct
 sources (closes #14377)

---
 youtube_dl/extractor/ketnet.py | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ketnet.py b/youtube_dl/extractor/ketnet.py
index fb9c2dbd4..93a98e1e0 100644
--- a/youtube_dl/extractor/ketnet.py
+++ b/youtube_dl/extractor/ketnet.py
@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 
+from .canvas import CanvasIE
 from .common import InfoExtractor
 
 
@@ -7,7 +8,7 @@ class KetnetIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?ketnet\.be/(?:[^/]+/)*(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'https://www.ketnet.be/kijken/zomerse-filmpjes',
-        'md5': 'd907f7b1814ef0fa285c0475d9994ed7',
+        'md5': '6bdeb65998930251bbd1c510750edba9',
         'info_dict': {
             'id': 'zomerse-filmpjes',
             'ext': 'mp4',
@@ -15,6 +16,20 @@ class KetnetIE(InfoExtractor):
             'description': 'Gluur mee met Ghost Rockers op de filmset',
             'thumbnail': r're:^https?://.*\.jpg$',
         }
+    }, {
+        # mzid in playerConfig instead of sources
+        'url': 'https://www.ketnet.be/kijken/nachtwacht/de-greystook',
+        'md5': '90139b746a0a9bd7bb631283f6e2a64e',
+        'info_dict': {
+            'id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
+            'display_id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
+            'ext': 'flv',
+            'title': 'Nachtwacht: De Greystook',
+            'description': 'md5:1db3f5dc4c7109c821261e7512975be7',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 1468.03,
+        },
+        'expected_warnings': ['is not a supported codec', 'Unknown MIME type'],
     }, {
         'url': 'https://www.ketnet.be/kijken/karrewiet/uitzending-8-september-2016',
         'only_matching': True,
@@ -38,6 +53,12 @@ class KetnetIE(InfoExtractor):
                 'player config'),
             video_id)
 
+        mzid = config.get('mzid')
+        if mzid:
+            return self.url_result(
+                'https://mediazone.vrt.be/api/v1/ketnet/assets/%s' % mzid,
+                CanvasIE.ie_key(), video_id=mzid)
+
         title = config['title']
 
         formats = []

From d2ae7e24e5c574fa45621771a134e58b21443e5f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Oct 2017 04:43:25 +0700
Subject: [PATCH 1376/1696] [postprocessor/ffmpeg] Convert to opus using
 libopus (closes #14381)

---
 youtube_dl/postprocessor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index f71d413b5..3ea1afcf3 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -44,7 +44,7 @@ ACODECS = {
     'aac': 'aac',
     'flac': 'flac',
     'm4a': 'aac',
-    'opus': 'opus',
+    'opus': 'libopus',
     'vorbis': 'libvorbis',
     'wav': None,
 }

From b7e14f06a4a4fbaafc593c2f118e4b0f5d8d7937 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matthias=20Ku=CC=88ch?= <mail@matthias-kuech.de>
Date: Tue, 3 Oct 2017 15:17:28 +0200
Subject: [PATCH 1377/1696] Fix for JSON meta data download

Added fixes according to #13651 and user @remitamine
---
 youtube_dl/extractor/nbc.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 836a41f06..35151f527 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 
 class NBCIE(AdobePassIE):
-    _VALID_URL = r'(?P<permalink>https?://(?:www\.)?nbc\.com/[^/]+/video/[^/]+/(?P<id>n?\d+))'
+    _VALID_URL = r'https?(?P<permalink>://(?:www\.)?nbc\.com/[^/]+/video/[^/]+/(?P<id>n?\d+))'
 
     _TESTS = [
         {
@@ -72,6 +72,7 @@ class NBCIE(AdobePassIE):
 
     def _real_extract(self, url):
         permalink, video_id = re.match(self._VALID_URL, url).groups()
+        permalink = 'http' + permalink
         video_data = self._download_json(
             'https://api.nbc.com/v3/videos', video_id, query={
                 'filter[permalink]': permalink,

From 3e4cedf9e8cd3157df2457df7274d0c842421945 Mon Sep 17 00:00:00 2001
From: Jakub Wilk <jwilk@jwilk.net>
Date: Tue, 3 Oct 2017 18:28:13 +0200
Subject: [PATCH 1378/1696] [tvn24] Relax _VALID_URL

---
 youtube_dl/extractor/tvn24.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tvn24.py b/youtube_dl/extractor/tvn24.py
index 12ed6039c..6590e1fd0 100644
--- a/youtube_dl/extractor/tvn24.py
+++ b/youtube_dl/extractor/tvn24.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class TVN24IE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:[^/]+)\.)?tvn24(?:bis)?\.pl/(?:[^/]+/)*(?P<id>[^/]+)\.html'
+    _VALID_URL = r'https?://(?:(?:[^/]+)\.)?tvn24(?:bis)?\.pl/(?:[^/]+/)*(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'http://www.tvn24.pl/wiadomosci-z-kraju,3/oredzie-artura-andrusa,702428.html',
         'md5': 'fbdec753d7bc29d96036808275f2130c',
@@ -18,7 +18,7 @@ class TVN24IE(InfoExtractor):
             'ext': 'mp4',
             'title': '"Święta mają być wesołe, dlatego, ludziska, wszyscy pod jemiołę"',
             'description': 'Wyjątkowe orędzie Artura Andrusa, jednego z gości "Szkła kontaktowego".',
-            'thumbnail': 're:http://.*[.]jpeg',
+            'thumbnail': 're:https?://.*[.]jpeg',
         }
     }, {
         'url': 'http://fakty.tvn24.pl/ogladaj-online,60/53-konferencja-bezpieczenstwa-w-monachium,716431.html',
@@ -29,6 +29,9 @@ class TVN24IE(InfoExtractor):
     }, {
         'url': 'http://tvn24bis.pl/poranek,146,m/gen-koziej-w-tvn24-bis-wracamy-do-czasow-zimnej-wojny,715660.html',
         'only_matching': True,
+    }, {
+        'url': 'https://www.tvn24.pl/magazyn-tvn24/angie-w-jednej-czwartej-polka-od-szarej-myszki-do-cesarzowej-europy,119,2158',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 9524dca3accc1e60cddd141f06924ed7404db9bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Oct 2017 02:53:20 +0700
Subject: [PATCH 1379/1696] [README.md] Use revision bound link to YoutubeDL
 options (closes #14401)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 7818e58df..2879aad24 100644
--- a/README.md
+++ b/README.md
@@ -1167,7 +1167,7 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl:
     ydl.download(['https://www.youtube.com/watch?v=BaW_jenozKc'])
 ```
 
-Most likely, you'll want to use various options. For a list of options available, have a look at [`youtube_dl/YoutubeDL.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L129-L279). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
+Most likely, you'll want to use various options. For a list of options available, have a look at [`youtube_dl/YoutubeDL.py`](https://github.com/rg3/youtube-dl/blob/3e4cedf9e8cd3157df2457df7274d0c842421945/youtube_dl/YoutubeDL.py#L137-L312). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
 
 Here's a more complete example of a program that outputs only errors (and a short message after the download is finished), and downloads/converts the video to an mp3 file:
 

From c110944fa2f21af733b4f3168764e1b008e11514 Mon Sep 17 00:00:00 2001
From: "M.K" <mk-pmb@users.noreply.github.com>
Date: Tue, 3 Oct 2017 22:50:27 +0200
Subject: [PATCH 1380/1696] [extractor/common] Fix typo in _parse_mpd_formats

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 2bbbf8f4d..a878550c2 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1920,7 +1920,7 @@ class InfoExtractor(object):
                             # can't be used at the same time
                             if '%(Number' in media_template and 's' not in representation_ms_info:
                                 segment_duration = None
-                                if 'total_number' not in representation_ms_info and 'segment_duration':
+                                if 'total_number' not in representation_ms_info and 'segment_duration' in representation_ms_info:
                                     segment_duration = float_or_none(representation_ms_info['segment_duration'], representation_ms_info['timescale'])
                                     representation_ms_info['total_number'] = int(math.ceil(float(period_duration) / segment_duration))
                                 representation_ms_info['fragments'] = [{

From 6e736d86e7d03d14279c403202fe2a632e6e5023 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Oct 2017 04:27:42 +0700
Subject: [PATCH 1381/1696] [beeg] Fix extraction (closes #14403)

---
 youtube_dl/extractor/beeg.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index bbeae4bac..bf22a41b7 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -60,9 +60,13 @@ class BeegIE(InfoExtractor):
         beeg_version = beeg_version or '2185'
         beeg_salt = beeg_salt or 'pmweAkq8lAYKdfWcFCUj0yoVgoPlinamH5UE1CB3H'
 
-        video = self._download_json(
-            'https://api.beeg.com/api/v6/%s/video/%s' % (beeg_version, video_id),
-            video_id)
+        for api_path in ('', 'api.'):
+            video = self._download_json(
+                'https://%sbeeg.com/api/v6/%s/video/%s'
+                % (api_path, beeg_version, video_id), video_id,
+                fatal=api_path == 'api.')
+            if video:
+                break
 
         def split(o, e):
             def cut(s, x):

From 6b46285e850f8bc8155a93adf13920752537e55d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 4 Oct 2017 07:45:13 +0200
Subject: [PATCH 1382/1696] [comedycentral] new shortcut :theopposition for
 "The Opposition" show

---
 youtube_dl/extractor/comedycentral.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 4cac29415..d08b909a6 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -120,13 +120,16 @@ class ComedyCentralTVIE(MTVServicesInfoExtractor):
 
 
 class ComedyCentralShortnameIE(InfoExtractor):
-    _VALID_URL = r'^:(?P<id>tds|thedailyshow)$'
+    _VALID_URL = r'^:(?P<id>tds|thedailyshow|theopposition)$'
     _TESTS = [{
         'url': ':tds',
         'only_matching': True,
     }, {
         'url': ':thedailyshow',
         'only_matching': True,
+    }, {
+        'url': ':theopposition',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -134,5 +137,6 @@ class ComedyCentralShortnameIE(InfoExtractor):
         shortcut_map = {
             'tds': 'http://www.cc.com/shows/the-daily-show-with-trevor-noah/full-episodes',
             'thedailyshow': 'http://www.cc.com/shows/the-daily-show-with-trevor-noah/full-episodes',
+            'theopposition': 'http://www.cc.com/shows/the-opposition-with-jordan-klepper/full-episodes',
         }
         return self.url_result(shortcut_map[video_id])

From cf5f6ed5be8bab252f3ded345b6b1fdf31426661 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Oct 2017 00:27:24 +0700
Subject: [PATCH 1383/1696] [xvideos] Add support for embed URLs and improve
 extraction (closes #14409)

---
 youtube_dl/extractor/xvideos.py | 42 ++++++++++++++++++++++++---------
 1 file changed, 31 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/xvideos.py b/youtube_dl/extractor/xvideos.py
index eca603028..085c8d4f3 100644
--- a/youtube_dl/extractor/xvideos.py
+++ b/youtube_dl/extractor/xvideos.py
@@ -14,8 +14,16 @@ from ..utils import (
 
 
 class XVideosIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?xvideos\.com/video(?P<id>[0-9]+)(?:.*)'
-    _TEST = {
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:
+                            (?:www\.)?xvideos\.com/video|
+                            flashservice\.xvideos\.com/embedframe/|
+                            static-hw\.xvideos\.com/swf/xv-player\.swf\?.*?\bid_video=
+                        )
+                        (?P<id>[0-9]+)
+                    '''
+    _TESTS = [{
         'url': 'http://www.xvideos.com/video4588838/biker_takes_his_girl',
         'md5': '14cea69fcb84db54293b1e971466c2e1',
         'info_dict': {
@@ -25,21 +33,33 @@ class XVideosIE(InfoExtractor):
             'duration': 108,
             'age_limit': 18,
         }
-    }
+    }, {
+        'url': 'https://flashservice.xvideos.com/embedframe/4588838',
+        'only_matching': True,
+    }, {
+        'url': 'http://static-hw.xvideos.com/swf/xv-player.swf?id_video=4588838',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+
+        webpage = self._download_webpage(
+            'http://www.xvideos.com/video%s/' % video_id, video_id)
 
         mobj = re.search(r'<h1 class="inlineError">(.+?)</h1>', webpage)
         if mobj:
             raise ExtractorError('%s said: %s' % (self.IE_NAME, clean_html(mobj.group(1))), expected=True)
 
-        video_title = self._html_search_regex(
-            r'<title>(.*?)\s+-\s+XVID', webpage, 'title')
-        video_thumbnail = self._search_regex(
+        title = self._html_search_regex(
+            (r'<title>(?P<title>.+?)\s+-\s+XVID',
+             r'setVideoTitle\s*\(\s*(["\'])(?P<title>(?:(?!\1).)+)\1'),
+            webpage, 'title', default=None,
+            group='title') or self._og_search_title(webpage)
+
+        thumbnail = self._search_regex(
             r'url_bigthumb=(.+?)&amp', webpage, 'thumbnail', fatal=False)
-        video_duration = int_or_none(self._og_search_property(
+        duration = int_or_none(self._og_search_property(
             'duration', webpage, default=None)) or parse_duration(
             self._search_regex(
                 r'<span[^>]+class=["\']duration["\'][^>]*>.*?(\d[^<]+)',
@@ -74,8 +94,8 @@ class XVideosIE(InfoExtractor):
         return {
             'id': video_id,
             'formats': formats,
-            'title': video_title,
-            'duration': video_duration,
-            'thumbnail': video_thumbnail,
+            'title': title,
+            'duration': duration,
+            'thumbnail': thumbnail,
             'age_limit': 18,
         }

From 6be08ce60205a65a6739667783eead56ccc34456 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Oct 2017 23:13:53 +0700
Subject: [PATCH 1384/1696] [utils] Use in OnDemandPagedList by default Not
 using cache results in redundant network I/O due to downloading the same
 pages while using --playlist-items n-m

---
 youtube_dl/extractor/mixcloud.py | 2 +-
 youtube_dl/extractor/nba.py      | 2 +-
 youtube_dl/utils.py              | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index f331db890..7b2bb6e20 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -291,7 +291,7 @@ class MixcloudUserIE(MixcloudPlaylistBaseIE):
             functools.partial(
                 self._tracks_page_func,
                 '%s/%s' % (user_id, list_type), video_id, 'list of %s' % list_type),
-            self._PAGE_SIZE, use_cache=True)
+            self._PAGE_SIZE)
 
         return self.playlist_result(
             entries, video_id, '%s (%s)' % (username, list_type), description)
diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 53561961c..be295a7a3 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -122,7 +122,7 @@ class NBAIE(TurnerBaseIE):
         playlist_title = self._og_search_title(webpage, fatal=False)
         entries = OnDemandPagedList(
             functools.partial(self._fetch_page, team, video_id),
-            self._PAGE_SIZE, use_cache=True)
+            self._PAGE_SIZE)
 
         return self.playlist_result(entries, team, playlist_title)
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 92b22e639..59fb33435 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1933,7 +1933,7 @@ class PagedList(object):
 
 
 class OnDemandPagedList(PagedList):
-    def __init__(self, pagefunc, pagesize, use_cache=False):
+    def __init__(self, pagefunc, pagesize, use_cache=True):
         self._pagefunc = pagefunc
         self._pagesize = pagesize
         self._use_cache = use_cache

From 7e85e8729f1e2ed9817466acef30fa6dc7e03e1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Oct 2017 23:34:46 +0700
Subject: [PATCH 1385/1696] [YoutubeDL] Fix out of range --playlist-items for
 iterable playlists and reduce code duplication (closes #14425)

---
 youtube_dl/YoutubeDL.py | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 474d6c95f..9036f0f94 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -911,12 +911,22 @@ class YoutubeDL(object):
                 playlistitems = iter_playlistitems(playlistitems_str)
 
             ie_entries = ie_result['entries']
+
+            def make_playlistitems_entries(list_ie_entries):
+                num_entries = len(list_ie_entries)
+                return [
+                    list_ie_entries[i - 1] for i in playlistitems
+                    if -num_entries <= i - 1 < num_entries]
+
+            def report_download(num_entries):
+                self.to_screen(
+                    '[%s] playlist %s: Downloading %d videos' %
+                    (ie_result['extractor'], playlist, num_entries))
+
             if isinstance(ie_entries, list):
                 n_all_entries = len(ie_entries)
                 if playlistitems:
-                    entries = [
-                        ie_entries[i - 1] for i in playlistitems
-                        if -n_all_entries <= i - 1 < n_all_entries]
+                    entries = make_playlistitems_entries(ie_entries)
                 else:
                     entries = ie_entries[playliststart:playlistend]
                 n_entries = len(entries)
@@ -934,20 +944,15 @@ class YoutubeDL(object):
                     entries = ie_entries.getslice(
                         playliststart, playlistend)
                 n_entries = len(entries)
-                self.to_screen(
-                    '[%s] playlist %s: Downloading %d videos' %
-                    (ie_result['extractor'], playlist, n_entries))
+                report_download(n_entries)
             else:  # iterable
                 if playlistitems:
-                    entry_list = list(ie_entries)
-                    entries = [entry_list[i - 1] for i in playlistitems]
+                    entries = make_playlistitems_entries(list(ie_entries))
                 else:
                     entries = list(itertools.islice(
                         ie_entries, playliststart, playlistend))
                 n_entries = len(entries)
-                self.to_screen(
-                    '[%s] playlist %s: Downloading %d videos' %
-                    (ie_result['extractor'], playlist, n_entries))
+                report_download(n_entries)
 
             if self.params.get('playlistreverse', False):
                 entries = entries[::-1]

From 86a15ed64b410336b2145f4a6b8e8a22cbf24f51 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Oct 2017 23:41:28 +0700
Subject: [PATCH 1386/1696] [test_YoutubeDL] Add test for #14425

---
 test/test_YoutubeDL.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index e70cbcd37..5ac34e663 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -770,6 +770,9 @@ class TestYoutubeDL(unittest.TestCase):
         result = get_ids({'playlist_items': '10'})
         self.assertEqual(result, [])
 
+        result = get_ids({'playlist_items': '3-10'})
+        self.assertEqual(result, [3, 4])
+
     def test_urlopen_no_file_protocol(self):
         # see https://github.com/rg3/youtube-dl/issues/8227
         ydl = YDL()

From cd6fc19ed76af6687fb2cb5f87d9ed4c3071c203 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Oct 2017 23:46:57 +0700
Subject: [PATCH 1387/1696] [YoutubeDL] Ignore duplicates in --playlist-items
 E.g. '--playlist-items 2-4,3-4,3' should result in '[2,3,4]', not
 '[2,3,4,3,4,3]'

---
 test/test_YoutubeDL.py  | 3 +++
 youtube_dl/YoutubeDL.py | 3 ++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 5ac34e663..db936bfb8 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -773,6 +773,9 @@ class TestYoutubeDL(unittest.TestCase):
         result = get_ids({'playlist_items': '3-10'})
         self.assertEqual(result, [3, 4])
 
+        result = get_ids({'playlist_items': '2-4,3-4,3'})
+        self.assertEqual(result, [2, 3, 4])
+
     def test_urlopen_no_file_protocol(self):
         # see https://github.com/rg3/youtube-dl/issues/8227
         ydl = YDL()
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 9036f0f94..855d6b8e5 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -65,6 +65,7 @@ from .utils import (
     locked_file,
     make_HTTPS_handler,
     MaxDownloadsReached,
+    orderedSet,
     PagedList,
     parse_filesize,
     PerRequestProxyHandler,
@@ -908,7 +909,7 @@ class YoutubeDL(object):
                                 yield int(item)
                         else:
                             yield int(string_segment)
-                playlistitems = iter_playlistitems(playlistitems_str)
+                playlistitems = orderedSet(iter_playlistitems(playlistitems_str))
 
             ie_entries = ie_result['entries']
 

From ac93c09ab2c4d099a628c1ed59670bef008db89c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Oct 2017 23:53:32 +0700
Subject: [PATCH 1388/1696] [xtube] Add support for embedded URLs (closes
 #14417)

---
 youtube_dl/extractor/xtube.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index bea9b87ad..c6c0b3291 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -18,7 +18,7 @@ class XTubeIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                         (?:
                             xtube:|
-                            https?://(?:www\.)?xtube\.com/(?:watch\.php\?.*\bv=|video-watch/(?P<display_id>[^/]+)-)
+                            https?://(?:www\.)?xtube\.com/(?:watch\.php\?.*\bv=|video-watch/(?:embedded/)?(?P<display_id>[^/]+)-)
                         )
                         (?P<id>[^/?&#]+)
                     '''
@@ -64,6 +64,9 @@ class XTubeIE(InfoExtractor):
     }, {
         'url': 'xtube:kVTUy_G222_',
         'only_matching': True,
+    }, {
+        'url': 'https://www.xtube.com/video-watch/embedded/milf-tara-and-teen-shared-and-cum-covered-extreme-bukkake-32203482?embedsize=big',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 2e2a8e97d53d7759d663be2a1dc3f4108342ea40 Mon Sep 17 00:00:00 2001
From: Jalaz Kumar <jaykay12@users.noreply.github.com>
Date: Fri, 6 Oct 2017 22:26:31 +0530
Subject: [PATCH 1389/1696] [pornflip] Extend _VALID_URL (closes #14405)

---
 youtube_dl/extractor/pornflip.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornflip.py b/youtube_dl/extractor/pornflip.py
index a4a5d390e..ee04936e1 100644
--- a/youtube_dl/extractor/pornflip.py
+++ b/youtube_dl/extractor/pornflip.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class PornFlipIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?pornflip\.com/(?:v|embed)/(?P<id>[0-9A-Za-z]{11})'
+    _VALID_URL = r'https?://(?:www\.)?pornflip\.com/(?:v|embed)/(?P<id>[0-9A-Za-z-]{11})'
     _TESTS = [{
         'url': 'https://www.pornflip.com/v/wz7DfNhMmep',
         'md5': '98c46639849145ae1fd77af532a9278c',
@@ -34,6 +34,12 @@ class PornFlipIE(InfoExtractor):
     }, {
         'url': 'https://www.pornflip.com/embed/wz7DfNhMmep',
         'only_matching': True,
+    }, {
+        'url': 'https://www.pornflip.com/v/EkRD6-vS2-s',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.pornflip.com/embed/EkRD6-vS2-s',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From b1a7bf44b95dee2803e8638b13c4467a253b5b8f Mon Sep 17 00:00:00 2001
From: remis <r-pankevicius@users.noreply.github.com>
Date: Fri, 6 Oct 2017 19:59:09 +0300
Subject: [PATCH 1390/1696] [lnkgo] Relax _VALID_URL

---
 youtube_dl/extractor/lnkgo.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/lnkgo.py b/youtube_dl/extractor/lnkgo.py
index 068378c9c..cfec0d3d0 100644
--- a/youtube_dl/extractor/lnkgo.py
+++ b/youtube_dl/extractor/lnkgo.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class LnkGoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?lnkgo\.alfa\.lt/visi-video/(?P<show>[^/]+)/ziurek-(?P<id>[A-Za-z0-9-]+)'
+    _VALID_URL = r'https?://(?:www\.)?lnkgo\.(?:alfa\.)?lt/visi-video/(?P<show>[^/]+)/ziurek-(?P<id>[A-Za-z0-9-]+)'
     _TESTS = [{
         'url': 'http://lnkgo.alfa.lt/visi-video/yra-kaip-yra/ziurek-yra-kaip-yra-162',
         'info_dict': {
@@ -42,6 +42,9 @@ class LnkGoIE(InfoExtractor):
         'params': {
             'skip_download': True,  # HLS download
         },
+    }, {
+        'url': 'http://www.lnkgo.lt/visi-video/aktualai-pratesimas/ziurek-putka-trys-klausimai',
+        'only_matching': True,
     }]
     _AGE_LIMITS = {
         'N-7': 7,

From e95284754192ea4b5b4e32f1a2274e5767d980b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Oct 2017 00:57:09 +0700
Subject: [PATCH 1391/1696] [PULL_REQUEST_TEMPLATE.md] Add explicit entry on
 flake8

---
 .github/PULL_REQUEST_TEMPLATE.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 46fa26f02..ba4ca7553 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -9,6 +9,7 @@
 ### Before submitting a *pull request* make sure you have:
 - [ ] At least skimmed through [adding new extractor tutorial](https://github.com/rg3/youtube-dl#adding-support-for-a-new-site) and [youtube-dl coding conventions](https://github.com/rg3/youtube-dl#youtube-dl-coding-conventions) sections
 - [ ] [Searched](https://github.com/rg3/youtube-dl/search?q=is%3Apr&type=Issues) the bugtracker for similar pull requests
+- [ ] Checked the code with [flake8](https://pypi.python.org/pypi/flake8)
 
 ### In order to be accepted and merged into youtube-dl each piece of code must be in public domain or released under [Unlicense](http://unlicense.org/). Check one of the following options:
 - [ ] I am the original author of this code and I am willing to release it under [Unlicense](http://unlicense.org/)

From 665f42d8c14626404372b349624632b8d39f3c0e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Oct 2017 01:40:00 +0700
Subject: [PATCH 1392/1696] [reddit] Sort formats (closes #14430)

---
 youtube_dl/extractor/reddit.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/reddit.py b/youtube_dl/extractor/reddit.py
index 01c85ee01..4d44b9d74 100644
--- a/youtube_dl/extractor/reddit.py
+++ b/youtube_dl/extractor/reddit.py
@@ -35,6 +35,8 @@ class RedditIE(InfoExtractor):
             'https://v.redd.it/%s/DASHPlaylist.mpd' % video_id, video_id,
             mpd_id='dash', fatal=False))
 
+        self._sort_formats(formats)
+
         return {
             'id': video_id,
             'title': video_id,

From 3fc8f5b7c239ddd366012d2f0da754fcfe247297 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Oct 2017 05:01:38 +0700
Subject: [PATCH 1393/1696] [ChangeLog] Actualize

---
 ChangeLog | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 80299d522..760d09047 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,27 @@
+version <unreleased>
+
+Core
+* [YoutubeDL] Ignore duplicates in --playlist-items
+* [YoutubeDL] Fix out of range --playlist-items for iterable playlists and
+  reduce code duplication (#14425)
++ [utils] Use cache in OnDemandPagedList by default
+* [postprocessor/ffmpeg] Convert to opus using libopus (#14381)
+
+Extractors
+* [reddit] Sort formats (#14430)
+* [lnkgo] Relax URL regular expression (#14423)
+* [pornflip] Extend URL regular expression (#14405, #14406)
++ [xtube] Add support for embed URLs (#14417)
++ [xvideos] Add support for embed URLs and improve extraction (#14409)
+* [beeg] Fix extraction (#14403)
+* [tvn24] Relax URL regular expression (#14395)
+* [nbc] Fix extraction (#13651, #13715, #14137, #14198, #14312, #14314, #14378,
+  #14392, #14414, #14419, #14431)
++ [ketnet] Add support for videos without direct sources (#14377)
+* [canvas] Generalize mediazone.vrt.be extractor and rework canvas and een
++ [afreecatv] Add support for adult videos (#14376)
+
+
 version 2017.10.01
 
 Core

From 8e751a185c53a81cd35900010118894fd8201b75 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Oct 2017 05:02:53 +0700
Subject: [PATCH 1394/1696] release 2017.10.07

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 3 ++-
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 3be306203..9fc425ca8 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.10.01*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.10.01**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.10.07*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.10.07**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.10.01
+[debug] youtube-dl version 2017.10.07
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 760d09047..2feb3dc2d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.10.07
 
 Core
 * [YoutubeDL] Ignore duplicates in --playlist-items
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index d36a07cf6..be9df7e31 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -130,7 +130,8 @@
  - **CamWithHer**
  - **canalc2.tv**
  - **Canalplus**: canalplus.fr, piwiplus.fr and d8.tv
- - **Canvas**: canvas.be and een.be
+ - **Canvas**
+ - **CanvasEen**: canvas.be and een.be
  - **CarambaTV**
  - **CarambaTVPage**
  - **CartoonNetwork**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 49fa02d43..705fcade6 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.10.01'
+__version__ = '2017.10.07'

From 8b561bfc9d15bf487be0e0bb5bb1d7248fd321d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Oct 2017 21:59:04 +0700
Subject: [PATCH 1395/1696] [youtube] Add support for hooktube.com (closes
 #14437)

---
 youtube_dl/extractor/youtube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index ad2e933ee..edd8713b7 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -332,6 +332,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/|
                             (?:www\.)?deturl\.com/www\.youtube\.com/|
                             (?:www\.)?pwnyoutube\.com/|
+                            (?:www\.)?hooktube\.com/|
                             (?:www\.)?yourepeat\.com/|
                             tube\.majestyc\.net/|
                             youtube\.googleapis\.com/)                        # the various hostnames, with wildcard subdomains

From a22ccac1f02b5957f1f083a54e1f83d22a7e6f69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Oct 2017 01:34:17 +0700
Subject: [PATCH 1396/1696] [fox] Delegate to uplynk:preplay (#14147)

---
 youtube_dl/extractor/fox.py | 33 +++++++++++++++++++++++----------
 1 file changed, 23 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/fox.py b/youtube_dl/extractor/fox.py
index facc665f6..5f98d017b 100644
--- a/youtube_dl/extractor/fox.py
+++ b/youtube_dl/extractor/fox.py
@@ -2,7 +2,10 @@
 from __future__ import unicode_literals
 
 from .adobepass import AdobePassIE
+from .uplynk import UplynkPreplayIE
+from ..compat import compat_str
 from ..utils import (
+    HEADRequest,
     int_or_none,
     parse_age_limit,
     parse_duration,
@@ -53,14 +56,7 @@ class FOXIE(AdobePassIE):
             })
 
         title = video['name']
-
-        m3u8_url = self._download_json(
-            video['videoRelease']['url'], video_id)['playURL']
-
-        formats = self._extract_m3u8_formats(
-            m3u8_url, video_id, 'mp4',
-            entry_protocol='m3u8_native', m3u8_id='hls')
-        self._sort_formats(formats)
+        release_url = video['videoRelease']['url']
 
         description = video.get('description')
         duration = int_or_none(video.get('durationInSeconds')) or int_or_none(
@@ -84,7 +80,7 @@ class FOXIE(AdobePassIE):
             # TODO: AP
             pass
 
-        return {
+        info = {
             'id': video_id,
             'title': title,
             'description': description,
@@ -97,5 +93,22 @@ class FOXIE(AdobePassIE):
             'episode': episode,
             'episode_number': episode_number,
             'release_year': release_year,
-            'formats': formats,
         }
+
+        urlh = self._request_webpage(HEADRequest(release_url), video_id)
+        video_url = compat_str(urlh.geturl())
+
+        if UplynkPreplayIE.suitable(video_url):
+            info.update({
+                '_type': 'url_transparent',
+                'url': video_url,
+                'ie_key': UplynkPreplayIE.ie_key(),
+            })
+        else:
+            m3u8_url = self._download_json(release_url, video_id)['playURL']
+            formats = self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4',
+                entry_protocol='m3u8_native', m3u8_id='hls')
+            self._sort_formats(formats)
+            info['formats'] = formats
+        return info

From b0dde6686c7110c9c2515a808d803239a81e6505 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleksandar=20Topuzovi=C4=87?=
 <aleksandar.topuzovic@gmail.com>
Date: Sat, 7 Oct 2017 23:40:08 +0100
Subject: [PATCH 1397/1696] [hrti] Relax _VALID_URL

---
 youtube_dl/extractor/hrti.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/hrti.py b/youtube_dl/extractor/hrti.py
index 656ce6d05..4f0369433 100644
--- a/youtube_dl/extractor/hrti.py
+++ b/youtube_dl/extractor/hrti.py
@@ -104,7 +104,7 @@ class HRTiIE(HRTiBaseIE):
                         (?:
                             hrti:(?P<short_id>[0-9]+)|
                             https?://
-                                hrti\.hrt\.hr/\#/video/show/(?P<id>[0-9]+)/(?P<display_id>[^/]+)?
+                                hrti\.hrt\.hr/(?:\#/)?video/show/(?P<id>[0-9]+)/(?P<display_id>[^/]+)?
                         )
                     '''
     _TESTS = [{
@@ -129,6 +129,9 @@ class HRTiIE(HRTiBaseIE):
     }, {
         'url': 'hrti:2181385',
         'only_matching': True,
+    }, {
+        'url': 'https://hrti.hrt.hr/video/show/3873068/cuvar-dvorca-dramska-serija-14',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 89923316210f8e17bb1a085278940e1c56fcff48 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Oct 2017 21:36:50 +0700
Subject: [PATCH 1398/1696] [wdr] Relax media link regex (closes #14447)

---
 youtube_dl/extractor/wdr.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index 8bb7362bb..621de1e1e 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -22,8 +22,13 @@ class WDRBaseIE(InfoExtractor):
         # for wdrmaus, in a tag with the class "videoButton" (previously a link
         # to the page in a multiline "videoLink"-tag)
         json_metadata = self._html_search_regex(
-            r'class=(?:"(?:mediaLink|wdrrPlayerPlayBtn|videoButton)\b[^"]*"[^>]+|"videoLink\b[^"]*"[\s]*>\n[^\n]*)data-extension="([^"]+)"',
-            webpage, 'media link', default=None, flags=re.MULTILINE)
+            r'''(?sx)class=
+                    (?:
+                        (["\'])(?:mediaLink|wdrrPlayerPlayBtn|videoButton)\b.*?\1[^>]+|
+                        (["\'])videoLink\b.*?\2[\s]*>\n[^\n]*
+                    )data-extension=(["\'])(?P<data>(?:(?!\3).)+)\3
+            ''',
+            webpage, 'media link', default=None, group='data')
 
         if not json_metadata:
             return

From 197224b7a4e37a6581bf1a0da18d0f67ea61a476 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Oct 2017 23:50:53 +0700
Subject: [PATCH 1399/1696] Fix some regexes

---
 youtube_dl/extractor/aenetworks.py         | 2 +-
 youtube_dl/extractor/appletrailers.py      | 4 ++--
 youtube_dl/extractor/ard.py                | 2 +-
 youtube_dl/extractor/bbc.py                | 2 +-
 youtube_dl/extractor/dailymotion.py        | 2 +-
 youtube_dl/extractor/deezer.py             | 2 +-
 youtube_dl/extractor/freespeech.py         | 2 +-
 youtube_dl/extractor/generic.py            | 2 +-
 youtube_dl/extractor/googleplus.py         | 2 +-
 youtube_dl/extractor/hrti.py               | 2 +-
 youtube_dl/extractor/ign.py                | 2 +-
 youtube_dl/extractor/infoq.py              | 6 +++---
 youtube_dl/extractor/jeuxvideo.py          | 2 +-
 youtube_dl/extractor/livestream.py         | 2 +-
 youtube_dl/extractor/makertv.py            | 2 +-
 youtube_dl/extractor/mangomolo.py          | 2 +-
 youtube_dl/extractor/meipai.py             | 2 +-
 youtube_dl/extractor/mtv.py                | 2 +-
 youtube_dl/extractor/myvideo.py            | 2 +-
 youtube_dl/extractor/nationalgeographic.py | 2 +-
 youtube_dl/extractor/naver.py              | 2 +-
 youtube_dl/extractor/npo.py                | 2 +-
 youtube_dl/extractor/ruhd.py               | 2 +-
 youtube_dl/extractor/stanfordoc.py         | 4 ++--
 youtube_dl/extractor/theplatform.py        | 2 +-
 youtube_dl/extractor/thisav.py             | 4 ++--
 youtube_dl/extractor/twitter.py            | 2 +-
 youtube_dl/extractor/vice.py               | 2 +-
 youtube_dl/extractor/videopremium.py       | 2 +-
 youtube_dl/extractor/youtube.py            | 2 +-
 30 files changed, 35 insertions(+), 35 deletions(-)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index 2dcdba9d2..da1b566c2 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -131,7 +131,7 @@ class AENetworksIE(AENetworksBaseIE):
              r'data-media-url=(["\'])(?P<url>(?:(?!\1).)+?)\1'],
             webpage, 'video url', group='url')
         theplatform_metadata = self._download_theplatform_metadata(self._search_regex(
-            r'https?://link.theplatform.com/s/([^?]+)', media_url, 'theplatform_path'), video_id)
+            r'https?://link\.theplatform\.com/s/([^?]+)', media_url, 'theplatform_path'), video_id)
         info = self._parse_theplatform_metadata(theplatform_metadata)
         if theplatform_metadata.get('AETN$isBehindWall'):
             requestor_id = self._DOMAIN_TO_REQUESTOR_ID[domain]
diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index b45b431e1..a9ef733e0 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -117,7 +117,7 @@ class AppleTrailersIE(InfoExtractor):
                             continue
                         formats.append({
                             'format_id': '%s-%s' % (version, size),
-                            'url': re.sub(r'_(\d+p.mov)', r'_h\1', src),
+                            'url': re.sub(r'_(\d+p\.mov)', r'_h\1', src),
                             'width': int_or_none(size_data.get('width')),
                             'height': int_or_none(size_data.get('height')),
                             'language': version[:2],
@@ -179,7 +179,7 @@ class AppleTrailersIE(InfoExtractor):
             formats = []
             for format in settings['metadata']['sizes']:
                 # The src is a file pointing to the real video file
-                format_url = re.sub(r'_(\d*p.mov)', r'_h\1', format['src'])
+                format_url = re.sub(r'_(\d*p\.mov)', r'_h\1', format['src'])
                 formats.append({
                     'url': format_url,
                     'format': format['type'],
diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 3f248b147..915f8862e 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -195,7 +195,7 @@ class ARDMediathekIE(InfoExtractor):
 
         title = self._html_search_regex(
             [r'<h1(?:\s+class="boxTopHeadline")?>(.*?)</h1>',
-             r'<meta name="dcterms.title" content="(.*?)"/>',
+             r'<meta name="dcterms\.title" content="(.*?)"/>',
              r'<h4 class="headline">(.*?)</h4>'],
             webpage, 'title')
         description = self._html_search_meta(
diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 8b20c03d6..5525f7c9b 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -386,7 +386,7 @@ class BBCCoUkIE(InfoExtractor):
                             m3u8_id=format_id, fatal=False))
                         if re.search(self._USP_RE, href):
                             usp_formats = self._extract_m3u8_formats(
-                                re.sub(self._USP_RE, r'/\1.ism/\1.m3u8', href),
+                                re.sub(self._USP_RE, r'/\1\.ism/\1\.m3u8', href),
                                 programme_id, ext='mp4', entry_protocol='m3u8_native',
                                 m3u8_id=format_id, fatal=False)
                             for f in usp_formats:
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index e9d0dd19c..21a2d0239 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -235,7 +235,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
 
         # vevo embed
         vevo_id = self._search_regex(
-            r'<link rel="video_src" href="[^"]*?vevo.com[^"]*?video=(?P<id>[\w]*)',
+            r'<link rel="video_src" href="[^"]*?vevo\.com[^"]*?video=(?P<id>[\w]*)',
             webpage, 'vevo embed', default=None)
         if vevo_id:
             return self.url_result('vevo:%s' % vevo_id, 'Vevo')
diff --git a/youtube_dl/extractor/deezer.py b/youtube_dl/extractor/deezer.py
index ec87b94db..a38b2683d 100644
--- a/youtube_dl/extractor/deezer.py
+++ b/youtube_dl/extractor/deezer.py
@@ -19,7 +19,7 @@ class DeezerPlaylistIE(InfoExtractor):
             'id': '176747451',
             'title': 'Best!',
             'uploader': 'Anonymous',
-            'thumbnail': r're:^https?://cdn-images.deezer.com/images/cover/.*\.jpg$',
+            'thumbnail': r're:^https?://cdn-images\.deezer\.com/images/cover/.*\.jpg$',
         },
         'playlist_count': 30,
         'skip': 'Only available in .de',
diff --git a/youtube_dl/extractor/freespeech.py b/youtube_dl/extractor/freespeech.py
index 0a70ca763..7fa271b51 100644
--- a/youtube_dl/extractor/freespeech.py
+++ b/youtube_dl/extractor/freespeech.py
@@ -27,7 +27,7 @@ class FreespeechIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         title = mobj.group('title')
         webpage = self._download_webpage(url, title)
-        info_json = self._search_regex(r'jQuery.extend\(Drupal.settings, ({.*?})\);', webpage, 'info')
+        info_json = self._search_regex(r'jQuery\.extend\(Drupal\.settings, ({.*?})\);', webpage, 'info')
         info = json.loads(info_json)
 
         return {
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 1721a3dbd..68b633839 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2206,7 +2206,7 @@ class GenericIE(InfoExtractor):
         # And then there are the jokers who advertise that they use RTA,
         # but actually don't.
         AGE_LIMIT_MARKERS = [
-            r'Proudly Labeled <a href="http://www.rtalabel.org/" title="Restricted to Adults">RTA</a>',
+            r'Proudly Labeled <a href="http://www\.rtalabel\.org/" title="Restricted to Adults">RTA</a>',
         ]
         if any(re.search(marker, webpage) for marker in AGE_LIMIT_MARKERS):
             age_limit = 18
diff --git a/youtube_dl/extractor/googleplus.py b/youtube_dl/extractor/googleplus.py
index 427499b11..6b927bb44 100644
--- a/youtube_dl/extractor/googleplus.py
+++ b/youtube_dl/extractor/googleplus.py
@@ -61,7 +61,7 @@ class GooglePlusIE(InfoExtractor):
             'width': int(width),
             'height': int(height),
         } for width, height, video_url in re.findall(
-            r'\d+,(\d+),(\d+),"(https?://[^.]+\.googleusercontent.com.*?)"', webpage)]
+            r'\d+,(\d+),(\d+),"(https?://[^.]+\.googleusercontent\.com.*?)"', webpage)]
         self._sort_formats(formats)
 
         return {
diff --git a/youtube_dl/extractor/hrti.py b/youtube_dl/extractor/hrti.py
index 4f0369433..7cef5f6ce 100644
--- a/youtube_dl/extractor/hrti.py
+++ b/youtube_dl/extractor/hrti.py
@@ -173,7 +173,7 @@ class HRTiIE(HRTiBaseIE):
 
 
 class HRTiPlaylistIE(HRTiBaseIE):
-    _VALID_URL = r'https?://hrti.hrt.hr/#/video/list/category/(?P<id>[0-9]+)/(?P<display_id>[^/]+)?'
+    _VALID_URL = r'https?://hrti\.hrt\.hr/#/video/list/category/(?P<id>[0-9]+)/(?P<display_id>[^/]+)?'
     _TESTS = [{
         'url': 'https://hrti.hrt.hr/#/video/list/category/212/ekumena',
         'info_dict': {
diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py
index c1367cf51..a96ea8010 100644
--- a/youtube_dl/extractor/ign.py
+++ b/youtube_dl/extractor/ign.py
@@ -203,7 +203,7 @@ class PCMagIE(IGNIE):
     _VALID_URL = r'https?://(?:www\.)?pcmag\.com/(?P<type>videos|article2)(/.+)?/(?P<name_or_id>.+)'
     IE_NAME = 'pcmag'
 
-    _EMBED_RE = r'iframe.setAttribute\("src",\s*__util.objToUrlString\("http://widgets\.ign\.com/video/embed/content.html?[^"]*url=([^"]+)["&]'
+    _EMBED_RE = r'iframe\.setAttribute\("src",\s*__util.objToUrlString\("http://widgets\.ign\.com/video/embed/content\.html?[^"]*url=([^"]+)["&]'
 
     _TESTS = [{
         'url': 'http://www.pcmag.com/videos/2015/01/06/010615-whats-new-now-is-gogo-snooping-on-your-data',
diff --git a/youtube_dl/extractor/infoq.py b/youtube_dl/extractor/infoq.py
index fe425e786..57c9b0cc4 100644
--- a/youtube_dl/extractor/infoq.py
+++ b/youtube_dl/extractor/infoq.py
@@ -69,9 +69,9 @@ class InfoQIE(BokeCCBaseIE):
         }]
 
     def _extract_cookies(self, webpage):
-        policy = self._search_regex(r'InfoQConstants.scp\s*=\s*\'([^\']+)\'', webpage, 'policy')
-        signature = self._search_regex(r'InfoQConstants.scs\s*=\s*\'([^\']+)\'', webpage, 'signature')
-        key_pair_id = self._search_regex(r'InfoQConstants.sck\s*=\s*\'([^\']+)\'', webpage, 'key-pair-id')
+        policy = self._search_regex(r'InfoQConstants\.scp\s*=\s*\'([^\']+)\'', webpage, 'policy')
+        signature = self._search_regex(r'InfoQConstants\.scs\s*=\s*\'([^\']+)\'', webpage, 'signature')
+        key_pair_id = self._search_regex(r'InfoQConstants\.sck\s*=\s*\'([^\']+)\'', webpage, 'key-pair-id')
         return 'CloudFront-Policy=%s; CloudFront-Signature=%s; CloudFront-Key-Pair-Id=%s' % (
             policy, signature, key_pair_id)
 
diff --git a/youtube_dl/extractor/jeuxvideo.py b/youtube_dl/extractor/jeuxvideo.py
index 1a4227f6b..e9f4ed738 100644
--- a/youtube_dl/extractor/jeuxvideo.py
+++ b/youtube_dl/extractor/jeuxvideo.py
@@ -30,7 +30,7 @@ class JeuxVideoIE(InfoExtractor):
         webpage = self._download_webpage(url, title)
         title = self._html_search_meta('name', webpage) or self._og_search_title(webpage)
         config_url = self._html_search_regex(
-            r'data-src(?:set-video)?="(/contenu/medias/video.php.*?)"',
+            r'data-src(?:set-video)?="(/contenu/medias/video\.php.*?)"',
             webpage, 'config URL')
         config_url = 'http://www.jeuxvideo.com' + config_url
 
diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 7f946c6ed..317ebbc4e 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -338,7 +338,7 @@ class LivestreamOriginalIE(InfoExtractor):
                 info = {
                     'title': self._og_search_title(webpage),
                     'description': self._og_search_description(webpage),
-                    'thumbnail': self._search_regex(r'channelLogo.src\s*=\s*"([^"]+)"', webpage, 'thumbnail', None),
+                    'thumbnail': self._search_regex(r'channelLogo\.src\s*=\s*"([^"]+)"', webpage, 'thumbnail', None),
                 }
             video_data = self._download_json(stream_url, content_id)
             is_live = video_data.get('isLive')
diff --git a/youtube_dl/extractor/makertv.py b/youtube_dl/extractor/makertv.py
index 3c34d4604..8eda69cfc 100644
--- a/youtube_dl/extractor/makertv.py
+++ b/youtube_dl/extractor/makertv.py
@@ -5,7 +5,7 @@ from .common import InfoExtractor
 
 
 class MakerTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www\.)?maker\.tv/(?:[^/]+/)*video|makerplayer.com/embed/maker)/(?P<id>[a-zA-Z0-9]{12})'
+    _VALID_URL = r'https?://(?:(?:www\.)?maker\.tv/(?:[^/]+/)*video|makerplayer\.com/embed/maker)/(?P<id>[a-zA-Z0-9]{12})'
     _TEST = {
         'url': 'http://www.maker.tv/video/Fh3QgymL9gsc',
         'md5': 'ca237a53a8eb20b6dc5bd60564d4ab3e',
diff --git a/youtube_dl/extractor/mangomolo.py b/youtube_dl/extractor/mangomolo.py
index 1885ac7df..dbd761a67 100644
--- a/youtube_dl/extractor/mangomolo.py
+++ b/youtube_dl/extractor/mangomolo.py
@@ -22,7 +22,7 @@ class MangomoloBaseIE(InfoExtractor):
 
         format_url = self._html_search_regex(
             [
-                r'file\s*:\s*"(https?://[^"]+?/playlist.m3u8)',
+                r'file\s*:\s*"(https?://[^"]+?/playlist\.m3u8)',
                 r'<a[^>]+href="(rtsp://[^"]+)"'
             ], webpage, 'format url')
         formats = self._extract_wowza_formats(
diff --git a/youtube_dl/extractor/meipai.py b/youtube_dl/extractor/meipai.py
index c8eacb4f4..2445b8b39 100644
--- a/youtube_dl/extractor/meipai.py
+++ b/youtube_dl/extractor/meipai.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 class MeipaiIE(InfoExtractor):
     IE_DESC = '美拍'
-    _VALID_URL = r'https?://(?:www\.)?meipai.com/media/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?meipai\.com/media/(?P<id>[0-9]+)'
     _TESTS = [{
         # regular uploaded video
         'url': 'http://www.meipai.com/media/531697625',
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 25af5ddfd..1154a3536 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -258,7 +258,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
 
         if mgid is None or ':' not in mgid:
             mgid = self._search_regex(
-                [r'data-mgid="(.*?)"', r'swfobject.embedSWF\(".*?(mgid:.*?)"'],
+                [r'data-mgid="(.*?)"', r'swfobject\.embedSWF\(".*?(mgid:.*?)"'],
                 webpage, 'mgid', default=None)
 
         if not mgid:
diff --git a/youtube_dl/extractor/myvideo.py b/youtube_dl/extractor/myvideo.py
index 6bb64eb63..367e811db 100644
--- a/youtube_dl/extractor/myvideo.py
+++ b/youtube_dl/extractor/myvideo.py
@@ -160,7 +160,7 @@ class MyVideoIE(InfoExtractor):
         else:
             video_playpath = ''
 
-        video_swfobj = self._search_regex(r'swfobject.embedSWF\(\'(.+?)\'', webpage, 'swfobj')
+        video_swfobj = self._search_regex(r'swfobject\.embedSWF\(\'(.+?)\'', webpage, 'swfobj')
         video_swfobj = compat_urllib_parse_unquote(video_swfobj)
 
         video_title = self._html_search_regex("<h1(?: class='globalHd')?>(.*?)</h1>",
diff --git a/youtube_dl/extractor/nationalgeographic.py b/youtube_dl/extractor/nationalgeographic.py
index b91d86528..9e8d28f48 100644
--- a/youtube_dl/extractor/nationalgeographic.py
+++ b/youtube_dl/extractor/nationalgeographic.py
@@ -111,7 +111,7 @@ class NationalGeographicIE(ThePlatformIE, AdobePassIE):
         release_url = self._search_regex(
             r'video_auth_playlist_url\s*=\s*"([^"]+)"',
             webpage, 'release url')
-        theplatform_path = self._search_regex(r'https?://link.theplatform.com/s/([^?]+)', release_url, 'theplatform path')
+        theplatform_path = self._search_regex(r'https?://link\.theplatform\.com/s/([^?]+)', release_url, 'theplatform path')
         video_id = theplatform_path.split('/')[-1]
         query = {
             'mbr': 'true',
diff --git a/youtube_dl/extractor/naver.py b/youtube_dl/extractor/naver.py
index e8131333f..2047d4402 100644
--- a/youtube_dl/extractor/naver.py
+++ b/youtube_dl/extractor/naver.py
@@ -43,7 +43,7 @@ class NaverIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        m_id = re.search(r'var rmcPlayer = new nhn.rmcnmv.RMCVideoPlayer\("(.+?)", "(.+?)"',
+        m_id = re.search(r'var rmcPlayer = new nhn\.rmcnmv\.RMCVideoPlayer\("(.+?)", "(.+?)"',
                          webpage)
         if m_id is None:
             error = self._html_search_regex(
diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index fa4ef20c5..b8fe24407 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -469,7 +469,7 @@ class SchoolTVIE(NPODataMidEmbedIE):
 
 class HetKlokhuisIE(NPODataMidEmbedIE):
     IE_NAME = 'hetklokhuis'
-    _VALID_URL = r'https?://(?:www\.)?hetklokhuis.nl/[^/]+/\d+/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?hetklokhuis\.nl/[^/]+/\d+/(?P<id>[^/?#&]+)'
 
     _TEST = {
         'url': 'http://hetklokhuis.nl/tv-uitzending/3471/Zwaartekrachtsgolven',
diff --git a/youtube_dl/extractor/ruhd.py b/youtube_dl/extractor/ruhd.py
index 2b830cf47..3c8053a26 100644
--- a/youtube_dl/extractor/ruhd.py
+++ b/youtube_dl/extractor/ruhd.py
@@ -25,7 +25,7 @@ class RUHDIE(InfoExtractor):
         video_url = self._html_search_regex(
             r'<param name="src" value="([^"]+)"', webpage, 'video url')
         title = self._html_search_regex(
-            r'<title>([^<]+)&nbsp;&nbsp; RUHD.ru - Видео Высокого качества №1 в России!</title>',
+            r'<title>([^<]+)&nbsp;&nbsp; RUHD\.ru - Видео Высокого качества №1 в России!</title>',
             webpage, 'title')
         description = self._html_search_regex(
             r'(?s)<div id="longdesc">(.+?)<span id="showlink">',
diff --git a/youtube_dl/extractor/stanfordoc.py b/youtube_dl/extractor/stanfordoc.py
index cce65fb10..ae3dd1380 100644
--- a/youtube_dl/extractor/stanfordoc.py
+++ b/youtube_dl/extractor/stanfordoc.py
@@ -66,7 +66,7 @@ class StanfordOpenClassroomIE(InfoExtractor):
                 r'(?s)<description>([^<]+)</description>',
                 coursepage, 'description', fatal=False)
 
-            links = orderedSet(re.findall(r'<a href="(VideoPage.php\?[^"]+)">', coursepage))
+            links = orderedSet(re.findall(r'<a href="(VideoPage\.php\?[^"]+)">', coursepage))
             info['entries'] = [self.url_result(
                 'http://openclassroom.stanford.edu/MainFolder/%s' % unescapeHTML(l)
             ) for l in links]
@@ -84,7 +84,7 @@ class StanfordOpenClassroomIE(InfoExtractor):
             rootpage = self._download_webpage(rootURL, info['id'],
                                               errnote='Unable to download course info page')
 
-            links = orderedSet(re.findall(r'<a href="(CoursePage.php\?[^"]+)">', rootpage))
+            links = orderedSet(re.findall(r'<a href="(CoursePage\.php\?[^"]+)">', rootpage))
             info['entries'] = [self.url_result(
                 'http://openclassroom.stanford.edu/MainFolder/%s' % unescapeHTML(l)
             ) for l in links]
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index de236bbba..b1a985ff6 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -216,7 +216,7 @@ class ThePlatformIE(ThePlatformBaseIE, AdobePassIE):
         def hex_to_bytes(hex):
             return binascii.a2b_hex(hex.encode('ascii'))
 
-        relative_path = re.match(r'https?://link.theplatform.com/s/([^?]+)', url).group(1)
+        relative_path = re.match(r'https?://link\.theplatform\.com/s/([^?]+)', url).group(1)
         clear_text = hex_to_bytes(flags + expiration_date + str_to_hex(relative_path))
         checksum = hmac.new(sig_key.encode('ascii'), clear_text, hashlib.sha1).hexdigest()
         sig = flags + expiration_date + checksum + str_to_hex(sig_secret)
diff --git a/youtube_dl/extractor/thisav.py b/youtube_dl/extractor/thisav.py
index 33683b139..dc3dd03c8 100644
--- a/youtube_dl/extractor/thisav.py
+++ b/youtube_dl/extractor/thisav.py
@@ -57,10 +57,10 @@ class ThisAVIE(InfoExtractor):
                 info_dict = self._extract_jwplayer_data(
                     webpage, video_id, require_title=False)
         uploader = self._html_search_regex(
-            r': <a href="http://www.thisav.com/user/[0-9]+/(?:[^"]+)">([^<]+)</a>',
+            r': <a href="http://www\.thisav\.com/user/[0-9]+/(?:[^"]+)">([^<]+)</a>',
             webpage, 'uploader name', fatal=False)
         uploader_id = self._html_search_regex(
-            r': <a href="http://www.thisav.com/user/[0-9]+/([^"]+)">(?:[^<]+)</a>',
+            r': <a href="http://www\.thisav\.com/user/[0-9]+/([^"]+)">(?:[^<]+)</a>',
             webpage, 'uploader id', fatal=False)
 
         info_dict.update({
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 0df3ad7c7..1b0b96371 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -174,7 +174,7 @@ class TwitterCardIE(TwitterBaseIE):
         webpage = self._download_webpage(url, video_id)
 
         iframe_url = self._html_search_regex(
-            r'<iframe[^>]+src="((?:https?:)?//(?:www.youtube.com/embed/[^"]+|(?:www\.)?vine\.co/v/\w+/card))"',
+            r'<iframe[^>]+src="((?:https?:)?//(?:www\.youtube\.com/embed/[^"]+|(?:www\.)?vine\.co/v/\w+/card))"',
             webpage, 'video iframe', default=None)
         if iframe_url:
             return self.url_result(iframe_url)
diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index b8b8bf979..bcc28693a 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -198,7 +198,7 @@ class ViceShowIE(InfoExtractor):
 
 class ViceArticleIE(InfoExtractor):
     IE_NAME = 'vice:article'
-    _VALID_URL = r'https://www.vice.com/[^/]+/article/(?P<id>[^?#]+)'
+    _VALID_URL = r'https://www\.vice\.com/[^/]+/article/(?P<id>[^?#]+)'
 
     _TESTS = [{
         'url': 'https://www.vice.com/en_us/article/on-set-with-the-woman-making-mormon-porn-in-utah',
diff --git a/youtube_dl/extractor/videopremium.py b/youtube_dl/extractor/videopremium.py
index 5de8273c3..cf690d7b0 100644
--- a/youtube_dl/extractor/videopremium.py
+++ b/youtube_dl/extractor/videopremium.py
@@ -26,7 +26,7 @@ class VideoPremiumIE(InfoExtractor):
         webpage_url = 'http://videopremium.tv/' + video_id
         webpage = self._download_webpage(webpage_url, video_id)
 
-        if re.match(r'^<html><head><script[^>]*>window.location\s*=', webpage):
+        if re.match(r'^<html><head><script[^>]*>window\.location\s*=', webpage):
             # Download again, we need a cookie
             webpage = self._download_webpage(
                 webpage_url, video_id,
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index edd8713b7..54f5d7279 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1683,7 +1683,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         video_uploader_id = None
         video_uploader_url = None
         mobj = re.search(
-            r'<link itemprop="url" href="(?P<uploader_url>https?://www.youtube.com/(?:user|channel)/(?P<uploader_id>[^"]+))">',
+            r'<link itemprop="url" href="(?P<uploader_url>https?://www\.youtube\.com/(?:user|channel)/(?P<uploader_id>[^"]+))">',
             video_webpage)
         if mobj is not None:
             video_uploader_id = mobj.group('uploader_id')

From ae5af89079558e0e128dc4d7f033459e68ad81e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Oct 2017 23:52:39 +0700
Subject: [PATCH 1400/1696] [hrti:playlist] Relax _VALID_URL

---
 youtube_dl/extractor/hrti.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/hrti.py b/youtube_dl/extractor/hrti.py
index 7cef5f6ce..6424d34ac 100644
--- a/youtube_dl/extractor/hrti.py
+++ b/youtube_dl/extractor/hrti.py
@@ -173,7 +173,7 @@ class HRTiIE(HRTiBaseIE):
 
 
 class HRTiPlaylistIE(HRTiBaseIE):
-    _VALID_URL = r'https?://hrti\.hrt\.hr/#/video/list/category/(?P<id>[0-9]+)/(?P<display_id>[^/]+)?'
+    _VALID_URL = r'https?://hrti\.hrt\.hr/(?:#/)?video/list/category/(?P<id>[0-9]+)/(?P<display_id>[^/]+)?'
     _TESTS = [{
         'url': 'https://hrti.hrt.hr/#/video/list/category/212/ekumena',
         'info_dict': {
@@ -185,6 +185,9 @@ class HRTiPlaylistIE(HRTiBaseIE):
     }, {
         'url': 'https://hrti.hrt.hr/#/video/list/category/212/',
         'only_matching': True,
+    }, {
+        'url': 'https://hrti.hrt.hr/video/list/category/212/ekumena',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 9e71f88105e546573b8615d49f9d0c064496d6e6 Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <infinisil@icloud.com>
Date: Mon, 9 Oct 2017 22:48:26 +0200
Subject: [PATCH 1401/1696] [vvvvid] Fix typo

---
 youtube_dl/extractor/vvvvid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vvvvid.py b/youtube_dl/extractor/vvvvid.py
index d44ec85fd..656a4b9e5 100644
--- a/youtube_dl/extractor/vvvvid.py
+++ b/youtube_dl/extractor/vvvvid.py
@@ -133,7 +133,7 @@ class VVVVIDIE(InfoExtractor):
             'season_id': season_id,
             'season_number': video_data.get('season_number'),
             'episode_id': str_or_none(video_data.get('id')),
-            'epidode_number': int_or_none(video_data.get('number')),
+            'episode_number': int_or_none(video_data.get('number')),
             'episode_title': video_data['title'],
             'view_count': int_or_none(video_data.get('views')),
             'like_count': int_or_none(video_data.get('video_likes')),

From 01c742ecd09be734ca4f3db08aba73424683ac1b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 10 Oct 2017 23:20:38 +0800
Subject: [PATCH 1402/1696] [facebook] Support thumbnails (closes #14416)

---
 ChangeLog                        |  6 ++++++
 youtube_dl/extractor/facebook.py | 18 ++++++++++++++----
 2 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 2feb3dc2d..2684cbcdd 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
++ [facebook] Support thumbnails (#14416)
+
+
 version 2017.10.07
 
 Core
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 4b3f6cc86..220ada3a6 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -67,9 +67,9 @@ class FacebookIE(InfoExtractor):
             'uploader': 'Tennis on Facebook',
             'upload_date': '20140908',
             'timestamp': 1410199200,
-        }
+        },
+        'skip': 'Requires logging in',
     }, {
-        'note': 'Video without discernible title',
         'url': 'https://www.facebook.com/video.php?v=274175099429670',
         'info_dict': {
             'id': '274175099429670',
@@ -78,6 +78,7 @@ class FacebookIE(InfoExtractor):
             'uploader': 'Asif Nawab Butt',
             'upload_date': '20140506',
             'timestamp': 1399398998,
+            'thumbnail': r're:^https?://.*',
         },
         'expected_warnings': [
             'title'
@@ -94,6 +95,7 @@ class FacebookIE(InfoExtractor):
             'upload_date': '20160110',
             'timestamp': 1452431627,
         },
+        'skip': 'Requires logging in',
     }, {
         'url': 'https://www.facebook.com/maxlayn/posts/10153807558977570',
         'md5': '037b1fa7f3c2d02b7a0d7bc16031ecc6',
@@ -121,7 +123,11 @@ class FacebookIE(InfoExtractor):
         'info_dict': {
             'id': '10153664894881749',
             'ext': 'mp4',
-            'title': 'Facebook video #10153664894881749',
+            'title': 'Average time to confirm recent Supreme Court nominees: 67 days Longest it\'s t...',
+            'thumbnail': r're:^https?://.*',
+            'timestamp': 1456259628,
+            'upload_date': '20160223',
+            'uploader': 'Barack Obama',
         },
     }, {
         # have 1080P, but only up to 720p in swf params
@@ -130,10 +136,11 @@ class FacebookIE(InfoExtractor):
         'info_dict': {
             'id': '10155529876156509',
             'ext': 'mp4',
-            'title': 'Holocaust survivor becomes US citizen',
+            'title': 'She survived the holocaust — and years later, she’s getting her citizenship s...',
             'timestamp': 1477818095,
             'upload_date': '20161030',
             'uploader': 'CNN',
+            'thumbnail': r're:^https?://.*',
         },
     }, {
         # bigPipe.onPageletArrive ... onPageletArrive pagelet_group_mall
@@ -158,6 +165,7 @@ class FacebookIE(InfoExtractor):
             'timestamp': 1477305000,
             'upload_date': '20161024',
             'uploader': 'La Guía Del Varón',
+            'thumbnail': r're:^https?://.*',
         },
         'params': {
             'skip_download': True,
@@ -376,6 +384,7 @@ class FacebookIE(InfoExtractor):
         timestamp = int_or_none(self._search_regex(
             r'<abbr[^>]+data-utime=["\'](\d+)', webpage,
             'timestamp', default=None))
+        thumbnail = self._og_search_thumbnail(webpage)
 
         info_dict = {
             'id': video_id,
@@ -383,6 +392,7 @@ class FacebookIE(InfoExtractor):
             'formats': formats,
             'uploader': uploader,
             'timestamp': timestamp,
+            'thumbnail': thumbnail,
         }
 
         return webpage, info_dict

From d0f2d6411406a35c9593bbb85375c2d7f8300c77 Mon Sep 17 00:00:00 2001
From: Jakub Wilk <jwilk@jwilk.net>
Date: Tue, 10 Oct 2017 18:45:10 +0200
Subject: [PATCH 1403/1696] [slideslive] Add extractor (closes #2680)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/slideslive.py | 34 ++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)
 create mode 100644 youtube_dl/extractor/slideslive.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 24e9acda6..d0f71aecd 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -943,6 +943,7 @@ from .skynewsarabia import (
 )
 from .skysports import SkySportsIE
 from .slideshare import SlideshareIE
+from .slideslive import SlidesLiveIE
 from .slutload import SlutloadIE
 from .smotri import (
     SmotriIE,
diff --git a/youtube_dl/extractor/slideslive.py b/youtube_dl/extractor/slideslive.py
new file mode 100644
index 000000000..104576033
--- /dev/null
+++ b/youtube_dl/extractor/slideslive.py
@@ -0,0 +1,34 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import ExtractorError
+
+
+class SlidesLiveIE(InfoExtractor):
+    _VALID_URL = r'https?://slideslive\.com/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'https://slideslive.com/38902413/gcc-ia16-backend',
+        'md5': 'b29fcd6c6952d0c79c5079b0e7a07e6f',
+        'info_dict': {
+            'id': 'LMtgR8ba0b0',
+            'ext': 'mp4',
+            'title': '38902413: external video',
+            'description': '3890241320170925-9-1yd6ech.mp4',
+            'uploader': 'SlidesLive Administrator',
+            'uploader_id': 'UC62SdArr41t_-_fX40QCLRw',
+            'upload_date': '20170925',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_data = self._download_json(
+            url, video_id, headers={'Accept': 'application/json'})
+        service_name = video_data['video_service_name']
+        if service_name == 'YOUTUBE':
+            yt_video_id = video_data['video_service_id']
+            return self.url_result(yt_video_id, 'Youtube', video_id=yt_video_id)
+        else:
+            raise ExtractorError(
+                'Unsupported service name: {0}'.format(service_name), expected=True)

From 04af3aca049588b6b3d4d4b57ee47224fdeee90f Mon Sep 17 00:00:00 2001
From: Khang Nguyen <khang.neon.1997@gmail.com>
Date: Thu, 5 Oct 2017 21:37:18 +0700
Subject: [PATCH 1404/1696] Remove YoutubeSharedVideoIE 
 https://github.com/rg3/youtube-dl/issues/14303

---
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/youtube.py    | 33 ------------------------------
 2 files changed, 34 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d0f71aecd..d96eafbc3 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1346,7 +1346,6 @@ from .youtube import (
     YoutubeSearchDateIE,
     YoutubeSearchIE,
     YoutubeSearchURLIE,
-    YoutubeSharedVideoIE,
     YoutubeShowIE,
     YoutubeSubscriptionsIE,
     YoutubeTruncatedIDIE,
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 54f5d7279..6e2d57d6a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2040,39 +2040,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         }
 
 
-class YoutubeSharedVideoIE(InfoExtractor):
-    _VALID_URL = r'(?:https?:)?//(?:www\.)?youtube\.com/shared\?.*\bci=(?P<id>[0-9A-Za-z_-]{11})'
-    IE_NAME = 'youtube:shared'
-
-    _TEST = {
-        'url': 'https://www.youtube.com/shared?ci=1nEzmT-M4fU',
-        'info_dict': {
-            'id': 'uPDB5I9wfp8',
-            'ext': 'webm',
-            'title': 'Pocoyo: 90 minutos de episódios completos Português para crianças - PARTE 3',
-            'description': 'md5:d9e4d9346a2dfff4c7dc4c8cec0f546d',
-            'upload_date': '20160219',
-            'uploader': 'Pocoyo - Português (BR)',
-            'uploader_id': 'PocoyoBrazil',
-        },
-        'add_ie': ['Youtube'],
-        'params': {
-            # There are already too many Youtube downloads
-            'skip_download': True,
-        },
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        real_video_id = self._html_search_meta(
-            'videoId', webpage, 'YouTube video id', fatal=True)
-
-        return self.url_result(real_video_id, YoutubeIE.ie_key())
-
-
 class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
     IE_DESC = 'YouTube.com playlists'
     _VALID_URL = r"""(?x)(?:

From dfc80bdd2e4ef3d30f161a93f99f3050537944ab Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 11 Oct 2017 02:03:00 +0800
Subject: [PATCH 1405/1696] [ChangeLog] Update after #14420

---
 ChangeLog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChangeLog b/ChangeLog
index 2684cbcdd..c541c4f62 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+- [youtube:shared] Removed extractor (#14420)
 + [facebook] Support thumbnails (#14416)
 
 

From cdab1df91242fb617b09a31c023822ef31ea37b8 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 11 Oct 2017 10:04:46 +0000
Subject: [PATCH 1406/1696] [afreecatv] remove AfreecaTVGlobalIE

the website now show this message
> Global AfreecaTV will be merged and integrated on July 20th, 2017.
Every user around the world are now able to interact with one another on
www.afreecatv.com!
---
 youtube_dl/extractor/afreecatv.py  | 104 -----------------------------
 youtube_dl/extractor/extractors.py |   5 +-
 2 files changed, 1 insertion(+), 108 deletions(-)

diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py
index 2c58f4617..e6513c7a4 100644
--- a/youtube_dl/extractor/afreecatv.py
+++ b/youtube_dl/extractor/afreecatv.py
@@ -271,107 +271,3 @@ class AfreecaTVIE(InfoExtractor):
             })
 
         return info
-
-
-class AfreecaTVGlobalIE(AfreecaTVIE):
-    IE_NAME = 'afreecatv:global'
-    _VALID_URL = r'https?://(?:www\.)?afreeca\.tv/(?P<channel_id>\d+)(?:/v/(?P<video_id>\d+))?'
-    _TESTS = [{
-        'url': 'http://afreeca.tv/36853014/v/58301',
-        'info_dict': {
-            'id': '58301',
-            'title': 'tryhard top100',
-            'uploader_id': '36853014',
-            'uploader': 'makgi Hearthstone Live!',
-        },
-        'playlist_count': 3,
-    }]
-
-    def _real_extract(self, url):
-        channel_id, video_id = re.match(self._VALID_URL, url).groups()
-        video_type = 'video' if video_id else 'live'
-        query = {
-            'pt': 'view',
-            'bid': channel_id,
-        }
-        if video_id:
-            query['vno'] = video_id
-        video_data = self._download_json(
-            'http://api.afreeca.tv/%s/view_%s.php' % (video_type, video_type),
-            video_id or channel_id, query=query)['channel']
-
-        if video_data.get('result') != 1:
-            raise ExtractorError('%s said: %s' % (self.IE_NAME, video_data['remsg']))
-
-        title = video_data['title']
-
-        info = {
-            'thumbnail': video_data.get('thumb'),
-            'view_count': int_or_none(video_data.get('vcnt')),
-            'age_limit': int_or_none(video_data.get('grade')),
-            'uploader_id': channel_id,
-            'uploader': video_data.get('cname'),
-        }
-
-        if video_id:
-            entries = []
-            for i, f in enumerate(video_data.get('flist', [])):
-                video_key = self.parse_video_key(f.get('key', ''))
-                f_url = f.get('file')
-                if not video_key or not f_url:
-                    continue
-                entries.append({
-                    'id': '%s_%s' % (video_id, video_key.get('part', i + 1)),
-                    'title': title,
-                    'upload_date': video_key.get('upload_date'),
-                    'duration': int_or_none(f.get('length')),
-                    'url': f_url,
-                    'protocol': 'm3u8_native',
-                    'ext': 'mp4',
-                })
-
-            info.update({
-                'id': video_id,
-                'title': title,
-                'duration': int_or_none(video_data.get('length')),
-            })
-            if len(entries) > 1:
-                info['_type'] = 'multi_video'
-                info['entries'] = entries
-            elif len(entries) == 1:
-                i = entries[0].copy()
-                i.update(info)
-                info = i
-        else:
-            formats = []
-            for s in video_data.get('strm', []):
-                s_url = s.get('purl')
-                if not s_url:
-                    continue
-                stype = s.get('stype')
-                if stype == 'HLS':
-                    formats.extend(self._extract_m3u8_formats(
-                        s_url, channel_id, 'mp4', m3u8_id=stype, fatal=False))
-                elif stype == 'RTMP':
-                    format_id = [stype]
-                    label = s.get('label')
-                    if label:
-                        format_id.append(label)
-                    formats.append({
-                        'format_id': '-'.join(format_id),
-                        'url': s_url,
-                        'tbr': int_or_none(s.get('bps')),
-                        'height': int_or_none(s.get('brt')),
-                        'ext': 'flv',
-                        'rtmp_live': True,
-                    })
-            self._sort_formats(formats)
-
-            info.update({
-                'id': channel_id,
-                'title': self._live_title(title),
-                'is_live': True,
-                'formats': formats,
-            })
-
-        return info
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d96eafbc3..a363d95bf 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -31,10 +31,7 @@ from .aenetworks import (
     AENetworksIE,
     HistoryTopicIE,
 )
-from .afreecatv import (
-    AfreecaTVIE,
-    AfreecaTVGlobalIE,
-)
+from .afreecatv import AfreecaTVIE
 from .airmozilla import AirMozillaIE
 from .aljazeera import AlJazeeraIE
 from .alphaporno import AlphaPornoIE

From 4fe4bda287f4b506850f0baa6ff86445423751e7 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 11 Oct 2017 11:36:05 +0000
Subject: [PATCH 1407/1696] [tubitv] add support for new url format(fixes
 #14460)

---
 youtube_dl/extractor/tubitv.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tubitv.py b/youtube_dl/extractor/tubitv.py
index c44018aec..36f6c1673 100644
--- a/youtube_dl/extractor/tubitv.py
+++ b/youtube_dl/extractor/tubitv.py
@@ -13,11 +13,11 @@ from ..utils import (
 
 
 class TubiTvIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tubitv\.com/video/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?tubitv\.com/(?:video|movies|tv-shows)/(?P<id>[0-9]+)'
     _LOGIN_URL = 'http://tubitv.com/login'
     _NETRC_MACHINE = 'tubitv'
     _GEO_COUNTRIES = ['US']
-    _TEST = {
+    _TESTS = [{
         'url': 'http://tubitv.com/video/283829/the_comedian_at_the_friday',
         'md5': '43ac06be9326f41912dc64ccf7a80320',
         'info_dict': {
@@ -27,7 +27,13 @@ class TubiTvIE(InfoExtractor):
             'description': 'A stand up comedian is forced to look at the decisions in his life while on a one week trip to the west coast.',
             'uploader_id': 'bc168bee0d18dd1cb3b86c68706ab434',
         },
-    }
+    }, {
+        'url': 'http://tubitv.com/tv-shows/321886/s01_e01_on_nom_stories',
+        'only_matching': True,
+    }, {
+        'url': 'http://tubitv.com/movies/383676/tracker',
+        'only_matching': True,
+    }]
 
     def _login(self):
         (username, password) = self._get_login_info()

From 5fe75f976f6c8da76bfea68e073e5f35c4300442 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 11 Oct 2017 14:14:51 +0000
Subject: [PATCH 1408/1696] [tva] fix extraction(fixes #14328)

---
 youtube_dl/extractor/tva.py | 48 ++++++++++++++++++-------------------
 1 file changed, 23 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/tva.py b/youtube_dl/extractor/tva.py
index 3ced098f9..b57abeaa4 100644
--- a/youtube_dl/extractor/tva.py
+++ b/youtube_dl/extractor/tva.py
@@ -3,52 +3,50 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
-    int_or_none,
-    parse_iso8601,
+    float_or_none,
     smuggle_url,
 )
 
 
 class TVAIE(InfoExtractor):
-    _VALID_URL = r'https?://videos\.tva\.ca/episode/(?P<id>\d+)'
+    _VALID_URL = r'https?://videos\.tva\.ca/details/_(?P<id>\d+)'
     _TEST = {
-        'url': 'http://videos.tva.ca/episode/85538',
+        'url': 'https://videos.tva.ca/details/_5596811470001',
         'info_dict': {
-            'id': '85538',
+            'id': '5596811470001',
             'ext': 'mp4',
-            'title': 'Épisode du 25 janvier 2017',
-            'description': 'md5:e9e7fb5532ab37984d2dc87229cadf98',
-            'upload_date': '20170126',
-            'timestamp': 1485442329,
+            'title': 'Un extrait de l\'épisode du dimanche 8 octobre 2017 !',
+            'uploader_id': '5481942443001',
+            'upload_date': '20171003',
+            'timestamp': 1507064617,
         },
         'params': {
             # m3u8 download
             'skip_download': True,
         }
     }
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/5481942443001/default_default/index.html?videoId=%s'
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         video_data = self._download_json(
-            "https://d18jmrhziuoi7p.cloudfront.net/isl/api/v1/dataservice/Items('%s')" % video_id,
-            video_id, query={
-                '$expand': 'Metadata,CustomId',
-                '$select': 'Metadata,Id,Title,ShortDescription,LongDescription,CreatedDate,CustomId,AverageUserRating,Categories,ShowName',
-                '$format': 'json',
+            'https://videos.tva.ca/proxy/item/_' + video_id, video_id, headers={
+                'Accept': 'application/json',
             })
-        metadata = video_data.get('Metadata', {})
+
+        def get_attribute(key):
+            for attribute in video_data.get('attributes', []):
+                if attribute.get('key') == key:
+                    return attribute.get('value')
+            return None
 
         return {
             '_type': 'url_transparent',
             'id': video_id,
-            'title': video_data['Title'],
-            'url': smuggle_url('ooyala:' + video_data['CustomId'], {'supportedformats': 'm3u8,hds'}),
-            'description': video_data.get('LongDescription') or video_data.get('ShortDescription'),
-            'series': video_data.get('ShowName'),
-            'episode': metadata.get('EpisodeTitle'),
-            'episode_number': int_or_none(metadata.get('EpisodeNumber')),
-            'categories': video_data.get('Categories'),
-            'average_rating': video_data.get('AverageUserRating'),
-            'timestamp': parse_iso8601(video_data.get('CreatedDate')),
-            'ie_key': 'Ooyala',
+            'title': get_attribute('title'),
+            'url': smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % video_id, {'geo_countries': ['CA']}),
+            'description': get_attribute('description'),
+            'thumbnail': get_attribute('image-background') or get_attribute('image-landscape'),
+            'duration': float_or_none(get_attribute('video-duration'), 1000),
+            'ie_key': 'BrightcoveNew',
         }

From 26bae2d96509b5c5ec80c27c1ea754b53c53818c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Oct 2017 21:59:30 +0700
Subject: [PATCH 1409/1696] [generic] Add support for channel9 embeds (closes
 #14469)

---
 youtube_dl/extractor/channel9.py | 6 ++++++
 youtube_dl/extractor/generic.py  | 6 ++++++
 2 files changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py
index e92894246..81108e704 100644
--- a/youtube_dl/extractor/channel9.py
+++ b/youtube_dl/extractor/channel9.py
@@ -81,6 +81,12 @@ class Channel9IE(InfoExtractor):
 
     _RSS_URL = 'http://channel9.msdn.com/%s/RSS'
 
+    @staticmethod
+    def _extract_urls(webpage):
+        return re.findall(
+            r'<iframe[^>]+src=["\'](https?://channel9\.msdn\.com/(?:[^/]+/)+)player\b',
+            webpage)
+
     def _extract_list(self, video_id, rss_url=None):
         if not rss_url:
             rss_url = self._RSS_URL % video_id
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 68b633839..6dab4c7f4 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -101,6 +101,7 @@ from .mediaset import MediasetIE
 from .joj import JojIE
 from .megaphone import MegaphoneIE
 from .vzaar import VzaarIE
+from .channel9 import Channel9IE
 
 
 class GenericIE(InfoExtractor):
@@ -2871,6 +2872,11 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 vzaar_urls, video_id, video_title, ie=VzaarIE.ie_key())
 
+        channel9_urls = Channel9IE._extract_urls(webpage)
+        if channel9_urls:
+            return self.playlist_from_matches(
+                channel9_urls, video_id, video_title, ie=Channel9IE.ie_key())
+
         def merge_dicts(dict1, dict2):
             merged = {}
             for k, v in dict1.items():

From 782195a9d417aab21cff4abe35ad9d705f4d8d83 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 11 Oct 2017 15:48:32 +0000
Subject: [PATCH 1410/1696] [once] add support for vmap urls

---
 youtube_dl/extractor/gamespot.py | 2 +-
 youtube_dl/extractor/once.py     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index 00d311158..02804d297 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -105,7 +105,7 @@ class GameSpotIE(OnceIE):
             onceux_url = self._parse_json(unescapeHTML(onceux_json), page_id).get('metadataUri')
             if onceux_url:
                 formats.extend(self._extract_once_formats(re.sub(
-                    r'https?://[^/]+', 'http://once.unicornmedia.com', onceux_url).replace('ads/vmap/', '')))
+                    r'https?://[^/]+', 'http://once.unicornmedia.com', onceux_url)))
 
         if not formats:
             for quality in ['sd', 'hd']:
diff --git a/youtube_dl/extractor/once.py b/youtube_dl/extractor/once.py
index 1bf96ea56..a637c8ecf 100644
--- a/youtube_dl/extractor/once.py
+++ b/youtube_dl/extractor/once.py
@@ -7,7 +7,7 @@ from .common import InfoExtractor
 
 
 class OnceIE(InfoExtractor):
-    _VALID_URL = r'https?://.+?\.unicornmedia\.com/now/[^/]+/[^/]+/(?P<domain_id>[^/]+)/(?P<application_id>[^/]+)/(?:[^/]+/)?(?P<media_item_id>[^/]+)/content\.(?:once|m3u8|mp4)'
+    _VALID_URL = r'https?://.+?\.unicornmedia\.com/now/(?:ads/vmap/)?[^/]+/[^/]+/(?P<domain_id>[^/]+)/(?P<application_id>[^/]+)/(?:[^/]+/)?(?P<media_item_id>[^/]+)/content\.(?:once|m3u8|mp4)'
     ADAPTIVE_URL_TEMPLATE = 'http://once.unicornmedia.com/now/master/playlist/%s/%s/%s/content.m3u8'
     PROGRESSIVE_URL_TEMPLATE = 'http://once.unicornmedia.com/now/media/progressive/%s/%s/%s/%s/content.mp4'
 

From 9e38dbb19ca6874c7350647ea2883d5bbb3b50a1 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 11 Oct 2017 15:50:00 +0000
Subject: [PATCH 1411/1696] [voxmedia] add support for recode.net(fixes #14173)

---
 youtube_dl/extractor/extractors.py |  5 ++-
 youtube_dl/extractor/voxmedia.py   | 66 ++++++++++++++++++++++++------
 2 files changed, 57 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index a363d95bf..5629c7623 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1244,7 +1244,10 @@ from .vodpl import VODPlIE
 from .vodplatform import VODPlatformIE
 from .voicerepublic import VoiceRepublicIE
 from .voot import VootIE
-from .voxmedia import VoxMediaIE
+from .voxmedia import (
+    VoxMediaVolumeIE,
+    VoxMediaIE,
+)
 from .vporn import VpornIE
 from .vrt import VRTIE
 from .vrak import VrakIE
diff --git a/youtube_dl/extractor/voxmedia.py b/youtube_dl/extractor/voxmedia.py
index f8e331493..c7a0a88fe 100644
--- a/youtube_dl/extractor/voxmedia.py
+++ b/youtube_dl/extractor/voxmedia.py
@@ -2,11 +2,44 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from .once import OnceIE
 from ..compat import compat_urllib_parse_unquote
+from ..utils import ExtractorError
+
+
+class VoxMediaVolumeIE(OnceIE):
+    _VALID_URL = r'https?://volume\.vox-cdn\.com/embed/(?P<id>[0-9a-f]{9})'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        video_data = self._parse_json(self._search_regex(
+            r'Volume\.createVideo\(({.+})\s*,\s*{.*}\s*,\s*\[.*\]\s*,\s*{.*}\);', webpage, 'video data'), video_id)
+        for provider_video_type in ('ooyala', 'youtube', 'brightcove'):
+            provider_video_id = video_data.get('%s_id' % provider_video_type)
+            if not provider_video_id:
+                continue
+            info = {
+                'id': video_id,
+                'title': video_data.get('title_short'),
+                'description': video_data.get('description_long') or video_data.get('description_short'),
+                'thumbnail': video_data.get('brightcove_thumbnail')
+            }
+            if provider_video_type == 'brightcove':
+                info['formats'] = self._extract_once_formats(provider_video_id)
+                self._sort_formats(info['formats'])
+            else:
+                info.update({
+                    '_type': 'url_transparent',
+                    'url': provider_video_id if provider_video_type == 'youtube' else '%s:%s' % (provider_video_type, provider_video_id),
+                    'ie_key': provider_video_type.capitalize(),
+                })
+            return info
+        raise ExtractorError('Unable to find provider video id')
 
 
 class VoxMediaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?:theverge|vox|sbnation|eater|polygon|curbed|racked)\.com/(?:[^/]+/)*(?P<id>[^/?]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:(?:theverge|vox|sbnation|eater|polygon|curbed|racked)\.com|recode\.net)/(?:[^/]+/)*(?P<id>[^/?]+)'
     _TESTS = [{
         'url': 'http://www.theverge.com/2014/6/27/5849272/material-world-how-google-discovered-what-software-is-made-of',
         'info_dict': {
@@ -31,6 +64,7 @@ class VoxMediaIE(InfoExtractor):
             'description': 'md5:87a51fe95ff8cea8b5bdb9ac7ae6a6af',
         },
         'add_ie': ['Ooyala'],
+        'skip': 'Video Not Found',
     }, {
         # volume embed
         'url': 'http://www.vox.com/2016/3/31/11336640/mississippi-lgbt-religious-freedom-bill',
@@ -84,6 +118,17 @@ class VoxMediaIE(InfoExtractor):
                 'description': 'md5:e02d56b026d51aa32c010676765a690d',
             },
         }],
+    }, {
+        # volume embed, Brightcove Once
+        'url': 'https://www.recode.net/2014/6/17/11628066/post-post-pc-ceo-the-full-code-conference-video-of-microsofts-satya',
+        'md5': '01571a896281f77dc06e084138987ea2',
+        'info_dict': {
+            'id': '1231c973d',
+            'ext': 'mp4',
+            'title': 'Post-Post-PC CEO: The Full Code Conference Video of Microsoft\'s Satya Nadella',
+            'description': 'The longtime veteran was chosen earlier this year as the software giant\'s third leader in its history.',
+        },
+        'add_ie': ['VoxMediaVolume'],
     }]
 
     def _real_extract(self, url):
@@ -91,9 +136,14 @@ class VoxMediaIE(InfoExtractor):
         webpage = compat_urllib_parse_unquote(self._download_webpage(url, display_id))
 
         def create_entry(provider_video_id, provider_video_type, title=None, description=None):
+            video_url = {
+                'youtube': '%s',
+                'ooyala': 'ooyala:%s',
+                'volume': 'http://volume.vox-cdn.com/embed/%s',
+            }[provider_video_type] % provider_video_id
             return {
                 '_type': 'url_transparent',
-                'url': provider_video_id if provider_video_type == 'youtube' else '%s:%s' % (provider_video_type, provider_video_id),
+                'url': video_url,
                 'title': title or self._og_search_title(webpage),
                 'description': description or self._og_search_description(webpage),
             }
@@ -124,17 +174,7 @@ class VoxMediaIE(InfoExtractor):
         volume_uuid = self._search_regex(
             r'data-volume-uuid="([^"]+)"', webpage, 'volume uuid', default=None)
         if volume_uuid:
-            volume_webpage = self._download_webpage(
-                'http://volume.vox-cdn.com/embed/%s' % volume_uuid, volume_uuid)
-            video_data = self._parse_json(self._search_regex(
-                r'Volume\.createVideo\(({.+})\s*,\s*{.*}\s*,\s*\[.*\]\s*,\s*{.*}\);', volume_webpage, 'video data'), volume_uuid)
-            for provider_video_type in ('ooyala', 'youtube'):
-                provider_video_id = video_data.get('%s_id' % provider_video_type)
-                if provider_video_id:
-                    description = video_data.get('description_long') or video_data.get('description_short')
-                    entries.append(create_entry(
-                        provider_video_id, provider_video_type, video_data.get('title_short'), description))
-                    break
+            entries.append(create_entry(volume_uuid, 'volume'))
 
         if len(entries) == 1:
             return entries[0]

From af0f74288dc1b46147bc8f6b5692d2a21c6e178b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Oct 2017 23:45:03 +0700
Subject: [PATCH 1412/1696] [YoutubeDL] Improve _default_format_spec (closes
 #14461)

---
 test/test_YoutubeDL.py  | 10 ++++++++--
 youtube_dl/YoutubeDL.py | 31 ++++++++++++++++++-------------
 2 files changed, 26 insertions(+), 15 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index db936bfb8..4af92fbd4 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -466,12 +466,18 @@ class TestFormatSelection(unittest.TestCase):
         ydl = YDL({'simulate': True})
         self.assertEqual(ydl._default_format_spec({}), 'bestvideo+bestaudio/best')
 
+        ydl = YDL({'is_live': True})
+        self.assertEqual(ydl._default_format_spec({}), 'best/bestvideo+bestaudio')
+
+        ydl = YDL({'simulate': True, 'is_live': True})
+        self.assertEqual(ydl._default_format_spec({}), 'bestvideo+bestaudio/best')
+
         ydl = YDL({'outtmpl': '-'})
-        self.assertEqual(ydl._default_format_spec({}), 'best')
+        self.assertEqual(ydl._default_format_spec({}), 'best/bestvideo+bestaudio')
 
         ydl = YDL({})
         self.assertEqual(ydl._default_format_spec({}, download=False), 'bestvideo+bestaudio/best')
-        self.assertEqual(ydl._default_format_spec({'is_live': True}), 'best')
+        self.assertEqual(ydl._default_format_spec({'is_live': True}), 'best/bestvideo+bestaudio')
 
 
 class TestYoutubeDL(unittest.TestCase):
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 855d6b8e5..342d6b47c 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1078,22 +1078,27 @@ class YoutubeDL(object):
         return _filter
 
     def _default_format_spec(self, info_dict, download=True):
-        req_format_list = []
 
-        def can_have_partial_formats():
-            if self.params.get('simulate', False):
-                return True
-            if not download:
-                return True
-            if self.params.get('outtmpl', DEFAULT_OUTTMPL) == '-':
-                return False
-            if info_dict.get('is_live'):
-                return False
+        def can_merge():
             merger = FFmpegMergerPP(self)
             return merger.available and merger.can_merge()
-        if can_have_partial_formats():
-            req_format_list.append('bestvideo+bestaudio')
-        req_format_list.append('best')
+
+        def prefer_best():
+            if self.params.get('simulate', False):
+                return False
+            if not download:
+                return False
+            if self.params.get('outtmpl', DEFAULT_OUTTMPL) == '-':
+                return True
+            if info_dict.get('is_live'):
+                return True
+            if not can_merge():
+                return True
+            return False
+
+        req_format_list = ['bestvideo+bestaudio', 'best']
+        if prefer_best():
+            req_format_list.reverse()
         return '/'.join(req_format_list)
 
     def build_format_selector(self, format_spec):

From 694b61545cc3fa37c31d5f62d26101fb2620a01d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 12 Oct 2017 00:41:20 +0700
Subject: [PATCH 1413/1696] [nexx] Add support for shortcuts and relax domain
 id extraction

---
 youtube_dl/extractor/nexx.py | 28 +++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/nexx.py b/youtube_dl/extractor/nexx.py
index d0235fdfe..071879ba4 100644
--- a/youtube_dl/extractor/nexx.py
+++ b/youtube_dl/extractor/nexx.py
@@ -18,7 +18,13 @@ from ..utils import (
 
 
 class NexxIE(InfoExtractor):
-    _VALID_URL = r'https?://api\.nexx(?:\.cloud|cdn\.com)/v3/(?P<domain_id>\d+)/videos/byid/(?P<id>\d+)'
+    _VALID_URL = r'''(?x)
+                        (?:
+                            https?://api\.nexx(?:\.cloud|cdn\.com)/v3/(?P<domain_id>\d+)/videos/byid/|
+                            nexx:(?P<domain_id_s>\d+):
+                        )
+                        (?P<id>\d+)
+                    '''
     _TESTS = [{
         # movie
         'url': 'https://api.nexx.cloud/v3/748/videos/byid/128907',
@@ -62,8 +68,18 @@ class NexxIE(InfoExtractor):
     }, {
         'url': 'https://api.nexxcdn.com/v3/748/videos/byid/128907',
         'only_matching': True,
+    }, {
+        'url': 'nexx:748:128907',
+        'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_domain_id(webpage):
+        mobj = re.search(
+            r'<script\b[^>]+\bsrc=["\'](?:https?:)?//require\.nexx(?:\.cloud|cdn\.com)/(?P<id>\d+)',
+            webpage)
+        return mobj.group('id') if mobj else None
+
     @staticmethod
     def _extract_urls(webpage):
         # Reference:
@@ -72,11 +88,8 @@ class NexxIE(InfoExtractor):
         entries = []
 
         # JavaScript Integration
-        mobj = re.search(
-            r'<script\b[^>]+\bsrc=["\']https?://require\.nexx(?:\.cloud|cdn\.com)/(?P<id>\d+)',
-            webpage)
-        if mobj:
-            domain_id = mobj.group('id')
+        domain_id = NexxIE._extract_domain_id(webpage)
+        if domain_id:
             for video_id in re.findall(
                     r'(?is)onPLAYReady.+?_play\.init\s*\(.+?\s*,\s*["\']?(\d+)',
                     webpage):
@@ -112,7 +125,8 @@ class NexxIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        domain_id, video_id = mobj.group('domain_id', 'id')
+        domain_id = mobj.group('domain_id') or mobj.group('domain_id_s')
+        video_id = mobj.group('id')
 
         # Reverse engineered from JS code (see getDeviceID function)
         device_id = '%d:%d:%d%d' % (

From ff3f1a62f087332fa6409b5cbc39871d49e74f37 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 12 Oct 2017 00:44:13 +0700
Subject: [PATCH 1414/1696] [funk] Add extractor (closes #14464)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/funk.py       | 43 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/generic.py    | 16 -----------
 3 files changed, 44 insertions(+), 16 deletions(-)
 create mode 100644 youtube_dl/extractor/funk.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 5629c7623..ecb33bc9e 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -381,6 +381,7 @@ from .freesound import FreesoundIE
 from .freespeech import FreespeechIE
 from .freshlive import FreshLiveIE
 from .funimation import FunimationIE
+from .funk import FunkIE
 from .funnyordie import FunnyOrDieIE
 from .fusion import FusionIE
 from .fxnetworks import FXNetworksIE
diff --git a/youtube_dl/extractor/funk.py b/youtube_dl/extractor/funk.py
new file mode 100644
index 000000000..ce5c67fbb
--- /dev/null
+++ b/youtube_dl/extractor/funk.py
@@ -0,0 +1,43 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from .nexx import NexxIE
+from ..utils import extract_attributes
+
+
+class FunkIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?funk\.net/(?:mix|channel)/(?:[^/]+/)*(?P<id>[^?/#]+)'
+    _TESTS = [{
+        'url': 'https://www.funk.net/mix/59d65d935f8b160001828b5b/0/59d517e741dca10001252574/',
+        'md5': '4d40974481fa3475f8bccfd20c5361f8',
+        'info_dict': {
+            'id': '716599',
+            'ext': 'mp4',
+            'title': 'Neue Rechte Welle',
+            'description': 'md5:a30a53f740ffb6bfd535314c2cc5fb69',
+            'timestamp': 1501337639,
+            'upload_date': '20170729',
+        },
+        'params': {
+            'format': 'bestvideo',
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://www.funk.net/channel/59d5149841dca100012511e3/0/59d52049999264000182e79d/',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        domain_id = NexxIE._extract_domain_id(webpage) or '741'
+        nexx_id = extract_attributes(self._search_regex(
+            r'(<div[^>]id=["\']mediaplayer-funk[^>]+>)',
+            webpage, 'media player'))['data-id']
+
+        return self.url_result(
+            'nexx:%s:%s' % (domain_id, nexx_id), ie=NexxIE.ie_key(),
+            video_id=nexx_id)
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6dab4c7f4..39630b6f6 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1613,22 +1613,6 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['BrightcoveLegacy'],
         },
-        # Nexx embed
-        {
-            'url': 'https://www.funk.net/serien/5940e15073f6120001657956/items/593efbb173f6120001657503',
-            'info_dict': {
-                'id': '247746',
-                'ext': 'mp4',
-                'title': "Yesterday's Jam (OV)",
-                'description': 'md5:09bc0984723fed34e2581624a84e05f0',
-                'timestamp': 1492594816,
-                'upload_date': '20170419',
-            },
-            'params': {
-                'format': 'bestvideo',
-                'skip_download': True,
-            },
-        },
         # Facebook <iframe> embed
         {
             'url': 'https://www.hostblogger.de/blog/archives/6181-Auto-jagt-Betonmischer.html',

From 7fee3377dced16cd16660aeb78d18d81508aa28b Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 11 Oct 2017 17:49:48 +0000
Subject: [PATCH 1415/1696] [steam] fix extraction(fixes #14067)

---
 youtube_dl/extractor/steam.py | 131 ++++++++++++++++++++--------------
 1 file changed, 77 insertions(+), 54 deletions(-)

diff --git a/youtube_dl/extractor/steam.py b/youtube_dl/extractor/steam.py
index 1a831ef6d..e5ac586a7 100644
--- a/youtube_dl/extractor/steam.py
+++ b/youtube_dl/extractor/steam.py
@@ -4,8 +4,10 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    extract_attributes,
     ExtractorError,
-    unescapeHTML,
+    get_element_by_class,
+    js_to_json,
 )
 
 
@@ -25,35 +27,39 @@ class SteamIE(InfoExtractor):
         'url': 'http://store.steampowered.com/video/105600/',
         'playlist': [
             {
-                'md5': 'f870007cee7065d7c76b88f0a45ecc07',
+                'md5': '6a294ee0c4b1f47f5bb76a65e31e3592',
                 'info_dict': {
-                    'id': '81300',
-                    'ext': 'flv',
-                    'title': 'Terraria 1.1 Trailer',
+                    'id': '2040428',
+                    'ext': 'mp4',
+                    'title': 'Terraria 1.3 Trailer',
                     'playlist_index': 1,
                 }
             },
             {
-                'md5': '61aaf31a5c5c3041afb58fb83cbb5751',
+                'md5': '911672b20064ca3263fa89650ba5a7aa',
                 'info_dict': {
-                    'id': '80859',
-                    'ext': 'flv',
-                    'title': 'Terraria Trailer',
+                    'id': '2029566',
+                    'ext': 'mp4',
+                    'title': 'Terraria 1.2 Trailer',
                     'playlist_index': 2,
                 }
             }
         ],
+        'info_dict': {
+            'id': '105600',
+            'title': 'Terraria',
+        },
         'params': {
             'playlistend': 2,
         }
     }, {
         'url': 'http://steamcommunity.com/sharedfiles/filedetails/?id=242472205',
         'info_dict': {
-            'id': 'WB5DvDOOvAY',
+            'id': 'X8kpJBlzD2E',
             'ext': 'mp4',
-            'upload_date': '20140329',
-            'title': 'FRONTIERS - Final Greenlight Trailer',
-            'description': 'md5:dc96a773669d0ca1b36c13c1f30250d9',
+            'upload_date': '20140617',
+            'title': 'FRONTIERS - Trapping',
+            'description': 'md5:bf6f7f773def614054089e5769c12a6e',
             'uploader': 'AAD Productions',
             'uploader_id': 'AtomicAgeDogGames',
         }
@@ -76,48 +82,65 @@ class SteamIE(InfoExtractor):
             self.report_age_confirmation()
             webpage = self._download_webpage(videourl, playlist_id)
 
+        flash_vars = self._parse_json(self._search_regex(
+            r'(?s)rgMovieFlashvars\s*=\s*({.+?});', webpage,
+            'flash vars'), playlist_id, js_to_json)
+
+        playlist_title = None
+        entries = []
         if fileID:
-            playlist_title = self._html_search_regex(
-                r'<div class="workshopItemTitle">(.+)</div>', webpage, 'title')
-            mweb = re.finditer(r'''(?x)
-                'movie_(?P<videoID>[0-9]+)':\s*\{\s*
-                YOUTUBE_VIDEO_ID:\s*"(?P<youtube_id>[^"]+)",
-                ''', webpage)
-            videos = [{
-                '_type': 'url',
-                'url': vid.group('youtube_id'),
-                'ie_key': 'Youtube',
-            } for vid in mweb]
-        else:
-            playlist_title = self._html_search_regex(
-                r'<h2 class="pageheader">(.*?)</h2>', webpage, 'game title')
-
-            mweb = re.finditer(r'''(?x)
-                'movie_(?P<videoID>[0-9]+)':\s*\{\s*
-                FILENAME:\s*"(?P<videoURL>[\w:/\.\?=]+)"
-                (,\s*MOVIE_NAME:\s*\"(?P<videoName>[\w:/\.\?=\+-]+)\")?\s*\},
-                ''', webpage)
-            titles = re.finditer(
-                r'<span class="title">(?P<videoName>.+?)</span>', webpage)
-            thumbs = re.finditer(
-                r'<img class="movie_thumb" src="(?P<thumbnail>.+?)">', webpage)
-            videos = []
-
-            for vid, vtitle, thumb in zip(mweb, titles, thumbs):
-                video_id = vid.group('videoID')
-                title = vtitle.group('videoName')
-                video_url = vid.group('videoURL')
-                video_thumb = thumb.group('thumbnail')
-                if not video_url:
-                    raise ExtractorError('Cannot find video url for %s' % video_id)
-                videos.append({
-                    'id': video_id,
-                    'url': video_url,
-                    'ext': 'flv',
-                    'title': unescapeHTML(title),
-                    'thumbnail': video_thumb
+            playlist_title = get_element_by_class('workshopItemTitle', webpage)
+            for movie in flash_vars.values():
+                if not movie:
+                    continue
+                youtube_id = movie.get('YOUTUBE_VIDEO_ID')
+                if not youtube_id:
+                    continue
+                entries.append({
+                    '_type': 'url',
+                    'url': youtube_id,
+                    'ie_key': 'Youtube',
                 })
-        if not videos:
+        else:
+            playlist_title = get_element_by_class('apphub_AppName', webpage)
+            for movie_id, movie in flash_vars.items():
+                if not movie:
+                    continue
+                video_id = self._search_regex(r'movie_(\d+)', movie_id, 'video id', fatal=False)
+                title = movie.get('MOVIE_NAME')
+                if not title or not video_id:
+                    continue
+                entry = {
+                    'id': video_id,
+                    'title': title.replace('+', ' '),
+                }
+                formats = []
+                flv_url = movie.get('FILENAME')
+                if flv_url:
+                    formats.append({
+                        'format_id': 'flv',
+                        'url': flv_url,
+                    })
+                highlight_element = self._search_regex(
+                    r'(<div[^>]+id="highlight_movie_%s"[^>]+>)' % video_id,
+                    webpage, 'highlight element', fatal=False)
+                if highlight_element:
+                    highlight_attribs = extract_attributes(highlight_element)
+                    if highlight_attribs:
+                        entry['thumbnail'] = highlight_attribs.get('data-poster')
+                        for quality in ('', '-hd'):
+                            for ext in ('webm', 'mp4'):
+                                video_url = highlight_attribs.get('data-%s%s-source' % (ext, quality))
+                                if video_url:
+                                    formats.append({
+                                        'format_id': ext + quality,
+                                        'url': video_url,
+                                    })
+                if not formats:
+                    continue
+                entry['formats'] = formats
+                entries.append(entry)
+        if not entries:
             raise ExtractorError('Could not find any videos')
 
-        return self.playlist_result(videos, playlist_id, playlist_title)
+        return self.playlist_result(entries, playlist_id, playlist_title)

From 2f0eb0a68a0711e7ac2c31b6177273433fb42b94 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 12 Oct 2017 01:05:14 +0700
Subject: [PATCH 1416/1696] [ChangeLog] Actualize

---
 ChangeLog | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index c541c4f62..7477443cc 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,8 +1,27 @@
 version <unreleased>
 
+Core
+* [YoutubeDL] Improve _default_format_spec (#14461)
+
 Extractors
+* [steam] Fix extraction (#14067)
++ [funk] Add support for funk.net (#14464)
++ [nexx] Add support for shortcuts and relax domain id extraction
++ [voxmedia] Add support for recode.net (#14173)
++ [once] Add support for vmap URLs
++ [generic] Add support for channel9 embeds (#14469)
+* [tva] Fix extraction (#14328)
++ [tubitv] Add support for new URL format (#14460)
+- [afreecatv:global] Remove extractor
 - [youtube:shared] Removed extractor (#14420)
++ [slideslive] Add support for slideslive.com (#2680)
 + [facebook] Support thumbnails (#14416)
+* [vvvvid] Fix episode number extraction (#14456)
+* [hrti:playlist] Relax URL regular expression
+* [wdr] Relax media link regular expression (#14447)
+* [hrti] Relax URL regular expression (#14443)
+* [fox] Delegate extraction to uplynk:preplay (#14147)
++ [youtube] Add support for hooktube.com (#14437)
 
 
 version 2017.10.07

From 76581082f6fd32787c7fe5c3e5be8442ba9f50a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 12 Oct 2017 01:06:28 +0700
Subject: [PATCH 1417/1696] release 2017.10.12

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 5 +++--
 youtube_dl/version.py     | 2 +-
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 9fc425ca8..5ef087328 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.10.07*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.10.07**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.10.12*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.10.12**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.10.07
+[debug] youtube-dl version 2017.10.12
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 7477443cc..9b57adcac 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.10.12
 
 Core
 * [YoutubeDL] Improve _default_format_spec (#14461)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index be9df7e31..7071450d4 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -36,7 +36,6 @@
  - **AdultSwim**
  - **aenetworks**: A+E Networks: A&E, Lifetime, History.com, FYI Network
  - **afreecatv**: afreecatv.com
- - **afreecatv:global**: afreecatv.com
  - **AirMozilla**
  - **AliExpressLive**
  - **AlJazeera**
@@ -296,6 +295,7 @@
  - **freespeech.org**
  - **FreshLive**
  - **Funimation**
+ - **Funk**
  - **FunnyOrDie**
  - **Fusion**
  - **Fux**
@@ -738,6 +738,7 @@
  - **skynewsarabia:video**
  - **SkySports**
  - **Slideshare**
+ - **SlidesLive**
  - **Slutload**
  - **smotri**: Smotri.com
  - **smotri:broadcast**: Smotri.com broadcasts
@@ -969,6 +970,7 @@
  - **VoiceRepublic**
  - **Voot**
  - **VoxMedia**
+ - **VoxMediaVolume**
  - **Vporn**
  - **vpro**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
  - **Vrak**
@@ -1044,7 +1046,6 @@
  - **youtube:search**: YouTube.com searches
  - **youtube:search:date**: YouTube.com searches, newest videos first
  - **youtube:search_url**: YouTube.com search URLs
- - **youtube:shared**
  - **youtube:show**: YouTube.com (multi-season) shows
  - **youtube:subscriptions**: YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)
  - **youtube:user**: YouTube.com user videos (URL or "ytuser" keyword)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 705fcade6..36e6e99da 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.10.07'
+__version__ = '2017.10.12'

From 344d1a6794270e67b8bb21b78995a1b116286a07 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 11 Oct 2017 20:51:57 +0000
Subject: [PATCH 1418/1696] [vh1] fix extraction(fixes #9613)

---
 youtube_dl/extractor/vh1.py | 139 +++++++-----------------------------
 1 file changed, 24 insertions(+), 115 deletions(-)

diff --git a/youtube_dl/extractor/vh1.py b/youtube_dl/extractor/vh1.py
index 570fa45ea..2341e5f71 100644
--- a/youtube_dl/extractor/vh1.py
+++ b/youtube_dl/extractor/vh1.py
@@ -1,131 +1,40 @@
 from __future__ import unicode_literals
 
-from .mtv import MTVIE
-
-import re
-from ..utils import fix_xml_ampersands
+from .mtv import MTVServicesInfoExtractor
 
 
-class VH1IE(MTVIE):
+class VH1IE(MTVServicesInfoExtractor):
     IE_NAME = 'vh1.com'
-    _FEED_URL = 'http://www.vh1.com/player/embed/AS3/fullepisode/rss/'
+    _FEED_URL = 'http://www.vh1.com/feeds/mrss/'
     _TESTS = [{
-        'url': 'http://www.vh1.com/video/metal-evolution/full-episodes/progressive-metal/1678612/playlist.jhtml',
-        'playlist': [
-            {
-                'md5': '7827a7505f59633983165bbd2c119b52',
-                'info_dict': {
-                    'id': '731565',
-                    'ext': 'mp4',
-                    'title': 'Metal Evolution: Ep. 11 Act 1',
-                    'description': 'Many rock academics have proclaimed that the truly progressive musicianship of the last 20 years has been found right here in the world of heavy metal, rather than obvious locales such as jazz, fusion or progressive rock. It stands to reason then, that much of this jaw-dropping virtuosity occurs within what\'s known as progressive metal, a genre that takes root with the likes of Rush in the \'70s, Queensryche and Fates Warning in the \'80s, and Dream Theater in the \'90s. Since then, the genre has exploded with creativity, spawning mind-bending, genre-defying acts such as Tool, Mastodon, Coheed And Cambria, Porcupine Tree, Meshuggah, A Perfect Circle and Opeth. Episode 12 looks at the extreme musicianship of these bands, as well as their often extreme literary prowess and conceptual strength, the end result being a rich level of respect and attention such challenging acts have brought upon the world of heavy metal, from a critical community usually dismissive of the form.'
-                }
-            },
-            {
-                'md5': '34fb4b7321c546b54deda2102a61821f',
-                'info_dict': {
-                    'id': '731567',
-                    'ext': 'mp4',
-                    'title': 'Metal Evolution: Ep. 11 Act 2',
-                    'description': 'Many rock academics have proclaimed that the truly progressive musicianship of the last 20 years has been found right here in the world of heavy metal, rather than obvious locales such as jazz, fusion or progressive rock. It stands to reason then, that much of this jaw-dropping virtuosity occurs within what\'s known as progressive metal, a genre that takes root with the likes of Rush in the \'70s, Queensryche and Fates Warning in the \'80s, and Dream Theater in the \'90s. Since then, the genre has exploded with creativity, spawning mind-bending, genre-defying acts such as Tool, Mastodon, Coheed And Cambria, Porcupine Tree, Meshuggah, A Perfect Circle and Opeth. Episode 11 looks at the extreme musicianship of these bands, as well as their often extreme literary prowess and conceptual strength, the end result being a rich level of respect and attention such challenging acts have brought upon the world of heavy metal, from a critical community usually dismissive of the form.'
-                }
-            },
-            {
-                'md5': '813f38dba4c1b8647196135ebbf7e048',
-                'info_dict': {
-                    'id': '731568',
-                    'ext': 'mp4',
-                    'title': 'Metal Evolution: Ep. 11 Act 3',
-                    'description': 'Many rock academics have proclaimed that the truly progressive musicianship of the last 20 years has been found right here in the world of heavy metal, rather than obvious locales such as jazz, fusion or progressive rock. It stands to reason then, that much of this jaw-dropping virtuosity occurs within what\'s known as progressive metal, a genre that takes root with the likes of Rush in the \'70s, Queensryche and Fates Warning in the \'80s, and Dream Theater in the \'90s. Since then, the genre has exploded with creativity, spawning mind-bending, genre-defying acts such as Tool, Mastodon, Coheed And Cambria, Porcupine Tree, Meshuggah, A Perfect Circle and Opeth. Episode 11 looks at the extreme musicianship of these bands, as well as their often extreme literary prowess and conceptual strength, the end result being a rich level of respect and attention such challenging acts have brought upon the world of heavy metal, from a critical community usually dismissive of the form.'
-                }
-            },
-            {
-                'md5': '51adb72439dfaed11c799115d76e497f',
-                'info_dict': {
-                    'id': '731569',
-                    'ext': 'mp4',
-                    'title': 'Metal Evolution: Ep. 11 Act 4',
-                    'description': 'Many rock academics have proclaimed that the truly progressive musicianship of the last 20 years has been found right here in the world of heavy metal, rather than obvious locales such as jazz, fusion or progressive rock. It stands to reason then, that much of this jaw-dropping virtuosity occurs within what\'s known as progressive metal, a genre that takes root with the likes of Rush in the \'70s, Queensryche and Fates Warning in the \'80s, and Dream Theater in the \'90s. Since then, the genre has exploded with creativity, spawning mind-bending, genre-defying acts such as Tool, Mastodon, Coheed And Cambria, Porcupine Tree, Meshuggah, A Perfect Circle and Opeth. Episode 11 looks at the extreme musicianship of these bands, as well as their often extreme literary prowess and conceptual strength, the end result being a rich level of respect and attention such challenging acts have brought upon the world of heavy metal, from a critical community usually dismissive of the form.'
-                }
-            },
-            {
-                'md5': '93d554aaf79320703b73a95288c76a6e',
-                'info_dict': {
-                    'id': '731570',
-                    'ext': 'mp4',
-                    'title': 'Metal Evolution: Ep. 11 Act 5',
-                    'description': 'Many rock academics have proclaimed that the truly progressive musicianship of the last 20 years has been found right here in the world of heavy metal, rather than obvious locales such as jazz, fusion or progressive rock. It stands to reason then, that much of this jaw-dropping virtuosity occurs within what\'s known as progressive metal, a genre that takes root with the likes of Rush in the \'70s, Queensryche and Fates Warning in the \'80s, and Dream Theater in the \'90s. Since then, the genre has exploded with creativity, spawning mind-bending, genre-defying acts such as Tool, Mastodon, Coheed And Cambria, Porcupine Tree, Meshuggah, A Perfect Circle and Opeth. Episode 11 looks at the extreme musicianship of these bands, as well as their often extreme literary prowess and conceptual strength, the end result being a rich level of respect and attention such challenging acts have brought upon the world of heavy metal, from a critical community usually dismissive of the form.'
-                }
-            }
-        ],
-        'skip': 'Blocked outside the US',
+        'url': 'http://www.vh1.com/episodes/0umwpq/hip-hop-squares-kent-jones-vs-nick-young-season-1-ep-120',
+        'info_dict': {
+            'title': 'Kent Jones vs. Nick Young',
+            'description': 'Come to Play. Stay to Party. With Mike Epps, TIP, O’Shea Jackson Jr., T-Pain, Tisha Campbell-Martin and more.',
+        },
+        'playlist_mincount': 4,
     }, {
         # Clip
-        'url': 'http://www.vh1.com/video/misc/706675/metal-evolution-episode-1-pre-metal-show-clip.jhtml#id=1674118',
-        'md5': '7d67cf6d9cdc6b4f3d3ac97a55403844',
+        'url': 'http://www.vh1.com/video-clips/t74mif/scared-famous-scared-famous-extended-preview',
         'info_dict': {
-            'id': '706675',
+            'id': '0a50c2d2-a86b-4141-9565-911c7e2d0b92',
             'ext': 'mp4',
-            'title': 'Metal Evolution: Episode 1 Pre-Metal Show Clip',
-            'description': 'The greatest documentary ever made about Heavy Metal begins as our host Sam Dunn travels the globe to seek out the origins and influences that helped create Heavy Metal. Sam speaks to legends like Kirk Hammett, Alice Cooper, Slash, Bill Ward, Geezer Butler, Tom Morello, Ace Frehley, Lemmy Kilmister, Dave Davies, and many many more. This episode is the prologue for the 11 hour series, and Sam goes back to the very beginning to reveal how Heavy Metal was created.'
+            'title': 'Scared Famous|October 9, 2017|1|NO-EPISODE#|Scared Famous + Extended Preview',
+            'description': 'md5:eff5551a274c473a29463de40f7b09da',
+            'upload_date': '20171009',
+            'timestamp': 1507574700,
         },
-        'skip': 'Blocked outside the US',
-    }, {
-        # Short link
-        'url': 'http://www.vh1.com/video/play.jhtml?id=1678353',
-        'md5': '853192b87ad978732b67dd8e549b266a',
-        'info_dict': {
-            'id': '730355',
-            'ext': 'mp4',
-            'title': 'Metal Evolution: Episode 11 Progressive Metal Sneak',
-            'description': 'In Metal Evolution\'s finale sneak, Sam sits with Michael Giles of King Crimson and gets feedback from Metallica guitarist Kirk Hammett on why the group was influential.'
+        'params': {
+            # m3u8 download
+            'skip_download': True,
         },
-        'skip': 'Blocked outside the US',
-    }, {
-        'url': 'http://www.vh1.com/video/macklemore-ryan-lewis/900535/cant-hold-us-ft-ray-dalton.jhtml',
-        'md5': 'b1bcb5b4380c9d7f544065589432dee7',
-        'info_dict': {
-            'id': '900535',
-            'ext': 'mp4',
-            'title': 'Macklemore & Ryan Lewis - "Can\'t Hold Us ft. Ray Dalton"',
-            'description': 'The Heist'
-        },
-        'skip': 'Blocked outside the US',
     }]
 
-    _VALID_URL = r'''(?x)
-        https?://www\.vh1\.com/video/
-        (?:
-            .+?/full-episodes/.+?/(?P<playlist_id>[^/]+)/playlist\.jhtml
-        |
-            (?:
-            play.jhtml\?id=|
-            misc/.+?/.+?\.jhtml\#id=
-            )
-            (?P<video_id>[0-9]+)$
-        |
-            [^/]+/(?P<music_id>[0-9]+)/[^/]+?
-        )
-    '''
+    _VALID_URL = r'https?://(?:www\.)?vh1\.com/(?:video-clips|episodes)/(?P<id>[^/?#.]+)'
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        if mobj.group('music_id'):
-            id_field = 'vid'
-            video_id = mobj.group('music_id')
-        else:
-            video_id = mobj.group('playlist_id') or mobj.group('video_id')
-            id_field = 'id'
-        doc_url = '%s?%s=%s' % (self._FEED_URL, id_field, video_id)
-
-        idoc = self._download_xml(
-            doc_url, video_id,
-            'Downloading info', transform_source=fix_xml_ampersands)
-
-        entries = []
-        for item in idoc.findall('.//item'):
-            info = self._get_video_info(item)
-            if info:
-                entries.append(info)
-
-        return self.playlist_result(entries, playlist_id=video_id)
+        playlist_id = self._match_id(url)
+        webpage = self._download_webpage(url, playlist_id)
+        mgid = self._extract_triforce_mgid(webpage)
+        videos_info = self._get_videos_info(mgid)
+        return videos_info

From 1baba7f4a85652c3844661b48378f188a9a5631f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 12 Oct 2017 12:02:26 +0800
Subject: [PATCH 1419/1696] [vh1] Adding coding cookie

---
 youtube_dl/extractor/vh1.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/vh1.py b/youtube_dl/extractor/vh1.py
index 2341e5f71..dff94a2b8 100644
--- a/youtube_dl/extractor/vh1.py
+++ b/youtube_dl/extractor/vh1.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .mtv import MTVServicesInfoExtractor

From c0bddd6d6597642c7c4733d0fbf52fbe4137d5a0 Mon Sep 17 00:00:00 2001
From: nyuszika7h <nyuszika7h@gmail.com>
Date: Wed, 11 Oct 2017 22:24:17 +0200
Subject: [PATCH 1420/1696] [kaltura] Ignore Widevine encrypted video (.wvm)

There is currently no public method to decrypt this, and there may be
other streams available that can be downloaded.

Example URL, has `.wvm` and `.mp4` formats:
https://www.voot.com/shows/bigg-boss-s11/11/538936/bigg-boss-extra-dose-arshi-s-quirky-demand/541700
---
 youtube_dl/extractor/kaltura.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 138d4844d..bdac2df3e 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -287,6 +287,9 @@ class KalturaIE(InfoExtractor):
             # skip for now.
             if f.get('fileExt') == 'chun':
                 continue
+            # DRM-protected video, cannot be decrypted
+            if f.get('fileExt') == 'wvm':
+                continue
             if not f.get('fileExt'):
                 # QT indicates QuickTime; some videos have broken fileExt
                 if f.get('containerFormat') == 'qt':

From 645ed3e7c90881f4fd66a40e783f4b3deb270262 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 12 Oct 2017 12:12:37 +0800
Subject: [PATCH 1421/1696] [ChangeLog] Update after #14471

[skip ci]
---
 ChangeLog | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 9b57adcac..db2c2bf88 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,8 @@
+version <unreleased>
+
+Extractors
+* [kaltura] Ignore Widevine encrypted video (.wvm) (#14471)
+
 version 2017.10.12
 
 Core

From 414e709405fe8c653fb58982626284134a86167e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 12 Oct 2017 09:20:07 +0000
Subject: [PATCH 1422/1696] [shahid] fix extraction(fixes #14448)

---
 youtube_dl/extractor/shahid.py | 61 ++++++++++++++++++----------------
 1 file changed, 32 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index 62d41e88a..374f7faf9 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -18,46 +18,32 @@ from ..utils import (
 
 class ShahidIE(InfoExtractor):
     _NETRC_MACHINE = 'shahid'
-    _VALID_URL = r'https?://shahid\.mbc\.net/ar/(?P<type>episode|movie)/(?P<id>\d+)'
+    _VALID_URL = r'https?://shahid\.mbc\.net/ar/(?:serie|show|movie)s/[^/]+/(?P<type>episode|clip|movie)-(?P<id>\d+)'
     _TESTS = [{
-        'url': 'https://shahid.mbc.net/ar/episode/90574/%D8%A7%D9%84%D9%85%D9%84%D9%83-%D8%B9%D8%A8%D8%AF%D8%A7%D9%84%D9%84%D9%87-%D8%A7%D9%84%D8%A5%D9%86%D8%B3%D8%A7%D9%86-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D9%83%D9%84%D9%8A%D8%A8-3.html',
+        'url': 'https://shahid.mbc.net/ar/shows/%D9%85%D8%AC%D9%84%D8%B3-%D8%A7%D9%84%D8%B4%D8%A8%D8%A7%D8%A8-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D9%83%D9%84%D9%8A%D8%A8-1/clip-275286',
         'info_dict': {
-            'id': '90574',
+            'id': '275286',
             'ext': 'mp4',
-            'title': 'الملك عبدالله الإنسان الموسم 1 كليب 3',
-            'description': 'الفيلم الوثائقي - الملك عبد الله الإنسان',
-            'duration': 2972,
-            'timestamp': 1422057420,
-            'upload_date': '20150123',
+            'title': 'مجلس الشباب الموسم 1 كليب 1',
+            'timestamp': 1506988800,
+            'upload_date': '20171003',
         },
         'params': {
             # m3u8 download
             'skip_download': True,
         }
     }, {
-        'url': 'https://shahid.mbc.net/ar/movie/151746/%D8%A7%D9%84%D9%82%D9%86%D8%A7%D8%B5%D8%A9.html',
+        'url': 'https://shahid.mbc.net/ar/movies/%D8%A7%D9%84%D9%82%D9%86%D8%A7%D8%B5%D8%A9/movie-151746',
         'only_matching': True
     }, {
         # shahid plus subscriber only
-        'url': 'https://shahid.mbc.net/ar/episode/90511/%D9%85%D8%B1%D8%A7%D9%8A%D8%A7-2011-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1.html',
+        'url': 'https://shahid.mbc.net/ar/series/%D9%85%D8%B1%D8%A7%D9%8A%D8%A7-2011-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1/episode-90511',
         'only_matching': True
     }]
 
-    def _real_initialize(self):
-        email, password = self._get_login_info()
-        if email is None:
-            return
-
+    def _api2_request(self, *args, **kwargs):
         try:
-            user_data = self._download_json(
-                'https://shahid.mbc.net/wd/service/users/login',
-                None, 'Logging in', data=json.dumps({
-                    'email': email,
-                    'password': password,
-                    'basic': 'false',
-                }).encode('utf-8'), headers={
-                    'Content-Type': 'application/json; charset=UTF-8',
-                })['user']
+            return self._download_json(*args, **kwargs)
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError):
                 fail_data = self._parse_json(
@@ -69,6 +55,21 @@ class ShahidIE(InfoExtractor):
                         raise ExtractorError(faults_message, expected=True)
             raise
 
+    def _real_initialize(self):
+        email, password = self._get_login_info()
+        if email is None:
+            return
+
+        user_data = self._api2_request(
+            'https://shahid.mbc.net/wd/service/users/login',
+            None, 'Logging in', data=json.dumps({
+                'email': email,
+                'password': password,
+                'basic': 'false',
+            }).encode('utf-8'), headers={
+                'Content-Type': 'application/json; charset=UTF-8',
+            })['user']
+
         self._download_webpage(
             'https://shahid.mbc.net/populateContext',
             None, 'Populate Context', data=urlencode_postdata({
@@ -93,15 +94,17 @@ class ShahidIE(InfoExtractor):
 
     def _real_extract(self, url):
         page_type, video_id = re.match(self._VALID_URL, url).groups()
+        if page_type == 'clip':
+            page_type = 'episode'
 
-        player = self._get_api_data(self._download_json(
-            'https://shahid.mbc.net/arContent/getPlayerContent-param-.id-%s.type-player.html' % video_id,
-            video_id, 'Downloading player JSON'))
+        playout = self._api2_request(
+            'https://api2.shahid.net/proxy/v2/playout/url/' + video_id,
+            video_id, 'Downloading player JSON')['playout']
 
-        if player.get('drm'):
+        if playout.get('drm'):
             raise ExtractorError('This video is DRM protected.', expected=True)
 
-        formats = self._extract_m3u8_formats(player['url'], video_id, 'mp4')
+        formats = self._extract_m3u8_formats(playout['url'], video_id, 'mp4')
         self._sort_formats(formats)
 
         video = self._get_api_data(self._download_json(

From 81ce479f4dcc17b83672f5ca3ffcb2a2d1618c96 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 12 Oct 2017 16:04:41 +0000
Subject: [PATCH 1423/1696] [udn] fix extraction

---
 youtube_dl/extractor/udn.py | 32 +++++++++++++++++++-------------
 1 file changed, 19 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/udn.py b/youtube_dl/extractor/udn.py
index daf45d0b4..2c8e5c7b4 100644
--- a/youtube_dl/extractor/udn.py
+++ b/youtube_dl/extractor/udn.py
@@ -1,7 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import json
 import re
 
 from .common import InfoExtractor
@@ -29,6 +28,7 @@ class UDNEmbedIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         },
+        'expected_warnings': ['Failed to parse JSON Expecting value'],
     }, {
         'url': 'https://video.udn.com/embed/news/300040',
         'only_matching': True,
@@ -43,10 +43,21 @@ class UDNEmbedIE(InfoExtractor):
 
         page = self._download_webpage(url, video_id)
 
-        options = json.loads(js_to_json(self._html_search_regex(
-            r'var\s+options\s*=\s*([^;]+);', page, 'video urls dictionary')))
-
-        video_urls = options['video']
+        options_str = self._html_search_regex(
+            r'var\s+options\s*=\s*([^;]+);', page, 'options')
+        trans_options_str = js_to_json(options_str)
+        options = self._parse_json(trans_options_str, 'options', fatal=False) or {}
+        if options:
+            video_urls = options['video']
+            title = options['title']
+            poster = options.get('poster')
+        else:
+            video_urls = self._parse_json(self._html_search_regex(
+                r'"video"\s*:\s*({.+?})\s*,', trans_options_str, 'video urls'), 'video urls')
+            title = self._html_search_regex(
+                r"title\s*:\s*'(.+?)'\s*,", options_str, 'title')
+            poster = self._html_search_regex(
+                r"poster\s*:\s*'(.+?)'\s*,", options_str, 'poster', default=None)
 
         if video_urls.get('youtube'):
             return self.url_result(video_urls.get('youtube'), 'Youtube')
@@ -68,7 +79,7 @@ class UDNEmbedIE(InfoExtractor):
                 formats.extend(self._extract_f4m_formats(
                     video_url, video_id, f4m_id='hds'))
             else:
-                mobj = re.search(r'_(?P<height>\d+)p_(?P<tbr>\d+).mp4', video_url)
+                mobj = re.search(r'_(?P<height>\d+)p_(?P<tbr>\d+)\.mp4', video_url)
                 a_format = {
                     'url': video_url,
                     # video_type may be 'mp4', which confuses YoutubeDL
@@ -83,14 +94,9 @@ class UDNEmbedIE(InfoExtractor):
 
         self._sort_formats(formats)
 
-        thumbnails = [{
-            'url': img_url,
-            'id': img_type,
-        } for img_type, img_url in options.get('gallery', [{}])[0].items() if img_url]
-
         return {
             'id': video_id,
             'formats': formats,
-            'title': options['title'],
-            'thumbnails': thumbnails,
+            'title': title,
+            'thumbnail': poster,
         }

From b0def2c297293dfee3ec48c73bb92116a0ee0154 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 12 Oct 2017 16:05:25 +0000
Subject: [PATCH 1424/1696] [onionstudios] add support for bulbs-video embeds

---
 youtube_dl/extractor/onionstudios.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/onionstudios.py b/youtube_dl/extractor/onionstudios.py
index 1d336cf30..c6e3d5640 100644
--- a/youtube_dl/extractor/onionstudios.py
+++ b/youtube_dl/extractor/onionstudios.py
@@ -13,11 +13,11 @@ from ..utils import (
 
 
 class OnionStudiosIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?onionstudios\.com/(?:videos/[^/]+-|embed\?.*\bid=)(?P<id>\d+)(?!-)'
+    _VALID_URL = r'https?://(?:www\.)?onionstudios\.com/(?:video(?:s/[^/]+-|/)|embed\?.*\bid=)(?P<id>\d+)(?!-)'
 
     _TESTS = [{
         'url': 'http://www.onionstudios.com/videos/hannibal-charges-forward-stops-for-a-cocktail-2937',
-        'md5': 'e49f947c105b8a78a675a0ee1bddedfe',
+        'md5': '719d1f8c32094b8c33902c17bcae5e34',
         'info_dict': {
             'id': '2937',
             'ext': 'mp4',
@@ -29,12 +29,15 @@ class OnionStudiosIE(InfoExtractor):
     }, {
         'url': 'http://www.onionstudios.com/embed?id=2855&autoplay=true',
         'only_matching': True,
+    }, {
+        'url': 'http://www.onionstudios.com/video/6139.json',
+        'only_matching': True,
     }]
 
     @staticmethod
     def _extract_url(webpage):
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?onionstudios\.com/embed.+?)\1', webpage)
+            r'(?s)<(?:iframe|bulbs-video)[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?onionstudios\.com/(?:embed.+?|video/\d+\.json))\1', webpage)
         if mobj:
             return mobj.group('url')
 

From 7a64c33aee81e3a495c162af61bcd030a514f109 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 12 Oct 2017 16:09:06 +0000
Subject: [PATCH 1425/1696] [rtlnl] add support for another type of embeds

---
 youtube_dl/extractor/rtlnl.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index 3e22998c6..bba25a233 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -12,10 +12,10 @@ class RtlNlIE(InfoExtractor):
     IE_NAME = 'rtl.nl'
     IE_DESC = 'rtl.nl and rtlxl.nl'
     _VALID_URL = r'''(?x)
-        https?://(?:www\.)?
+        https?://(?:(?:www|static)\.)?
         (?:
             rtlxl\.nl/[^\#]*\#!/[^/]+/|
-            rtl\.nl/(?:system/videoplayer/(?:[^/]+/)+(?:video_)?embed\.html\b.+?\buuid=|video/)
+            rtl\.nl/(?:(?:system/videoplayer/(?:[^/]+/)+(?:video_)?embed\.html|embed)\b.+?\buuid=|video/)
         )
         (?P<id>[0-9a-f-]+)'''
 
@@ -73,6 +73,9 @@ class RtlNlIE(InfoExtractor):
     }, {
         'url': 'https://www.rtl.nl/video/c603c9c2-601d-4b5e-8175-64f1e942dc7d/',
         'only_matching': True,
+    }, {
+        'url': 'https://static.rtl.nl/embed/?uuid=1a2970fc-5c0b-43ff-9fdc-927e39e6d1bc&autoplay=false&publicatiepunt=rtlnieuwsnl',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 50d808f5c942b454b3febbf1ef5d78bd204ce469 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 12 Oct 2017 16:12:47 +0000
Subject: [PATCH 1426/1696] [common] add support for jwplayer youtube embeds

---
 youtube_dl/extractor/common.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a878550c2..a69240693 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2322,7 +2322,6 @@ class InfoExtractor(object):
             formats = self._parse_jwplayer_formats(
                 video_data['sources'], video_id=this_video_id, m3u8_id=m3u8_id,
                 mpd_id=mpd_id, rtmp_params=rtmp_params, base_url=base_url)
-            self._sort_formats(formats)
 
             subtitles = {}
             tracks = video_data.get('tracks')
@@ -2339,16 +2338,25 @@ class InfoExtractor(object):
                         'url': self._proto_relative_url(track_url)
                     })
 
-            entries.append({
+            entry = {
                 'id': this_video_id,
-                'title': video_data['title'] if require_title else video_data.get('title'),
+                'title': unescapeHTML(video_data['title'] if require_title else video_data.get('title')),
                 'description': video_data.get('description'),
                 'thumbnail': self._proto_relative_url(video_data.get('image')),
                 'timestamp': int_or_none(video_data.get('pubdate')),
                 'duration': float_or_none(jwplayer_data.get('duration') or video_data.get('duration')),
                 'subtitles': subtitles,
-                'formats': formats,
-            })
+            }
+            # https://github.com/jwplayer/jwplayer/blob/master/src/js/utils/validator.js#L32
+            if len(formats) == 1 and re.search(r'^(?:http|//).*(?:youtube\.com|youtu\.be)/.+', formats[0]['url']):
+                entry.update({
+                    '_type': 'url_transparent',
+                    'url': formats[0]['url'],
+                })
+            else:
+                self._sort_formats(formats)
+                entry['formats'] = formats
+            entries.append(entry)
         if len(entries) == 1:
             return entries[0]
         else:

From 2637fadc38566abdc84c62a05d1a6573e60799bf Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 12 Oct 2017 16:14:43 +0000
Subject: [PATCH 1427/1696] [generic] fix some of the tests

---
 youtube_dl/extractor/generic.py | 36 ++++++++++++++++++++++++---------
 1 file changed, 27 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 39630b6f6..2a9c3e2de 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1091,7 +1091,7 @@ class GenericIE(InfoExtractor):
                 'ext': 'mp4',
                 'upload_date': '20150212',
                 'uploader': 'The National Archives UK',
-                'description': 'md5:a236581cd2449dd2df4f93412f3f01c6',
+                'description': 'md5:8078af856dca76edc42910b61273dbbf',
                 'uploader_id': 'NationalArchives08',
                 'title': 'Webinar: Using Discovery, The National Archives’ online catalogue',
             },
@@ -1107,7 +1107,8 @@ class GenericIE(InfoExtractor):
             },
             'params': {
                 'skip_download': True,
-            }
+            },
+            'skip': 'does not contain a video anymore',
         },
         # Complex jwplayer
         {
@@ -1116,6 +1117,7 @@ class GenericIE(InfoExtractor):
                 'id': 'videos',
                 'ext': 'mp4',
                 'title': 'king machine trailer 1',
+                'description': 'Browse King Machine videos & audio for sweet media. Your eyes will thank you.',
                 'thumbnail': r're:^https?://.*\.jpg$',
             },
         },
@@ -1168,7 +1170,7 @@ class GenericIE(InfoExtractor):
             'playlist_mincount': 5,
             'info_dict': {
                 'id': 'aanslagen-kopenhagen',
-                'title': 'Aanslagen Kopenhagen | RTL Nieuws',
+                'title': 'Aanslagen Kopenhagen',
             }
         },
         # Zapiks embed
@@ -1300,6 +1302,7 @@ class GenericIE(InfoExtractor):
             'params': {
                 'skip_download': True,
             },
+            'skip': 'This video is unavailable.',
         },
         # Pladform embed
         {
@@ -1313,6 +1316,7 @@ class GenericIE(InfoExtractor):
                 'duration': 694,
                 'age_limit': 0,
             },
+            'skip': 'HTTP Error 404: Not Found',
         },
         # Playwire embed
         {
@@ -1333,6 +1337,14 @@ class GenericIE(InfoExtractor):
                 'id': '518726732',
                 'ext': 'mp4',
                 'title': 'Facebook Creates "On This Day" | Crunch Report',
+                'description': 'Amazon updates Fire TV line, Tesla\'s Model X spotted in the wild',
+                'timestamp': 1427237531,
+                'uploader': 'Crunch Report',
+                'upload_date': '20150324',
+            },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
             },
         },
         # SVT embed
@@ -1384,16 +1396,20 @@ class GenericIE(InfoExtractor):
                 'upload_date': '20140107',
                 'timestamp': 1389118457,
             },
+            'skip': 'Invalid Page URL',
         },
         # NBC News embed
         {
             'url': 'http://www.vulture.com/2016/06/letterman-couldnt-care-less-about-late-night.html',
             'md5': '1aa589c675898ae6d37a17913cf68d66',
             'info_dict': {
-                'id': '701714499682',
+                'id': 'x_dtl_oa_LettermanliftPR_160608',
                 'ext': 'mp4',
-                'title': 'PREVIEW: On Assignment: David Letterman',
+                'title': 'David Letterman: A Preview',
                 'description': 'A preview of Tom Brokaw\'s interview with David Letterman as part of the On Assignment series powered by Dateline. Airs Sunday June 12 at 7/6c.',
+                'upload_date': '20160609',
+                'timestamp': 1465431544,
+                'uploader': 'NBCU-NEWS',
             },
         },
         # UDN embed
@@ -1410,6 +1426,7 @@ class GenericIE(InfoExtractor):
                 # m3u8 download
                 'skip_download': True,
             },
+            'expected_warnings': ['Failed to parse JSON Expecting value'],
         },
         # Ooyala embed
         {
@@ -1417,7 +1434,7 @@ class GenericIE(InfoExtractor):
             'info_dict': {
                 'id': '50YnY4czr4ms1vJ7yz3xzq0excz_pUMs',
                 'ext': 'mp4',
-                'description': 'VIDEO: INDEX/MATCH versus VLOOKUP.',
+                'description': 'Index/Match versus VLOOKUP.',
                 'title': 'This is what separates the Excel masters from the wannabes',
                 'duration': 191.933,
             },
@@ -1455,7 +1472,8 @@ class GenericIE(InfoExtractor):
                 'upload_date': '20150622',
                 'uploader': 'Public Sénat',
                 'uploader_id': 'xa9gza',
-            }
+            },
+            'skip': 'File not found.',
         },
         # OnionStudios embed
         {
@@ -2253,7 +2271,7 @@ class GenericIE(InfoExtractor):
 
         # Look for embedded rtl.nl player
         matches = re.findall(
-            r'<iframe[^>]+?src="((?:https?:)?//(?:www\.)?rtl\.nl/system/videoplayer/[^"]+(?:video_)?embed[^"]+)"',
+            r'<iframe[^>]+?src="((?:https?:)?//(?:(?:www|static)\.)?rtl\.nl/(?:system/videoplayer/[^"]+(?:video_)?)?embed[^"]+)"',
             webpage)
         if matches:
             return self.playlist_from_matches(matches, video_id, video_title, ie='RtlNl')
@@ -2652,7 +2670,7 @@ class GenericIE(InfoExtractor):
 
         # Look for UDN embeds
         mobj = re.search(
-            r'<iframe[^>]+src="(?P<url>%s)"' % UDNEmbedIE._PROTOCOL_RELATIVE_VALID_URL, webpage)
+            r'<iframe[^>]+src="(?:https?:)?(?P<url>%s)"' % UDNEmbedIE._PROTOCOL_RELATIVE_VALID_URL, webpage)
         if mobj is not None:
             return self.url_result(
                 compat_urlparse.urljoin(url, mobj.group('url')), 'UDNEmbed')

From e650659b94c526362be0cf963b0f2241934bcbbe Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 12 Oct 2017 17:39:51 +0000
Subject: [PATCH 1428/1696] [infoq] fix http format downloading

---
 youtube_dl/extractor/infoq.py | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/infoq.py b/youtube_dl/extractor/infoq.py
index 57c9b0cc4..c3e892feb 100644
--- a/youtube_dl/extractor/infoq.py
+++ b/youtube_dl/extractor/infoq.py
@@ -8,7 +8,10 @@ from ..compat import (
     compat_urllib_parse_unquote,
     compat_urlparse,
 )
-from ..utils import determine_ext
+from ..utils import (
+    determine_ext,
+    update_url_query,
+)
 from .bokecc import BokeCCBaseIE
 
 
@@ -68,21 +71,22 @@ class InfoQIE(BokeCCBaseIE):
             'play_path': playpath,
         }]
 
-    def _extract_cookies(self, webpage):
+    def _extract_cf_auth(self, webpage):
         policy = self._search_regex(r'InfoQConstants\.scp\s*=\s*\'([^\']+)\'', webpage, 'policy')
         signature = self._search_regex(r'InfoQConstants\.scs\s*=\s*\'([^\']+)\'', webpage, 'signature')
         key_pair_id = self._search_regex(r'InfoQConstants\.sck\s*=\s*\'([^\']+)\'', webpage, 'key-pair-id')
-        return 'CloudFront-Policy=%s; CloudFront-Signature=%s; CloudFront-Key-Pair-Id=%s' % (
-            policy, signature, key_pair_id)
+        return {
+            'Policy': policy,
+            'Signature': signature,
+            'Key-Pair-Id': key_pair_id,
+        }
 
     def _extract_http_video(self, webpage):
         http_video_url = self._search_regex(r'P\.s\s*=\s*\'([^\']+)\'', webpage, 'video URL')
+        http_video_url = update_url_query(http_video_url, self._extract_cf_auth(webpage))
         return [{
             'format_id': 'http_video',
             'url': http_video_url,
-            'http_headers': {
-                'Cookie': self._extract_cookies(webpage)
-            },
         }]
 
     def _extract_http_audio(self, webpage, video_id):
@@ -91,22 +95,20 @@ class InfoQIE(BokeCCBaseIE):
         if not http_audio_url:
             return []
 
-        cookies_header = {'Cookie': self._extract_cookies(webpage)}
-
         # base URL is found in the Location header in the response returned by
         # GET https://www.infoq.com/mp3download.action?filename=... when logged in.
         http_audio_url = compat_urlparse.urljoin('http://res.infoq.com/downloads/mp3downloads/', http_audio_url)
+        http_audio_url = update_url_query(http_audio_url, self._extract_cf_auth(webpage))
 
         # audio file seem to be missing some times even if there is a download link
         # so probe URL to make sure
-        if not self._is_valid_url(http_audio_url, video_id, headers=cookies_header):
+        if not self._is_valid_url(http_audio_url, video_id):
             return []
 
         return [{
             'format_id': 'http_audio',
             'url': http_audio_url,
             'vcodec': 'none',
-            'http_headers': cookies_header,
         }]
 
     def _real_extract(self, url):

From 94a530c6cb65d555d8e3289de155c2f8f9326dea Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 12 Oct 2017 19:03:47 +0000
Subject: [PATCH 1429/1696] [howstuffworks] add support for more domains

---
 youtube_dl/extractor/howstuffworks.py | 39 +++++----------------------
 1 file changed, 7 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/howstuffworks.py b/youtube_dl/extractor/howstuffworks.py
index 2be68abad..cf90ab3c9 100644
--- a/youtube_dl/extractor/howstuffworks.py
+++ b/youtube_dl/extractor/howstuffworks.py
@@ -11,45 +11,20 @@ from ..utils import (
 
 
 class HowStuffWorksIE(InfoExtractor):
-    _VALID_URL = r'https?://[\da-z-]+\.howstuffworks\.com/(?:[^/]+/)*(?:\d+-)?(?P<id>.+?)-video\.htm'
+    _VALID_URL = r'https?://[\da-z-]+\.(?:howstuffworks|stuff(?:(?:youshould|theydontwantyouto)know|toblowyourmind|momnevertoldyou)|(?:brain|car)stuffshow|fwthinking|geniusstuff)\.com/(?:[^/]+/)*(?:\d+-)?(?P<id>.+?)-video\.htm'
     _TESTS = [
         {
-            'url': 'http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm',
+            'url': 'http://www.stufftoblowyourmind.com/videos/optical-illusions-video.htm',
+            'md5': '76646a5acc0c92bf7cd66751ca5db94d',
             'info_dict': {
-                'id': '450221',
-                'ext': 'flv',
-                'title': 'Cool Jobs - Iditarod Musher',
-                'description': 'Cold sleds, freezing temps and warm dog breath... an Iditarod musher\'s dream. Kasey-Dee Gardner jumps on a sled to find out what the big deal is.',
-                'display_id': 'cool-jobs-iditarod-musher',
-                'thumbnail': r're:^https?://.*\.jpg$',
-                'duration': 161,
-            },
-            'skip': 'Video broken',
-        },
-        {
-            'url': 'http://adventure.howstuffworks.com/7199-survival-zone-food-and-water-in-the-savanna-video.htm',
-            'info_dict': {
-                'id': '453464',
+                'id': '855410',
                 'ext': 'mp4',
-                'title': 'Survival Zone: Food and Water In the Savanna',
-                'description': 'Learn how to find both food and water while trekking in the African savannah. In this video from the Discovery Channel.',
-                'display_id': 'survival-zone-food-and-water-in-the-savanna',
-                'thumbnail': r're:^https?://.*\.jpg$',
+                'title': 'Your Trickster Brain: Optical Illusions -- Science on the Web',
+                'description': 'md5:e374ff9561f6833ad076a8cc0a5ab2fb',
             },
         },
         {
-            'url': 'http://entertainment.howstuffworks.com/arts/2706-sword-swallowing-1-by-dan-meyer-video.htm',
-            'info_dict': {
-                'id': '440011',
-                'ext': 'mp4',
-                'title': 'Sword Swallowing #1 by Dan Meyer',
-                'description': 'Video footage (1 of 3) used by permission of the owner Dan Meyer through Sword Swallowers Association International <www.swordswallow.org>',
-                'display_id': 'sword-swallowing-1-by-dan-meyer',
-                'thumbnail': r're:^https?://.*\.jpg$',
-            },
-        },
-        {
-            'url': 'http://shows.howstuffworks.com/stuff-to-blow-your-mind/optical-illusions-video.htm',
+            'url': 'http://shows.howstuffworks.com/more-shows/why-does-balloon-stick-to-hair-video.htm',
             'only_matching': True,
         }
     ]

From c9bd503e7de28e9d4ba9ebee5e509a8abb90fc72 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 13 Oct 2017 08:41:57 +0000
Subject: [PATCH 1430/1696] [spike] bypass geo restriction

---
 youtube_dl/extractor/spike.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/spike.py b/youtube_dl/extractor/spike.py
index c59896a17..a7b1b3b5f 100644
--- a/youtube_dl/extractor/spike.py
+++ b/youtube_dl/extractor/spike.py
@@ -44,6 +44,7 @@ class SpikeIE(MTVServicesInfoExtractor):
     _FEED_URL = 'http://www.spike.com/feeds/mrss/'
     _MOBILE_TEMPLATE = 'http://m.spike.com/videos/video.rbml?id=%s'
     _CUSTOM_URL_REGEX = re.compile(r'spikenetworkapp://([^/]+/[-a-fA-F0-9]+)')
+    _GEO_COUNTRIES = ['US']
 
     def _extract_mgid(self, webpage):
         mgid = super(SpikeIE, self)._extract_mgid(webpage)

From 685e87b61f785b096745cda5ea64ea0b950f56d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Oct 2017 20:26:24 +0700
Subject: [PATCH 1431/1696] [youtube] Fix relative URLs in description

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 6e2d57d6a..4e8db240d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1630,7 +1630,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     class="[^"]*"[^>]*>
                 [^<]+\.{3}\s*
                 </a>
-            ''', r'\1', video_description)
+            ''', lambda m: compat_urlparse.urljoin(url, m.group(1)), video_description)
             video_description = clean_html(video_description)
         else:
             fd_mobj = re.search(r'<meta name="description" content="([^"]+)"', video_webpage)

From 210a2720bcdaf4f98561fefea021f42cae39462d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Oct 2017 01:44:57 +0700
Subject: [PATCH 1432/1696] [anvato] Process master m3u8 manifests >>>
 Individual m3u8 manifests are not always present, e.g.
 anvato:anvato_scripps_app_web_prod_0837996dbe373629133857ae9eb72e740424d80a:4173834

---
 youtube_dl/extractor/anvato.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/anvato.py b/youtube_dl/extractor/anvato.py
index 8023da702..e443ecff6 100644
--- a/youtube_dl/extractor/anvato.py
+++ b/youtube_dl/extractor/anvato.py
@@ -197,12 +197,16 @@ class AnvatoIE(InfoExtractor):
                 'tbr': tbr if tbr != 0 else None,
             }
 
-            if ext == 'm3u8' or media_format in ('m3u8', 'm3u8-variant'):
-                if tbr is not None:
-                    a_format.update({
-                        'format_id': '-'.join(filter(None, ['hls', compat_str(tbr)])),
-                        'ext': 'mp4',
-                    })
+            if media_format == 'm3u8' and tbr is not None:
+                a_format.update({
+                    'format_id': '-'.join(filter(None, ['hls', compat_str(tbr)])),
+                    'ext': 'mp4',
+                })
+            elif media_format == 'm3u8-variant' or ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    video_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls', fatal=False))
+                continue
             elif ext == 'mp3' or media_format == 'mp3':
                 a_format['vcodec'] = 'none'
             else:

From b21ab85088345323d1e6d988b2cdce8e02fe6bdf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Oct 2017 01:46:05 +0700
Subject: [PATCH 1433/1696] [scrippsnetworks:watch] Fix extraction (closes
 #14389)

---
 youtube_dl/extractor/scrippsnetworks.py | 209 +++++++++++++++++++-----
 1 file changed, 166 insertions(+), 43 deletions(-)

diff --git a/youtube_dl/extractor/scrippsnetworks.py b/youtube_dl/extractor/scrippsnetworks.py
index 597d6f543..30bb31d69 100644
--- a/youtube_dl/extractor/scrippsnetworks.py
+++ b/youtube_dl/extractor/scrippsnetworks.py
@@ -1,60 +1,183 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .adobepass import AdobePassIE
+import datetime
+import json
+import hashlib
+import hmac
+import re
+
+from .common import InfoExtractor
+from .anvato import AnvatoIE
 from ..utils import (
-    int_or_none,
-    smuggle_url,
-    update_url_query,
+    urlencode_postdata,
+    xpath_text,
 )
 
 
-class ScrippsNetworksWatchIE(AdobePassIE):
+class ScrippsNetworksWatchIE(InfoExtractor):
     IE_NAME = 'scrippsnetworks:watch'
-    _VALID_URL = r'https?://watch\.(?:hgtv|foodnetwork|travelchannel|diynetwork|cookingchanneltv)\.com/player\.[A-Z0-9]+\.html#(?P<id>\d+)'
-    _TEST = {
-        'url': 'http://watch.hgtv.com/player.HNT.html#0256538',
+    _VALID_URL = r'''(?x)
+                    https?://
+                        watch\.
+                        (?P<site>hgtv|foodnetwork|travelchannel|diynetwork|cookingchanneltv)\.com/
+                        (?:
+                            player\.[A-Z0-9]+\.html\#|
+                            show/(?:[^/]+/){2}
+                        )
+                        (?P<id>\d+)
+                    '''
+    _TESTS = [{
+        'url': 'http://watch.hgtv.com/show/HGTVE/Best-Ever-Treehouses/2241515/Best-Ever-Treehouses/',
         'md5': '26545fd676d939954c6808274bdb905a',
         'info_dict': {
-            'id': '0256538',
+            'id': '4173834',
             'ext': 'mp4',
-            'title': 'Seeking a Wow House',
-            'description': 'Buyers retiring in Palm Springs, California, want a modern house with major wow factor. They\'re also looking for a pool and a large, open floorplan with tall windows looking out at the views.',
-            'uploader': 'SCNI',
-            'upload_date': '20170207',
-            'timestamp': 1486450493,
+            'title': 'Best Ever Treehouses',
+            'description': "We're searching for the most over the top treehouses.",
+            'uploader': 'ANV',
+            'upload_date': '20170922',
+            'timestamp': 1506056400,
         },
-        'skip': 'requires TV provider authentication',
+        'params': {
+            'skip_download': True,
+        },
+        'add_ie': [AnvatoIE.ie_key()],
+    }, {
+        'url': 'http://watch.diynetwork.com/show/DSAL/Salvage-Dawgs/2656646/Covington-Church/',
+        'only_matching': True,
+    }, {
+        'url': 'http://watch.diynetwork.com/player.HNT.html#2656646',
+        'only_matching': True,
+    }]
+
+    _SNI_TABLE = {
+        'hgtv': 'hgtv',
+        'diynetwork': 'diy',
+        'foodnetwork': 'food',
+        'cookingchanneltv': 'cook',
+        'travelchannel': 'trav',
+        'geniuskitchen': 'geniuskitchen',
     }
+    _SNI_HOST = 'web.api.video.snidigital.com'
+
+    _AWS_REGION = 'us-east-1'
+    _AWS_IDENTITY_ID_JSON = json.dumps({
+        'IdentityId': '%s:7655847c-0ae7-4d9b-80d6-56c062927eb3' % _AWS_REGION
+    })
+    _AWS_USER_AGENT = 'aws-sdk-js/2.80.0 callback'
+    _AWS_API_KEY = 'E7wSQmq0qK6xPrF13WmzKiHo4BQ7tip4pQcSXVl1'
+    _AWS_SERVICE = 'execute-api'
+    _AWS_REQUEST = 'aws4_request'
+    _AWS_SIGNED_HEADERS = ';'.join([
+        'host', 'x-amz-date', 'x-amz-security-token', 'x-api-key'])
+    _AWS_CANONICAL_REQUEST_TEMPLATE = '''GET
+%(uri)s
+
+host:%(host)s
+x-amz-date:%(date)s
+x-amz-security-token:%(token)s
+x-api-key:%(key)s
+
+%(signed_headers)s
+%(payload_hash)s'''
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        channel = self._parse_json(self._search_regex(
-            r'"channels"\s*:\s*(\[.+\])',
-            webpage, 'channels'), video_id)[0]
-        video_data = next(v for v in channel['videos'] if v.get('nlvid') == video_id)
-        title = video_data['title']
-        release_url = video_data['releaseUrl']
-        if video_data.get('restricted'):
-            requestor_id = self._search_regex(
-                r'requestorId\s*=\s*"([^"]+)";', webpage, 'requestor id')
-            resource = self._get_mvpd_resource(
-                requestor_id, title, video_id,
-                video_data.get('ratings', [{}])[0].get('rating'))
-            auth = self._extract_mvpd_auth(
-                url, video_id, requestor_id, resource)
-            release_url = update_url_query(release_url, {'auth': auth})
+        mobj = re.match(self._VALID_URL, url)
+        site_id, video_id = mobj.group('site', 'id')
 
-        return {
-            '_type': 'url_transparent',
-            'id': video_id,
-            'title': title,
-            'url': smuggle_url(release_url, {'force_smil_url': True}),
-            'description': video_data.get('description'),
-            'thumbnail': video_data.get('thumbnailUrl'),
-            'series': video_data.get('showTitle'),
-            'season_number': int_or_none(video_data.get('season')),
-            'episode_number': int_or_none(video_data.get('episodeNumber')),
-            'ie_key': 'ThePlatform',
+        def aws_hash(s):
+            return hashlib.sha256(s.encode('utf-8')).hexdigest()
+
+        token = self._download_json(
+            'https://cognito-identity.us-east-1.amazonaws.com/', video_id,
+            data=self._AWS_IDENTITY_ID_JSON.encode('utf-8'),
+            headers={
+                'Accept': '*/*',
+                'Content-Type': 'application/x-amz-json-1.1',
+                'Referer': url,
+                'X-Amz-Content-Sha256': aws_hash(self._AWS_IDENTITY_ID_JSON),
+                'X-Amz-Target': 'AWSCognitoIdentityService.GetOpenIdToken',
+                'X-Amz-User-Agent': self._AWS_USER_AGENT,
+            })['Token']
+
+        sts = self._download_xml(
+            'https://sts.amazonaws.com/', video_id, data=urlencode_postdata({
+                'Action': 'AssumeRoleWithWebIdentity',
+                'RoleArn': 'arn:aws:iam::710330595350:role/Cognito_WebAPIUnauth_Role',
+                'RoleSessionName': 'web-identity',
+                'Version': '2011-06-15',
+                'WebIdentityToken': token,
+            }), headers={
+                'Referer': url,
+                'X-Amz-User-Agent': self._AWS_USER_AGENT,
+                'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8',
+            })
+
+        def get(key):
+            return xpath_text(
+                sts, './/{https://sts.amazonaws.com/doc/2011-06-15/}%s' % key,
+                fatal=True)
+
+        access_key_id = get('AccessKeyId')
+        secret_access_key = get('SecretAccessKey')
+        session_token = get('SessionToken')
+
+        # Task 1: http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+        uri = '/1/web/brands/%s/episodes/scrid/%s' % (self._SNI_TABLE[site_id], video_id)
+        datetime_now = datetime.datetime.utcnow().strftime('%Y%m%dT%H%M%SZ')
+        date = datetime_now[:8]
+        canonical_string = self._AWS_CANONICAL_REQUEST_TEMPLATE % {
+            'uri': uri,
+            'host': self._SNI_HOST,
+            'date': datetime_now,
+            'token': session_token,
+            'key': self._AWS_API_KEY,
+            'signed_headers': self._AWS_SIGNED_HEADERS,
+            'payload_hash': aws_hash(''),
         }
+
+        # Task 2: http://docs.aws.amazon.com/general/latest/gr/sigv4-create-string-to-sign.html
+        credential_string = '/'.join([date, self._AWS_REGION, self._AWS_SERVICE, self._AWS_REQUEST])
+        string_to_sign = '\n'.join([
+            'AWS4-HMAC-SHA256', datetime_now, credential_string,
+            aws_hash(canonical_string)])
+
+        # Task 3: http://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
+        def aws_hmac(key, msg):
+            return hmac.new(key, msg.encode('utf-8'), hashlib.sha256)
+
+        def aws_hmac_digest(key, msg):
+            return aws_hmac(key, msg).digest()
+
+        def aws_hmac_hexdigest(key, msg):
+            return aws_hmac(key, msg).hexdigest()
+
+        k_secret = 'AWS4' + secret_access_key
+        k_date = aws_hmac_digest(k_secret.encode('utf-8'), date)
+        k_region = aws_hmac_digest(k_date, self._AWS_REGION)
+        k_service = aws_hmac_digest(k_region, self._AWS_SERVICE)
+        k_signing = aws_hmac_digest(k_service, self._AWS_REQUEST)
+
+        signature = aws_hmac_hexdigest(k_signing, string_to_sign)
+
+        auth_header = ', '.join([
+            'AWS4-HMAC-SHA256 Credential=%s' % '/'.join(
+                [access_key_id, date, self._AWS_REGION, self._AWS_SERVICE, self._AWS_REQUEST]),
+            'SignedHeaders=%s' % self._AWS_SIGNED_HEADERS,
+            'Signature=%s' % signature,
+        ])
+
+        mcp_id = self._download_json(
+            'https://%s%s' % (self._SNI_HOST, uri), video_id, headers={
+                'Accept': '*/*',
+                'Referer': url,
+                'Authorization': auth_header,
+                'X-Amz-Date': datetime_now,
+                'X-Amz-Security-Token': session_token,
+                'X-Api-Key': self._AWS_API_KEY,
+            })['results'][0]['mcpId']
+
+        return self.url_result(
+            'anvato:anvato_scripps_app_web_prod_0837996dbe373629133857ae9eb72e740424d80a:%s' % mcp_id,
+            AnvatoIE.ie_key(), video_id=mcp_id)

From 57eb45b11110bdc6d99b02b738129565239a12b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Oct 2017 02:01:16 +0700
Subject: [PATCH 1434/1696] [scrippsnetworks:watch] Add support for
 geniuskitchen.com

---
 youtube_dl/extractor/scrippsnetworks.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/scrippsnetworks.py b/youtube_dl/extractor/scrippsnetworks.py
index 30bb31d69..411fef84c 100644
--- a/youtube_dl/extractor/scrippsnetworks.py
+++ b/youtube_dl/extractor/scrippsnetworks.py
@@ -20,10 +20,11 @@ class ScrippsNetworksWatchIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                     https?://
                         watch\.
-                        (?P<site>hgtv|foodnetwork|travelchannel|diynetwork|cookingchanneltv)\.com/
+                        (?P<site>hgtv|foodnetwork|travelchannel|diynetwork|cookingchanneltv|geniuskitchen)\.com/
                         (?:
                             player\.[A-Z0-9]+\.html\#|
-                            show/(?:[^/]+/){2}
+                            show/(?:[^/]+/){2}|
+                            player/
                         )
                         (?P<id>\d+)
                     '''
@@ -49,6 +50,9 @@ class ScrippsNetworksWatchIE(InfoExtractor):
     }, {
         'url': 'http://watch.diynetwork.com/player.HNT.html#2656646',
         'only_matching': True,
+    }, {
+        'url': 'http://watch.geniuskitchen.com/player/3787617/Ample-Hills-Ice-Cream-Bike/',
+        'only_matching': True,
     }]
 
     _SNI_TABLE = {
@@ -57,7 +61,7 @@ class ScrippsNetworksWatchIE(InfoExtractor):
         'foodnetwork': 'food',
         'cookingchanneltv': 'cook',
         'travelchannel': 'trav',
-        'geniuskitchen': 'geniuskitchen',
+        'geniuskitchen': 'genius',
     }
     _SNI_HOST = 'web.api.video.snidigital.com'
 

From ee88c1cbc684a3ef09cf31a33b3adac7b299b670 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Oct 2017 02:26:10 +0700
Subject: [PATCH 1435/1696] [ChangeLog] Actualize

---
 ChangeLog | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index db2c2bf88..0c720b2cc 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,22 @@
 version <unreleased>
 
+Core
++ [common] Add support for jwplayer youtube embeds
+
 Extractors
+* [scrippsnetworks:watch] Fix extraction (#14389)
+* [anvato] Process master m3u8 manifests
+* [youtube] Fix relative URLs in description
+* [spike] Bypass geo restriction
++ [howstuffworks] Add support for more domains
+* [infoq] Fix http format downloading
++ [rtlnl] Add support for another type of embeds
++ [onionstudios] Add support for bulbs-video embeds
+* [udn] Fix extraction
+* [shahid] Fix extraction (#14448)
 * [kaltura] Ignore Widevine encrypted video (.wvm) (#14471)
+* [vh1] Fix extraction (#9613)
+
 
 version 2017.10.12
 

From 715534083dda7ecfd7aa0156c34422b394672b5e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Oct 2017 02:26:58 +0700
Subject: [PATCH 1436/1696] release 2017.10.15

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 5ef087328..2feb22dd0 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.10.12*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.10.12**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.10.15*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.10.15**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.10.12
+[debug] youtube-dl version 2017.10.15
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 0c720b2cc..3b3ff459b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.10.15
 
 Core
 + [common] Add support for jwplayer youtube embeds
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 36e6e99da..49d475e85 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.10.12'
+__version__ = '2017.10.15'

From 9bb2c7673e45aee95023d980ff307d7a90ac58c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Oct 2017 03:38:34 +0700
Subject: [PATCH 1437/1696] [redditr] Fix extraction for URLs with query
 (closes #14495)

---
 youtube_dl/extractor/reddit.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/reddit.py b/youtube_dl/extractor/reddit.py
index 4d44b9d74..f36bc648c 100644
--- a/youtube_dl/extractor/reddit.py
+++ b/youtube_dl/extractor/reddit.py
@@ -1,5 +1,7 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
@@ -45,7 +47,7 @@ class RedditIE(InfoExtractor):
 
 
 class RedditRIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?reddit\.com/r/[^/]+/comments/(?P<id>[^/]+)'
+    _VALID_URL = r'(?P<url>https?://(?:www\.)?reddit\.com/r/[^/]+/comments/(?P<id>[^/?#&]+))'
     _TESTS = [{
         'url': 'https://www.reddit.com/r/videos/comments/6rrwyj/that_small_heart_attack/',
         'info_dict': {
@@ -83,10 +85,13 @@ class RedditRIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        url, video_id = mobj.group('url', 'id')
+
         video_id = self._match_id(url)
 
         data = self._download_json(
-            url + '.json', video_id)[0]['data']['children'][0]['data']
+            url + '/.json', video_id)[0]['data']['children'][0]['data']
 
         video_url = data['url']
 

From ee093a0ea04d973cc6dbd0d53b57c976a9e68dad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Oct 2017 06:11:02 +0700
Subject: [PATCH 1438/1696] [anvato] Add ability to bypass geo restriction

---
 youtube_dl/extractor/anvato.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/anvato.py b/youtube_dl/extractor/anvato.py
index e443ecff6..7a29cd2c6 100644
--- a/youtube_dl/extractor/anvato.py
+++ b/youtube_dl/extractor/anvato.py
@@ -18,6 +18,7 @@ from ..utils import (
     int_or_none,
     strip_jsonp,
     unescapeHTML,
+    unsmuggle_url,
 )
 
 
@@ -275,6 +276,9 @@ class AnvatoIE(InfoExtractor):
             anvplayer_data['accessKey'], anvplayer_data['video'])
 
     def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
+        self._initialize_geo_bypass(smuggled_data.get('geo_countries'))
+
         mobj = re.match(self._VALID_URL, url)
         access_key, video_id = mobj.group('access_key_or_mcp', 'id')
         if access_key not in self._ANVACK_TABLE:

From 4827270526621cdabe74275deb38b04c2ef1b0d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Oct 2017 06:11:35 +0700
Subject: [PATCH 1439/1696] [scrippsnetworks:watch] Bypass geo restriction

---
 youtube_dl/extractor/scrippsnetworks.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/scrippsnetworks.py b/youtube_dl/extractor/scrippsnetworks.py
index 411fef84c..b446a02ba 100644
--- a/youtube_dl/extractor/scrippsnetworks.py
+++ b/youtube_dl/extractor/scrippsnetworks.py
@@ -10,6 +10,7 @@ import re
 from .common import InfoExtractor
 from .anvato import AnvatoIE
 from ..utils import (
+    smuggle_url,
     urlencode_postdata,
     xpath_text,
 )
@@ -183,5 +184,7 @@ x-api-key:%(key)s
             })['results'][0]['mcpId']
 
         return self.url_result(
-            'anvato:anvato_scripps_app_web_prod_0837996dbe373629133857ae9eb72e740424d80a:%s' % mcp_id,
+            smuggle_url(
+                'anvato:anvato_scripps_app_web_prod_0837996dbe373629133857ae9eb72e740424d80a:%s' % mcp_id,
+                {'geo_countries': ['US']}),
             AnvatoIE.ie_key(), video_id=mcp_id)

From 5efaf43c93247446c8616454bb3f59b7ae13fcb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Oct 2017 06:13:07 +0700
Subject: [PATCH 1440/1696] [downloader/fragment] Output ad fragment count

---
 youtube_dl/downloader/fragment.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 6f6fb4a77..7e891b92a 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -117,9 +117,15 @@ class FragmentFD(FileDownloader):
     def _prepare_frag_download(self, ctx):
         if 'live' not in ctx:
             ctx['live'] = False
+        if not ctx['live']:
+            total_frags_str = '%d' % ctx['total_frags']
+            ad_frags = ctx.get('ad_frags', 0)
+            if ad_frags:
+                total_frags_str += ' (not including %d ad)' % ad_frags
+        else:
+            total_frags_str = 'unknown (live)'
         self.to_screen(
-            '[%s] Total fragments: %s'
-            % (self.FD_NAME, ctx['total_frags'] if not ctx['live'] else 'unknown (live)'))
+            '[%s] Total fragments: %s' % (self.FD_NAME, total_frags_str))
         self.report_destination(ctx['filename'])
         dl = HttpQuietDownloader(
             self.ydl,

From 74c42d9ec3c6e0f0f67923fd151ce2531637a3db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Oct 2017 06:13:48 +0700
Subject: [PATCH 1441/1696] [downloader/hls] Ignore anvato ad fragments (closes
 #14496)

---
 youtube_dl/downloader/hls.py | 28 ++++++++++++++++++++++++----
 1 file changed, 24 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 46308cf07..7955ca510 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -75,15 +75,29 @@ class HlsFD(FragmentFD):
                 fd.add_progress_hook(ph)
             return fd.real_download(filename, info_dict)
 
-        total_frags = 0
+        def anvato_ad(s):
+            return s.startswith('#ANVATO-SEGMENT-INFO') and 'type=ad' in s
+
+        media_frags = 0
+        ad_frags = 0
+        ad_frag_next = False
         for line in s.splitlines():
             line = line.strip()
-            if line and not line.startswith('#'):
-                total_frags += 1
+            if not line:
+                continue
+            if line.startswith('#'):
+                if anvato_ad(line):
+                    ad_frags += 1
+                continue
+            if ad_frag_next:
+                ad_frag_next = False
+                continue
+            media_frags += 1
 
         ctx = {
             'filename': filename,
-            'total_frags': total_frags,
+            'total_frags': media_frags,
+            'ad_frags': ad_frags,
         }
 
         self._prepare_and_start_frag_download(ctx)
@@ -101,10 +115,14 @@ class HlsFD(FragmentFD):
         decrypt_info = {'METHOD': 'NONE'}
         byte_range = {}
         frag_index = 0
+        ad_frag_next = False
         for line in s.splitlines():
             line = line.strip()
             if line:
                 if not line.startswith('#'):
+                    if ad_frag_next:
+                        ad_frag_next = False
+                        continue
                     frag_index += 1
                     if frag_index <= ctx['fragment_index']:
                         continue
@@ -175,6 +193,8 @@ class HlsFD(FragmentFD):
                         'start': sub_range_start,
                         'end': sub_range_start + int(splitted_byte_range[0]),
                     }
+                elif anvato_ad(line):
+                    ad_frag_next = True
 
         self._finish_frag_download(ctx)
 

From bd7e1406b31ed18c48d51486262465c88bd92866 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Oct 2017 06:15:37 +0700
Subject: [PATCH 1442/1696] [ChangeLog] Actualize

---
 ChangeLog | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 3b3ff459b..4eb41a7a6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,15 @@
+version <unreleased>
+
+Core
+* [downloader/hls] Ignore anvato ad fragments (#14496)
+* [downloader/fragment] Output ad fragment count
+
+Extractors
+* [scrippsnetworks:watch] Bypass geo restriction
++ [anvato] Add ability to bypass geo restriction
+* [redditr] Fix extraction for URLs with query (#14495)
+
+
 version 2017.10.15
 
 Core

From 7e721e35daa954f2c12f1113950ae07f1302f49e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Oct 2017 06:16:41 +0700
Subject: [PATCH 1443/1696] release 2017.10.15.1

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 2feb22dd0..1a72ab291 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.10.15*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.10.15**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.10.15.1*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.10.15.1**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.10.15
+[debug] youtube-dl version 2017.10.15.1
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 4eb41a7a6..d728e4d03 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.10.15.1
 
 Core
 * [downloader/hls] Ignore anvato ad fragments (#14496)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 49d475e85..d01ba3095 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.10.15'
+__version__ = '2017.10.15.1'

From aaab8c5e711c773885f67fd53788c89d94d48031 Mon Sep 17 00:00:00 2001
From: Pawit Pornkitprasan <p.pawit@gmail.com>
Date: Sun, 15 Oct 2017 10:40:57 +0700
Subject: [PATCH 1444/1696] [niconico] Improve uploader metadata extraction
 robustness (closes #14135)

---
 youtube_dl/extractor/niconico.py | 29 +++++++++++++++++++++++++----
 1 file changed, 25 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 026329d3e..df7f528be 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -40,7 +40,7 @@ class NiconicoIE(InfoExtractor):
             'uploader': 'takuya0301',
             'uploader_id': '2698420',
             'upload_date': '20131123',
-            'timestamp': 1385182762,
+            'timestamp': int,  # timestamp is unstable
             'description': '(c) copyright 2008, Blender Foundation / www.bigbuckbunny.org',
             'duration': 33,
             'view_count': int,
@@ -115,8 +115,8 @@ class NiconicoIE(InfoExtractor):
         'skip': 'Requires an account',
     }, {
         # "New" HTML5 video
+        # md5 is unstable
         'url': 'http://www.nicovideo.jp/watch/sm31464864',
-        'md5': '351647b4917660986dc0fa8864085135',
         'info_dict': {
             'id': 'sm31464864',
             'ext': 'mp4',
@@ -124,7 +124,7 @@ class NiconicoIE(InfoExtractor):
             'description': 'md5:e52974af9a96e739196b2c1ca72b5feb',
             'timestamp': 1498514060,
             'upload_date': '20170626',
-            'uploader': 'ゲス',
+            'uploader': 'ゲスト',
             'uploader_id': '40826363',
             'thumbnail': r're:https?://.*',
             'duration': 198,
@@ -132,6 +132,25 @@ class NiconicoIE(InfoExtractor):
             'comment_count': int,
         },
         'skip': 'Requires an account',
+    }, {
+        # Video without owner
+        'url': 'http://www.nicovideo.jp/watch/sm18238488',
+        'md5': 'd265680a1f92bdcbbd2a507fc9e78a9e',
+        'info_dict': {
+            'id': 'sm18238488',
+            'ext': 'mp4',
+            'title': '【実写版】ミュータントタートルズ',
+            'description': 'md5:15df8988e47a86f9e978af2064bf6d8e',
+            'timestamp': 1341160408,
+            'upload_date': '20120701',
+            'uploader': None,
+            'uploader_id': None,
+            'thumbnail': r're:https?://.*',
+            'duration': 5271,
+            'view_count': int,
+            'comment_count': int,
+        },
+        'skip': 'Requires an account',
     }, {
         'url': 'http://sp.nicovideo.jp/watch/sm28964488?ss_pos=1&cp_in=wt_tg',
         'only_matching': True,
@@ -395,7 +414,9 @@ class NiconicoIE(InfoExtractor):
 
         webpage_url = get_video_info('watch_url') or url
 
-        owner = api_data.get('owner', {})
+        # Note: cannot use api_data.get('owner', {}) because owner may be set to "null"
+        # in the JSON, which will cause None to be returned instead of {}.
+        owner = try_get(api_data, lambda x: x.get('owner'), dict) or {}
         uploader_id = get_video_info(['ch_id', 'user_id']) or owner.get('id')
         uploader = get_video_info(['ch_name', 'user_nickname']) or owner.get('nickname')
 

From a9ee4f6e49a5910b432d0ba4d8dc60231aa7deba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Oct 2017 11:03:54 +0700
Subject: [PATCH 1445/1696] [downloader/hls] Fix total fragments count when ad
 fragments exist

---
 youtube_dl/downloader/hls.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 7955ca510..1a6e226c8 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -88,6 +88,7 @@ class HlsFD(FragmentFD):
             if line.startswith('#'):
                 if anvato_ad(line):
                     ad_frags += 1
+                    ad_frag_next = True
                 continue
             if ad_frag_next:
                 ad_frag_next = False

From 8cc1840ccb2f17837fc7f12ceea91826eb925ecc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Oct 2017 22:12:34 +0700
Subject: [PATCH 1446/1696] [arte] Capture and output error message

---
 youtube_dl/extractor/arte.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 5cde90c5b..ffc321821 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -6,6 +6,7 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_parse_qs,
+    compat_str,
     compat_urllib_parse_urlparse,
 )
 from ..utils import (
@@ -15,6 +16,7 @@ from ..utils import (
     int_or_none,
     NO_DEFAULT,
     qualities,
+    try_get,
     unified_strdate,
 )
 
@@ -80,12 +82,15 @@ class ArteTVBaseIE(InfoExtractor):
         info = self._download_json(json_url, video_id)
         player_info = info['videoJsonPlayer']
 
-        vsr = player_info['VSR']
-
+        vsr = try_get(player_info, lambda x: x['VSR'], dict)
         if not vsr:
-            raise ExtractorError(
-                'Video %s is not available' % player_info.get('VID') or video_id,
-                expected=True)
+            error = None
+            if try_get(player_info, lambda x: x['custom_msg']['type']) == 'error':
+                error = try_get(
+                    player_info, lambda x: x['custom_msg']['msg'], compat_str)
+            if not error:
+                error = 'Video %s is not available' % player_info.get('VID') or video_id
+            raise ExtractorError(error, expected=True)
 
         upload_date_str = player_info.get('shootingDate')
         if not upload_date_str:

From acc4ea6237f7a2c7d923eca5b004aefbb6a6a766 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 16 Oct 2017 05:11:25 +0700
Subject: [PATCH 1447/1696] [eporner] Add support for embed URLs (closes
 #14507)

---
 youtube_dl/extractor/eporner.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/eporner.py b/youtube_dl/extractor/eporner.py
index f3734e9f8..81f2e2ee1 100644
--- a/youtube_dl/extractor/eporner.py
+++ b/youtube_dl/extractor/eporner.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 
 class EpornerIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?eporner\.com/hd-porn/(?P<id>\w+)(?:/(?P<display_id>[\w-]+))?'
+    _VALID_URL = r'https?://(?:www\.)?eporner\.com/(?:hd-porn|embed)/(?P<id>\w+)(?:/(?P<display_id>[\w-]+))?'
     _TESTS = [{
         'url': 'http://www.eporner.com/hd-porn/95008/Infamous-Tiffany-Teen-Strip-Tease-Video/',
         'md5': '39d486f046212d8e1b911c52ab4691f8',
@@ -35,6 +35,9 @@ class EpornerIE(InfoExtractor):
     }, {
         'url': 'http://www.eporner.com/hd-porn/3YRUtzMcWn0',
         'only_matching': True,
+    }, {
+        'url': 'http://www.eporner.com/hd-porn/3YRUtzMcWn0',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 83fcf19e2d517a2862fd89dffb1d0abad024a5b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 16 Oct 2017 05:48:45 +0700
Subject: [PATCH 1448/1696] [drtv] Respect preference for direct http formats
 (#14509)

---
 youtube_dl/extractor/drtv.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index 69effba58..f757745ba 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -138,6 +138,7 @@ class DRTVIE(InfoExtractor):
                             'tbr': int_or_none(bitrate),
                             'ext': link.get('FileFormat'),
                             'vcodec': 'none' if kind == 'AudioResource' else None,
+                            'preference': preference,
                         })
                 subtitles_list = asset.get('SubtitlesList')
                 if isinstance(subtitles_list, list):

From c233003afe34ba9db7861bf44d5097ade870c739 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 17 Oct 2017 15:39:06 +0800
Subject: [PATCH 1449/1696] [megaphone] Fix deprecated escape sequence

---
 youtube_dl/extractor/megaphone.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/megaphone.py b/youtube_dl/extractor/megaphone.py
index 60e3caf0d..5bafa6cf4 100644
--- a/youtube_dl/extractor/megaphone.py
+++ b/youtube_dl/extractor/megaphone.py
@@ -18,7 +18,7 @@ class MegaphoneIE(InfoExtractor):
             'id': 'GLT9749789991',
             'ext': 'mp3',
             'title': '#97 What Kind Of Idiot Gets Phished?',
-            'thumbnail': 're:^https://.*\.png.*$',
+            'thumbnail': r're:^https://.*\.png.*$',
             'duration': 1776.26375,
             'author': 'Reply All',
         },

From 6b9cbd023f1206f90e60cbed4497e6b107438542 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 17 Oct 2017 09:22:43 +0000
Subject: [PATCH 1450/1696] [pbs] restrict direct video url regex(fixes #14519)

---
 youtube_dl/extractor/pbs.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 8889e4a1a..b51dcbe10 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -187,7 +187,7 @@ class PBSIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://
         (?:
            # Direct video URL
-           (?:%s)/(?:viralplayer|video)/(?P<id>[0-9]+)/? |
+           (?:%s)/(?:(?:vir|port)alplayer|video)/(?P<id>[0-9]+)(?:[?/]|$) |
            # Article with embedded player (or direct video)
            (?:www\.)?pbs\.org/(?:[^/]+/){1,5}(?P<presumptive_id>[^/]+?)(?:\.html)?/?(?:$|[?\#]) |
            # Player
@@ -367,6 +367,10 @@ class PBSIE(InfoExtractor):
         {
             'url': 'http://watch.knpb.org/video/2365616055/',
             'only_matching': True,
+        },
+        {
+            'url': 'https://player.pbs.org/portalplayer/3004638221/?uid=',
+            'only_matching': True,
         }
     ]
     _ERRORS = {

From fa4bc6e71261613cf530437a2407ff7b61ea6cb5 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 17 Oct 2017 10:07:37 +0000
Subject: [PATCH 1451/1696] [youtube] replace youtube redirect urls in
 description(fixes #14517)

---
 youtube_dl/extractor/youtube.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4e8db240d..5aef555fb 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1622,6 +1622,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         # description
         description_original = video_description = get_element_by_id("eow-description", video_webpage)
         if video_description:
+
+            def replace_url(m):
+                redir_url = compat_urlparse.urljoin(url, m.group(1))
+                parsed_redir_url = compat_urllib_parse_urlparse(redir_url)
+                if re.search(r'^(?:www\.)?(?:youtube(?:-nocookie)?\.com|youtu\.be)$', parsed_redir_url.netloc) and parsed_redir_url.path == '/redirect':
+                    qs = compat_parse_qs(parsed_redir_url.query)
+                    q = qs.get('q')
+                    if q and q[0]:
+                        return q[0]
+                return redir_url
+
             description_original = video_description = re.sub(r'''(?x)
                 <a\s+
                     (?:[a-zA-Z-]+="[^"]*"\s+)*?
@@ -1630,7 +1641,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     class="[^"]*"[^>]*>
                 [^<]+\.{3}\s*
                 </a>
-            ''', lambda m: compat_urlparse.urljoin(url, m.group(1)), video_description)
+            ''', replace_url, video_description)
             video_description = clean_html(video_description)
         else:
             fd_mobj = re.search(r'<meta name="description" content="([^"]+)"', video_webpage)

From 6f3b4a98c96c54eb0e3933b3cd6b4e6848582a02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Oct 2017 22:53:34 +0700
Subject: [PATCH 1452/1696] [downloader/fragment] Report warning instead of
 error on inconsistent download state

---
 youtube_dl/downloader/fragment.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 7e891b92a..93002e45a 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -158,7 +158,7 @@ class FragmentFD(FileDownloader):
             if os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename']))):
                 self._read_ytdl_file(ctx)
                 if ctx['fragment_index'] > 0 and resume_len == 0:
-                    self.report_error(
+                    self.report_warning(
                         'Inconsistent state of incomplete fragment download. '
                         'Restarting from the beginning...')
                     ctx['fragment_index'] = resume_len = 0

From ca1c9f26fa8277c0ea75c1d91a74f454cc9aaac5 Mon Sep 17 00:00:00 2001
From: Parmjit Virk <pvirk@mts.net>
Date: Wed, 18 Oct 2017 16:46:06 -0500
Subject: [PATCH 1453/1696] [unity] Add extractor (fixes #14528)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/unity.py      | 32 ++++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+)
 create mode 100644 youtube_dl/extractor/unity.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index ecb33bc9e..1624129d2 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1138,6 +1138,7 @@ from .udn import UDNEmbedIE
 from .uktvplay import UKTVPlayIE
 from .digiteka import DigitekaIE
 from .unistra import UnistraIE
+from .unity import UnityIE
 from .uol import UOLIE
 from .uplynk import (
     UplynkIE,
diff --git a/youtube_dl/extractor/unity.py b/youtube_dl/extractor/unity.py
new file mode 100644
index 000000000..73daacf29
--- /dev/null
+++ b/youtube_dl/extractor/unity.py
@@ -0,0 +1,32 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from .youtube import YoutubeIE
+
+
+class UnityIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?unity3d\.com/learn/tutorials/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'https://unity3d.com/learn/tutorials/topics/animation/animate-anything-mecanim',
+        'info_dict': {
+            'id': 'jWuNtik0C8E',
+            'ext': 'mp4',
+            'title': 'Live Training 22nd September 2014 -  Animate Anything',
+            'description': 'md5:e54913114bd45a554c56cdde7669636e',
+            'duration': 2893,
+            'uploader': 'Unity',
+            'uploader_id': 'Unity3D',
+            'upload_date': '20140926',
+        }
+    }, {
+        'url': 'https://unity3d.com/learn/tutorials/projects/2d-ufo-tutorial/following-player-camera?playlist=25844',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        youtube_id = self._search_regex(
+            r'data-video-id="([_0-9a-zA-Z-]+)"',
+            webpage, 'youtube ID')
+        return self.url_result(youtube_id, ie=YoutubeIE.ie_key(), video_id=video_id)

From e1d168e59243e740b83b02dd4f5b7b28c60ef7d3 Mon Sep 17 00:00:00 2001
From: Alex Seiler <seileralex@gmail.com>
Date: Thu, 19 Oct 2017 17:17:20 +0200
Subject: [PATCH 1454/1696] [servus] Add extractor (closes #14362)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/servus.py     | 43 ++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+)
 create mode 100644 youtube_dl/extractor/servus.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 1624129d2..09b20a39a 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -925,6 +925,7 @@ from .seeker import SeekerIE
 from .senateisvp import SenateISVPIE
 from .sendtonews import SendtoNewsIE
 from .servingsys import ServingSysIE
+from .servus import ServusIE
 from .sexu import SexuIE
 from .shahid import ShahidIE
 from .shared import (
diff --git a/youtube_dl/extractor/servus.py b/youtube_dl/extractor/servus.py
new file mode 100644
index 000000000..264e1dd8b
--- /dev/null
+++ b/youtube_dl/extractor/servus.py
@@ -0,0 +1,43 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class ServusIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?servus\.com/(?:at|de)/p/[^/]+/(?P<id>AA-\w+|\d+-\d+)'
+    _TESTS = [{
+        'url': 'https://www.servus.com/de/p/Die-Gr%C3%BCnen-aus-Sicht-des-Volkes/AA-1T6VBU5PW1W12/',
+        'md5': '046dee641cda1c4cabe13baef3be2c1c',
+        'info_dict': {
+            'id': 'AA-1T6VBU5PW1W12',
+            'ext': 'mp4',
+            'title': 'Die Grünen aus Volkssicht',
+            'description': 'md5:052b5da1cb2cd7d562ef1f19be5a5cba',
+            'thumbnail': r're:^https?://.*\.jpg$',
+        }
+    }, {
+        'url': 'https://www.servus.com/at/p/Wie-das-Leben-beginnt/1309984137314-381415152/',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        formats = self._extract_m3u8_formats(
+            'https://stv.rbmbtnx.net/api/v1/manifests/%s.m3u8' % video_id,
+            video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls')
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'formats': formats,
+        }

From 382fa456ea79d0f22d274195f63e7d49b3334697 Mon Sep 17 00:00:00 2001
From: Felix Yan <felixonmars@archlinux.org>
Date: Thu, 19 Oct 2017 11:36:32 -0500
Subject: [PATCH 1455/1696] [ChangeLog] Fix typo

---
 ChangeLog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index d728e4d03..38446fff9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -834,7 +834,7 @@ version 2017.04.14
 
 Core
 + [downloader/hls] Add basic support for EXT-X-BYTERANGE tag (#10955)
-+ [adobepass] Improve Comcast and Verison login code (#10803)
++ [adobepass] Improve Comcast and Verizon login code (#10803)
 + [adobepass] Add support for Verizon (#10803)
 
 Extractors

From a26a3c6d343a4da1a3739e3af39147545c51a05d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Oct 2017 21:43:34 +0700
Subject: [PATCH 1456/1696] [soundcloud] Update client id (closes #14546)

---
 youtube_dl/extractor/soundcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 1c6799d57..8894f4b0c 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -138,7 +138,7 @@ class SoundcloudIE(InfoExtractor):
         },
     ]
 
-    _CLIENT_ID = 'JlZIsxg2hY5WnBgtn3jfS0UYCl0K8DOg'
+    _CLIENT_ID = 'c6CU49JDMapyrQo06UxU9xouB9ZVzqCn'
     _IPHONE_CLIENT_ID = '376f225bf427445fc4bfb6b99b72e0bf'
 
     @staticmethod

From fc5c47d13cefab662fa031f77a07b94d39018055 Mon Sep 17 00:00:00 2001
From: Alex Seiler <seileralex@gmail.com>
Date: Fri, 20 Oct 2017 18:31:13 +0200
Subject: [PATCH 1457/1696] [parliamentliveuk] Fix extraction (closes #14524)

---
 youtube_dl/extractor/parliamentliveuk.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/parliamentliveuk.py b/youtube_dl/extractor/parliamentliveuk.py
index ebdab8db9..bdd5ff565 100644
--- a/youtube_dl/extractor/parliamentliveuk.py
+++ b/youtube_dl/extractor/parliamentliveuk.py
@@ -11,7 +11,7 @@ class ParliamentLiveUKIE(InfoExtractor):
     _TESTS = [{
         'url': 'http://parliamentlive.tv/Event/Index/c1e9d44d-fd6c-4263-b50f-97ed26cc998b',
         'info_dict': {
-            'id': 'c1e9d44d-fd6c-4263-b50f-97ed26cc998b',
+            'id': '1_af9nv9ym',
             'ext': 'mp4',
             'title': 'Home Affairs Committee',
             'uploader_id': 'FFMPEG-01',
@@ -28,14 +28,14 @@ class ParliamentLiveUKIE(InfoExtractor):
         webpage = self._download_webpage(
             'http://vodplayer.parliamentlive.tv/?mid=' + video_id, video_id)
         widget_config = self._parse_json(self._search_regex(
-            r'kWidgetConfig\s*=\s*({.+});',
+            r'(?s)kWidgetConfig\s*=\s*({.+});',
             webpage, 'kaltura widget config'), video_id)
-        kaltura_url = 'kaltura:%s:%s' % (widget_config['wid'][1:], widget_config['entry_id'])
+        kaltura_url = 'kaltura:%s:%s' % (
+            widget_config['wid'][1:], widget_config['entry_id'])
         event_title = self._download_json(
             'http://parliamentlive.tv/Event/GetShareVideo/' + video_id, video_id)['event']['title']
         return {
             '_type': 'url_transparent',
-            'id': video_id,
             'title': event_title,
             'description': '',
             'url': kaltura_url,

From c9dcd4b0c573cee16ee061f6598a5a9330858aab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Oct 2017 23:36:52 +0700
Subject: [PATCH 1458/1696] [ChangeLog] Actualize

---
 ChangeLog | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 38446fff9..27d03c1dd 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,23 @@
+version <unreleased>
+
+Core
+* [downloader/fragment] Report warning instead of error on inconsistent
+  download state
+* [downloader/hls] Fix total fragments count when ad fragments exist
+
+Extractors
+* [parliamentliveuk] Fix extraction (#14524)
+* [soundcloud] Update client id (#14546)
++ [servus] Add support for servus.com (#14362)
++ [unity] Add support for unity3d.com (#14528)
+* [youtube] Replace youtube redirect URLs in description (#14517)
+* [pbs] Restrict direct video URL regular expression (#14519)
+* [drtv] Respect preference for direct HTTP formats (#14509)
++ [eporner] Add support for embed URLs (#14507)
+* [arte] Capture and output error message
+* [niconico] Improve uploader metadata extraction robustness (#14135)
+
+
 version 2017.10.15.1
 
 Core

From fa3f0fd856a80137d873ab305c061af3e8fc5cb7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Oct 2017 23:40:25 +0700
Subject: [PATCH 1459/1696] release 2017.10.20

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 ++
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 1a72ab291..81fe10d54 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.10.15.1*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.10.15.1**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.10.20*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.10.20**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.10.15.1
+[debug] youtube-dl version 2017.10.20
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 27d03c1dd..547b55981 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.10.20
 
 Core
 * [downloader/fragment] Report warning instead of error on inconsistent
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 7071450d4..be5de22df 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -728,6 +728,7 @@
  - **SenateISVP**
  - **SendtoNews**
  - **ServingSys**
+ - **Servus**
  - **Sexu**
  - **Shahid**
  - **Shared**: shared.sx
@@ -887,6 +888,7 @@
  - **UDNEmbed**: 聯合影音
  - **UKTVPlay**
  - **Unistra**
+ - **Unity**
  - **uol.com.br**
  - **uplynk**
  - **uplynk:preplay**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d01ba3095..4d1686670 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.10.15.1'
+__version__ = '2017.10.20'

From 424505df7668234333b823e8ea3d7f2621fec9ae Mon Sep 17 00:00:00 2001
From: Alex Seiler <seileralex@gmail.com>
Date: Fri, 20 Oct 2017 20:10:56 +0200
Subject: [PATCH 1460/1696] [azmedien] Fix test

---
 youtube_dl/extractor/azmedien.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/azmedien.py b/youtube_dl/extractor/azmedien.py
index f4e07d901..68f26e2ca 100644
--- a/youtube_dl/extractor/azmedien.py
+++ b/youtube_dl/extractor/azmedien.py
@@ -47,7 +47,7 @@ class AZMedienIE(AZMedienBaseIE):
         'url': 'http://www.telezueri.ch/62-show-zuerinews/13772-episode-sonntag-18-dezember-2016/32419-segment-massenabweisungen-beim-hiltl-club-wegen-pelzboom',
         'info_dict': {
             'id': '1_2444peh4',
-            'ext': 'mov',
+            'ext': 'mp4',
             'title': 'Massenabweisungen beim Hiltl Club wegen Pelzboom',
             'description': 'md5:9ea9dd1b159ad65b36ddcf7f0d7c76a8',
             'uploader_id': 'TeleZ?ri',

From 9a9de2d7b2462c9a16bac0ead3f415ef64179833 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Oct 2017 01:32:01 +0700
Subject: [PATCH 1461/1696] [travis] Allow download tests to fail and fast
 finish

---
 .travis.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index f41e11137..b655738ec 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,6 +11,10 @@ sudo: false
 env:
   - YTDL_TEST_SET=core
   - YTDL_TEST_SET=download
+matrix:
+  fast_finish: true
+  allow_failures:
+    - env: YTDL_TEST_SET=download
 script: ./devscripts/run_tests.sh
 notifications:
   email:

From 5c0e5bc4df5beea3fe457e4c2d92643837f1e70e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Oct 2017 02:11:11 +0700
Subject: [PATCH 1462/1696] [README.md] Add build status bagde

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 2879aad24..ea321d536 100644
--- a/README.md
+++ b/README.md
@@ -1,3 +1,5 @@
+[![Build Status](https://travis-ci.org/rg3/youtube-dl.svg?branch=master)](https://travis-ci.org/rg3/youtube-dl)
+
 youtube-dl - download videos from youtube.com or other video platforms
 
 - [INSTALLATION](#installation)

From 21ce434051f104bd270c4d03ddf524adc161f582 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Oct 2017 02:14:25 +0700
Subject: [PATCH 1463/1696] [travis] Enable IRC notifications Let's see how is
 it verbose now

---
 .travis.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index b655738ec..7d77e6c70 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -20,7 +20,7 @@ notifications:
   email:
     - filippo.valsorda@gmail.com
     - yasoob.khld@gmail.com
-#  irc:
-#    channels:
-#      - "irc.freenode.org#youtube-dl"
-#    skip_join: true
+  irc:
+    channels:
+      - "irc.freenode.org#youtube-dl"
+    skip_join: true

From 3ebbd9991e1218a28dad89c13ffb47ebb7ce174c Mon Sep 17 00:00:00 2001
From: Alex Seiler <seileralex@gmail.com>
Date: Sat, 21 Oct 2017 20:26:58 +0200
Subject: [PATCH 1464/1696] [nick] Add support for more nickelodeon sites
 (closes #14553)

---
 youtube_dl/extractor/nick.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
index 510b1c41f..46662fef8 100644
--- a/youtube_dl/extractor/nick.py
+++ b/youtube_dl/extractor/nick.py
@@ -75,7 +75,7 @@ class NickIE(MTVServicesInfoExtractor):
 
 class NickDeIE(MTVServicesInfoExtractor):
     IE_NAME = 'nick.de'
-    _VALID_URL = r'https?://(?:www\.)?(?P<host>nick\.(?:de|com\.pl)|nickelodeon\.(?:nl|at))/[^/]+/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<host>nick\.(?:de|com\.pl)|nickelodeon\.(?:nl|at|dk|no|se))/[^/]+/(?:[^/]+/)*(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://www.nick.de/playlist/3773-top-videos/videos/episode/17306-zu-wasser-und-zu-land-rauchende-erdnusse',
         'only_matching': True,
@@ -91,6 +91,15 @@ class NickDeIE(MTVServicesInfoExtractor):
     }, {
         'url': 'http://www.nick.com.pl/seriale/474-spongebob-kanciastoporty/wideo/17412-teatr-to-jest-to-rodeo-oszolom',
         'only_matching': True,
+    }, {
+        'url': 'http://www.nickelodeon.no/program/2626-bulderhuset/videoer/90947-femteklasse-veronica-vs-vanzilla',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.nickelodeon.dk/serier/2626-hojs-hus/videoer/761-tissepause',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.nickelodeon.se/serier/2626-lugn-i-stormen/videos/998-',
+        'only_matching': True,
     }]
 
     def _extract_mrss_url(self, webpage, host):

From cf6bda312b0bbb821df5f26a1a1e879fda1c557b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Oct 2017 01:30:35 +0700
Subject: [PATCH 1465/1696] [nickde] Add support for nick.ch

---
 youtube_dl/extractor/nick.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
index 46662fef8..3f8a16f51 100644
--- a/youtube_dl/extractor/nick.py
+++ b/youtube_dl/extractor/nick.py
@@ -75,7 +75,7 @@ class NickIE(MTVServicesInfoExtractor):
 
 class NickDeIE(MTVServicesInfoExtractor):
     IE_NAME = 'nick.de'
-    _VALID_URL = r'https?://(?:www\.)?(?P<host>nick\.(?:de|com\.pl)|nickelodeon\.(?:nl|at|dk|no|se))/[^/]+/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<host>nick\.(?:de|com\.pl|ch)|nickelodeon\.(?:nl|at|dk|no|se))/[^/]+/(?:[^/]+/)*(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://www.nick.de/playlist/3773-top-videos/videos/episode/17306-zu-wasser-und-zu-land-rauchende-erdnusse',
         'only_matching': True,
@@ -100,6 +100,9 @@ class NickDeIE(MTVServicesInfoExtractor):
     }, {
         'url': 'http://www.nickelodeon.se/serier/2626-lugn-i-stormen/videos/998-',
         'only_matching': True,
+    }, {
+        'url': 'http://www.nick.ch/shows/2304-adventure-time-abenteuerzeit-mit-finn-und-jake',
+        'only_matching': True,
     }]
 
     def _extract_mrss_url(self, webpage, host):

From f03ee0b3725898f321f795d68f4f9c4630b188b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Oct 2017 01:40:05 +0700
Subject: [PATCH 1466/1696] [nickde] Add support for nickelodeon.be

---
 youtube_dl/extractor/nick.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
index 3f8a16f51..f0be29880 100644
--- a/youtube_dl/extractor/nick.py
+++ b/youtube_dl/extractor/nick.py
@@ -75,7 +75,7 @@ class NickIE(MTVServicesInfoExtractor):
 
 class NickDeIE(MTVServicesInfoExtractor):
     IE_NAME = 'nick.de'
-    _VALID_URL = r'https?://(?:www\.)?(?P<host>nick\.(?:de|com\.pl|ch)|nickelodeon\.(?:nl|at|dk|no|se))/[^/]+/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<host>nick\.(?:de|com\.pl|ch)|nickelodeon\.(?:nl|be|at|dk|no|se))/[^/]+/(?:[^/]+/)*(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://www.nick.de/playlist/3773-top-videos/videos/episode/17306-zu-wasser-und-zu-land-rauchende-erdnusse',
         'only_matching': True,
@@ -103,6 +103,9 @@ class NickDeIE(MTVServicesInfoExtractor):
     }, {
         'url': 'http://www.nick.ch/shows/2304-adventure-time-abenteuerzeit-mit-finn-und-jake',
         'only_matching': True,
+    }, {
+        'url': 'http://www.nickelodeon.be/afspeellijst/4530-top-videos/videos/episode/73917-inval-broodschapper-lariekoek-arie',
+        'only_matching': True,
     }]
 
     def _extract_mrss_url(self, webpage, host):

From 9ff6273caecaa3e169d69ed8c480ba36776c7434 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Oct 2017 01:51:01 +0700
Subject: [PATCH 1467/1696] [nickru] Add support for more sites

---
 youtube_dl/extractor/nick.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
index f0be29880..310eea2cf 100644
--- a/youtube_dl/extractor/nick.py
+++ b/youtube_dl/extractor/nick.py
@@ -147,13 +147,28 @@ class NickNightIE(NickDeIE):
 
 class NickRuIE(MTVServicesInfoExtractor):
     IE_NAME = 'nickelodeonru'
-    _VALID_URL = r'https?://(?:www\.)nickelodeon\.ru/(?:playlist|shows|videos)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)nickelodeon\.(?:ru|fr|es|pt|ro|hu)/[^/]+/(?:[^/]+/)*(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://www.nickelodeon.ru/shows/henrydanger/videos/episodes/3-sezon-15-seriya-licenziya-na-polyot/pmomfb#playlist/7airc6',
         'only_matching': True,
     }, {
         'url': 'http://www.nickelodeon.ru/videos/smotri-na-nickelodeon-v-iyule/g9hvh7',
         'only_matching': True,
+    }, {
+        'url': 'http://www.nickelodeon.fr/programmes/bob-l-eponge/videos/le-marathon-de-booh-kini-bottom-mardi-31-octobre/nfn7z0',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.nickelodeon.es/videos/nickelodeon-consejos-tortitas/f7w7xy',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.nickelodeon.pt/series/spongebob-squarepants/videos/a-bolha-de-tinta-gigante/xutq1b',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.nickelodeon.ro/emisiuni/shimmer-si-shine/video/nahal-din-bomboane/uw5u2k',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.nickelodeon.hu/musorok/spongyabob-kockanadrag/videok/episodes/buborekfujas-az-elszakadt-nadrag/q57iob#playlist/k6te4y',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From f7a5038305682a2f41ca08addc338198c3c2e27b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Oct 2017 02:46:28 +0700
Subject: [PATCH 1468/1696] [travis] Disable IRC notifications

---
 .travis.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 7d77e6c70..5f4f3922b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -20,7 +20,3 @@ notifications:
   email:
     - filippo.valsorda@gmail.com
     - yasoob.khld@gmail.com
-  irc:
-    channels:
-      - "irc.freenode.org#youtube-dl"
-    skip_join: true

From 36e2d3ca43145e5f13874a02447db572173cc057 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Oct 2017 08:16:30 +0700
Subject: [PATCH 1469/1696] [dramafever] Fix login error message extraction

---
 youtube_dl/extractor/dramafever.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 9a498d72a..95883a037 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -59,7 +59,7 @@ class DramaFeverBaseIE(AMPIE):
         if all(logout_pattern not in response
                for logout_pattern in ['href="/accounts/logout/"', '>Log out<']):
             error = self._html_search_regex(
-                r'(?s)class="hidden-xs prompt"[^>]*>(.+?)<',
+                r'(?s)<h\d[^>]+\bclass="hidden-xs prompt"[^>]*>(.+?)</h\d',
                 response, 'error message', default=None)
             if error:
                 raise ExtractorError('Unable to login: %s' % error, expected=True)

From 55c727a547a3873396c75b62d55393da79ce0fc9 Mon Sep 17 00:00:00 2001
From: rawcoder <rawcoder@users.noreply.github.com>
Date: Sun, 22 Oct 2017 07:02:20 +0530
Subject: [PATCH 1470/1696] [ndtv] Add support for sub-sites

---
 youtube_dl/extractor/ndtv.py | 101 ++++++++++++++++++++++++++++-------
 1 file changed, 81 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/ndtv.py b/youtube_dl/extractor/ndtv.py
index 255f60878..ddec89f2c 100644
--- a/youtube_dl/extractor/ndtv.py
+++ b/youtube_dl/extractor/ndtv.py
@@ -1,45 +1,106 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_parse_unquote_plus
+)
 from ..utils import (
-    int_or_none,
+    parse_duration,
     remove_end,
     unified_strdate,
+    urljoin
 )
 
 
 class NDTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ndtv\.com/video/(?:[^/]+/)+[^/?^&]+-(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:[^/]+\.)?ndtv\.com/(?:[^/]+/)*videos?/?(?:[^/]+/)*[^/?^&]+-(?P<id>\d+)'
 
-    _TEST = {
-        'url': 'http://www.ndtv.com/video/news/news/ndtv-exclusive-don-t-need-character-certificate-from-rahul-gandhi-says-arvind-kejriwal-300710',
-        'md5': '39f992dbe5fb531c395d8bbedb1e5e88',
-        'info_dict': {
-            'id': '300710',
-            'ext': 'mp4',
-            'title': "NDTV exclusive: Don't need character certificate from Rahul Gandhi, says Arvind Kejriwal",
-            'description': 'md5:ab2d4b4a6056c5cb4caa6d729deabf02',
-            'upload_date': '20131208',
-            'duration': 1327,
-            'thumbnail': r're:https?://.*\.jpg',
+    _TESTS = [
+        {
+            'url': 'https://khabar.ndtv.com/video/show/prime-time/prime-time-ill-system-and-poor-education-468818',
+            'md5': '78efcf3880ef3fd9b83d405ca94a38eb',
+            'info_dict': {
+                'id': '468818',
+                'ext': 'mp4',
+                'title': "प्राइम टाइम: सिस्टम बीमार, स्कूल बदहाल",
+                'description': 'md5:f410512f1b49672e5695dea16ef2731d',
+                'upload_date': '20170928',
+                'duration': 2218,
+                'thumbnail': r're:https?://.*\.jpg',
+            }
         },
-    }
+        {
+            # __filename is url
+            'url': 'http://movies.ndtv.com/videos/cracker-free-diwali-wishes-from-karan-johar-kriti-sanon-other-stars-470304',
+            'md5': 'f1d709352305b44443515ac56b45aa46',
+            'info_dict': {
+                'id': '470304',
+                'ext': 'mp4',
+                'title': "Cracker-Free Diwali Wishes From Karan Johar, Kriti Sanon & Other Stars",
+                'description': 'md5:f115bba1adf2f6433fa7c1ade5feb465',
+                'upload_date': '20171019',
+                'duration': 137,
+                'thumbnail': r're:https?://.*\.jpg',
+            }
+        },
+        {
+            'url': 'https://www.ndtv.com/video/news/news/delhi-s-air-quality-status-report-after-diwali-is-very-poor-470372',
+            'only_matching': True
+        },
+        {
+            'url': 'https://auto.ndtv.com/videos/the-cnb-daily-october-13-2017-469935',
+            'only_matching': True
+        },
+        {
+            'url': 'https://sports.ndtv.com/cricket/videos/2nd-t20i-rock-thrown-at-australia-cricket-team-bus-after-win-over-india-469764',
+            'only_matching': True
+        },
+        {
+            'url': 'http://gadgets.ndtv.com/videos/uncharted-the-lost-legacy-review-465568',
+            'only_matching': True
+        },
+        {
+            'url': 'http://profit.ndtv.com/videos/news/video-indian-economy-on-very-solid-track-international-monetary-fund-chief-470040',
+            'only_matching': True
+        },
+        {
+            'url': 'http://food.ndtv.com/video-basil-seeds-coconut-porridge-419083',
+            'only_matching': True
+        },
+        {
+            'url': 'https://doctor.ndtv.com/videos/top-health-stories-of-the-week-467396',
+            'only_matching': True
+        },
+        {
+            'url': 'https://swirlster.ndtv.com/video/how-to-make-friends-at-work-469324',
+            'only_matching': True
+        }
+    ]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        title = remove_end(self._og_search_title(webpage), ' - NDTV')
+        # '__title' does not contain extra words such as sub-site name, "Video" etc.
+        title = compat_urllib_parse_unquote_plus(
+            self._search_regex(r"__title\s*=\s*'([^']+)'", webpage, 'title', default=None) or
+            self._og_search_title(webpage))
 
         filename = self._search_regex(
-            r"__filename='([^']+)'", webpage, 'video filename')
-        video_url = 'http://bitcast-b.bitgravity.com/ndtvod/23372/ndtv/%s' % filename
+            r"(?:__)?filename\s*[:=]\s*'([^']+)'", webpage, 'video filename')
+        # in "movies" sub-site pages, filename is URL
+        video_url = urljoin('https://ndtvod.bc-ssl.cdn.bitgravity.com/23372/ndtv/', filename.lstrip('/'))
 
-        duration = int_or_none(self._search_regex(
-            r"__duration='([^']+)'", webpage, 'duration', fatal=False))
+        # "doctor" sub-site has MM:SS format
+        duration = parse_duration(self._search_regex(
+            r"(?:__)?duration\s*[:=]\s*'([^']+)'", webpage, 'duration', fatal=False))
 
+        # "sports", "doctor", "swirlster" sub-sites don't have 'publish-date'
         upload_date = unified_strdate(self._html_search_meta(
-            'publish-date', webpage, 'upload date', fatal=False))
+            'publish-date', webpage, 'upload date', default=None) or self._html_search_meta(
+            'uploadDate', webpage, 'upload date', default=None) or self._search_regex(
+            r'datePublished"\s*:\s*"([^"]+)"', webpage, 'upload date', fatal=False))
 
         description = remove_end(self._og_search_description(webpage), ' (Read more)')
 

From cdd1ce92c46167c12514a061609308112e79ea84 Mon Sep 17 00:00:00 2001
From: "J.D. Purcell" <jdpurcell@gmail.com>
Date: Mon, 23 Oct 2017 12:12:50 -0400
Subject: [PATCH 1471/1696] [twitch:clips] Fix title extraction

---
 youtube_dl/extractor/twitch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index c926c99a9..fefcd2807 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -609,7 +609,7 @@ class TwitchClipsIE(InfoExtractor):
                 r'(?s)clipInfo\s*=\s*({.+?});', webpage, 'clip info'),
             video_id, transform_source=js_to_json)
 
-        title = clip.get('channel_title') or self._og_search_title(webpage)
+        title = clip.get('title') or clip.get('channel_title') or self._og_search_title(webpage)
 
         formats = [{
             'url': option['source'],

From 7913e0fca7df6840e8434449f534f9744c9394f2 Mon Sep 17 00:00:00 2001
From: mrBliss <dewinant@gmail.com>
Date: Fri, 20 Oct 2017 14:41:07 +0200
Subject: [PATCH 1472/1696] [canvas] Add support for vrt.be/vrtnu (closes
 #11873)

---
 youtube_dl/extractor/canvas.py     | 142 ++++++++++++++++++++++++++++-
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/gigya.py      |  22 +++++
 youtube_dl/extractor/medialaan.py  |  17 +---
 4 files changed, 167 insertions(+), 15 deletions(-)
 create mode 100644 youtube_dl/extractor/gigya.py

diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py
index 6899f8431..c7e9b8ff9 100644
--- a/youtube_dl/extractor/canvas.py
+++ b/youtube_dl/extractor/canvas.py
@@ -1,16 +1,24 @@
 from __future__ import unicode_literals
 
 import re
+import json
 
 from .common import InfoExtractor
+from .gigya import GigyaBaseIE
+
+
+from ..compat import compat_HTTPError
 from ..utils import (
-    float_or_none,
+    ExtractorError,
     strip_or_none,
+    float_or_none,
+    int_or_none,
+    parse_iso8601,
 )
 
 
 class CanvasIE(InfoExtractor):
-    _VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?P<site_id>canvas|een|ketnet)/assets/(?P<id>m[dz]-ast-[^/?#&]+)'
+    _VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?P<site_id>canvas|een|ketnet|vrtvideo)/assets/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'https://mediazone.vrt.be/api/v1/ketnet/assets/md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
         'md5': '90139b746a0a9bd7bb631283f6e2a64e',
@@ -166,3 +174,133 @@ class CanvasEenIE(InfoExtractor):
             'title': title,
             'description': self._og_search_description(webpage),
         }
+
+
+class VrtNUIE(GigyaBaseIE):
+    IE_DESC = 'VrtNU.be'
+    _VALID_URL = r'https?://(?:www\.)?vrt\.be/(?P<site_id>vrtnu)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'https://www.vrt.be/vrtnu/a-z/postbus-x/1/postbus-x-s1a1/',
+        'info_dict': {
+            'id': 'pbs-pub-2e2d8c27-df26-45c9-9dc6-90c78153044d$vid-90c932b1-e21d-4fb8-99b1-db7b49cf74de',
+            'ext': 'flv',
+            'title': 'De zwarte weduwe',
+            'description': 'md5:d90c21dced7db869a85db89a623998d4',
+            'duration': 1457.04,
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'season': '1',
+            'season_number': 1,
+            'episode_number': 1,
+        },
+        # 'skip': 'This video is only available for registered users'
+    }]
+    _NETRC_MACHINE = 'vrtnu'
+    _APIKEY = '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy'
+    _CONTEXT_ID = 'R3595707040'
+
+    def _real_initialize(self):
+        self._login()
+
+    def _login(self):
+        username, password = self._get_login_info()
+        if username is None:
+            self.raise_login_required()
+
+        auth_data = {
+            'APIKey': self._APIKEY,
+            'targetEnv': 'jssdk',
+            'loginID': username,
+            'password': password,
+            'authMode': 'cookie',
+        }
+
+        auth_info = self._gigya_login(auth_data)
+
+        # Sometimes authentication fails for no good reason, retry
+        login_attempt = 1
+        while login_attempt <= 3:
+            try:
+                # When requesting a token, no actual token is returned, but the
+                # necessary cookies are set.
+                self._request_webpage(
+                    'https://token.vrt.be',
+                    None, note='Requesting a token', errnote='Could not get a token',
+                    headers={
+                        'Content-Type': 'application/json',
+                        'Referer': 'https://www.vrt.be/vrtnu/',
+                    },
+                    data=json.dumps({
+                        'uid': auth_info['UID'],
+                        'uidsig': auth_info['UIDSignature'],
+                        'ts': auth_info['signatureTimestamp'],
+                        'email': auth_info['profile']['email'],
+                    }).encode('utf-8'))
+            except ExtractorError as e:
+                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+                    login_attempt += 1
+                    self.report_warning('Authentication failed')
+                    self._sleep(1, None, msg_template='Waiting for %(timeout)s seconds before trying again')
+                else:
+                    raise e
+            else:
+                break
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        title = self._html_search_regex(
+            r'(?ms)<h1 class="content__heading">(.+?)</h1>',
+            webpage, 'title').strip()
+
+        description = self._html_search_regex(
+            r'(?ms)<div class="content__description">(.+?)</div>',
+            webpage, 'description', default=None)
+
+        season = self._html_search_regex(
+            [r'''(?xms)<div\ class="tabs__tab\ tabs__tab--active">\s*
+                    <span>seizoen\ (.+?)</span>\s*
+                </div>''',
+             r'<option value="seizoen (\d{1,3})" data-href="[^"]+?" selected>'],
+            webpage, 'season', default=None)
+
+        season_number = int_or_none(season)
+
+        episode_number = int_or_none(self._html_search_regex(
+            r'''(?xms)<div\ class="content__episode">\s*
+                    <abbr\ title="aflevering">afl</abbr>\s*<span>(\d+)</span>
+                </div>''',
+            webpage, 'episode_number', default=None))
+
+        release_date = parse_iso8601(self._html_search_regex(
+            r'(?ms)<div class="content__broadcastdate">\s*<time\ datetime="(.+?)"',
+            webpage, 'release_date', default=None))
+
+        # If there's a ? or a # in the URL, remove them and everything after
+        clean_url = url.split('?')[0].split('#')[0].strip('/')
+        securevideo_url = clean_url + '.mssecurevideo.json'
+
+        json = self._download_json(securevideo_url, display_id)
+        # We are dealing with a '../<show>.relevant' URL
+        redirect_url = json.get('url')
+        if redirect_url:
+            return self.url_result('https:' + redirect_url)
+        else:
+            # There is only one entry, but with an unknown key, so just get
+            # the first one
+            video_id = list(json.values())[0].get('videoid')
+
+        return {
+            '_type': 'url_transparent',
+            'url': 'https://mediazone.vrt.be/api/v1/vrtvideo/assets/%s' % video_id,
+            'ie_key': CanvasIE.ie_key(),
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'season': season,
+            'season_number': season_number,
+            'episode_number': episode_number,
+            'release_date': release_date,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 09b20a39a..42b312c42 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -150,6 +150,7 @@ from .canalc2 import Canalc2IE
 from .canvas import (
     CanvasIE,
     CanvasEenIE,
+    VrtNUIE,
 )
 from .carambatv import (
     CarambaTVIE,
diff --git a/youtube_dl/extractor/gigya.py b/youtube_dl/extractor/gigya.py
new file mode 100644
index 000000000..412178492
--- /dev/null
+++ b/youtube_dl/extractor/gigya.py
@@ -0,0 +1,22 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from ..utils import (
+    ExtractorError,
+    urlencode_postdata,
+)
+
+
+class GigyaBaseIE(InfoExtractor):
+    def _gigya_login(self, auth_data):
+        auth_info = self._download_json(
+            'https://accounts.eu1.gigya.com/accounts.login', None,
+            note='Logging in', errnote='Unable to log in',
+            data=urlencode_postdata(auth_data))
+
+        error_message = auth_info.get('errorDetails') or auth_info.get('errorMessage')
+        if error_message:
+            raise ExtractorError(
+                'Unable to login: %s' % error_message, expected=True)
+        return auth_info
diff --git a/youtube_dl/extractor/medialaan.py b/youtube_dl/extractor/medialaan.py
index 4c32fbc2c..f8c30052f 100644
--- a/youtube_dl/extractor/medialaan.py
+++ b/youtube_dl/extractor/medialaan.py
@@ -2,19 +2,18 @@ from __future__ import unicode_literals
 
 import re
 
-from .common import InfoExtractor
+from .gigya import GigyaBaseIE
+
 from ..compat import compat_str
 from ..utils import (
-    ExtractorError,
     int_or_none,
     parse_duration,
     try_get,
     unified_timestamp,
-    urlencode_postdata,
 )
 
 
-class MedialaanIE(InfoExtractor):
+class MedialaanIE(GigyaBaseIE):
     _VALID_URL = r'''(?x)
                     https?://
                         (?:www\.|nieuws\.)?
@@ -119,15 +118,7 @@ class MedialaanIE(InfoExtractor):
             'password': password,
         }
 
-        auth_info = self._download_json(
-            'https://accounts.eu1.gigya.com/accounts.login', None,
-            note='Logging in', errnote='Unable to log in',
-            data=urlencode_postdata(auth_data))
-
-        error_message = auth_info.get('errorDetails') or auth_info.get('errorMessage')
-        if error_message:
-            raise ExtractorError(
-                'Unable to login: %s' % error_message, expected=True)
+        auth_info = self._gigya_login(auth_data)
 
         self._uid = auth_info['UID']
         self._uid_signature = auth_info['UIDSignature']

From b8c6ffc51823a9ed6af69da0f9777c6504c1d027 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 25 Oct 2017 23:17:14 +0700
Subject: [PATCH 1473/1696] [vrtnu] Add support for cookies authentication and
 simplify (#11873)

---
 youtube_dl/extractor/canvas.py | 26 +++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py
index c7e9b8ff9..3faa76076 100644
--- a/youtube_dl/extractor/canvas.py
+++ b/youtube_dl/extractor/canvas.py
@@ -5,8 +5,6 @@ import json
 
 from .common import InfoExtractor
 from .gigya import GigyaBaseIE
-
-
 from ..compat import compat_HTTPError
 from ..utils import (
     ExtractorError,
@@ -192,7 +190,7 @@ class VrtNUIE(GigyaBaseIE):
             'season_number': 1,
             'episode_number': 1,
         },
-        # 'skip': 'This video is only available for registered users'
+        'skip': 'This video is only available for registered users'
     }]
     _NETRC_MACHINE = 'vrtnu'
     _APIKEY = '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy'
@@ -204,7 +202,7 @@ class VrtNUIE(GigyaBaseIE):
     def _login(self):
         username, password = self._get_login_info()
         if username is None:
-            self.raise_login_required()
+            return
 
         auth_data = {
             'APIKey': self._APIKEY,
@@ -281,15 +279,21 @@ class VrtNUIE(GigyaBaseIE):
         clean_url = url.split('?')[0].split('#')[0].strip('/')
         securevideo_url = clean_url + '.mssecurevideo.json'
 
-        json = self._download_json(securevideo_url, display_id)
+        try:
+            video = self._download_json(securevideo_url, display_id)
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+                self.raise_login_required()
+            raise
+
         # We are dealing with a '../<show>.relevant' URL
-        redirect_url = json.get('url')
+        redirect_url = video.get('url')
         if redirect_url:
-            return self.url_result('https:' + redirect_url)
-        else:
-            # There is only one entry, but with an unknown key, so just get
-            # the first one
-            video_id = list(json.values())[0].get('videoid')
+            return self.url_result(self._proto_relative_url(redirect_url, 'https:'))
+
+        # There is only one entry, but with an unknown key, so just get
+        # the first one
+        video_id = list(video.values())[0].get('videoid')
 
         return {
             '_type': 'url_transparent',

From d673ab65628e1c83e31d3396f5090dad26218232 Mon Sep 17 00:00:00 2001
From: Logan B <einstein95@users.noreply.github.com>
Date: Thu, 26 Oct 2017 05:23:27 +1300
Subject: [PATCH 1474/1696] [nbc] Add support for classic-tv videos

---
 youtube_dl/extractor/nbc.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 35151f527..554dec36e 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 
 class NBCIE(AdobePassIE):
-    _VALID_URL = r'https?(?P<permalink>://(?:www\.)?nbc\.com/[^/]+/video/[^/]+/(?P<id>n?\d+))'
+    _VALID_URL = r'https?(?P<permalink>://(?:www\.)?nbc\.com/(?:classic-tv/)?[^/]+/video/[^/]+/(?P<id>n?\d+))'
 
     _TESTS = [
         {
@@ -67,7 +67,11 @@ class NBCIE(AdobePassIE):
                 'skip_download': True,
             },
             'skip': 'Only works from US',
-        }
+        },
+        {
+            'url': 'https://www.nbc.com/classic-tv/charles-in-charge/video/charles-in-charge-pilot/n3310',
+            'only_matching': True,
+        },
     ]
 
     def _real_extract(self, url):

From dc24a7d4a2210d6bf282e234e49193d204c41124 Mon Sep 17 00:00:00 2001
From: Alex Seiler <seileralex@gmail.com>
Date: Wed, 25 Oct 2017 18:27:55 +0200
Subject: [PATCH 1475/1696] [myvideo] Remove extractor (closes #8557)

Redirects to store.maxdome.de
---
 youtube_dl/extractor/extractors.py |   1 -
 youtube_dl/extractor/myvideo.py    | 177 -----------------------------
 2 files changed, 178 deletions(-)
 delete mode 100644 youtube_dl/extractor/myvideo.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 42b312c42..18350810b 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -624,7 +624,6 @@ from .mwave import MwaveIE, MwaveMeetGreetIE
 from .myspace import MySpaceIE, MySpaceAlbumIE
 from .myspass import MySpassIE
 from .myvi import MyviIE
-from .myvideo import MyVideoIE
 from .myvidster import MyVidsterIE
 from .nationalgeographic import (
     NationalGeographicVideoIE,
diff --git a/youtube_dl/extractor/myvideo.py b/youtube_dl/extractor/myvideo.py
deleted file mode 100644
index 367e811db..000000000
--- a/youtube_dl/extractor/myvideo.py
+++ /dev/null
@@ -1,177 +0,0 @@
-from __future__ import unicode_literals
-
-import binascii
-import base64
-import hashlib
-import re
-import json
-
-from .common import InfoExtractor
-from ..compat import (
-    compat_ord,
-    compat_urllib_parse_unquote,
-    compat_urllib_parse_urlencode,
-)
-from ..utils import (
-    ExtractorError,
-    sanitized_Request,
-)
-
-
-class MyVideoIE(InfoExtractor):
-    _WORKING = False
-    _VALID_URL = r'https?://(?:www\.)?myvideo\.de/(?:[^/]+/)?watch/(?P<id>[0-9]+)/[^?/]+.*'
-    IE_NAME = 'myvideo'
-    _TEST = {
-        'url': 'http://www.myvideo.de/watch/8229274/bowling_fail_or_win',
-        'md5': '2d2753e8130479ba2cb7e0a37002053e',
-        'info_dict': {
-            'id': '8229274',
-            'ext': 'flv',
-            'title': 'bowling-fail-or-win',
-        }
-    }
-
-    # Original Code from: https://github.com/dersphere/plugin.video.myvideo_de.git
-    # Released into the Public Domain by Tristan Fischer on 2013-05-19
-    # https://github.com/rg3/youtube-dl/pull/842
-    def __rc4crypt(self, data, key):
-        x = 0
-        box = list(range(256))
-        for i in list(range(256)):
-            x = (x + box[i] + compat_ord(key[i % len(key)])) % 256
-            box[i], box[x] = box[x], box[i]
-        x = 0
-        y = 0
-        out = ''
-        for char in data:
-            x = (x + 1) % 256
-            y = (y + box[x]) % 256
-            box[x], box[y] = box[y], box[x]
-            out += chr(compat_ord(char) ^ box[(box[x] + box[y]) % 256])
-        return out
-
-    def __md5(self, s):
-        return hashlib.md5(s).hexdigest().encode()
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
-        GK = (
-            b'WXpnME1EZGhNRGhpTTJNM01XVmhOREU0WldNNVpHTTJOakpt'
-            b'TW1FMU5tVTBNR05pWkRaa05XRXhNVFJoWVRVd1ptSXhaVEV3'
-            b'TnpsbA0KTVRkbU1tSTRNdz09'
-        )
-
-        # Get video webpage
-        webpage_url = 'http://www.myvideo.de/watch/%s' % video_id
-        webpage = self._download_webpage(webpage_url, video_id)
-
-        mobj = re.search('source src=\'(.+?)[.]([^.]+)\'', webpage)
-        if mobj is not None:
-            self.report_extraction(video_id)
-            video_url = mobj.group(1) + '.flv'
-
-            video_title = self._html_search_regex('<title>([^<]+)</title>',
-                                                  webpage, 'title')
-
-            return {
-                'id': video_id,
-                'url': video_url,
-                'title': video_title,
-            }
-
-        mobj = re.search(r'data-video-service="/service/data/video/%s/config' % video_id, webpage)
-        if mobj is not None:
-            request = sanitized_Request('http://www.myvideo.de/service/data/video/%s/config' % video_id, '')
-            response = self._download_webpage(request, video_id,
-                                              'Downloading video info')
-            info = json.loads(base64.b64decode(response).decode('utf-8'))
-            return {
-                'id': video_id,
-                'title': info['title'],
-                'url': info['streaming_url'].replace('rtmpe', 'rtmpt'),
-                'play_path': info['filename'],
-                'ext': 'flv',
-                'thumbnail': info['thumbnail'][0]['url'],
-            }
-
-        # try encxml
-        mobj = re.search('var flashvars={(.+?)}', webpage)
-        if mobj is None:
-            raise ExtractorError('Unable to extract video')
-
-        params = {}
-        encxml = ''
-        sec = mobj.group(1)
-        for (a, b) in re.findall('(.+?):\'(.+?)\',?', sec):
-            if not a == '_encxml':
-                params[a] = b
-            else:
-                encxml = compat_urllib_parse_unquote(b)
-        if not params.get('domain'):
-            params['domain'] = 'www.myvideo.de'
-        xmldata_url = '%s?%s' % (encxml, compat_urllib_parse_urlencode(params))
-        if 'flash_playertype=MTV' in xmldata_url:
-            self._downloader.report_warning('avoiding MTV player')
-            xmldata_url = (
-                'http://www.myvideo.de/dynamic/get_player_video_xml.php'
-                '?flash_playertype=D&ID=%s&_countlimit=4&autorun=yes'
-            ) % video_id
-
-        # get enc data
-        enc_data = self._download_webpage(xmldata_url, video_id).split('=')[1]
-        enc_data_b = binascii.unhexlify(enc_data)
-        sk = self.__md5(
-            base64.b64decode(base64.b64decode(GK)) +
-            self.__md5(
-                str(video_id).encode('utf-8')
-            )
-        )
-        dec_data = self.__rc4crypt(enc_data_b, sk)
-
-        # extracting infos
-        self.report_extraction(video_id)
-
-        video_url = None
-        mobj = re.search('connectionurl=\'(.*?)\'', dec_data)
-        if mobj:
-            video_url = compat_urllib_parse_unquote(mobj.group(1))
-            if 'myvideo2flash' in video_url:
-                self.report_warning(
-                    'Rewriting URL to use unencrypted rtmp:// ...',
-                    video_id)
-                video_url = video_url.replace('rtmpe://', 'rtmp://')
-
-        if not video_url:
-            # extract non rtmp videos
-            mobj = re.search('path=\'(http.*?)\' source=\'(.*?)\'', dec_data)
-            if mobj is None:
-                raise ExtractorError('unable to extract url')
-            video_url = compat_urllib_parse_unquote(mobj.group(1)) + compat_urllib_parse_unquote(mobj.group(2))
-
-        video_file = self._search_regex('source=\'(.*?)\'', dec_data, 'video file')
-        video_file = compat_urllib_parse_unquote(video_file)
-
-        if not video_file.endswith('f4m'):
-            ppath, prefix = video_file.split('.')
-            video_playpath = '%s:%s' % (prefix, ppath)
-        else:
-            video_playpath = ''
-
-        video_swfobj = self._search_regex(r'swfobject\.embedSWF\(\'(.+?)\'', webpage, 'swfobj')
-        video_swfobj = compat_urllib_parse_unquote(video_swfobj)
-
-        video_title = self._html_search_regex("<h1(?: class='globalHd')?>(.*?)</h1>",
-                                              webpage, 'title')
-
-        return {
-            'id': video_id,
-            'url': video_url,
-            'tc_url': video_url,
-            'title': video_title,
-            'ext': 'flv',
-            'play_path': video_playpath,
-            'player_url': video_swfobj,
-        }

From 30e6161799dfdf9f53d3c8eaa9e10afe615bc5dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 26 Oct 2017 23:16:16 +0700
Subject: [PATCH 1476/1696] [soundgasm] Improve extraction (closes #14588)

---
 youtube_dl/extractor/soundgasm.py | 35 +++++++++++++++++++++----------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/soundgasm.py b/youtube_dl/extractor/soundgasm.py
index e004e2c5a..3d78a9d76 100644
--- a/youtube_dl/extractor/soundgasm.py
+++ b/youtube_dl/extractor/soundgasm.py
@@ -8,36 +8,49 @@ from .common import InfoExtractor
 
 class SoundgasmIE(InfoExtractor):
     IE_NAME = 'soundgasm'
-    _VALID_URL = r'https?://(?:www\.)?soundgasm\.net/u/(?P<user>[0-9a-zA-Z_\-]+)/(?P<title>[0-9a-zA-Z_\-]+)'
+    _VALID_URL = r'https?://(?:www\.)?soundgasm\.net/u/(?P<user>[0-9a-zA-Z_-]+)/(?P<display_id>[0-9a-zA-Z_-]+)'
     _TEST = {
         'url': 'http://soundgasm.net/u/ytdl/Piano-sample',
         'md5': '010082a2c802c5275bb00030743e75ad',
         'info_dict': {
             'id': '88abd86ea000cafe98f96321b23cc1206cbcbcc9',
             'ext': 'm4a',
-            'title': 'ytdl_Piano-sample',
-            'description': 'Royalty Free Sample Music'
+            'title': 'Piano sample',
+            'description': 'Royalty Free Sample Music',
+            'uploader': 'ytdl',
         }
     }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('title')
-        audio_title = mobj.group('user') + '_' + mobj.group('title')
+        display_id = mobj.group('display_id')
+
         webpage = self._download_webpage(url, display_id)
+
         audio_url = self._html_search_regex(
-            r'(?s)m4a\:\s"([^"]+)"', webpage, 'audio URL')
-        audio_id = re.split(r'\/|\.', audio_url)[-2]
+            r'(?s)m4a\s*:\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage,
+            'audio URL', group='url')
+
+        title = self._search_regex(
+            r'<div[^>]+\bclass=["\']jp-title[^>]+>([^<]+)',
+            webpage, 'title', default=display_id)
+
         description = self._html_search_regex(
-            r'(?s)<li>Description:\s(.*?)<\/li>', webpage, 'description',
-            fatal=False)
+            (r'(?s)<div[^>]+\bclass=["\']jp-description[^>]+>(.+?)</div>',
+             r'(?s)<li>Description:\s(.*?)<\/li>'),
+            webpage, 'description', fatal=False)
+
+        audio_id = self._search_regex(
+            r'/([^/]+)\.m4a', audio_url, 'audio id', default=display_id)
 
         return {
             'id': audio_id,
             'display_id': display_id,
             'url': audio_url,
-            'title': audio_title,
-            'description': description
+            'vcodec': 'none',
+            'title': title,
+            'description': description,
+            'uploader': mobj.group('user'),
         }
 
 

From 7c1f419341ac2dec123eaa0075212edc6af3302b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 27 Oct 2017 22:21:47 +0700
Subject: [PATCH 1477/1696] [vimeo] Restrict iframe embed regex (closes #14600)

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index c3f71b45e..cedb54876 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -412,7 +412,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         urls = []
         # Look for embedded (iframe) Vimeo player
         for mobj in re.finditer(
-                r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//player\.vimeo\.com/video/.+?)\1',
+                r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//player\.vimeo\.com/video/\d+.*?)\1',
                 webpage):
             urls.append(VimeoIE._smuggle_referrer(unescapeHTML(mobj.group('url')), url))
         PLAIN_EMBED_RE = (

From f2332f18e66fc5255d11a2762bfaff02f8221251 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 27 Oct 2017 22:26:43 +0700
Subject: [PATCH 1478/1696] [youtube] Restrict embed regex (#14600)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 5aef555fb..9943dddc1 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1391,7 +1391,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             )
             (["\'])
                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
-                (?:embed|v|p)/.+?)
+                (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
             \1''', webpage)]
 
         # lazyYT YouTube embed

From 8e01f3ca811e15aae04c7f2c5345c5eca38f99d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Oct 2017 22:58:01 +0700
Subject: [PATCH 1479/1696] [dctptv] Fix extraction (closes #14599)

---
 youtube_dl/extractor/dctp.py | 68 ++++++++++++++++++++++++++----------
 1 file changed, 50 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index 00fbbff2f..3a6d0560e 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -2,53 +2,85 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import unified_strdate
+from ..compat import compat_str
+from ..utils import (
+    float_or_none,
+    unified_strdate,
+)
 
 
 class DctpTvIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?dctp\.tv/(#/)?filme/(?P<id>.+?)/$'
+    _VALID_URL = r'https?://(?:www\.)?dctp\.tv/(?:#/)?filme/(?P<id>[^/?#&]+)'
     _TEST = {
         'url': 'http://www.dctp.tv/filme/videoinstallation-fuer-eine-kaufhausfassade/',
-        'md5': '174dd4a8a6225cf5655952f969cfbe24',
         'info_dict': {
             'id': '95eaa4f33dad413aa17b4ee613cccc6c',
             'display_id': 'videoinstallation-fuer-eine-kaufhausfassade',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Videoinstallation für eine Kaufhausfassade',
             'description': 'Kurzfilm',
             'upload_date': '20110407',
             'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 71.24,
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
         },
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        display_id = self._match_id(url)
 
-        object_id = self._html_search_meta('DC.identifier', webpage)
+        webpage = self._download_webpage(url, display_id)
 
-        servers_json = self._download_json(
-            'http://www.dctp.tv/elastic_streaming_client/get_streaming_server/',
-            video_id, note='Downloading server list')
-        server = servers_json[0]['server']
-        m3u8_path = self._search_regex(
-            r'\'([^\'"]+/playlist\.m3u8)"', webpage, 'm3u8 path')
-        formats = self._extract_m3u8_formats(
-            'http://%s%s' % (server, m3u8_path), video_id, ext='mp4',
-            entry_protocol='m3u8_native')
+        video_id = self._html_search_meta(
+            'DC.identifier', webpage, 'video id',
+            default=None) or self._search_regex(
+            r'id=["\']uuid[^>]+>([^<]+)<', webpage, 'video id')
 
         title = self._og_search_title(webpage)
+
+        servers = self._download_json(
+            'http://www.dctp.tv/streaming_servers/', display_id,
+            note='Downloading server list', fatal=False)
+
+        if servers:
+            endpoint = next(
+                server['endpoint']
+                for server in servers
+                if isinstance(server.get('endpoint'), compat_str) and
+                'cloudfront' in server['endpoint'])
+        else:
+            endpoint = 'rtmpe://s2pqqn4u96e4j8.cloudfront.net/cfx/st/'
+
+        app = self._search_regex(
+            r'^rtmpe?://[^/]+/(?P<app>.*)$', endpoint, 'app')
+
+        formats = [{
+            'url': endpoint,
+            'app': app,
+            'play_path': 'mp4:%s_dctp_0500_4x3.m4v' % video_id,
+            'page_url': url,
+            'player_url': 'http://svm-prod-dctptv-static.s3.amazonaws.com/dctptv-relaunch2012-109.swf',
+            'ext': 'flv',
+        }]
+
         description = self._html_search_meta('DC.description', webpage)
         upload_date = unified_strdate(
             self._html_search_meta('DC.date.created', webpage))
         thumbnail = self._og_search_thumbnail(webpage)
+        duration = float_or_none(self._search_regex(
+            r'id=["\']duration_in_ms[^+]>(\d+)', webpage, 'duration',
+            default=None), scale=1000)
 
         return {
-            'id': object_id,
+            'id': video_id,
             'title': title,
             'formats': formats,
-            'display_id': video_id,
+            'display_id': display_id,
             'description': description,
             'upload_date': upload_date,
             'thumbnail': thumbnail,
+            'duration': duration,
         }

From 47a8587915668ef82632a7a75f8bc9862679623a Mon Sep 17 00:00:00 2001
From: Andrew Udvare <audvare@gmail.com>
Date: Sun, 12 Mar 2017 18:19:32 -0400
Subject: [PATCH 1480/1696] [younow] Add extractor

---
 youtube_dl/extractor/extractors.py |   5 +
 youtube_dl/extractor/younow.py     | 197 +++++++++++++++++++++++++++++
 2 files changed, 202 insertions(+)
 create mode 100644 youtube_dl/extractor/younow.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 18350810b..b6ad50ec7 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1335,6 +1335,11 @@ from .youku import (
     YoukuIE,
     YoukuShowIE,
 )
+from .younow import (
+    YouNowIE,
+    YouNowChannelIE,
+    YouNowMomentIE,
+)
 from .youporn import YouPornIE
 from .yourupload import YourUploadIE
 from .youtube import (
diff --git a/youtube_dl/extractor/younow.py b/youtube_dl/extractor/younow.py
new file mode 100644
index 000000000..99abd66a8
--- /dev/null
+++ b/youtube_dl/extractor/younow.py
@@ -0,0 +1,197 @@
+# coding: utf-8
+from __future__ import unicode_literals
+from datetime import date, datetime
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import int_or_none, UnsupportedError
+
+MOMENT_URL_FORMAT = 'https://cdn.younow.com/php/api/moment/fetch/id=%s'
+STREAM_URL_FORMAT = 'https://hls.younow.com/momentsplaylists/live/%s/%s.m3u8'
+
+
+class YouNowIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?younow\.com/(?P<id>[^/]+)'
+    _TEST = {
+        'url': 'https://www.younow.com/AmandaPadeezy',
+        'info_dict': {
+            'id': 'AmandaPadeezy',
+            'ext': 'mp4',
+            'is_live': True,
+            'title': 'March 26, 2017',
+            'description': 'YouNow is the best way to broadcast live and get an audience to watch you.',
+            'thumbnail': 'https://ynassets.s3.amazonaws.com/broadcast/live/157869188/157869188.jpg',
+            'tags': ['girls'],
+            'categories': ['girls'],
+            'uploader': 'AmandaPadeezy',
+            'uploader_id': '6716501',
+            'uploader_url': 'https://www.younow.com/AmandaPadeezy',
+            'creator': 'AmandaPadeezy',
+            'formats': [{
+                'url': 'https://cdn.younow.com/php/api/broadcast/videoPath/hls=1/broadcastId=157869188/channelId=6716501',
+                'ext': 'mp4',
+                'protocol': 'm3u8',
+            }],
+        }
+    }
+
+    def _real_extract(self, url):
+        username = self._match_id(url)
+        data = self._download_json('https://api.younow.com/php/api/broadcast/info/curId=0/user=%s' % (username), username)
+
+        if data.get('media'):
+            stream_url = 'https://cdn.younow.com/php/api/broadcast/videoPath/hls=1/broadcastId=%s/channelId=%s' % (
+                data.get('broadcastId'),
+                data.get('userId'),
+            )
+        else:
+            raise UnsupportedError('Unsupported stream or user is not streaming at this time')
+
+        webpage = self._download_webpage(url, username)
+        try:
+            uploader = data['user']['profileUrlString']
+        except KeyError:
+            uploader = username
+        try:
+            title = data['title']
+        except KeyError:
+            title = date.today().strftime('%B %d, %Y')
+
+        return {
+            'id': uploader,
+            'is_live': True,
+            'title': title,
+            'description': self._og_search_description(webpage),
+            'thumbnail': data.get('awsUrl'),
+            'tags': data.get('tags'),
+            'categories': data.get('tags'),
+            'uploader': uploader,
+            'uploader_id': data.get('userId'),
+            'uploader_url': 'https://www.younow.com/%s' % (data['user']['profileUrlString'],),
+            'creator': uploader,
+            'view_count': int_or_none(data.get('viewers')),
+            'like_count': int_or_none(data.get('likes')),
+            'formats': [{
+                'url': stream_url,
+                'ext': 'mp4',
+                'protocol': 'm3u8',
+            }],
+        }
+
+
+def _moment_to_entry(item):
+    title = item.get('text')
+    title_type = item.get('titleType')
+    if not title:
+        if title_type:
+            title = 'YouNow %s' % item.get('titleType')
+        else:
+            title = 'YouNow moment'
+
+    entry = {
+        'id': compat_str(item['momentId']),
+        'title': title,
+        'view_count': int_or_none(item.get('views')),
+        'like_count': int_or_none(item.get('likes')),
+        'timestamp': int_or_none(item.get('created')),
+        'formats': [{
+            'url': STREAM_URL_FORMAT % (item['momentId'], item['momentId']),
+            'ext': 'mp4',
+            'protocol': 'm3u8',
+        }],
+    }
+
+    try:
+        entry['uploader'] = entry['creator'] = item['owner']['name']
+        entry['uploader_url'] = 'https://www.younow.com/%s' % (item['owner']['name'],)
+        entry['uploader_id'] = item['owner']['userId']
+    except KeyError:
+        pass
+
+    return entry
+
+
+class YouNowChannelIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?younow\.com/(?P<id>[^/]+)/channel'
+    _TEST = {
+        'url': 'https://www.younow.com/Kate_Swiz/channel',
+        'info_dict': {
+            'title': 'Kate_Swiz moments'
+        },
+        'playlist_count': 6,
+    }
+
+    MOMENTS_URL_FORMAT = 'https://cdn.younow.com/php/api/moment/profile/channelId=%s/createdBefore=%d/records=20'
+
+    def _real_extract(self, url):
+        entries = []
+        username = self._match_id(url)
+        user_info = self._download_json('https://api.younow.com/php/api/broadcast/info/curId=0/user=%s' % (username), username, note='Downloading user information')
+        channel_id = user_info['userId']
+        created_before = 0
+        moment_ids = []
+        moment_ids_processed = []
+        err = False
+
+        while True:
+            if created_before:
+                cb = datetime.fromtimestamp(created_before)
+            else:
+                cb = datetime.now()
+            info = self._download_json(self.MOMENTS_URL_FORMAT % (channel_id, created_before), username, note='Downloading moments data (created before %s)' % (cb))
+
+            for item in info['items']:
+                if item['type'] == 'moment':
+                    entry = _moment_to_entry(item)
+                    moment_ids_processed.append(entry['id'])
+                    entries.append(entry)
+                elif item['type'] == 'collection':
+                    moment_ids += [compat_str(x) for x in item['momentsIds']]
+
+                try:
+                    created_before = int_or_none(item['created'])
+                except KeyError:
+                    err = True
+                    break
+
+            if (err or
+                    not info['hasMore'] or
+                    'items' not in info or
+                    not info['items']):
+                break
+
+        for mid in set(moment_ids):
+            if mid in moment_ids_processed:
+                continue
+            item = self._download_json(MOMENT_URL_FORMAT % (mid), mid)
+            entries.append(_moment_to_entry(item['item']))
+
+        return self.playlist_result(entries, playlist_title='%s moments' % (username))
+
+
+class YouNowMomentIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?younow\.com/[^/]+/(?P<id>[^/]+)/[^/]+'
+    _TEST = {
+        'url': 'https://www.younow.com/GABO.../20712117/36319236/3b316doc/m',
+        'info_dict': {
+            'id': '20712117',
+            'ext': 'mp4',
+            'title': 'YouNow capture',
+            'view_count': 19,
+            'like_count': 0,
+            'timestamp': 1490432040,
+            'formats': [{
+                'url': 'https://hls.younow.com/momentsplaylists/live/20712117/20712117.m3u8',
+                'ext': 'mp4',
+                'protocol': 'm3u8',
+            }],
+            'upload_date': '20170325',
+            'uploader': 'GABO...',
+            'uploader_id': 35917228,
+        },
+    }
+
+    def _real_extract(self, url):
+        mid = self._match_id(url)
+        item = self._download_json(MOMENT_URL_FORMAT % (mid), mid)
+        return _moment_to_entry(item['item'])

From eb4b5818e2a297bd001eb1b4962d709b1245fd24 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Oct 2017 04:16:07 +0700
Subject: [PATCH 1481/1696] [younow] Fix issues and improve extraction (closes
 #9255, closes #9432, closes #12436)

---
 youtube_dl/extractor/extractors.py |   2 +-
 youtube_dl/extractor/younow.py     | 221 +++++++++++++++--------------
 2 files changed, 114 insertions(+), 109 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index b6ad50ec7..2eed706f9 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1336,7 +1336,7 @@ from .youku import (
     YoukuShowIE,
 )
 from .younow import (
-    YouNowIE,
+    YouNowLiveIE,
     YouNowChannelIE,
     YouNowMomentIE,
 )
diff --git a/youtube_dl/extractor/younow.py b/youtube_dl/extractor/younow.py
index 99abd66a8..04dbc87fc 100644
--- a/youtube_dl/extractor/younow.py
+++ b/youtube_dl/extractor/younow.py
@@ -1,17 +1,22 @@
 # coding: utf-8
 from __future__ import unicode_literals
-from datetime import date, datetime
+
+import itertools
 
 from .common import InfoExtractor
 from ..compat import compat_str
-from ..utils import int_or_none, UnsupportedError
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    try_get,
+)
 
-MOMENT_URL_FORMAT = 'https://cdn.younow.com/php/api/moment/fetch/id=%s'
-STREAM_URL_FORMAT = 'https://hls.younow.com/momentsplaylists/live/%s/%s.m3u8'
+CDN_API_BASE = 'https://cdn.younow.com/php/api'
+MOMENT_URL_FORMAT = '%s/moment/fetch/id=%%s' % CDN_API_BASE
 
 
-class YouNowIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?younow\.com/(?P<id>[^/]+)'
+class YouNowLiveIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?younow\.com/(?P<id>[^/?#&]+)'
     _TEST = {
         'url': 'https://www.younow.com/AmandaPadeezy',
         'info_dict': {
@@ -19,179 +24,179 @@ class YouNowIE(InfoExtractor):
             'ext': 'mp4',
             'is_live': True,
             'title': 'March 26, 2017',
-            'description': 'YouNow is the best way to broadcast live and get an audience to watch you.',
-            'thumbnail': 'https://ynassets.s3.amazonaws.com/broadcast/live/157869188/157869188.jpg',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'tags': ['girls'],
             'categories': ['girls'],
             'uploader': 'AmandaPadeezy',
             'uploader_id': '6716501',
             'uploader_url': 'https://www.younow.com/AmandaPadeezy',
             'creator': 'AmandaPadeezy',
-            'formats': [{
-                'url': 'https://cdn.younow.com/php/api/broadcast/videoPath/hls=1/broadcastId=157869188/channelId=6716501',
-                'ext': 'mp4',
-                'protocol': 'm3u8',
-            }],
-        }
+        },
+        'skip': True,
     }
 
+    @classmethod
+    def suitable(cls, url):
+        return (False
+                if YouNowChannelIE.suitable(url) or YouNowMomentIE.suitable(url)
+                else super(YouNowLiveIE, cls).suitable(url))
+
     def _real_extract(self, url):
         username = self._match_id(url)
-        data = self._download_json('https://api.younow.com/php/api/broadcast/info/curId=0/user=%s' % (username), username)
 
-        if data.get('media'):
-            stream_url = 'https://cdn.younow.com/php/api/broadcast/videoPath/hls=1/broadcastId=%s/channelId=%s' % (
-                data.get('broadcastId'),
-                data.get('userId'),
-            )
-        else:
-            raise UnsupportedError('Unsupported stream or user is not streaming at this time')
+        data = self._download_json(
+            'https://api.younow.com/php/api/broadcast/info/curId=0/user=%s'
+            % username, username)
 
-        webpage = self._download_webpage(url, username)
-        try:
-            uploader = data['user']['profileUrlString']
-        except KeyError:
-            uploader = username
-        try:
-            title = data['title']
-        except KeyError:
-            title = date.today().strftime('%B %d, %Y')
+        if data.get('errorCode') != 0:
+            raise ExtractorError(data['errorMsg'], expected=True)
+
+        uploader = try_get(
+            data, lambda x: x['user']['profileUrlString'],
+            compat_str) or username
 
         return {
             'id': uploader,
             'is_live': True,
-            'title': title,
-            'description': self._og_search_description(webpage),
+            'title': self._live_title(uploader),
             'thumbnail': data.get('awsUrl'),
             'tags': data.get('tags'),
             'categories': data.get('tags'),
             'uploader': uploader,
             'uploader_id': data.get('userId'),
-            'uploader_url': 'https://www.younow.com/%s' % (data['user']['profileUrlString'],),
+            'uploader_url': 'https://www.younow.com/%s' % username,
             'creator': uploader,
             'view_count': int_or_none(data.get('viewers')),
             'like_count': int_or_none(data.get('likes')),
             'formats': [{
-                'url': stream_url,
+                'url': '%s/broadcast/videoPath/hls=1/broadcastId=%s/channelId=%s'
+                       % (CDN_API_BASE, data['broadcastId'], data['userId']),
                 'ext': 'mp4',
                 'protocol': 'm3u8',
             }],
         }
 
 
-def _moment_to_entry(item):
+def _extract_moment(item, fatal=True):
+    moment_id = item.get('momentId')
+    if not moment_id:
+        if not fatal:
+            return
+        raise ExtractorError('Unable to extract moment id')
+
+    moment_id = compat_str(moment_id)
+
     title = item.get('text')
-    title_type = item.get('titleType')
     if not title:
-        if title_type:
-            title = 'YouNow %s' % item.get('titleType')
-        else:
-            title = 'YouNow moment'
+        title = 'YouNow %s' % (
+            item.get('momentType') or item.get('titleType') or 'moment')
+
+    uploader = try_get(item, lambda x: x['owner']['name'], compat_str)
+    uploader_id = try_get(item, lambda x: x['owner']['userId'])
+    uploader_url = 'https://www.younow.com/%s' % uploader if uploader else None
 
     entry = {
-        'id': compat_str(item['momentId']),
+        'extractor_key': 'YouNowMoment',
+        'id': moment_id,
         'title': title,
         'view_count': int_or_none(item.get('views')),
         'like_count': int_or_none(item.get('likes')),
         'timestamp': int_or_none(item.get('created')),
+        'creator': uploader,
+        'uploader': uploader,
+        'uploader_id': uploader_id,
+        'uploader_url': uploader_url,
         'formats': [{
-            'url': STREAM_URL_FORMAT % (item['momentId'], item['momentId']),
+            'url': 'https://hls.younow.com/momentsplaylists/live/%s/%s.m3u8'
+                   % (moment_id, moment_id),
             'ext': 'mp4',
-            'protocol': 'm3u8',
+            'protocol': 'm3u8_native',
         }],
     }
 
-    try:
-        entry['uploader'] = entry['creator'] = item['owner']['name']
-        entry['uploader_url'] = 'https://www.younow.com/%s' % (item['owner']['name'],)
-        entry['uploader_id'] = item['owner']['userId']
-    except KeyError:
-        pass
-
     return entry
 
 
 class YouNowChannelIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?younow\.com/(?P<id>[^/]+)/channel'
     _TEST = {
-        'url': 'https://www.younow.com/Kate_Swiz/channel',
+        'url': 'https://www.younow.com/its_Kateee_/channel',
         'info_dict': {
-            'title': 'Kate_Swiz moments'
+            'id': '14629760',
+            'title': 'its_Kateee_ moments'
         },
-        'playlist_count': 6,
+        'playlist_mincount': 8,
     }
 
-    MOMENTS_URL_FORMAT = 'https://cdn.younow.com/php/api/moment/profile/channelId=%s/createdBefore=%d/records=20'
+    def _entries(self, username, channel_id):
+        created_before = 0
+        for page_num in itertools.count(1):
+            if created_before is None:
+                break
+            info = self._download_json(
+                '%s/moment/profile/channelId=%s/createdBefore=%d/records=20'
+                % (CDN_API_BASE, channel_id, created_before), username,
+                note='Downloading moments page %d' % page_num)
+            items = info.get('items')
+            if not items or not isinstance(items, list):
+                break
+            for item in items:
+                if not isinstance(item, dict):
+                    continue
+                item_type = item.get('type')
+                if item_type == 'moment':
+                    entry = _extract_moment(item, fatal=False)
+                    if entry:
+                        yield entry
+                elif item_type == 'collection':
+                    moments = item.get('momentsIds')
+                    if isinstance(moments, list):
+                        for moment_id in moments:
+                            m = self._download_json(
+                                MOMENT_URL_FORMAT % moment_id, username,
+                                note='Downloading %s moment JSON' % moment_id,
+                                fatal=False)
+                            if m and isinstance(m, dict) and m.get('item'):
+                                entry = _extract_moment(m['item'])
+                                if entry:
+                                    yield entry
+                created_before = int_or_none(item.get('created'))
 
     def _real_extract(self, url):
-        entries = []
         username = self._match_id(url)
-        user_info = self._download_json('https://api.younow.com/php/api/broadcast/info/curId=0/user=%s' % (username), username, note='Downloading user information')
-        channel_id = user_info['userId']
-        created_before = 0
-        moment_ids = []
-        moment_ids_processed = []
-        err = False
-
-        while True:
-            if created_before:
-                cb = datetime.fromtimestamp(created_before)
-            else:
-                cb = datetime.now()
-            info = self._download_json(self.MOMENTS_URL_FORMAT % (channel_id, created_before), username, note='Downloading moments data (created before %s)' % (cb))
-
-            for item in info['items']:
-                if item['type'] == 'moment':
-                    entry = _moment_to_entry(item)
-                    moment_ids_processed.append(entry['id'])
-                    entries.append(entry)
-                elif item['type'] == 'collection':
-                    moment_ids += [compat_str(x) for x in item['momentsIds']]
-
-                try:
-                    created_before = int_or_none(item['created'])
-                except KeyError:
-                    err = True
-                    break
-
-            if (err or
-                    not info['hasMore'] or
-                    'items' not in info or
-                    not info['items']):
-                break
-
-        for mid in set(moment_ids):
-            if mid in moment_ids_processed:
-                continue
-            item = self._download_json(MOMENT_URL_FORMAT % (mid), mid)
-            entries.append(_moment_to_entry(item['item']))
-
-        return self.playlist_result(entries, playlist_title='%s moments' % (username))
+        channel_id = compat_str(self._download_json(
+            'https://api.younow.com/php/api/broadcast/info/curId=0/user=%s'
+            % username, username, note='Downloading user information')['userId'])
+        return self.playlist_result(
+            self._entries(username, channel_id), channel_id,
+            '%s moments' % username)
 
 
 class YouNowMomentIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?younow\.com/[^/]+/(?P<id>[^/]+)/[^/]+'
+    _VALID_URL = r'https?://(?:www\.)?younow\.com/[^/]+/(?P<id>[^/?#&]+)'
     _TEST = {
         'url': 'https://www.younow.com/GABO.../20712117/36319236/3b316doc/m',
+        'md5': 'a30c70eadb9fb39a1aa3c8c0d22a0807',
         'info_dict': {
             'id': '20712117',
             'ext': 'mp4',
             'title': 'YouNow capture',
-            'view_count': 19,
-            'like_count': 0,
+            'view_count': int,
+            'like_count': int,
             'timestamp': 1490432040,
-            'formats': [{
-                'url': 'https://hls.younow.com/momentsplaylists/live/20712117/20712117.m3u8',
-                'ext': 'mp4',
-                'protocol': 'm3u8',
-            }],
             'upload_date': '20170325',
             'uploader': 'GABO...',
             'uploader_id': 35917228,
         },
     }
 
+    @classmethod
+    def suitable(cls, url):
+        return (False
+                if YouNowChannelIE.suitable(url)
+                else super(YouNowMomentIE, cls).suitable(url))
+
     def _real_extract(self, url):
-        mid = self._match_id(url)
-        item = self._download_json(MOMENT_URL_FORMAT % (mid), mid)
-        return _moment_to_entry(item['item'])
+        video_id = self._match_id(url)
+        item = self._download_json(MOMENT_URL_FORMAT % video_id, video_id)
+        return _extract_moment(item['item'])

From c3206d02e94ce98c6467762a228a9e58616c6d8f Mon Sep 17 00:00:00 2001
From: enigmaquip <enigmaquip@users.noreply.github.com>
Date: Sat, 28 Oct 2017 16:20:18 -0600
Subject: [PATCH 1482/1696] [fxnetworks] Extract series metadata

---
 youtube_dl/extractor/fxnetworks.py | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/fxnetworks.py b/youtube_dl/extractor/fxnetworks.py
index 629897317..37549fb01 100644
--- a/youtube_dl/extractor/fxnetworks.py
+++ b/youtube_dl/extractor/fxnetworks.py
@@ -3,27 +3,31 @@ from __future__ import unicode_literals
 
 from .adobepass import AdobePassIE
 from ..utils import (
-    update_url_query,
     extract_attributes,
+    int_or_none,
     parse_age_limit,
     smuggle_url,
+    update_url_query,
 )
 
 
 class FXNetworksIE(AdobePassIE):
     _VALID_URL = r'https?://(?:www\.)?(?:fxnetworks|simpsonsworld)\.com/video/(?P<id>\d+)'
     _TESTS = [{
-        'url': 'http://www.fxnetworks.com/video/719841347694',
-        'md5': '1447d4722e42ebca19e5232ab93abb22',
+        'url': 'http://www.fxnetworks.com/video/1032565827847',
+        'md5': '8d99b97b4aa7a202f55b6ed47ea7e703',
         'info_dict': {
-            'id': '719841347694',
+            'id': 'dRzwHC_MMqIv',
             'ext': 'mp4',
-            'title': 'Vanpage',
-            'description': 'F*ck settling down. You\'re the Worst returns for an all new season August 31st on FXX.',
+            'title': 'First Look: Better Things - Season 2',
+            'description': 'Because real life is like a fart. Watch this FIRST LOOK to see what inspired the new season of Better Things.',
             'age_limit': 14,
             'uploader': 'NEWA-FNG-FX',
-            'upload_date': '20160706',
-            'timestamp': 1467844741,
+            'upload_date': '20170825',
+            'timestamp': 1503686274,
+            'episode_number': 0,
+            'season_number': 2,
+            'series': 'Better Things',
         },
         'add_ie': ['ThePlatform'],
     }, {
@@ -64,6 +68,9 @@ class FXNetworksIE(AdobePassIE):
             'id': video_id,
             'title': title,
             'url': smuggle_url(update_url_query(release_url, query), {'force_smil_url': True}),
+            'series': video_data.get('data-show-title'),
+            'episode_number': int_or_none(video_data.get('data-episode')),
+            'season_number': int_or_none(video_data.get('data-season')),
             'thumbnail': video_data.get('data-large-thumb'),
             'age_limit': parse_age_limit(rating),
             'ie_key': 'ThePlatform',

From 056653bbb1b94ba04f331ed4c27a1c0d24fe1e30 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Oct 2017 07:04:48 +0700
Subject: [PATCH 1483/1696] [utils] Add support for zero years and months in
 parse_duration

---
 test/test_utils.py  |  1 +
 youtube_dl/utils.py | 12 +++++++++++-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index efa73d0f4..cc13f795c 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -540,6 +540,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_duration('87 Min.'), 5220)
         self.assertEqual(parse_duration('PT1H0.040S'), 3600.04)
         self.assertEqual(parse_duration('PT00H03M30SZ'), 210)
+        self.assertEqual(parse_duration('P0Y0M0DT0H4M20.880S'), 260.88)
 
     def test_fix_xml_ampersands(self):
         self.assertEqual(
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 59fb33435..34866a54b 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1835,10 +1835,20 @@ def parse_duration(s):
         days, hours, mins, secs, ms = m.groups()
     else:
         m = re.match(
-            r'''(?ix)(?:P?T)?
+            r'''(?ix)(?:P?
+                (?:
+                    [0-9]+\s*y(?:ears?)?\s*
+                )?
+                (?:
+                    [0-9]+\s*m(?:onths?)?\s*
+                )?
+                (?:
+                    [0-9]+\s*w(?:eeks?)?\s*
+                )?
                 (?:
                     (?P<days>[0-9]+)\s*d(?:ays?)?\s*
                 )?
+                T)?
                 (?:
                     (?P<hours>[0-9]+)\s*h(?:ours?)?\s*
                 )?

From 9211e3319e6006373d8b5055f7a3d0bbd734c57b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Oct 2017 07:05:55 +0700
Subject: [PATCH 1484/1696] [extractor/common] Prefix format id for audio only
 HLS formats

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a69240693..52f2055b5 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1401,7 +1401,7 @@ class InfoExtractor(object):
             media_url = media.get('URI')
             if media_url:
                 format_id = []
-                for v in (group_id, name):
+                for v in (m3u8_id, group_id, name):
                     if v:
                         format_id.append(v)
                 f = {

From 514e8aefd488b385122fa989e937e5f9ae62d136 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Oct 2017 07:11:37 +0700
Subject: [PATCH 1485/1696] [egghead] Fix extraction (closes #14388)

---
 youtube_dl/extractor/egghead.py | 81 +++++++++++++++++++++++++--------
 1 file changed, 62 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/egghead.py b/youtube_dl/extractor/egghead.py
index e4a3046af..edabaafe6 100644
--- a/youtube_dl/extractor/egghead.py
+++ b/youtube_dl/extractor/egghead.py
@@ -2,7 +2,9 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
+    determine_ext,
     int_or_none,
     try_get,
     unified_timestamp,
@@ -17,7 +19,7 @@ class EggheadCourseIE(InfoExtractor):
         'url': 'https://egghead.io/courses/professor-frisby-introduces-composable-functional-javascript',
         'playlist_count': 29,
         'info_dict': {
-            'id': 'professor-frisby-introduces-composable-functional-javascript',
+            'id': '72',
             'title': 'Professor Frisby Introduces Composable Functional JavaScript',
             'description': 're:(?s)^This course teaches the ubiquitous.*You\'ll start composing functionality before you know it.$',
         },
@@ -26,14 +28,28 @@ class EggheadCourseIE(InfoExtractor):
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
 
-        course = self._download_json(
-            'https://egghead.io/api/v1/series/%s' % playlist_id, playlist_id)
+        lessons = self._download_json(
+            'https://egghead.io/api/v1/series/%s/lessons' % playlist_id,
+            playlist_id, 'Downloading course lessons JSON')
 
-        entries = [
-            self.url_result(
-                'wistia:%s' % lesson['wistia_id'], ie='Wistia',
-                video_id=lesson['wistia_id'], video_title=lesson.get('title'))
-            for lesson in course['lessons'] if lesson.get('wistia_id')]
+        entries = []
+        for lesson in lessons:
+            lesson_url = lesson.get('http_url')
+            if not lesson_url or not isinstance(lesson_url, compat_str):
+                continue
+            lesson_id = lesson.get('id')
+            if lesson_id:
+                lesson_id = compat_str(lesson_id)
+            entries.append(self.url_result(
+                lesson_url, ie=EggheadLessonIE.ie_key(), video_id=lesson_id))
+
+        course = self._download_json(
+            'https://egghead.io/api/v1/series/%s' % playlist_id,
+            playlist_id, 'Downloading course JSON', fatal=False) or {}
+
+        playlist_id = course.get('id')
+        if playlist_id:
+            playlist_id = compat_str(playlist_id)
 
         return self.playlist_result(
             entries, playlist_id, course.get('title'),
@@ -43,11 +59,12 @@ class EggheadCourseIE(InfoExtractor):
 class EggheadLessonIE(InfoExtractor):
     IE_DESC = 'egghead.io lesson'
     IE_NAME = 'egghead:lesson'
-    _VALID_URL = r'https://egghead\.io/lessons/(?P<id>[^/?#&]+)'
-    _TEST = {
+    _VALID_URL = r'https://egghead\.io/(?:api/v1/)?lessons/(?P<id>[^/?#&]+)'
+    _TESTS = [{
         'url': 'https://egghead.io/lessons/javascript-linear-data-flow-with-container-style-types-box',
         'info_dict': {
-            'id': 'fv5yotjxcg',
+            'id': '1196',
+            'display_id': 'javascript-linear-data-flow-with-container-style-types-box',
             'ext': 'mp4',
             'title': 'Create linear data flow with container style types (Box)',
             'description': 'md5:9aa2cdb6f9878ed4c39ec09e85a8150e',
@@ -60,25 +77,51 @@ class EggheadLessonIE(InfoExtractor):
         },
         'params': {
             'skip_download': True,
+            'format': 'bestvideo',
         },
-    }
+    }, {
+        'url': 'https://egghead.io/api/v1/lessons/react-add-redux-to-a-react-application',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        lesson_id = self._match_id(url)
+        display_id = self._match_id(url)
 
         lesson = self._download_json(
-            'https://egghead.io/api/v1/lessons/%s' % lesson_id, lesson_id)
+            'https://egghead.io/api/v1/lessons/%s' % display_id, display_id)
+
+        lesson_id = compat_str(lesson['id'])
+        title = lesson['title']
+
+        formats = []
+        for _, format_url in lesson['media_urls'].items():
+            if not format_url or not isinstance(format_url, compat_str):
+                continue
+            ext = determine_ext(format_url)
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, lesson_id, 'mp4', entry_protocol='m3u8',
+                    m3u8_id='hls', fatal=False))
+            elif ext == 'mpd':
+                formats.extend(self._extract_mpd_formats(
+                    format_url, lesson_id, mpd_id='dash', fatal=False))
+            else:
+                formats.append({
+                    'url': format_url,
+                })
+        self._sort_formats(formats)
 
         return {
-            '_type': 'url_transparent',
-            'ie_key': 'Wistia',
-            'url': 'wistia:%s' % lesson['wistia_id'],
-            'id': lesson['wistia_id'],
-            'title': lesson.get('title'),
+            'id': lesson_id,
+            'display_id': display_id,
+            'title': title,
             'description': lesson.get('summary'),
             'thumbnail': lesson.get('thumb_nail'),
             'timestamp': unified_timestamp(lesson.get('published_at')),
             'duration': int_or_none(lesson.get('duration')),
             'view_count': int_or_none(lesson.get('plays_count')),
             'tags': try_get(lesson, lambda x: x['tag_list'], list),
+            'series': try_get(
+                lesson, lambda x: x['series']['title'], compat_str),
+            'formats': formats,
         }

From 518d357b46cb840224d69cc543667be3a87b9b9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Oct 2017 07:21:33 +0700
Subject: [PATCH 1486/1696] [ChangeLog] Actualize

---
 ChangeLog | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 547b55981..795491d34 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,28 @@
+version <unreleased>
+
+Core
+* [extractor/common] Prefix format id for audio only HLS formats
++ [utils] Add support for zero years and months in parse_duration
+
+Extractors
+* [egghead] Fix extraction (#14388)
++ [fxnetworks] Extract series metadata (#14603)
++ [younow] Add support for younow.com (#9255, #9432, #12436)
+* [dctptv] Fix extraction (#14599)
+* [youtube] Restrict embed regex (#14600)
+* [vimeo] Restrict iframe embed regex (#14600)
+* [soundgasm] Improve extraction (#14588)
+- [myvideo] Remove extractor (#8557)
++ [nbc] Add support for classic-tv videos (#14575)
++ [vrtnu] Add support for cookies authentication and simplify (#11873)
++ [canvas] Add support for vrt.be/vrtnu (#11873)
+* [twitch:clips] Fix title extraction (#14566)
++ [ndtv] Add support for sub-sites (#14534)
+* [dramafever] Fix login error message extraction
++ [nick] Add support for more nickelodeon sites (no, dk, se, ch, fr, es, pt,
+  ro, hu) (#14553)
+
+
 version 2017.10.20
 
 Core

From 6d0630d8801fd3278a05fa7e55a73bd454403e5a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Oct 2017 07:22:53 +0700
Subject: [PATCH 1487/1696] release 2017.10.29

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 5 ++++-
 youtube_dl/version.py     | 2 +-
 4 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 81fe10d54..881475878 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.10.20*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.10.20**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.10.29*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.10.29**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.10.20
+[debug] youtube-dl version 2017.10.29
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 795491d34..d33a710fb 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.10.29
 
 Core
 * [extractor/common] Prefix format id for audio only HLS formats
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index be5de22df..7b8e7403a 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -498,7 +498,6 @@
  - **MySpace:album**
  - **MySpass**
  - **Myvi**
- - **myvideo** (Currently broken)
  - **MyVidster**
  - **n-tv.de**
  - **natgeo**
@@ -977,6 +976,7 @@
  - **vpro**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
  - **Vrak**
  - **VRT**: deredactie.be, sporza.be, cobra.be and cobra.canvas.be
+ - **VrtNU**: VrtNU.be
  - **vrv**
  - **vrv:series**
  - **VShare**
@@ -1035,6 +1035,9 @@
  - **YouJizz**
  - **youku**: 优酷
  - **youku:show**
+ - **YouNowChannel**
+ - **YouNowLive**
+ - **YouNowMoment**
  - **YouPorn**
  - **YourUpload**
  - **youtube**: YouTube.com
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4d1686670..43f080bc3 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.10.20'
+__version__ = '2017.10.29'

From 8fe767e07261abb8013b18ca2ed31ebb8d95c7d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 31 Oct 2017 23:05:25 +0700
Subject: [PATCH 1488/1696] [spankbang] Detect unavailable videos (closes
 #14644)

---
 youtube_dl/extractor/spankbang.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index 3394c7e6b..2863e53b5 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import ExtractorError
 
 
 class SpankBangIE(InfoExtractor):
@@ -33,6 +34,10 @@ class SpankBangIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
+        if re.search(r'<[^>]+\bid=["\']video_removed', webpage):
+            raise ExtractorError(
+                'Video %s is not available' % video_id, expected=True)
+
         stream_key = self._html_search_regex(
             r'''var\s+stream_key\s*=\s*['"](.+?)['"]''',
             webpage, 'stream key')

From 044eeb145556cb41485b6b6644f40b2161a4e0f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 1 Nov 2017 23:39:26 +0700
Subject: [PATCH 1489/1696] [extractor/common] Respect URL query in
 _extract_wowza_formats (closes #14645)

---
 youtube_dl/extractor/common.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 52f2055b5..a67ac4411 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2233,27 +2233,35 @@ class InfoExtractor(object):
         return formats
 
     def _extract_wowza_formats(self, url, video_id, m3u8_entry_protocol='m3u8_native', skip_protocols=[]):
+        query = compat_urlparse.urlparse(url).query
         url = re.sub(r'/(?:manifest|playlist|jwplayer)\.(?:m3u8|f4m|mpd|smil)', '', url)
         url_base = self._search_regex(
             r'(?:(?:https?|rtmp|rtsp):)?(//[^?]+)', url, 'format url')
         http_base_url = '%s:%s' % ('http', url_base)
         formats = []
+
+        def manifest_url(manifest):
+            m_url = '%s/%s' % (http_base_url, manifest)
+            if query:
+                m_url += '?%s' % query
+            return m_url
+
         if 'm3u8' not in skip_protocols:
             formats.extend(self._extract_m3u8_formats(
-                http_base_url + '/playlist.m3u8', video_id, 'mp4',
+                manifest_url('playlist.m3u8'), video_id, 'mp4',
                 m3u8_entry_protocol, m3u8_id='hls', fatal=False))
         if 'f4m' not in skip_protocols:
             formats.extend(self._extract_f4m_formats(
-                http_base_url + '/manifest.f4m',
+                manifest_url('manifest.f4m'),
                 video_id, f4m_id='hds', fatal=False))
         if 'dash' not in skip_protocols:
             formats.extend(self._extract_mpd_formats(
-                http_base_url + '/manifest.mpd',
+                manifest_url('manifest.mpd'),
                 video_id, mpd_id='dash', fatal=False))
         if re.search(r'(?:/smil:|\.smil)', url_base):
             if 'smil' not in skip_protocols:
                 rtmp_formats = self._extract_smil_formats(
-                    http_base_url + '/jwplayer.smil',
+                    manifest_url('jwplayer.smil'),
                     video_id, fatal=False)
                 for rtmp_format in rtmp_formats:
                     rtsp_format = rtmp_format.copy()

From b0f4331002798522621aff55deaf18406d17e081 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 2 Nov 2017 13:30:01 +0100
Subject: [PATCH 1490/1696] [gamespot] extract formats referenced with new data
 fields(#14652)

---
 youtube_dl/extractor/gamespot.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index 02804d297..6d177cbaf 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class GameSpotIE(OnceIE):
-    _VALID_URL = r'https?://(?:www\.)?gamespot\.com/.*-(?P<id>\d+)/?'
+    _VALID_URL = r'https?://(?:www\.)?gamespot\.com/videos/(?:[^/]+/\d+-|embed/)(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.gamespot.com/videos/arma-3-community-guide-sitrep-i/2300-6410818/',
         'md5': 'b2a30deaa8654fcccd43713a6b6a4825',
@@ -35,6 +35,9 @@ class GameSpotIE(OnceIE):
         'params': {
             'skip_download': True,  # m3u8 downloads
         },
+    }, {
+        'url': 'https://www.gamespot.com/videos/embed/6439218/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -52,7 +55,7 @@ class GameSpotIE(OnceIE):
             manifest_url = f4m_url
             formats.extend(self._extract_f4m_formats(
                 f4m_url + '?hdcore=3.7.0', page_id, f4m_id='hds', fatal=False))
-        m3u8_url = streams.get('m3u8_stream')
+        m3u8_url = dict_get(streams, ('m3u8_stream', 'adaptive_stream'))
         if m3u8_url:
             manifest_url = m3u8_url
             m3u8_formats = self._extract_m3u8_formats(
@@ -60,7 +63,7 @@ class GameSpotIE(OnceIE):
                 m3u8_id='hls', fatal=False)
             formats.extend(m3u8_formats)
         progressive_url = dict_get(
-            streams, ('progressive_hd', 'progressive_high', 'progressive_low'))
+            streams, ('progressive_hd', 'progressive_high', 'progressive_low', 'other_lr'))
         if progressive_url and manifest_url:
             qualities_basename = self._search_regex(
                 r'/([^/]+)\.csmil/',

From 44cca168cc36a71ea77d8635a44f6ee9d2c33a99 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 2 Nov 2017 14:16:15 +0100
Subject: [PATCH 1491/1696] [skysport] add support ooyala embed_token protected
 videos(fixes #14641)

---
 youtube_dl/extractor/skysports.py | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/skysports.py b/youtube_dl/extractor/skysports.py
index 4ca9f6b3c..efcbb36a9 100644
--- a/youtube_dl/extractor/skysports.py
+++ b/youtube_dl/extractor/skysports.py
@@ -2,7 +2,12 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import strip_or_none
+from ..utils import (
+    extract_attributes,
+    smuggle_url,
+    strip_or_none,
+    urljoin,
+)
 
 
 class SkySportsIE(InfoExtractor):
@@ -22,12 +27,22 @@ class SkySportsIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+        video_data = extract_attributes(self._search_regex(
+            r'(<div.+?class="sdc-article-video__media-ooyala"[^>]+>)', webpage, 'video data'))
+
+        video_url = 'ooyala:%s' % video_data['data-video-id']
+        if video_data.get('data-token-required') == 'true':
+            token_fetch_options = self._parse_json(video_data.get('data-token-fetch-options', '{}'), video_id, fatal=False) or {}
+            token_fetch_url = token_fetch_options.get('url')
+            if token_fetch_url:
+                embed_token = self._download_webpage(urljoin(url, token_fetch_url), video_id, fatal=False)
+                if embed_token:
+                    video_url = smuggle_url(video_url, {'embed_token': embed_token.strip('"')})
 
         return {
             '_type': 'url_transparent',
             'id': video_id,
-            'url': 'ooyala:%s' % self._search_regex(
-                r'data-video-id="([^"]+)"', webpage, 'ooyala id'),
+            'url': video_url,
             'title': self._og_search_title(webpage),
             'description': strip_or_none(self._og_search_description(webpage)),
             'ie_key': 'Ooyala',

From cd670befc4c823a38a88fffbaa6c493e539dd79d Mon Sep 17 00:00:00 2001
From: Jimbolino <Jimbolino@users.noreply.github.com>
Date: Thu, 2 Nov 2017 17:48:43 +0100
Subject: [PATCH 1492/1696] [22tracks] Remove extractor (closes #11024)

---
 youtube_dl/extractor/extractors.py      |  4 --
 youtube_dl/extractor/twentytwotracks.py | 86 -------------------------
 2 files changed, 90 deletions(-)
 delete mode 100644 youtube_dl/extractor/twentytwotracks.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 2eed706f9..92f7e9027 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1110,10 +1110,6 @@ from .tvplayer import TVPlayerIE
 from .tweakers import TweakersIE
 from .twentyfourvideo import TwentyFourVideoIE
 from .twentymin import TwentyMinutenIE
-from .twentytwotracks import (
-    TwentyTwoTracksIE,
-    TwentyTwoTracksGenreIE
-)
 from .twitch import (
     TwitchVideoIE,
     TwitchChapterIE,
diff --git a/youtube_dl/extractor/twentytwotracks.py b/youtube_dl/extractor/twentytwotracks.py
deleted file mode 100644
index d6c0ab184..000000000
--- a/youtube_dl/extractor/twentytwotracks.py
+++ /dev/null
@@ -1,86 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import int_or_none
-
-# 22Tracks regularly replace the audio tracks that can be streamed on their
-# site. The tracks usually expire after 1 months, so we can't add tests.
-
-
-class TwentyTwoTracksIE(InfoExtractor):
-    _VALID_URL = r'https?://22tracks\.com/(?P<city>[a-z]+)/(?P<genre>[\da-z]+)/(?P<id>\d+)'
-    IE_NAME = '22tracks:track'
-
-    _API_BASE = 'http://22tracks.com/api'
-
-    def _extract_info(self, city, genre_name, track_id=None):
-        item_id = track_id if track_id else genre_name
-
-        cities = self._download_json(
-            '%s/cities' % self._API_BASE, item_id,
-            'Downloading cities info',
-            'Unable to download cities info')
-        city_id = [x['id'] for x in cities if x['slug'] == city][0]
-
-        genres = self._download_json(
-            '%s/genres/%s' % (self._API_BASE, city_id), item_id,
-            'Downloading %s genres info' % city,
-            'Unable to download %s genres info' % city)
-        genre = [x for x in genres if x['slug'] == genre_name][0]
-        genre_id = genre['id']
-
-        tracks = self._download_json(
-            '%s/tracks/%s' % (self._API_BASE, genre_id), item_id,
-            'Downloading %s genre tracks info' % genre_name,
-            'Unable to download track info')
-
-        return [x for x in tracks if x['id'] == item_id][0] if track_id else [genre['title'], tracks]
-
-    def _get_track_url(self, filename, track_id):
-        token = self._download_json(
-            'http://22tracks.com/token.php?desktop=true&u=/128/%s' % filename,
-            track_id, 'Downloading token', 'Unable to download token')
-        return 'http://audio.22tracks.com%s?st=%s&e=%d' % (token['filename'], token['st'], token['e'])
-
-    def _extract_track_info(self, track_info, track_id):
-        download_url = self._get_track_url(track_info['filename'], track_id)
-        title = '%s - %s' % (track_info['artist'].strip(), track_info['title'].strip())
-        return {
-            'id': track_id,
-            'url': download_url,
-            'ext': 'mp3',
-            'title': title,
-            'duration': int_or_none(track_info.get('duration')),
-            'timestamp': int_or_none(track_info.get('published_at') or track_info.get('created'))
-        }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-
-        city = mobj.group('city')
-        genre = mobj.group('genre')
-        track_id = mobj.group('id')
-
-        track_info = self._extract_info(city, genre, track_id)
-        return self._extract_track_info(track_info, track_id)
-
-
-class TwentyTwoTracksGenreIE(TwentyTwoTracksIE):
-    _VALID_URL = r'https?://22tracks\.com/(?P<city>[a-z]+)/(?P<genre>[\da-z]+)/?$'
-    IE_NAME = '22tracks:genre'
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-
-        city = mobj.group('city')
-        genre = mobj.group('genre')
-
-        genre_title, tracks = self._extract_info(city, genre)
-
-        entries = [
-            self._extract_track_info(track_info, track_info['id'])
-            for track_info in tracks]
-
-        return self.playlist_result(entries, genre, genre_title)

From 48107c198bd76e611e3d4c2486cdc5403829a05a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Nov 2017 22:10:55 +0700
Subject: [PATCH 1493/1696] [f4m] Prefer baseURL for relative URLs (closes
 #14660)

---
 youtube_dl/downloader/f4m.py   | 25 +++++++++++++++++--------
 youtube_dl/extractor/common.py | 14 +++++++-------
 2 files changed, 24 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index c8fde9a89..fdb80f42a 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -243,8 +243,17 @@ def remove_encrypted_media(media):
                        media))
 
 
-def _add_ns(prop):
-    return '{http://ns.adobe.com/f4m/1.0}%s' % prop
+def _add_ns(prop, ver=1):
+    return '{http://ns.adobe.com/f4m/%d.0}%s' % (ver, prop)
+
+
+def get_base_url(manifest):
+    base_url = xpath_text(
+        manifest, [_add_ns('baseURL'), _add_ns('baseURL', 2)],
+        'base URL', default=None)
+    if base_url:
+        base_url = base_url.strip()
+    return base_url
 
 
 class F4mFD(FragmentFD):
@@ -330,13 +339,13 @@ class F4mFD(FragmentFD):
             rate, media = list(filter(
                 lambda f: int(f[0]) == requested_bitrate, formats))[0]
 
-        base_url = compat_urlparse.urljoin(man_url, media.attrib['url'])
+        # Prefer baseURL for relative URLs as per 11.2 of F4M 3.0 spec.
+        man_base_url = get_base_url(doc) or man_url
+
+        base_url = compat_urlparse.urljoin(man_base_url, media.attrib['url'])
         bootstrap_node = doc.find(_add_ns('bootstrapInfo'))
-        # From Adobe F4M 3.0 spec:
-        # The <baseURL> element SHALL be the base URL for all relative
-        # (HTTP-based) URLs in the manifest. If <baseURL> is not present, said
-        # URLs should be relative to the location of the containing document.
-        boot_info, bootstrap_url = self._parse_bootstrap_node(bootstrap_node, man_url)
+        boot_info, bootstrap_url = self._parse_bootstrap_node(
+            bootstrap_node, man_base_url)
         live = boot_info['live']
         metadata_node = media.find(_add_ns('metadata'))
         if metadata_node is not None:
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a67ac4411..64fb869aa 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -29,7 +29,10 @@ from ..compat import (
     compat_urlparse,
     compat_xml_parse_error,
 )
-from ..downloader.f4m import remove_encrypted_media
+from ..downloader.f4m import (
+    get_base_url,
+    remove_encrypted_media,
+)
 from ..utils import (
     NO_DEFAULT,
     age_restricted,
@@ -1239,11 +1242,8 @@ class InfoExtractor(object):
         media_nodes = remove_encrypted_media(media_nodes)
         if not media_nodes:
             return formats
-        base_url = xpath_text(
-            manifest, ['{http://ns.adobe.com/f4m/1.0}baseURL', '{http://ns.adobe.com/f4m/2.0}baseURL'],
-            'base URL', default=None)
-        if base_url:
-            base_url = base_url.strip()
+
+        manifest_base_url = get_base_url(manifest)
 
         bootstrap_info = xpath_element(
             manifest, ['{http://ns.adobe.com/f4m/1.0}bootstrapInfo', '{http://ns.adobe.com/f4m/2.0}bootstrapInfo'],
@@ -1275,7 +1275,7 @@ class InfoExtractor(object):
                     continue
                 manifest_url = (
                     media_url if media_url.startswith('http://') or media_url.startswith('https://')
-                    else ((base_url or '/'.join(manifest_url.split('/')[:-1])) + '/' + media_url))
+                    else ((manifest_base_url or '/'.join(manifest_url.split('/')[:-1])) + '/' + media_url))
                 # If media_url is itself a f4m manifest do the recursive extraction
                 # since bitrates in parent manifest (this one) and media_url manifest
                 # may differ leading to inability to resolve the format by requested

From 187ee66c941d9c397a46ffa490375e2c405500e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Nov 2017 22:11:39 +0700
Subject: [PATCH 1494/1696] [extractor/common] Add protocol for f4m formats

---
 youtube_dl/extractor/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 64fb869aa..e2d9f52b0 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1310,6 +1310,7 @@ class InfoExtractor(object):
                 'url': manifest_url,
                 'manifest_url': manifest_url,
                 'ext': 'flv' if bootstrap_info is not None else None,
+                'protocol': 'f4m',
                 'tbr': tbr,
                 'width': width,
                 'height': height,

From 181e381fda4ddb9083f3834a8bd1bab72c937545 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Nov 2017 22:12:20 +0700
Subject: [PATCH 1495/1696] [test_InfoExtractor] Add test for #14660

---
 test/test_InfoExtractor.py            | 26 ++++++++++++++++++++++++++
 test/testdata/f4m/custom_base_url.f4m | 10 ++++++++++
 2 files changed, 36 insertions(+)
 create mode 100644 test/testdata/f4m/custom_base_url.f4m

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index f18a823fc..686c63efa 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -574,6 +574,32 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 self.ie._sort_formats(formats)
                 expect_value(self, formats, expected_formats, None)
 
+    def test_parse_f4m_formats(self):
+        _TEST_CASES = [
+            (
+                # https://github.com/rg3/youtube-dl/issues/14660
+                'custom_base_url',
+                'http://api.new.livestream.com/accounts/6115179/events/6764928/videos/144884262.f4m',
+                [{
+                    'manifest_url': 'http://api.new.livestream.com/accounts/6115179/events/6764928/videos/144884262.f4m',
+                    'ext': 'flv',
+                    'format_id': '2148',
+                    'protocol': 'f4m',
+                    'tbr': 2148,
+                    'width': 1280,
+                    'height': 720,
+                }]
+            ),
+        ]
+
+        for f4m_file, f4m_url, expected_formats in _TEST_CASES:
+            with io.open('./test/testdata/f4m/%s.f4m' % f4m_file,
+                         mode='r', encoding='utf-8') as f:
+                formats = self.ie._parse_f4m_formats(
+                    compat_etree_fromstring(f.read().encode('utf-8')),
+                    f4m_url, None)
+                self.ie._sort_formats(formats)
+                expect_value(self, formats, expected_formats, None)
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/testdata/f4m/custom_base_url.f4m b/test/testdata/f4m/custom_base_url.f4m
new file mode 100644
index 000000000..74e1539e8
--- /dev/null
+++ b/test/testdata/f4m/custom_base_url.f4m
@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<manifest xmlns="http://ns.adobe.com/f4m/1.0">
+    <streamType>recorded</streamType>
+    <baseURL>http://vod.livestream.com/events/0000000000673980/</baseURL>
+    <duration>269.293</duration>
+    <bootstrapInfo profile="named" id="bootstrap_1">AAAAm2Fic3QAAAAAAAAAAQAAAAPoAAAAAAAEG+0AAAAAAAAAAAAAAAAAAQAAABlhc3J0AAAAAAAAAAABAAAAAQAAAC4BAAAAVmFmcnQAAAAAAAAD6AAAAAAEAAAAAQAAAAAAAAAAAAAXcAAAAC0AAAAAAAQHQAAAE5UAAAAuAAAAAAAEGtUAAAEYAAAAAAAAAAAAAAAAAAAAAAA=</bootstrapInfo>
+    <media url="b90f532f-b0f6-4f4e-8289-706d490b2fd8_2292" bootstrapInfoId="bootstrap_1" bitrate="2148" width="1280" height="720" videoCodec="avc1.4d401f" audioCodec="mp4a.40.2">
+        <metadata>AgAKb25NZXRhRGF0YQgAAAAIAAhkdXJhdGlvbgBAcNSwIMSbpgAFd2lkdGgAQJQAAAAAAAAABmhlaWdodABAhoAAAAAAAAAJZnJhbWVyYXRlAEA4/7DoLwW3AA12aWRlb2RhdGFyYXRlAECe1DLgjcobAAx2aWRlb2NvZGVjaWQAQBwAAAAAAAAADWF1ZGlvZGF0YXJhdGUAQGSimlvaPKQADGF1ZGlvY29kZWNpZABAJAAAAAAAAAAACQ==</metadata>
+    </media>
+</manifest>

From 6e71bbf4abc729cae3b0e428c3bb321690c9e485 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Nov 2017 16:12:56 +0700
Subject: [PATCH 1496/1696] [hotstar] Bypass geo restriction (closes #14672)

---
 youtube_dl/extractor/hotstar.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/hotstar.py b/youtube_dl/extractor/hotstar.py
index 3a7a66a34..9be958be6 100644
--- a/youtube_dl/extractor/hotstar.py
+++ b/youtube_dl/extractor/hotstar.py
@@ -11,6 +11,7 @@ from ..utils import (
 
 class HotStarIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?hotstar\.com/(?:.+?[/-])?(?P<id>\d{10})'
+    _GEO_COUNTRIES = ['IN']
     _TESTS = [{
         'url': 'http://www.hotstar.com/on-air-with-aib--english-1000076273',
         'info_dict': {

From 477c97f86b5451f384a84a7a8d8237cfd1bec1d2 Mon Sep 17 00:00:00 2001
From: Alpesh Valia <alpeshvalia727@gmail.com>
Date: Thu, 16 Mar 2017 22:00:11 +0530
Subject: [PATCH 1497/1696] [hotstar:playlist] Add extractor

---
 youtube_dl/extractor/extractors.py |  5 ++-
 youtube_dl/extractor/hotstar.py    | 58 +++++++++++++++++++++++++++++-
 2 files changed, 61 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 92f7e9027..d084707ee 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -432,7 +432,10 @@ from .hitbox import HitboxIE, HitboxLiveIE
 from .hitrecord import HitRecordIE
 from .hornbunny import HornBunnyIE
 from .hotnewhiphop import HotNewHipHopIE
-from .hotstar import HotStarIE
+from .hotstar import (
+    HotStarIE,
+    HotStarPlaylistIE,
+)
 from .howcast import HowcastIE
 from .howstuffworks import HowStuffWorksIE
 from .hrti import (
diff --git a/youtube_dl/extractor/hotstar.py b/youtube_dl/extractor/hotstar.py
index 9be958be6..8d8a80a82 100644
--- a/youtube_dl/extractor/hotstar.py
+++ b/youtube_dl/extractor/hotstar.py
@@ -7,6 +7,7 @@ from ..utils import (
     determine_ext,
     int_or_none,
 )
+import re
 
 
 class HotStarIE(InfoExtractor):
@@ -17,7 +18,7 @@ class HotStarIE(InfoExtractor):
         'info_dict': {
             'id': '1000076273',
             'ext': 'mp4',
-            'title': 'On Air With AIB - English',
+            'title': 'On Air With AIB',
             'description': 'md5:c957d8868e9bc793ccb813691cc4c434',
             'timestamp': 1447227000,
             'upload_date': '20151111',
@@ -100,3 +101,58 @@ class HotStarIE(InfoExtractor):
             'episode_number': int_or_none(video_data.get('episodeNumber')),
             'series': video_data.get('contentTitle'),
         }
+
+
+class HotStarPlaylistIE(InfoExtractor):
+    IE_NAME = 'hotstar:playlist'
+    _VALID_URL = r'https?://(?:www\.)?hotstar\.com/tv/(?P<playlist_title>.+)/(?P<series_id>\d+)/episodes/(?P<playlist_id>\d{1,})'
+
+    _TESTS = [{
+        'url': 'http://www.hotstar.com/tv/pow-bandi-yuddh-ke/10999/episodes/10856/9993',
+        'info_dict': {
+            'id': '10856',
+            'title': 'pow-bandi-yuddh-ke',
+        },
+        'playlist_mincount': 0,
+    }, {
+        'url': 'http://www.hotstar.com/tv/pow-bandi-yuddh-ke/10999/episodes/10856/9993',
+        'only_matching': True,
+    }]
+
+    def _extract_episode_info(self, series_id, playlist_title, video):
+
+        picture_url = video.get('urlPictures')
+        thumbnail = ''
+        if picture_url:
+            thumbnail = 'http://media0-starag.startv.in/r1/thumbs/PCTV/%s/%s/PCTV-%s-hs.jpg' % (picture_url[-2:], picture_url, picture_url)
+
+        episode_title = video.get('episodeTitle', '')
+        episode_title = episode_title.lower().replace(' ', '-')
+        url = "http://www.hotstar.com/tv/%s/%s/%s/%s" % (playlist_title, series_id, episode_title, video.get('contentId'))
+
+        info_dict = {
+            'id': video.get('contentId'),
+            'title': video.get('episodeTitle'),
+            'description': video.get('longDescription'),
+            'thumbnail': thumbnail,
+            'url': url,
+            '_type': 'url',
+        }
+        return info_dict
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        series_id = mobj.group('series_id')
+        playlist_id = mobj.group('playlist_id')
+        playlist_title = mobj.group('playlist_title')
+
+        collection = self._download_json(
+            "http://search.hotstar.com/AVS/besc?action=SearchContents&appVersion=5.0.39&channel=PCTV&moreFilters=series:%s;&query=*&searchOrder=last_broadcast_date+desc,year+asc,title+asc&type=EPISODE" % playlist_id,
+            playlist_id
+        )
+
+        videos = collection.get('resultObj', {}).get('response', {}).get('docs', [])
+        entries = [
+            self._extract_episode_info(series_id, playlist_title, video)
+            for video in videos if video.get('contentId')]
+        return self.playlist_result(entries, playlist_id, playlist_title)

From 909191de9154bf289b333cfe01b8e88e3ac1fefc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Nov 2017 19:14:48 +0700
Subject: [PATCH 1498/1696] [hotstar:playlist] Fix issues and improve (closes
 #12465)

---
 youtube_dl/extractor/hotstar.py | 128 ++++++++++++++++----------------
 1 file changed, 66 insertions(+), 62 deletions(-)

diff --git a/youtube_dl/extractor/hotstar.py b/youtube_dl/extractor/hotstar.py
index 8d8a80a82..d28af36ec 100644
--- a/youtube_dl/extractor/hotstar.py
+++ b/youtube_dl/extractor/hotstar.py
@@ -1,18 +1,41 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    determine_ext,
-    int_or_none,
-)
 import re
 
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    determine_ext,
+    ExtractorError,
+    int_or_none,
+)
 
-class HotStarIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?hotstar\.com/(?:.+?[/-])?(?P<id>\d{10})'
+
+class HotStarBaseIE(InfoExtractor):
     _GEO_COUNTRIES = ['IN']
+
+    def _download_json(self, *args, **kwargs):
+        response = super(HotStarBaseIE, self)._download_json(*args, **kwargs)
+        if response['resultCode'] != 'OK':
+            if kwargs.get('fatal'):
+                raise ExtractorError(
+                    response['errorDescription'], expected=True)
+            return None
+        return response['resultObj']
+
+    def _download_content_info(self, content_id):
+        return self._download_json(
+            'https://account.hotstar.com/AVS/besc', content_id, query={
+                'action': 'GetAggregatedContentDetails',
+                'appVersion': '5.0.40',
+                'channel': 'PCTV',
+                'contentId': content_id,
+            })['contentInfo'][0]
+
+
+class HotStarIE(HotStarBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?hotstar\.com/(?:.+?[/-])?(?P<id>\d{10})'
     _TESTS = [{
         'url': 'http://www.hotstar.com/on-air-with-aib--english-1000076273',
         'info_dict': {
@@ -36,23 +59,11 @@ class HotStarIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata', fatal=True, query=None):
-        json_data = super(HotStarIE, self)._download_json(
-            url_or_request, video_id, note, fatal=fatal, query=query)
-        if json_data['resultCode'] != 'OK':
-            if fatal:
-                raise ExtractorError(json_data['errorDescription'])
-            return None
-        return json_data['resultObj']
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        video_data = self._download_json(
-            'http://account.hotstar.com/AVS/besc', video_id, query={
-                'action': 'GetAggregatedContentDetails',
-                'channel': 'PCTV',
-                'contentId': video_id,
-            })['contentInfo'][0]
+
+        video_data = self._download_content_info(video_id)
+
         title = video_data['episodeTitle']
 
         if video_data.get('encrypted') == 'Y':
@@ -103,56 +114,49 @@ class HotStarIE(InfoExtractor):
         }
 
 
-class HotStarPlaylistIE(InfoExtractor):
+class HotStarPlaylistIE(HotStarBaseIE):
     IE_NAME = 'hotstar:playlist'
-    _VALID_URL = r'https?://(?:www\.)?hotstar\.com/tv/(?P<playlist_title>.+)/(?P<series_id>\d+)/episodes/(?P<playlist_id>\d{1,})'
-
+    _VALID_URL = r'(?P<url>https?://(?:www\.)?hotstar\.com/tv/[^/]+/(?P<content_id>\d+))/(?P<type>[^/]+)/(?P<id>\d+)'
     _TESTS = [{
-        'url': 'http://www.hotstar.com/tv/pow-bandi-yuddh-ke/10999/episodes/10856/9993',
+        'url': 'http://www.hotstar.com/tv/pratidaan/14982/episodes/14812/9993',
         'info_dict': {
-            'id': '10856',
-            'title': 'pow-bandi-yuddh-ke',
+            'id': '14812',
         },
-        'playlist_mincount': 0,
+        'playlist_mincount': 75,
     }, {
-        'url': 'http://www.hotstar.com/tv/pow-bandi-yuddh-ke/10999/episodes/10856/9993',
+        'url': 'http://www.hotstar.com/tv/pratidaan/14982/popular-clips/9998/9998',
         'only_matching': True,
     }]
-
-    def _extract_episode_info(self, series_id, playlist_title, video):
-
-        picture_url = video.get('urlPictures')
-        thumbnail = ''
-        if picture_url:
-            thumbnail = 'http://media0-starag.startv.in/r1/thumbs/PCTV/%s/%s/PCTV-%s-hs.jpg' % (picture_url[-2:], picture_url, picture_url)
-
-        episode_title = video.get('episodeTitle', '')
-        episode_title = episode_title.lower().replace(' ', '-')
-        url = "http://www.hotstar.com/tv/%s/%s/%s/%s" % (playlist_title, series_id, episode_title, video.get('contentId'))
-
-        info_dict = {
-            'id': video.get('contentId'),
-            'title': video.get('episodeTitle'),
-            'description': video.get('longDescription'),
-            'thumbnail': thumbnail,
-            'url': url,
-            '_type': 'url',
-        }
-        return info_dict
+    _ITEM_TYPES = {
+        'episodes': 'EPISODE',
+        'popular-clips': 'CLIPS',
+    }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        series_id = mobj.group('series_id')
-        playlist_id = mobj.group('playlist_id')
-        playlist_title = mobj.group('playlist_title')
+        base_url = mobj.group('url')
+        content_id = mobj.group('content_id')
+        playlist_type = mobj.group('type')
+
+        content_info = self._download_content_info(content_id)
+        playlist_id = compat_str(content_info['categoryId'])
 
         collection = self._download_json(
-            "http://search.hotstar.com/AVS/besc?action=SearchContents&appVersion=5.0.39&channel=PCTV&moreFilters=series:%s;&query=*&searchOrder=last_broadcast_date+desc,year+asc,title+asc&type=EPISODE" % playlist_id,
-            playlist_id
-        )
+            'https://search.hotstar.com/AVS/besc', playlist_id, query={
+                'action': 'SearchContents',
+                'appVersion': '5.0.40',
+                'channel': 'PCTV',
+                'moreFilters': 'series:%s;' % playlist_id,
+                'query': '*',
+                'searchOrder': 'last_broadcast_date desc,year desc,title asc',
+                'type': self._ITEM_TYPES.get(playlist_type, 'EPISODE'),
+            })
 
-        videos = collection.get('resultObj', {}).get('response', {}).get('docs', [])
         entries = [
-            self._extract_episode_info(series_id, playlist_title, video)
-            for video in videos if video.get('contentId')]
-        return self.playlist_result(entries, playlist_id, playlist_title)
+            self.url_result(
+                '%s/_/%s' % (base_url, video['contentId']),
+                ie=HotStarIE.ie_key(), video_id=video['contentId'])
+            for video in collection['response']['docs']
+            if video.get('contentId')]
+
+        return self.playlist_result(entries, playlist_id)

From e0998333fac2238eff8880992c11f76402c4007c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 6 Nov 2017 22:36:46 +0700
Subject: [PATCH 1499/1696] [ChangeLog] Actualize

---
 ChangeLog | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index d33a710fb..3cbbdda97 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,19 @@
+version <unreleased>
+
+Core
++ [extractor/common] Add protocol for f4m formats
+* [f4m] Prefer baseURL for relative URLs (#14660)
+* [extractor/common] Respect URL query in _extract_wowza_formats (14645)
+
+Extractors
++ [hotstar:playlist] Add support for playlists (#12465)
+* [hotstar] Bypass geo restriction (#14672)
+- [22tracks] Remove extractor (#11024, #14628)
++ [skysport] Sdd support ooyala videos protected with embed_token (#14641)
+* [gamespot] Extract formats referenced with new data fields (#14652)
+* [spankbang] Detect unavailable videos (#14644)
+
+
 version 2017.10.29
 
 Core

From f34b841b51be6872914ffe17b210c54b0d823c3c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 6 Nov 2017 22:39:24 +0700
Subject: [PATCH 1500/1696] release 2017.11.06

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 3 +--
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 881475878..be6e6ddab 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.10.29*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.10.29**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.11.06*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.11.06**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.10.29
+[debug] youtube-dl version 2017.11.06
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 3cbbdda97..8af368274 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.11.06
 
 Core
 + [extractor/common] Add protocol for f4m formats
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 7b8e7403a..6009df571 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -3,8 +3,6 @@
  - **1up.com**
  - **20min**
  - **220.ro**
- - **22tracks:genre**
- - **22tracks:track**
  - **24video**
  - **3qsdn**: 3Q SDN
  - **3sat**
@@ -342,6 +340,7 @@
  - **HornBunny**
  - **HotNewHipHop**
  - **HotStar**
+ - **hotstar:playlist**
  - **Howcast**
  - **HowStuffWorks**
  - **HRTi**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 43f080bc3..8b67d23fe 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.10.29'
+__version__ = '2017.11.06'

From cc6a960e134614f8af2a42dcd8bf146d63638a3c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 8 Nov 2017 20:30:05 +0100
Subject: [PATCH 1501/1696] use older login method(closes #11572)

---
 youtube_dl/extractor/crunchyroll.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 8bdaf0c2c..18ef3da10 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -43,6 +43,17 @@ class CrunchyrollBaseIE(InfoExtractor):
         if username is None:
             return
 
+        self._download_webpage(
+            'https://www.crunchyroll.com/?a=formhandler',
+            None, 'Logging in', 'Wrong login info',
+            data=urlencode_postdata({
+                'formname': 'RpcApiUser_Login',
+                'next_url': 'https://www.crunchyroll.com/acct/membership',
+                'name': username,
+                'password': password,
+            }))
+
+        '''
         login_page = self._download_webpage(
             self._LOGIN_URL, None, 'Downloading login page')
 
@@ -86,6 +97,7 @@ class CrunchyrollBaseIE(InfoExtractor):
             raise ExtractorError('Unable to login: %s' % error, expected=True)
 
         raise ExtractorError('Unable to log in')
+        '''
 
     def _real_initialize(self):
         self._login()

From 4222346fb2f42af10ac902cd46469d23923cf114 Mon Sep 17 00:00:00 2001
From: hcwhan <hcwhan@gmail.com>
Date: Tue, 7 Nov 2017 17:59:09 +0800
Subject: [PATCH 1502/1696] [pandatv] Update API URL and sign format URLs

---
 youtube_dl/extractor/pandatv.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pandatv.py b/youtube_dl/extractor/pandatv.py
index c86d70771..c99a1bb1f 100644
--- a/youtube_dl/extractor/pandatv.py
+++ b/youtube_dl/extractor/pandatv.py
@@ -6,6 +6,7 @@ from ..utils import (
     ExtractorError,
     qualities,
 )
+import json
 
 
 class PandaTVIE(InfoExtractor):
@@ -33,7 +34,7 @@ class PandaTVIE(InfoExtractor):
         video_id = self._match_id(url)
 
         config = self._download_json(
-            'https://www.panda.tv/api_room?roomid=%s' % video_id, video_id)
+            'https://www.panda.tv/api_room_v2?roomid=%s' % video_id, video_id)
 
         error_code = config.get('errno', 0)
         if error_code is not 0:
@@ -66,6 +67,11 @@ class PandaTVIE(InfoExtractor):
             plflag1 = '4'
         live_panda = 'live_panda' if plflag0 < 1 else ''
 
+        plflag_auth = json.loads(video_info["plflag_list"])
+        sign = plflag_auth["auth"]["sign"]
+        ts = plflag_auth["auth"]["time"]
+        rid = plflag_auth["auth"]["rid"]
+
         quality_key = qualities(['OD', 'HD', 'SD'])
         suffix = ['_small', '_mid', '']
         formats = []
@@ -77,8 +83,8 @@ class PandaTVIE(InfoExtractor):
                 continue
             for pref, (ext, pl) in enumerate((('m3u8', '-hls'), ('flv', ''))):
                 formats.append({
-                    'url': 'https://pl%s%s.live.panda.tv/live_panda/%s%s%s.%s'
-                    % (pl, plflag1, room_key, live_panda, suffix[quality], ext),
+                    'url': 'https://pl%s%s.live.panda.tv/live_panda/%s%s%s.%s?sign=%s&ts=%s&rid=%s'
+                    % (pl, plflag1, room_key, live_panda, suffix[quality], ext, sign, ts, rid),
                     'format_id': '%s-%s' % (k, ext),
                     'quality': quality,
                     'source_preference': pref,

From 61fb07e156671159353ae19a152926cab277ac87 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Nov 2017 23:25:43 +0700
Subject: [PATCH 1503/1696] [pandatv] Modernize (closes #14693)

---
 youtube_dl/extractor/pandatv.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/pandatv.py b/youtube_dl/extractor/pandatv.py
index c99a1bb1f..13a2e7efc 100644
--- a/youtube_dl/extractor/pandatv.py
+++ b/youtube_dl/extractor/pandatv.py
@@ -6,7 +6,6 @@ from ..utils import (
     ExtractorError,
     qualities,
 )
-import json
 
 
 class PandaTVIE(InfoExtractor):
@@ -67,10 +66,10 @@ class PandaTVIE(InfoExtractor):
             plflag1 = '4'
         live_panda = 'live_panda' if plflag0 < 1 else ''
 
-        plflag_auth = json.loads(video_info["plflag_list"])
-        sign = plflag_auth["auth"]["sign"]
-        ts = plflag_auth["auth"]["time"]
-        rid = plflag_auth["auth"]["rid"]
+        plflag_auth = self._parse_json(video_info['plflag_list'], video_id)
+        sign = plflag_auth['auth']['sign']
+        ts = plflag_auth['auth']['time']
+        rid = plflag_auth['auth']['rid']
 
         quality_key = qualities(['OD', 'HD', 'SD'])
         suffix = ['_small', '_mid', '']

From a9543e37c8e460e69a8556c8e5004ebd8e9b4da4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Nov 2017 00:29:08 +0800
Subject: [PATCH 1504/1696] [wsj] Recognize another URL pattern (closes #14704)

---
 ChangeLog                   | 6 ++++++
 youtube_dl/extractor/wsj.py | 5 ++++-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 8af368274..cedab4723 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
++ [wsj] Recognize another URL pattern (#14704)
+
+
 version 2017.11.06
 
 Core
diff --git a/youtube_dl/extractor/wsj.py b/youtube_dl/extractor/wsj.py
index 9b5487710..67236f377 100644
--- a/youtube_dl/extractor/wsj.py
+++ b/youtube_dl/extractor/wsj.py
@@ -13,7 +13,7 @@ class WSJIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                         (?:
                             https?://video-api\.wsj\.com/api-video/player/iframe\.html\?.*?\bguid=|
-                            https?://(?:www\.)?(?:wsj|barrons)\.com/video/[^/]+/|
+                            https?://(?:www\.)?(?:wsj|barrons)\.com/video/(?:[^/]+/)+|
                             wsj:
                         )
                         (?P<id>[a-fA-F0-9-]{36})
@@ -38,6 +38,9 @@ class WSJIE(InfoExtractor):
     }, {
         'url': 'http://www.barrons.com/video/capitalism-deserves-more-respect-from-millennials/F301217E-6F46-43AE-B8D2-B7180D642EE9.html',
         'only_matching': True,
+    }, {
+        'url': 'https://www.wsj.com/video/series/a-brief-history-of/the-modern-cell-carrier-how-we-got-here/980E2187-401D-48A1-B82B-1486CEE06CB9',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 59d2e6d04f621f41a72a232b8c93250991b4ae5c Mon Sep 17 00:00:00 2001
From: gkoelln <gkoelln7@gmail.com>
Date: Fri, 10 Nov 2017 15:59:48 -0600
Subject: [PATCH 1505/1696] [cartoonnetwork] Update tokenizer_src (closes
 #14666)

---
 youtube_dl/extractor/cartoonnetwork.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cartoonnetwork.py b/youtube_dl/extractor/cartoonnetwork.py
index 086ec90c9..6aeebd7b3 100644
--- a/youtube_dl/extractor/cartoonnetwork.py
+++ b/youtube_dl/extractor/cartoonnetwork.py
@@ -31,7 +31,7 @@ class CartoonNetworkIE(TurnerBaseIE):
             'http://www.cartoonnetwork.com/video-seo-svc/episodeservices/getCvpPlaylist?networkName=CN2&' + query, video_id, {
                 'secure': {
                     'media_src': 'http://androidhls-secure.cdn.turner.com/toon/big',
-                    'tokenizer_src': 'http://www.cartoonnetwork.com/cntv/mvpd/processors/services/token_ipadAdobe.do',
+                    'tokenizer_src': 'https://token.vgtf.net/token/token_mobile',
                 },
             }, {
                 'url': url,

From a5203935d6cb753bafaf67164553027b62c01781 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 11 Nov 2017 12:41:15 +0100
Subject: [PATCH 1506/1696] [gamespot] skip Brightcove Once http
 formats(#14652)

---
 youtube_dl/extractor/gamespot.py | 3 ++-
 youtube_dl/extractor/once.py     | 4 ++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index 6d177cbaf..be1ed8b4a 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -108,7 +108,8 @@ class GameSpotIE(OnceIE):
             onceux_url = self._parse_json(unescapeHTML(onceux_json), page_id).get('metadataUri')
             if onceux_url:
                 formats.extend(self._extract_once_formats(re.sub(
-                    r'https?://[^/]+', 'http://once.unicornmedia.com', onceux_url)))
+                    r'https?://[^/]+', 'http://once.unicornmedia.com', onceux_url),
+                    skip_http_formats=True))
 
         if not formats:
             for quality in ['sd', 'hd']:
diff --git a/youtube_dl/extractor/once.py b/youtube_dl/extractor/once.py
index a637c8ecf..6ba6fe5d3 100644
--- a/youtube_dl/extractor/once.py
+++ b/youtube_dl/extractor/once.py
@@ -11,7 +11,7 @@ class OnceIE(InfoExtractor):
     ADAPTIVE_URL_TEMPLATE = 'http://once.unicornmedia.com/now/master/playlist/%s/%s/%s/content.m3u8'
     PROGRESSIVE_URL_TEMPLATE = 'http://once.unicornmedia.com/now/media/progressive/%s/%s/%s/%s/content.mp4'
 
-    def _extract_once_formats(self, url):
+    def _extract_once_formats(self, url, skip_http_formats=False):
         domain_id, application_id, media_item_id = re.match(
             OnceIE._VALID_URL, url).groups()
         formats = self._extract_m3u8_formats(
@@ -27,7 +27,7 @@ class OnceIE(InfoExtractor):
             rendition_id = self._search_regex(
                 r'/now/media/playlist/[^/]+/[^/]+/([^/]+)',
                 adaptive_format['url'], 'redition id', default=None)
-            if rendition_id:
+            if rendition_id and not skip_http_formats:
                 progressive_format = adaptive_format.copy()
                 progressive_format.update({
                     'url': self.PROGRESSIVE_URL_TEMPLATE % (

From 79d1f8ed6803b6097f0f3cd57f72e0378bdc1f34 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 11 Nov 2017 13:02:39 +0100
Subject: [PATCH 1507/1696] [gamespot] add support for article URLS(closes
 #14652)

---
 youtube_dl/extractor/gamespot.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index be1ed8b4a..e6d6d9b1d 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class GameSpotIE(OnceIE):
-    _VALID_URL = r'https?://(?:www\.)?gamespot\.com/videos/(?:[^/]+/\d+-|embed/)(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?gamespot\.com/(?:video|article)s/(?:[^/]+/\d+-|embed/)(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.gamespot.com/videos/arma-3-community-guide-sitrep-i/2300-6410818/',
         'md5': 'b2a30deaa8654fcccd43713a6b6a4825',

From e4d9586562d24cbbea6ee07162290ec602399f37 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Nov 2017 20:49:03 +0700
Subject: [PATCH 1508/1696] Remove sensitive data from logging in messages

---
 youtube_dl/extractor/animeondemand.py | 2 +-
 youtube_dl/extractor/atresplayer.py   | 2 +-
 youtube_dl/extractor/bambuser.py      | 2 +-
 youtube_dl/extractor/dramafever.py    | 2 +-
 youtube_dl/extractor/funimation.py    | 2 +-
 youtube_dl/extractor/noco.py          | 2 +-
 youtube_dl/extractor/patreon.py       | 2 +-
 youtube_dl/extractor/pluralsight.py   | 2 +-
 youtube_dl/extractor/roosterteeth.py  | 2 +-
 youtube_dl/extractor/safari.py        | 2 +-
 youtube_dl/extractor/twitch.py        | 2 +-
 youtube_dl/extractor/udemy.py         | 2 +-
 youtube_dl/extractor/viki.py          | 2 +-
 youtube_dl/extractor/vk.py            | 2 +-
 14 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index 69d363311..34c2b363e 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -78,7 +78,7 @@ class AnimeOnDemandIE(InfoExtractor):
             post_url = urljoin(self._LOGIN_URL, post_url)
 
         response = self._download_webpage(
-            post_url, None, 'Logging in as %s' % username,
+            post_url, None, 'Logging in',
             data=urlencode_postdata(login_form), headers={
                 'Referer': self._LOGIN_URL,
             })
diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 01fa308ff..1a31ebe08 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -87,7 +87,7 @@ class AtresPlayerIE(InfoExtractor):
             self._LOGIN_URL, urlencode_postdata(login_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         response = self._download_webpage(
-            request, None, 'Logging in as %s' % username)
+            request, None, 'Logging in')
 
         error = self._html_search_regex(
             r'(?s)<ul[^>]+class="[^"]*\blist_error\b[^"]*">(.+?)</ul>',
diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py
index 0eb1930c2..633c57553 100644
--- a/youtube_dl/extractor/bambuser.py
+++ b/youtube_dl/extractor/bambuser.py
@@ -59,7 +59,7 @@ class BambuserIE(InfoExtractor):
             self._LOGIN_URL, urlencode_postdata(login_form))
         request.add_header('Referer', self._LOGIN_URL)
         response = self._download_webpage(
-            request, None, 'Logging in as %s' % username)
+            request, None, 'Logging in')
 
         login_error = self._html_search_regex(
             r'(?s)<div class="messages error">(.+?)</div>',
diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 95883a037..6b60e542b 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -54,7 +54,7 @@ class DramaFeverBaseIE(AMPIE):
         request = sanitized_Request(
             self._LOGIN_URL, urlencode_postdata(login_form))
         response = self._download_webpage(
-            request, None, 'Logging in as %s' % username)
+            request, None, 'Logging in')
 
         if all(logout_pattern not in response
                for logout_pattern in ['href="/accounts/logout/"', '>Log out<']):
diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 8c37509ec..107f658ba 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -57,7 +57,7 @@ class FunimationIE(InfoExtractor):
         try:
             data = self._download_json(
                 'https://prod-api-funimationnow.dadcdigital.com/api/auth/login/',
-                None, 'Logging in as %s' % username, data=urlencode_postdata({
+                None, 'Logging in', data=urlencode_postdata({
                     'username': username,
                     'password': password,
                 }))
diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index 8b83e1f76..a9f9b10c4 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -70,7 +70,7 @@ class NocoIE(InfoExtractor):
             return
 
         login = self._download_json(
-            self._LOGIN_URL, None, 'Logging in as %s' % username,
+            self._LOGIN_URL, None, 'Logging in',
             data=urlencode_postdata({
                 'a': 'login',
                 'cookie': '1',
diff --git a/youtube_dl/extractor/patreon.py b/youtube_dl/extractor/patreon.py
index a6a2c273f..d4b1d34ca 100644
--- a/youtube_dl/extractor/patreon.py
+++ b/youtube_dl/extractor/patreon.py
@@ -67,7 +67,7 @@ class PatreonIE(InfoExtractor):
             'https://www.patreon.com/processLogin',
             compat_urllib_parse_urlencode(login_form).encode('utf-8')
         )
-        login_page = self._download_webpage(request, None, note='Logging in as %s' % username)
+        login_page = self._download_webpage(request, None, note='Logging in')
 
         if re.search(r'onLoginFailed', login_page):
             raise ExtractorError('Unable to login, incorrect username and/or password', expected=True)
diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index f6a9131b1..4bf0aa786 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -116,7 +116,7 @@ class PluralsightIE(PluralsightBaseIE):
             post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
 
         response = self._download_webpage(
-            post_url, None, 'Logging in as %s' % username,
+            post_url, None, 'Logging in',
             data=urlencode_postdata(login_form),
             headers={'Content-Type': 'application/x-www-form-urlencoded'})
 
diff --git a/youtube_dl/extractor/roosterteeth.py b/youtube_dl/extractor/roosterteeth.py
index 46dfc78f5..8b703800e 100644
--- a/youtube_dl/extractor/roosterteeth.py
+++ b/youtube_dl/extractor/roosterteeth.py
@@ -68,7 +68,7 @@ class RoosterTeethIE(InfoExtractor):
 
         login_request = self._download_webpage(
             self._LOGIN_URL, None,
-            note='Logging in as %s' % username,
+            note='Logging in',
             data=urlencode_postdata(login_form),
             headers={
                 'Referer': self._LOGIN_URL,
diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index 909a6ba97..cc6698f88 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -61,7 +61,7 @@ class SafariBaseIE(InfoExtractor):
         request = sanitized_Request(
             self._LOGIN_URL, urlencode_postdata(login_form), headers=headers)
         login_page = self._download_webpage(
-            request, None, 'Logging in as %s' % username)
+            request, None, 'Logging in')
 
         if not is_logged(login_page):
             raise ExtractorError(
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index fefcd2807..bf57eac01 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -101,7 +101,7 @@ class TwitchBaseIE(InfoExtractor):
             fail(clean_html(login_page))
 
         redirect_page, handle = login_step(
-            login_page, handle, 'Logging in as %s' % username, {
+            login_page, handle, 'Logging in', {
                 'username': username,
                 'password': password,
             })
diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 207c4a6a7..c248ea727 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -164,7 +164,7 @@ class UdemyIE(InfoExtractor):
         })
 
         response = self._download_webpage(
-            self._LOGIN_URL, None, 'Logging in as %s' % username,
+            self._LOGIN_URL, None, 'Logging in',
             data=urlencode_postdata(login_form),
             headers={
                 'Referer': self._ORIGIN_URL,
diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 853e5c75f..ad2a2a4b7 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -99,7 +99,7 @@ class VikiBaseIE(InfoExtractor):
 
         login = self._call_api(
             'sessions.json', None,
-            'Logging in as %s' % username, post_data=login_form)
+            'Logging in', post_data=login_form)
 
         self._token = login.get('token')
         if not self._token:
diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 105e172d5..0d8376522 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -67,7 +67,7 @@ class VKBaseIE(InfoExtractor):
 
         login_page = self._download_webpage(
             'https://login.vk.com/?act=login', None,
-            note='Logging in as %s' % username,
+            note='Logging in',
             data=urlencode_postdata(login_form))
 
         if re.search(r'onLoginFailed', login_page):

From af85ce29c61749676ab934a2b297505ab33bf4c7 Mon Sep 17 00:00:00 2001
From: Bob Poekert <bob@poekert.com>
Date: Sat, 11 Nov 2017 22:25:21 -0800
Subject: [PATCH 1509/1696] [ccma] Fix typo

---
 youtube_dl/extractor/ccma.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ccma.py b/youtube_dl/extractor/ccma.py
index 39938c9ac..bec0a825a 100644
--- a/youtube_dl/extractor/ccma.py
+++ b/youtube_dl/extractor/ccma.py
@@ -93,7 +93,7 @@ class CCMAIE(InfoExtractor):
             'description': clean_html(informacio.get('descripcio')),
             'duration': duration,
             'timestamp': timestamp,
-            'thumnails': thumbnails,
+            'thumbnails': thumbnails,
             'subtitles': subtitles,
             'formats': formats,
         }

From 5fc12b954971f5f63d1e87b05e8b01a9ae0e3b01 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Nov 2017 18:35:17 +0700
Subject: [PATCH 1510/1696] [instagram:user] Fix extraction (closes #14699)

---
 youtube_dl/extractor/instagram.py | 121 +++++++++++++++---------------
 1 file changed, 59 insertions(+), 62 deletions(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index 4667335e0..20db31f86 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 
+import itertools
 import re
 
 from .common import InfoExtractor
@@ -7,7 +8,6 @@ from ..compat import compat_str
 from ..utils import (
     get_element_by_attribute,
     int_or_none,
-    limit_length,
     lowercase_escape,
     try_get,
 )
@@ -212,7 +212,7 @@ class InstagramIE(InfoExtractor):
 
 
 class InstagramUserIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?instagram\.com/(?P<username>[^/]{2,})/?(?:$|[?#])'
+    _VALID_URL = r'https?://(?:www\.)?instagram\.com/(?P<id>[^/]{2,})/?(?:$|[?#])'
     IE_DESC = 'Instagram user profile'
     IE_NAME = 'instagram:user'
     _TEST = {
@@ -221,82 +221,79 @@ class InstagramUserIE(InfoExtractor):
             'id': 'porsche',
             'title': 'porsche',
         },
-        'playlist_mincount': 2,
-        'playlist': [{
-            'info_dict': {
-                'id': '614605558512799803_462752227',
-                'ext': 'mp4',
-                'title': '#Porsche Intelligent Performance.',
-                'thumbnail': r're:^https?://.*\.jpg',
-                'uploader': 'Porsche',
-                'uploader_id': 'porsche',
-                'timestamp': 1387486713,
-                'upload_date': '20131219',
-            },
-        }],
+        'playlist_count': 5,
         'params': {
             'extract_flat': True,
             'skip_download': True,
+            'playlistend': 5,
         }
     }
 
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        uploader_id = mobj.group('username')
+    def _entries(self, uploader_id):
+        query = {
+            '__a': 1,
+        }
 
-        entries = []
-        page_count = 0
-        media_url = 'http://instagram.com/%s/media' % uploader_id
-        while True:
+        def get_count(kind):
+            return int_or_none(try_get(
+                node, lambda x: x['%ss' % kind]['count']))
+
+        for page_num in itertools.count(1):
             page = self._download_json(
-                media_url, uploader_id,
-                note='Downloading page %d ' % (page_count + 1),
-            )
-            page_count += 1
+                'https://instagram.com/%s/' % uploader_id, uploader_id,
+                note='Downloading page %d' % page_num,
+                fatal=False, query=query)
+            if not page:
+                break
 
-            for it in page['items']:
-                if it.get('type') != 'video':
+            nodes = try_get(page, lambda x: x['user']['media']['nodes'], list)
+            if not nodes:
+                break
+
+            max_id = None
+
+            for node in nodes:
+                node_id = node.get('id')
+                if node_id:
+                    max_id = node_id
+
+                if node.get('__typename') != 'GraphVideo' and node.get('is_video') is not True:
+                    continue
+                video_id = node.get('code')
+                if not video_id:
                     continue
-                like_count = int_or_none(it.get('likes', {}).get('count'))
-                user = it.get('user', {})
 
-                formats = [{
-                    'format_id': k,
-                    'height': v.get('height'),
-                    'width': v.get('width'),
-                    'url': v['url'],
-                } for k, v in it['videos'].items()]
-                self._sort_formats(formats)
+                info = self.url_result(
+                    'https://instagram.com/p/%s/' % video_id,
+                    ie=InstagramIE.ie_key(), video_id=video_id)
 
-                thumbnails_el = it.get('images', {})
-                thumbnail = thumbnails_el.get('thumbnail', {}).get('url')
+                description = try_get(
+                    node, [lambda x: x['caption'], lambda x: x['text']['id']],
+                    compat_str)
+                thumbnail = node.get('thumbnail_src') or node.get('display_src')
+                timestamp = int_or_none(node.get('date'))
 
-                # In some cases caption is null, which corresponds to None
-                # in python. As a result, it.get('caption', {}) gives None
-                title = (it.get('caption') or {}).get('text', it['id'])
+                comment_count = get_count('comment')
+                like_count = get_count('like')
+                view_count = int_or_none(node.get('video_views'))
 
-                entries.append({
-                    'id': it['id'],
-                    'title': limit_length(title, 80),
-                    'formats': formats,
+                info.update({
+                    'description': description,
                     'thumbnail': thumbnail,
-                    'webpage_url': it.get('link'),
-                    'uploader': user.get('full_name'),
-                    'uploader_id': user.get('username'),
+                    'timestamp': timestamp,
+                    'comment_count': comment_count,
                     'like_count': like_count,
-                    'timestamp': int_or_none(it.get('created_time')),
+                    'view_count': view_count,
                 })
 
-            if not page['items']:
-                break
-            max_id = page['items'][-1]['id'].split('_')[0]
-            media_url = (
-                'http://instagram.com/%s/media?max_id=%s' % (
-                    uploader_id, max_id))
+                yield info
 
-        return {
-            '_type': 'playlist',
-            'entries': entries,
-            'id': uploader_id,
-            'title': uploader_id,
-        }
+            if not max_id:
+                break
+
+            query['max_id'] = max_id
+
+    def _real_extract(self, url):
+        uploader_id = self._match_id(url)
+        return self.playlist_result(
+            self._entries(uploader_id), uploader_id, uploader_id)

From d4e31b72b971172ffdee7fbe3070d20e4454259c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 13 Nov 2017 10:24:35 +0100
Subject: [PATCH 1511/1696] [gamespot] lower the preference of http
 formats(#14652)

---
 youtube_dl/extractor/gamespot.py | 2 +-
 youtube_dl/extractor/once.py     | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index e6d6d9b1d..a9606a02c 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -109,7 +109,7 @@ class GameSpotIE(OnceIE):
             if onceux_url:
                 formats.extend(self._extract_once_formats(re.sub(
                     r'https?://[^/]+', 'http://once.unicornmedia.com', onceux_url),
-                    skip_http_formats=True))
+                    http_formats_preference=-1))
 
         if not formats:
             for quality in ['sd', 'hd']:
diff --git a/youtube_dl/extractor/once.py b/youtube_dl/extractor/once.py
index 6ba6fe5d3..8ae5fadd8 100644
--- a/youtube_dl/extractor/once.py
+++ b/youtube_dl/extractor/once.py
@@ -11,7 +11,7 @@ class OnceIE(InfoExtractor):
     ADAPTIVE_URL_TEMPLATE = 'http://once.unicornmedia.com/now/master/playlist/%s/%s/%s/content.m3u8'
     PROGRESSIVE_URL_TEMPLATE = 'http://once.unicornmedia.com/now/media/progressive/%s/%s/%s/%s/content.mp4'
 
-    def _extract_once_formats(self, url, skip_http_formats=False):
+    def _extract_once_formats(self, url, http_formats_preference=None):
         domain_id, application_id, media_item_id = re.match(
             OnceIE._VALID_URL, url).groups()
         formats = self._extract_m3u8_formats(
@@ -27,7 +27,7 @@ class OnceIE(InfoExtractor):
             rendition_id = self._search_regex(
                 r'/now/media/playlist/[^/]+/[^/]+/([^/]+)',
                 adaptive_format['url'], 'redition id', default=None)
-            if rendition_id and not skip_http_formats:
+            if rendition_id:
                 progressive_format = adaptive_format.copy()
                 progressive_format.update({
                     'url': self.PROGRESSIVE_URL_TEMPLATE % (
@@ -35,6 +35,7 @@ class OnceIE(InfoExtractor):
                     'format_id': adaptive_format['format_id'].replace(
                         'hls', 'http'),
                     'protocol': 'http',
+                    'preference': http_formats_preference,
                 })
                 progressive_formats.append(progressive_format)
         self._check_formats(progressive_formats, media_item_id)

From 388beb86e0c8e3f76958aa8a258bd396b8b1e0fe Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 13 Nov 2017 10:30:12 +0100
Subject: [PATCH 1512/1696] [gamespot] add test for #14652

---
 youtube_dl/extractor/gamespot.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index a9606a02c..ab647dd41 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -38,6 +38,9 @@ class GameSpotIE(OnceIE):
     }, {
         'url': 'https://www.gamespot.com/videos/embed/6439218/',
         'only_matching': True,
+    }, {
+        'url': 'https://www.gamespot.com/articles/the-last-of-us-2-receives-new-ps4-trailer/1100-6454469/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 27adc9ec65be412e07f6e55d9d9b56c1c224d1db Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 13 Nov 2017 11:24:15 +0100
Subject: [PATCH 1513/1696] [tva] fix extraction(closes #14736)

---
 youtube_dl/extractor/tva.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/tva.py b/youtube_dl/extractor/tva.py
index b57abeaa4..0b863df2f 100644
--- a/youtube_dl/extractor/tva.py
+++ b/youtube_dl/extractor/tva.py
@@ -32,6 +32,8 @@ class TVAIE(InfoExtractor):
         video_data = self._download_json(
             'https://videos.tva.ca/proxy/item/_' + video_id, video_id, headers={
                 'Accept': 'application/json',
+            }, query={
+                'appId': '5955fc5f23eec60006c951f1',
             })
 
         def get_attribute(key):

From 05dee6c520eb959316a2e58203cbd5d30e908bc3 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 13 Nov 2017 19:15:16 +0100
Subject: [PATCH 1514/1696] [crunchyroll] extract old rtmp formats

---
 youtube_dl/extractor/crunchyroll.py | 154 ++++++++++++++++------------
 1 file changed, 91 insertions(+), 63 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 18ef3da10..b53f2d705 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -38,6 +38,16 @@ class CrunchyrollBaseIE(InfoExtractor):
     _LOGIN_FORM = 'login_form'
     _NETRC_MACHINE = 'crunchyroll'
 
+    def _call_rpc_api(self, method, video_id, note=None, data=None):
+        data = data or {}
+        data['req'] = 'RpcApi' + method
+        data = compat_urllib_parse_urlencode(data).encode('utf-8')
+        return self._download_xml(
+            'http://www.crunchyroll.com/xml/',
+            video_id, note, fatal=False, data=data, headers={
+                'Content-Type': 'application/x-www-form-urlencoded',
+            })
+
     def _login(self):
         (username, password) = self._get_login_info()
         if username is None:
@@ -377,15 +387,19 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
     def _get_subtitles(self, video_id, webpage):
         subtitles = {}
         for sub_id, sub_name in re.findall(r'\bssid=([0-9]+)"[^>]+?\btitle="([^"]+)', webpage):
-            sub_page = self._download_webpage(
-                'http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id=' + sub_id,
-                video_id, note='Downloading subtitles for ' + sub_name)
-            id = self._search_regex(r'id=\'([0-9]+)', sub_page, 'subtitle_id', fatal=False)
-            iv = self._search_regex(r'<iv>([^<]+)', sub_page, 'subtitle_iv', fatal=False)
-            data = self._search_regex(r'<data>([^<]+)', sub_page, 'subtitle_data', fatal=False)
-            if not id or not iv or not data:
+            sub_doc = self._call_rpc_api(
+                'Subtitle_GetXml', video_id,
+                'Downloading subtitles for ' + sub_name, data={
+                    'subtitle_script_id': sub_id,
+                })
+            if not sub_doc:
                 continue
-            subtitle = self._decrypt_subtitles(data, iv, id).decode('utf-8')
+            sid = sub_doc.get('id')
+            iv = xpath_text(sub_doc, 'iv', 'subtitle iv')
+            data = xpath_text(sub_doc, 'data', 'subtitle data')
+            if not sid or not iv or not data:
+                continue
+            subtitle = self._decrypt_subtitles(data, iv, sid).decode('utf-8')
             lang_code = self._search_regex(r'lang_code=["\']([^"\']+)', subtitle, 'subtitle_lang_code', fatal=False)
             if not lang_code:
                 continue
@@ -456,65 +470,79 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         for fmt in available_fmts:
             stream_quality, stream_format = self._FORMAT_IDS[fmt]
             video_format = fmt + 'p'
-            streamdata_req = sanitized_Request(
-                'http://www.crunchyroll.com/xml/?req=RpcApiVideoPlayer_GetStandardConfig&media_id=%s&video_format=%s&video_quality=%s'
-                % (video_id, stream_format, stream_quality),
-                compat_urllib_parse_urlencode({'current_page': url}).encode('utf-8'))
-            streamdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
-            streamdata = self._download_xml(
-                streamdata_req, video_id,
-                note='Downloading media info for %s' % video_format)
-            stream_info = streamdata.find('./{default}preload/stream_info')
-            video_encode_id = xpath_text(stream_info, './video_encode_id')
-            if video_encode_id in video_encode_ids:
-                continue
-            video_encode_ids.append(video_encode_id)
+            stream_infos = []
+            streamdata = self._call_rpc_api(
+                'VideoPlayer_GetStandardConfig', video_id,
+                'Downloading media info for %s' % video_format, data={
+                    'media_id': video_id,
+                    'video_format': stream_format,
+                    'video_quality': stream_quality,
+                    'current_page': url,
+                })
+            if streamdata:
+                stream_info = streamdata.find('./{default}preload/stream_info')
+                if stream_info:
+                    stream_infos.append(stream_info)
+            stream_info = self._call_rpc_api(
+                'VideoEncode_GetStreamInfo', video_id,
+                'Downloading stream info for %s' % video_format, data={
+                    'media_id': video_id,
+                    'video_format': stream_format,
+                    'video_encode_quality': stream_quality,
+                })
+            if stream_info:
+                stream_infos.append(stream_info)
+            for stream_info in stream_infos:
+                video_encode_id = xpath_text(stream_info, './video_encode_id')
+                if video_encode_id in video_encode_ids:
+                    continue
+                video_encode_ids.append(video_encode_id)
 
-            video_file = xpath_text(stream_info, './file')
-            if not video_file:
-                continue
-            if video_file.startswith('http'):
-                formats.extend(self._extract_m3u8_formats(
-                    video_file, video_id, 'mp4', entry_protocol='m3u8_native',
-                    m3u8_id='hls', fatal=False))
-                continue
-
-            video_url = xpath_text(stream_info, './host')
-            if not video_url:
-                continue
-            metadata = stream_info.find('./metadata')
-            format_info = {
-                'format': video_format,
-                'format_id': video_format,
-                'height': int_or_none(xpath_text(metadata, './height')),
-                'width': int_or_none(xpath_text(metadata, './width')),
-            }
-
-            if '.fplive.net/' in video_url:
-                video_url = re.sub(r'^rtmpe?://', 'http://', video_url.strip())
-                parsed_video_url = compat_urlparse.urlparse(video_url)
-                direct_video_url = compat_urlparse.urlunparse(parsed_video_url._replace(
-                    netloc='v.lvlt.crcdn.net',
-                    path='%s/%s' % (remove_end(parsed_video_url.path, '/'), video_file.split(':')[-1])))
-                if self._is_valid_url(direct_video_url, video_id, video_format):
-                    format_info.update({
-                        'url': direct_video_url,
-                    })
-                    formats.append(format_info)
+                video_file = xpath_text(stream_info, './file')
+                if not video_file:
+                    continue
+                if video_file.startswith('http'):
+                    formats.extend(self._extract_m3u8_formats(
+                        video_file, video_id, 'mp4', entry_protocol='m3u8_native',
+                        m3u8_id='hls', fatal=False))
                     continue
 
-            format_info.update({
-                'url': video_url,
-                'play_path': video_file,
-                'ext': 'flv',
-            })
-            formats.append(format_info)
-        self._sort_formats(formats)
+                video_url = xpath_text(stream_info, './host')
+                if not video_url:
+                    continue
+                metadata = stream_info.find('./metadata')
+                format_info = {
+                    'format': video_format,
+                    'height': int_or_none(xpath_text(metadata, './height')),
+                    'width': int_or_none(xpath_text(metadata, './width')),
+                }
 
-        metadata = self._download_xml(
-            'http://www.crunchyroll.com/xml', video_id,
-            note='Downloading media info', query={
-                'req': 'RpcApiVideoPlayer_GetMediaMetadata',
+                if '.fplive.net/' in video_url:
+                    video_url = re.sub(r'^rtmpe?://', 'http://', video_url.strip())
+                    parsed_video_url = compat_urlparse.urlparse(video_url)
+                    direct_video_url = compat_urlparse.urlunparse(parsed_video_url._replace(
+                        netloc='v.lvlt.crcdn.net',
+                        path='%s/%s' % (remove_end(parsed_video_url.path, '/'), video_file.split(':')[-1])))
+                    if self._is_valid_url(direct_video_url, video_id, video_format):
+                        format_info.update({
+                            'format_id': 'http-' + video_format,
+                            'url': direct_video_url,
+                        })
+                        formats.append(format_info)
+                        continue
+
+                format_info.update({
+                    'format_id': 'rtmp-' + video_format,
+                    'url': video_url,
+                    'play_path': video_file,
+                    'ext': 'flv',
+                })
+                formats.append(format_info)
+        self._sort_formats(formats, ('height', 'width', 'tbr', 'fps'))
+
+        metadata = self._call_rpc_api(
+            'VideoPlayer_GetMediaMetadata', video_id,
+            note='Downloading media info', data={
                 'media_id': video_id,
             })
 

From 5871ebac473e723376722a37baecf51d6ae7d781 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Nov 2017 01:43:20 +0700
Subject: [PATCH 1515/1696] [YoutubeDL] Fix playlist range optimization for
 --playlist-items (closes #14740)

---
 youtube_dl/YoutubeDL.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 342d6b47c..68721e9ab 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -948,7 +948,8 @@ class YoutubeDL(object):
                 report_download(n_entries)
             else:  # iterable
                 if playlistitems:
-                    entries = make_playlistitems_entries(list(ie_entries))
+                    entries = make_playlistitems_entries(list(itertools.islice(
+                        ie_entries, 0, max(playlistitems))))
                 else:
                     entries = list(itertools.islice(
                         ie_entries, playliststart, playlistend))

From 0987f2ddb27a27506c697ad9dae2ccbf24fc786d Mon Sep 17 00:00:00 2001
From: Timendum <timedum@gmail.com>
Date: Tue, 14 Nov 2017 16:34:45 +0100
Subject: [PATCH 1516/1696] [vshare] Fix extraction (closes #14473)

---
 youtube_dl/extractor/generic.py | 16 +++++++++++++++
 youtube_dl/extractor/vshare.py  | 36 +++++++++++++++++++++++++--------
 2 files changed, 44 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 2a9c3e2de..31564e550 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -102,6 +102,7 @@ from .joj import JojIE
 from .megaphone import MegaphoneIE
 from .vzaar import VzaarIE
 from .channel9 import Channel9IE
+from .vshare import VShareIE
 
 
 class GenericIE(InfoExtractor):
@@ -1921,6 +1922,16 @@ class GenericIE(InfoExtractor):
                 'title': 'Rescue Kit 14 Free Edition - Getting started',
             },
             'playlist_count': 4,
+        },
+        {
+            # vshare embed
+            'url': 'https://youtube-dl-demo.neocities.org/vshare.html',
+            'md5': '17b39f55b5497ae8b59f5fbce8e35886',
+            'info_dict': {
+                'id': '0f64ce6',
+                'title': 'vl14062007715967',
+                'ext': 'mp4',
+            }
         }
         # {
         #     # TODO: find another test
@@ -2879,6 +2890,11 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 channel9_urls, video_id, video_title, ie=Channel9IE.ie_key())
 
+        vshare_urls = VShareIE._extract_urls(webpage)
+        if vshare_urls:
+            return self.playlist_from_matches(
+                vshare_urls, video_id, video_title, ie=VShareIE.ie_key())
+
         def merge_dicts(dict1, dict2):
             merged = {}
             for k, v in dict1.items():
diff --git a/youtube_dl/extractor/vshare.py b/youtube_dl/extractor/vshare.py
index 5addbc280..ea39a9051 100644
--- a/youtube_dl/extractor/vshare.py
+++ b/youtube_dl/extractor/vshare.py
@@ -1,14 +1,18 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..compat import compat_chr
+from ..utils import decode_packed_codes
 
 
 class VShareIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?vshare\.io/[dv]/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'https://vshare.io/d/0f64ce6',
-        'md5': '16d7b8fef58846db47419199ff1ab3e7',
+        'md5': '17b39f55b5497ae8b59f5fbce8e35886',
         'info_dict': {
             'id': '0f64ce6',
             'title': 'vl14062007715967',
@@ -19,20 +23,36 @@ class VShareIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    def _extract_packed(self, webpage):
+        packed = self._search_regex(r'(eval\(function.+)', webpage, 'packed code')
+        unpacked = decode_packed_codes(packed)
+        digits = self._search_regex(r'\[((?:\d+,?)+)\]', unpacked, 'digits')
+        digits = digits.split(',')
+        digits = [int(digit) for digit in digits]
+        key_digit = self._search_regex(r'fromCharCode\(.+?(\d+)\)}', unpacked, 'key digit')
+        chars = [compat_chr(d - int(key_digit)) for d in digits]
+        return ''.join(chars)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(
-            'https://vshare.io/d/%s' % video_id, video_id)
+            'https://vshare.io/v/%s/width-650/height-430/1' % video_id, video_id)
 
-        title = self._html_search_regex(
-            r'(?s)<div id="root-container">(.+?)<br/>', webpage, 'title')
-        video_url = self._search_regex(
-            r'<a[^>]+href=(["\'])(?P<url>(?:https?:)?//.+?)\1[^>]*>[Cc]lick\s+here',
-            webpage, 'video url', group='url')
+        title = self._html_search_regex(r'<title>([^<]+)</title>', webpage, 'title')
+        title = title.split(' - ')[0]
 
+        unpacked = self._extract_packed(webpage)
+        video_urls = re.findall(r'<source src="([^"]+)', unpacked)
+        formats = [{'url': video_url} for video_url in video_urls]
         return {
             'id': video_id,
             'title': title,
-            'url': video_url,
+            'formats': formats,
         }
+
+    @staticmethod
+    def _extract_urls(webpage):
+        return re.findall(
+            r'<iframe[^>]+?src=["\'](?P<url>(?:https?:)?//(?:www\.)?vshare\.io/v/[^/?#&]+)',
+            webpage)

From ff31f2d5c3750364b013a9bf59b85cebd0cee1fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Nov 2017 22:39:54 +0700
Subject: [PATCH 1517/1696] [vshare] Capture and output error message

---
 youtube_dl/extractor/vshare.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vshare.py b/youtube_dl/extractor/vshare.py
index ea39a9051..20ce22e16 100644
--- a/youtube_dl/extractor/vshare.py
+++ b/youtube_dl/extractor/vshare.py
@@ -5,7 +5,10 @@ import re
 
 from .common import InfoExtractor
 from ..compat import compat_chr
-from ..utils import decode_packed_codes
+from ..utils import (
+    decode_packed_codes,
+    ExtractorError,
+)
 
 
 class VShareIE(InfoExtractor):
@@ -42,6 +45,12 @@ class VShareIE(InfoExtractor):
         title = self._html_search_regex(r'<title>([^<]+)</title>', webpage, 'title')
         title = title.split(' - ')[0]
 
+        error = self._html_search_regex(
+            r'(?s)<div[^>]+\bclass=["\']xxx-error[^>]+>(.+?)</div', webpage,
+            'error', default=None)
+        if error:
+            raise ExtractorError(error, expected=True)
+
         unpacked = self._extract_packed(webpage)
         video_urls = re.findall(r'<source src="([^"]+)', unpacked)
         formats = [{'url': video_url} for video_url in video_urls]

From a2b6aba8ded704f55a6650fc3e20b2e6c8aec094 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Nov 2017 22:49:25 +0700
Subject: [PATCH 1518/1696] [vshare] Improve extraction, fix formats sorting
 and carry long lines

---
 youtube_dl/extractor/vshare.py | 41 ++++++++++++++++++++--------------
 1 file changed, 24 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/vshare.py b/youtube_dl/extractor/vshare.py
index 20ce22e16..e4ec77889 100644
--- a/youtube_dl/extractor/vshare.py
+++ b/youtube_dl/extractor/vshare.py
@@ -26,13 +26,20 @@ class VShareIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_urls(webpage):
+        return re.findall(
+            r'<iframe[^>]+?src=["\'](?P<url>(?:https?:)?//(?:www\.)?vshare\.io/v/[^/?#&]+)',
+            webpage)
+
     def _extract_packed(self, webpage):
-        packed = self._search_regex(r'(eval\(function.+)', webpage, 'packed code')
+        packed = self._search_regex(
+            r'(eval\(function.+)', webpage, 'packed code')
         unpacked = decode_packed_codes(packed)
         digits = self._search_regex(r'\[((?:\d+,?)+)\]', unpacked, 'digits')
-        digits = digits.split(',')
-        digits = [int(digit) for digit in digits]
-        key_digit = self._search_regex(r'fromCharCode\(.+?(\d+)\)}', unpacked, 'key digit')
+        digits = [int(digit) for digit in digits.split(',')]
+        key_digit = self._search_regex(
+            r'fromCharCode\(.+?(\d+)\)}', unpacked, 'key digit')
         chars = [compat_chr(d - int(key_digit)) for d in digits]
         return ''.join(chars)
 
@@ -40,9 +47,11 @@ class VShareIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(
-            'https://vshare.io/v/%s/width-650/height-430/1' % video_id, video_id)
+            'https://vshare.io/v/%s/width-650/height-430/1' % video_id,
+            video_id)
 
-        title = self._html_search_regex(r'<title>([^<]+)</title>', webpage, 'title')
+        title = self._html_search_regex(
+            r'<title>([^<]+)</title>', webpage, 'title')
         title = title.split(' - ')[0]
 
         error = self._html_search_regex(
@@ -51,17 +60,15 @@ class VShareIE(InfoExtractor):
         if error:
             raise ExtractorError(error, expected=True)
 
-        unpacked = self._extract_packed(webpage)
-        video_urls = re.findall(r'<source src="([^"]+)', unpacked)
-        formats = [{'url': video_url} for video_url in video_urls]
-        return {
+        info = self._parse_html5_media_entries(
+            url, '<video>%s</video>' % self._extract_packed(webpage),
+            video_id)[0]
+
+        self._sort_formats(info['formats'])
+
+        info.update({
             'id': video_id,
             'title': title,
-            'formats': formats,
-        }
+        })
 
-    @staticmethod
-    def _extract_urls(webpage):
-        return re.findall(
-            r'<iframe[^>]+?src=["\'](?P<url>(?:https?:)?//(?:www\.)?vshare\.io/v/[^/?#&]+)',
-            webpage)
+        return info

From ea2295842f79c9efff3a9abce1d0eee7de4953d6 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 14 Nov 2017 17:41:30 +0100
Subject: [PATCH 1519/1696] [common] skip Apple FairPlay m3u8 manifests(closes
 #14741)

---
 youtube_dl/extractor/common.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e2d9f52b0..a9d68fc0c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1356,6 +1356,9 @@ class InfoExtractor(object):
         if '#EXT-X-FAXS-CM:' in m3u8_doc:  # Adobe Flash Access
             return []
 
+        if re.search(r'#EXT-X-SESSION-KEY:.*?URI="skd://', m3u8_doc):  # Apple FairPlay
+            return []
+
         formats = []
 
         format_url = lambda u: (

From fae0eb42ec4309fe7fb8476d30621ba1d60fa168 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Nov 2017 23:59:30 +0700
Subject: [PATCH 1520/1696] [ChangeLog] Actualize

---
 ChangeLog | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index cedab4723..6ed0f011f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,25 @@
 version <unreleased>
 
+Core
+* [common] Skip Apple FairPlay m3u8 manifests (#14741)
+* [YoutubeDL] Fix playlist range optimization for --playlist-items (#14740)
+
 Extractors
+* [vshare] Capture and output error message
+* [vshare] Fix extraction (#14473)
+* [crunchyroll] Extract old RTMP formats
+* [tva] Fix extraction (#14736)
+* [gamespot] Lower preference of HTTP formats (#14652)
+* [instagram:user] Fix extraction (#14699)
+* [ccma] Fix typo (#14730)
+- Remove sensitive data from logging in messages
+* [instagram:user] Fix extraction (#14699)
++ [gamespot] Add support for article URLs (#14652)
+* [gamespot] Skip Brightcove Once HTTP formats (#14652)
+* [cartoonnetwork] Update tokenizer_src (#14666)
 + [wsj] Recognize another URL pattern (#14704)
+* [pandatv] Update API URL and sign format URLs (#14693)
+* [crunchyroll] Use old login method (#11572)
 
 
 version 2017.11.06

From 08e45b39e76419f63aa43d5008257789d8a30bf8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 15 Nov 2017 00:15:42 +0700
Subject: [PATCH 1521/1696] release 2017.11.15

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index be6e6ddab..4dd1a6e59 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.11.06*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.11.06**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.11.15*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.11.15**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.11.06
+[debug] youtube-dl version 2017.11.15
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 6ed0f011f..6b4befb8f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.11.15
 
 Core
 * [common] Skip Apple FairPlay m3u8 manifests (#14741)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 8b67d23fe..1c3cbefeb 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.11.06'
+__version__ = '2017.11.15'

From 9cbd4dda10ad248a5268ec1e0e563cf97024a8b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 15 Nov 2017 22:14:54 +0700
Subject: [PATCH 1522/1696] [instagram] Fix description, timestamp and counters
 extraction (closes #14755)

---
 youtube_dl/extractor/instagram.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index 20db31f86..a77f619d2 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -130,13 +130,21 @@ class InstagramIE(InfoExtractor):
                 video_url = media.get('video_url')
                 height = int_or_none(media.get('dimensions', {}).get('height'))
                 width = int_or_none(media.get('dimensions', {}).get('width'))
-                description = media.get('caption')
+                description = try_get(
+                    media, lambda x: x['edge_media_to_caption']['edges'][0]['node']['text'],
+                    compat_str) or media.get('caption')
                 thumbnail = media.get('display_src')
-                timestamp = int_or_none(media.get('date'))
+                timestamp = int_or_none(media.get('taken_at_timestamp') or media.get('date'))
                 uploader = media.get('owner', {}).get('full_name')
                 uploader_id = media.get('owner', {}).get('username')
-                like_count = int_or_none(media.get('likes', {}).get('count'))
-                comment_count = int_or_none(media.get('comments', {}).get('count'))
+
+                def get_count(key, kind):
+                    return int_or_none(try_get(
+                        media, (lambda x: x['edge_media_%s' % key]['count'],
+                                lambda x: x['%ss' % kind]['count'])))
+                like_count = get_count('preview_like', 'like')
+                comment_count = get_count('to_comment', 'comment')
+
                 comments = [{
                     'author': comment.get('user', {}).get('username'),
                     'author_id': comment.get('user', {}).get('id'),

From 3192d4bc7a063983c3a82bc4320c16d65679307a Mon Sep 17 00:00:00 2001
From: Windom <windom@users.noreply.github.com>
Date: Thu, 16 Nov 2017 20:05:04 +0200
Subject: [PATCH 1523/1696] [spankbang] Add support for mobile URLs and fix
 test

---
 youtube_dl/extractor/spankbang.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index 2863e53b5..e6c2dcfc4 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -7,7 +7,7 @@ from ..utils import ExtractorError
 
 
 class SpankBangIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www|[a-z]{2})\.)?spankbang\.com/(?P<id>[\da-z]+)/video'
+    _VALID_URL = r'https?://(?:(?:www|m|[a-z]{2})\.)?spankbang\.com/(?P<id>[\da-z]+)/video'
     _TESTS = [{
         'url': 'http://spankbang.com/3vvn/video/fantasy+solo',
         'md5': '1cc433e1d6aa14bc376535b8679302f7',
@@ -15,7 +15,7 @@ class SpankBangIE(InfoExtractor):
             'id': '3vvn',
             'ext': 'mp4',
             'title': 'fantasy solo',
-            'description': 'Watch fantasy solo free HD porn video - 05 minutes - dillion harper masturbates on a bed free adult movies.',
+            'description': 'Watch fantasy solo free HD porn video - 05 minutes -  Babe,Masturbation,Solo,Toy  - dillion harper masturbates on a bed free adult movies sexy clips.',
             'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': 'silly2587',
             'age_limit': 18,
@@ -28,6 +28,10 @@ class SpankBangIE(InfoExtractor):
         # no uploader
         'url': 'http://spankbang.com/lklg/video/sex+with+anyone+wedding+edition+2',
         'only_matching': True,
+    }, {
+        # mobile page
+        'url': 'http://m.spankbang.com/1o2de/video/can+t+remember+her+name',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 38db52adf35c2134444e5b6b601e9567797e9195 Mon Sep 17 00:00:00 2001
From: Windom <windom@users.noreply.github.com>
Date: Thu, 16 Nov 2017 20:50:07 +0200
Subject: [PATCH 1524/1696] [drtuber] Add support for mobile URLs

---
 youtube_dl/extractor/drtuber.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/drtuber.py b/youtube_dl/extractor/drtuber.py
index c5d56a9ad..c88b3126b 100644
--- a/youtube_dl/extractor/drtuber.py
+++ b/youtube_dl/extractor/drtuber.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class DrTuberIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?drtuber\.com/(?:video|embed)/(?P<id>\d+)(?:/(?P<display_id>[\w-]+))?'
+    _VALID_URL = r'https?://(?:(?:www|m)\.)?drtuber\.com/(?:video|embed)/(?P<id>\d+)(?:/(?P<display_id>[\w-]+))?'
     _TESTS = [{
         'url': 'http://www.drtuber.com/video/1740434/hot-perky-blonde-naked-golf',
         'md5': '93e680cf2536ad0dfb7e74d94a89facd',
@@ -28,6 +28,9 @@ class DrTuberIE(InfoExtractor):
     }, {
         'url': 'http://www.drtuber.com/embed/489939',
         'only_matching': True,
+    }, {
+        'url': 'http://m.drtuber.com/video/3893529/lingerie-blowjob-from-beautiful-teen',
+        'only_matching': True,
     }]
 
     @staticmethod

From f610dbb05f8d17cc95437958835a437c3777b38c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Nov 2017 19:02:56 +0700
Subject: [PATCH 1525/1696] [extractor/common] Use final URL when dumping
 request (closes #14769)

---
 youtube_dl/extractor/common.py | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a9d68fc0c..8e4ee0deb 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -592,19 +592,11 @@ class InfoExtractor(object):
         if not encoding:
             encoding = self._guess_encoding_from_content(content_type, webpage_bytes)
         if self._downloader.params.get('dump_intermediate_pages', False):
-            try:
-                url = url_or_request.get_full_url()
-            except AttributeError:
-                url = url_or_request
-            self.to_screen('Dumping request to ' + url)
+            self.to_screen('Dumping request to ' + urlh.geturl())
             dump = base64.b64encode(webpage_bytes).decode('ascii')
             self._downloader.to_screen(dump)
         if self._downloader.params.get('write_pages', False):
-            try:
-                url = url_or_request.get_full_url()
-            except AttributeError:
-                url = url_or_request
-            basen = '%s_%s' % (video_id, url)
+            basen = '%s_%s' % (video_id, urlh.geturl())
             if len(basen) > 240:
                 h = '___' + hashlib.md5(basen.encode('utf-8')).hexdigest()
                 basen = basen[:240 - len(h)] + h

From a9efdf3d4a18ec5657ea50f31715e1b88a945820 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 19 Nov 2017 12:59:31 +0100
Subject: [PATCH 1526/1696] [livestream] make smil extraction non fatal(fixes
 #14792)

---
 youtube_dl/extractor/livestream.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 317ebbc4e..c4776bbf3 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -114,7 +114,7 @@ class LivestreamIE(InfoExtractor):
 
         smil_url = video_data.get('smil_url')
         if smil_url:
-            formats.extend(self._extract_smil_formats(smil_url, video_id))
+            formats.extend(self._extract_smil_formats(smil_url, video_id, fatal=False))
 
         m3u8_url = video_data.get('m3u8_url')
         if m3u8_url:

From 8f639411042d35cd3be6eeff485e3015bafce4d7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 22 Nov 2017 22:49:48 +0800
Subject: [PATCH 1527/1696] [youku] Fix extraction; update ccode (closes
 #14815)

---
 ChangeLog                     | 6 ++++++
 youtube_dl/extractor/youku.py | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 6b4befb8f..e3b7750f6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [youku] Fix extraction; update ccode (#14815)
+
+
 version 2017.11.15
 
 Core
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 0c4bc2eda..6822a30bc 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -154,7 +154,7 @@ class YoukuIE(InfoExtractor):
         # request basic data
         basic_data_params = {
             'vid': video_id,
-            'ccode': '0402' if 'tudou.com' in url else '0401',
+            'ccode': '0502',
             'client_ip': '192.168.1.1',
             'utid': cna,
             'client_ts': time.time() / 1000,

From 2688664762f406b1ba2913af25ee3a2d2ba58038 Mon Sep 17 00:00:00 2001
From: enigmaquip <enigmaquip@users.noreply.github.com>
Date: Wed, 22 Nov 2017 16:39:11 -0700
Subject: [PATCH 1528/1696] [culturebox] Fix extraction (closes #14827)

---
 youtube_dl/extractor/francetv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 2bcbb3e39..037e538cc 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -363,6 +363,6 @@ class CultureboxIE(FranceTVBaseInfoExtractor):
             raise ExtractorError('Video %s is not available' % name, expected=True)
 
         video_id, catalogue = self._search_regex(
-            r'"http://videos\.francetv\.fr/video/([^@]+@[^"]+)"', webpage, 'video id').split('@')
+            r'"https?://videos\.francetv\.fr/video/([^@]+@[^"]+)"', webpage, 'video id').split('@')
 
         return self._extract_video(video_id, catalogue)

From 939be9adfe810ada7dbd5e9032bcfec19fafa14b Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Wed, 22 Nov 2017 11:47:02 -0500
Subject: [PATCH 1529/1696] [JWPlatform] Support iframes

Support content.jwplatform... src attributes inside <iframe> tags in
addition to <script> tags. Just a regexp change.

Add a test (currently Generic_72).
---
 youtube_dl/extractor/generic.py    | 12 ++++++++++++
 youtube_dl/extractor/jwplatform.py |  2 +-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 31564e550..8efe68106 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1136,6 +1136,18 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             }
         },
+        {
+            # JWPlatform iframe
+            'url': 'https://www.mediaite.com/tv/dem-senator-claims-gary-cohn-faked-a-bad-connection-during-trump-call-to-get-him-off-the-phone/',
+            'md5': 'ca00a040364b5b439230e7ebfd02c4e9',
+            'info_dict': {
+                'id': 'O0c5JcKT',
+                'ext': 'mp4',
+                'upload_date': '20171122',
+                'timestamp': 1511366290,
+                'title': 'Dem Senator Claims Gary Cohn Faked a Bad Connection During Trump Call to Get Him Off the Phone',
+            },
+        },
         {
             # Video.js embed, multiple formats
             'url': 'http://ortcam.com/solidworks-урок-6-настройка-чертежа_33f9b7351.html',
diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index 33d55f770..4e74eafac 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -24,7 +24,7 @@ class JWPlatformIE(InfoExtractor):
     @staticmethod
     def _extract_url(webpage):
         mobj = re.search(
-            r'<script[^>]+?src=["\'](?P<url>(?:https?:)?//content.jwplatform.com/players/[a-zA-Z0-9]{8})',
+            r'<(script|iframe)[^>]+?src=["\'](?P<url>(?:https?:)?//content.jwplatform.com/players/[a-zA-Z0-9]{8})',
             webpage)
         if mobj:
             return mobj.group('url')

From 6899b1d9e87857e0ae2d36f710e0e12885b29c09 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Wed, 22 Nov 2017 11:56:47 -0500
Subject: [PATCH 1530/1696] [Generic] Update test 69 (suffolk/sjc)

suffolk.edu/sjc => suffolk.edu/sjc/live.php

Unfortunately it only transmits video a few mornings per month, so
leaving the 'skip' is probably appropriate. Updating the 'skip- to
include the calendar information though.
---
 youtube_dl/extractor/generic.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8efe68106..d2e4551e3 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1099,9 +1099,9 @@ class GenericIE(InfoExtractor):
         },
         # jwplayer rtmp
         {
-            'url': 'http://www.suffolk.edu/sjc/',
+            'url': 'http://www.suffolk.edu/sjc/live.php',
             'info_dict': {
-                'id': 'sjclive',
+                'id': 'live',
                 'ext': 'flv',
                 'title': 'Massachusetts Supreme Judicial Court Oral Arguments',
                 'uploader': 'www.suffolk.edu',
@@ -1109,7 +1109,7 @@ class GenericIE(InfoExtractor):
             'params': {
                 'skip_download': True,
             },
-            'skip': 'does not contain a video anymore',
+            'skip': 'Only has video a few mornings per month, see http://www.suffolk.edu/sjc/',
         },
         # Complex jwplayer
         {

From 32ad4f3faf33d7ffedf08b3f8e39acb084b361fa Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Fri, 24 Nov 2017 08:10:47 -0500
Subject: [PATCH 1531/1696] [JWPlatform] Use non-capturing group in RE

Per @yan12125.
---
 youtube_dl/extractor/jwplatform.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index 4e74eafac..c9bcbb08f 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -24,7 +24,7 @@ class JWPlatformIE(InfoExtractor):
     @staticmethod
     def _extract_url(webpage):
         mobj = re.search(
-            r'<(script|iframe)[^>]+?src=["\'](?P<url>(?:https?:)?//content.jwplatform.com/players/[a-zA-Z0-9]{8})',
+            r'<(?:script|iframe)[^>]+?src=["\'](?P<url>(?:https?:)?//content.jwplatform.com/players/[a-zA-Z0-9]{8})',
             webpage)
         if mobj:
             return mobj.group('url')

From 805f5bf75978ba24d072d040af81bd8d1ec3fed9 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Fri, 24 Nov 2017 08:11:32 -0500
Subject: [PATCH 1532/1696] [Generic] ie_key in JWPlatform test 72

---
 youtube_dl/extractor/generic.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index d2e4551e3..56df2ab47 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1147,6 +1147,7 @@ class GenericIE(InfoExtractor):
                 'timestamp': 1511366290,
                 'title': 'Dem Senator Claims Gary Cohn Faked a Bad Connection During Trump Call to Get Him Off the Phone',
             },
+            'add_ie': [JWPlatformIE.ie_key()],
         },
         {
             # Video.js embed, multiple formats

From 6049176471b4c567c1e9bd8f77b298e323d5b8e7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 24 Nov 2017 21:40:51 +0800
Subject: [PATCH 1533/1696] [ChangeLog] Update after #14828

[skip ci]
---
 ChangeLog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index e3b7750f6..c4a84c597 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -2,7 +2,7 @@ version <unreleased>
 
 Extractors
 * [youku] Fix extraction; update ccode (#14815)
-
++ [JWPlatform] support iframes (#14828)
 
 version 2017.11.15
 

From e0a8686f48d10ed86f7be92132dd37481981adf3 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 24 Nov 2017 18:42:41 +0100
Subject: [PATCH 1534/1696] [faz] fix extraction and add support for Perform
 Group embeds(fixes #14714)

---
 youtube_dl/extractor/extractors.py   |  1 +
 youtube_dl/extractor/faz.py          | 34 +++++++++---
 youtube_dl/extractor/performgroup.py | 83 ++++++++++++++++++++++++++++
 3 files changed, 111 insertions(+), 7 deletions(-)
 create mode 100644 youtube_dl/extractor/performgroup.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d084707ee..aecb84b18 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -789,6 +789,7 @@ from .patreon import PatreonIE
 from .pbs import PBSIE
 from .pearvideo import PearVideoIE
 from .people import PeopleIE
+from .performgroup import PerformGroupIE
 from .periscope import (
     PeriscopeIE,
     PeriscopeUserIE,
diff --git a/youtube_dl/extractor/faz.py b/youtube_dl/extractor/faz.py
index 4bc8fc512..312ee2aee 100644
--- a/youtube_dl/extractor/faz.py
+++ b/youtube_dl/extractor/faz.py
@@ -1,7 +1,10 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..compat import compat_etree_fromstring
 from ..utils import (
     xpath_element,
     xpath_text,
@@ -43,10 +46,15 @@ class FazIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
         description = self._og_search_description(webpage)
-        config_xml_url = self._search_regex(
-            r'videoXMLURL\s*=\s*"([^"]+)', webpage, 'config xml url')
-        config = self._download_xml(
-            config_xml_url, video_id, 'Downloading config xml')
+        media = self._html_search_regex(
+            r"data-videojs-media='([^']+)",
+            webpage, 'media')
+        if media == 'extern':
+            perform_url = self._search_regex(
+                r"<iframe[^>]+?src='((?:http:)?//player\.performgroup\.com/eplayer/eplayer\.html#/?[0-9a-f]{26}\.[0-9a-z]{26})",
+                webpage, 'perform url')
+            return self.url_result(perform_url)
+        config = compat_etree_fromstring(media)
 
         encodings = xpath_element(config, 'ENCODINGS', 'encodings', True)
         formats = []
@@ -55,12 +63,24 @@ class FazIE(InfoExtractor):
             if encoding is not None:
                 encoding_url = xpath_text(encoding, 'FILENAME')
                 if encoding_url:
-                    formats.append({
+                    tbr = xpath_text(encoding, 'AVERAGEBITRATE', 1000)
+                    if tbr:
+                        tbr = int_or_none(tbr.replace(',', '.'))
+                    f = {
                         'url': encoding_url,
                         'format_id': code.lower(),
                         'quality': pref,
-                        'tbr': int_or_none(xpath_text(encoding, 'AVERAGEBITRATE')),
-                    })
+                        'tbr': tbr,
+                        'vcodec': xpath_text(encoding, 'CODEC'),
+                    }
+                    mobj = re.search(r'(\d+)x(\d+)_(\d+)\.mp4', encoding_url)
+                    if mobj:
+                        f.update({
+                            'width': int(mobj.group(1)),
+                            'height': int(mobj.group(2)),
+                            'tbr': tbr or int(mobj.group(3)),
+                        })
+                    formats.append(f)
         self._sort_formats(formats)
 
         return {
diff --git a/youtube_dl/extractor/performgroup.py b/youtube_dl/extractor/performgroup.py
new file mode 100644
index 000000000..26942bfb3
--- /dev/null
+++ b/youtube_dl/extractor/performgroup.py
@@ -0,0 +1,83 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class PerformGroupIE(InfoExtractor):
+    _VALID_URL = r'https?://player\.performgroup\.com/eplayer(?:/eplayer\.html|\.js)#/?(?P<id>[0-9a-f]{26})\.(?P<auth_token>[0-9a-z]{26})'
+    _TESTS = [{
+        # http://www.faz.net/aktuell/sport/fussball/wm-2018-playoffs-schweiz-besiegt-nordirland-1-0-15286104.html
+        'url': 'http://player.performgroup.com/eplayer/eplayer.html#d478c41c5d192f56b9aa859de8.1w4crrej5w14e1ed4s1ce4ykab',
+        'md5': '259cb03d142e2e52471e8837ecacb29f',
+        'info_dict': {
+            'id': 'xgrwobuzumes1lwjxtcdpwgxd',
+            'ext': 'mp4',
+            'title': 'Liga MX: Keine Einsicht nach Horrorfoul',
+            'description': 'md5:7cd3b459c82725b021e046ab10bf1c5b',
+            'timestamp': 1511533477,
+            'upload_date': '20171124',
+        }
+    }]
+
+    def _call_api(self, service, auth_token, content_id, referer_url):
+        return self._download_json(
+            'http://ep3.performfeeds.com/ep%s/%s/%s/' % (service, auth_token, content_id),
+            content_id, headers={
+                'Referer': referer_url,
+                'Origin': 'http://player.performgroup.com',
+            }, query={
+                '_fmt': 'json',
+            })
+
+    def _real_extract(self, url):
+        player_id, auth_token = re.search(self._VALID_URL, url).groups()
+        bootstrap = self._call_api('bootstrap', auth_token, player_id, url)
+        video = bootstrap['config']['dataSource']['sourceItems'][0]['videos'][0]
+        video_id = video['uuid']
+        vod = self._call_api('vod', auth_token, video_id, url)
+        media = vod['videos']['video'][0]['media']
+
+        formats = []
+        hls_url = media.get('hls', {}).get('url')
+        if hls_url:
+            formats.extend(self._extract_m3u8_formats(hls_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+
+        hds_url = media.get('hds', {}).get('url')
+        if hds_url:
+            formats.extend(self._extract_f4m_formats(hds_url + '?hdcore', video_id, f4m_id='hds', fatal=False))
+
+        for c in media.get('content', []):
+            c_url = c.get('url')
+            if not c_url:
+                continue
+            tbr = int_or_none(c.get('bitrate'), 1000)
+            format_id = 'http'
+            if tbr:
+                format_id += '-%d' % tbr
+            formats.append({
+                'format_id': format_id,
+                'url': c_url,
+                'tbr': tbr,
+                'width': int_or_none(c.get('width')),
+                'height': int_or_none(c.get('height')),
+                'filesize': int_or_none(c.get('fileSize')),
+                'vcodec': c.get('type'),
+                'fps': int_or_none(c.get('videoFrameRate')),
+                'vbr': int_or_none(c.get('videoRate'), 1000),
+                'abr': int_or_none(c.get('audioRate'), 1000),
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': video['title'],
+            'description': video.get('description'),
+            'thumbnail': video.get('poster'),
+            'duration': int_or_none(video.get('duration')),
+            'timestamp': int_or_none(video.get('publishedTime'), 1000),
+            'formats': formats,
+        }

From c6c6a64aa58af54d3f1d4ebdd21547e708b29bf6 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 24 Nov 2017 19:00:56 +0100
Subject: [PATCH 1535/1696] [fox9] fix extraction

---
 youtube_dl/extractor/fox9.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/fox9.py b/youtube_dl/extractor/fox9.py
index 56d9975d0..17dfffa7b 100644
--- a/youtube_dl/extractor/fox9.py
+++ b/youtube_dl/extractor/fox9.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 from .anvato import AnvatoIE
-from ..utils import js_to_json
 
 
 class FOX9IE(AnvatoIE):
@@ -34,9 +33,9 @@ class FOX9IE(AnvatoIE):
 
         video_id = self._parse_json(
             self._search_regex(
-                r'AnvatoPlaylist\s*\(\s*(\[.+?\])\s*\)\s*;',
+                r"this\.videosJson\s*=\s*'(\[.+?\])';",
                 webpage, 'anvato playlist'),
-            video_id, transform_source=js_to_json)[0]['video']
+            video_id)[0]['video']
 
         return self._get_anvato_videos(
             'anvato_epfox_app_web_prod_b3373168e12f423f41504f207000188daf88251b',

From cd9ff4ec5be86b4a16afc792ff9cf0d5c3eacd47 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 24 Nov 2017 20:00:01 +0100
Subject: [PATCH 1536/1696] [massengeschmacktv] Add support for Massengeschmack
 TV(replaces Fernseh Kritik TV)

---
 youtube_dl/extractor/extractors.py        |  2 +-
 youtube_dl/extractor/fktv.py              | 51 ---------------
 youtube_dl/extractor/massengeschmacktv.py | 77 +++++++++++++++++++++++
 3 files changed, 78 insertions(+), 52 deletions(-)
 delete mode 100644 youtube_dl/extractor/fktv.py
 create mode 100644 youtube_dl/extractor/massengeschmacktv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index aecb84b18..f1a464970 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -348,7 +348,6 @@ from .firstpost import FirstpostIE
 from .firsttv import FirstTVIE
 from .fivemin import FiveMinIE
 from .fivetv import FiveTVIE
-from .fktv import FKTVIE
 from .flickr import FlickrIE
 from .flipagram import FlipagramIE
 from .folketinget import FolketingetIE
@@ -572,6 +571,7 @@ from .mangomolo import (
     MangomoloLiveIE,
 )
 from .manyvids import ManyVidsIE
+from .massengeschmacktv import MassengeschmackTVIE
 from .matchtv import MatchTVIE
 from .mdr import MDRIE
 from .mediaset import MediasetIE
diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
deleted file mode 100644
index 2958452f4..000000000
--- a/youtube_dl/extractor/fktv.py
+++ /dev/null
@@ -1,51 +0,0 @@
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import (
-    clean_html,
-    determine_ext,
-    js_to_json,
-)
-
-
-class FKTVIE(InfoExtractor):
-    IE_NAME = 'fernsehkritik.tv'
-    _VALID_URL = r'https?://(?:www\.)?fernsehkritik\.tv/folge-(?P<id>[0-9]+)(?:/.*)?'
-
-    _TEST = {
-        'url': 'http://fernsehkritik.tv/folge-1',
-        'md5': '21f0b0c99bce7d5b524eb1b17b1c6d79',
-        'info_dict': {
-            'id': '1',
-            'ext': 'mp4',
-            'title': 'Folge 1 vom 10. April 2007',
-            'thumbnail': r're:^https?://.*\.jpg$',
-        },
-    }
-
-    def _real_extract(self, url):
-        episode = self._match_id(url)
-
-        webpage = self._download_webpage(
-            'http://fernsehkritik.tv/folge-%s/play' % episode, episode)
-        title = clean_html(self._html_search_regex(
-            '<h3>([^<]+)</h3>', webpage, 'title'))
-        thumbnail = self._search_regex(r'POSTER\s*=\s*"([^"]+)', webpage, 'thumbnail', fatal=False)
-        sources = self._parse_json(self._search_regex(r'(?s)MEDIA\s*=\s*(\[.+?\]);', webpage, 'media'), episode, js_to_json)
-
-        formats = []
-        for source in sources:
-            furl = source.get('src')
-            if furl:
-                formats.append({
-                    'url': furl,
-                    'format_id': determine_ext(furl),
-                })
-        self._sort_formats(formats)
-
-        return {
-            'id': episode,
-            'title': title,
-            'formats': formats,
-            'thumbnail': thumbnail,
-        }
diff --git a/youtube_dl/extractor/massengeschmacktv.py b/youtube_dl/extractor/massengeschmacktv.py
new file mode 100644
index 000000000..cfcc6b224
--- /dev/null
+++ b/youtube_dl/extractor/massengeschmacktv.py
@@ -0,0 +1,77 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    determine_ext,
+    int_or_none,
+    js_to_json,
+    mimetype2ext,
+    parse_filesize,
+)
+
+
+class MassengeschmackTVIE(InfoExtractor):
+    IE_NAME = 'massengeschmack.tv'
+    _VALID_URL = r'https?://(?:www\.)?massengeschmack\.tv/play/(?P<id>[^?&#]+)'
+
+    _TEST = {
+        'url': 'https://massengeschmack.tv/play/fktv202',
+        'md5': 'a9e054db9c2b5a08f0a0527cc201e8d3',
+        'info_dict': {
+            'id': 'fktv202',
+            'ext': 'mp4',
+            'title': 'Fernsehkritik-TV - Folge 202',
+        },
+    }
+
+    def _real_extract(self, url):
+        episode = self._match_id(url)
+
+        webpage = self._download_webpage(url, episode)
+        title = clean_html(self._html_search_regex(
+            '<h3>([^<]+)</h3>', webpage, 'title'))
+        thumbnail = self._search_regex(r'POSTER\s*=\s*"([^"]+)', webpage, 'thumbnail', fatal=False)
+        sources = self._parse_json(self._search_regex(r'(?s)MEDIA\s*=\s*(\[.+?\]);', webpage, 'media'), episode, js_to_json)
+
+        formats = []
+        for source in sources:
+            furl = source.get('src')
+            if not furl:
+                continue
+            furl = self._proto_relative_url(furl)
+            ext = determine_ext(furl) or mimetype2ext(source.get('type'))
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    furl, episode, 'mp4', 'm3u8_native',
+                    m3u8_id='hls', fatal=False))
+            else:
+                formats.append({
+                    'url': furl,
+                    'format_id': determine_ext(furl),
+                })
+
+        for (durl, format_id, width, height, filesize) in re.findall(r'''(?x)
+                                   <a[^>]+?href="(?P<url>(?:https:)?//[^"]+)".*?
+                                   <strong>(?P<format_id>.+?)</strong>.*?
+                                   <small>(?:(?P<width>\d+)x(?P<height>\d+))?\s+?\((?P<filesize>[\d,]+\s*[GM]iB)\)</small>
+                                ''', webpage):
+            formats.append({
+                'url': durl,
+                'format_id': format_id,
+                'width': int_or_none(width),
+                'height': int_or_none(height),
+                'filesize': parse_filesize(filesize),
+                'vcodec': 'none' if format_id.startswith('Audio') else None,
+            })
+
+        self._sort_formats(formats, ('width', 'height', 'filesize', 'tbr'))
+
+        return {
+            'id': episode,
+            'title': title,
+            'formats': formats,
+            'thumbnail': thumbnail,
+        }

From 6f5c598a28b2b400ffb4d2bc2d228c2802ab7fd7 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 25 Nov 2017 15:49:49 +0100
Subject: [PATCH 1537/1696] [france2.fr:generation-what] fix extraction

---
 youtube_dl/extractor/francetv.py | 38 ++++++++++++++++----------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 037e538cc..5a3abeaff 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -3,7 +3,6 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
 from ..compat import compat_urlparse
@@ -308,31 +307,32 @@ class FranceTVInfoIE(FranceTVBaseInfoExtractor):
         return self._extract_video(video_id, catalogue)
 
 
-class GenerationQuoiIE(InfoExtractor):
-    IE_NAME = 'france2.fr:generation-quoi'
-    _VALID_URL = r'https?://generation-quoi\.france2\.fr/portrait/(?P<id>[^/?#]+)'
+class GenerationWhatIE(InfoExtractor):
+    IE_NAME = 'france2.fr:generation-what'
+    _VALID_URL = r'https?://generation-what\.francetv\.fr/[^/]+/video/(?P<id>[^/?#]+)'
 
-    _TEST = {
-        'url': 'http://generation-quoi.france2.fr/portrait/garde-a-vous',
+    _TESTS = [{
+        'url': 'http://generation-what.francetv.fr/portrait/video/present-arms',
         'info_dict': {
-            'id': 'k7FJX8VBcvvLmX4wA5Q',
+            'id': 'wtvKYUG45iw',
             'ext': 'mp4',
-            'title': 'Génération Quoi - Garde à Vous',
-            'uploader': 'Génération Quoi',
+            'title': 'Generation What - Garde à vous - FRA',
+            'uploader': 'Generation What',
+            'uploader_id': 'UCHH9p1eetWCgt4kXBYCb3_w',
+            'upload_date': '20160411',
         },
-        'params': {
-            # It uses Dailymotion
-            'skip_download': True,
-        },
-    }
+    }, {
+        'url': 'http://generation-what.francetv.fr/europe/video/present-arms',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
-        info_url = compat_urlparse.urljoin(url, '/medias/video/%s.json' % display_id)
-        info_json = self._download_webpage(info_url, display_id)
-        info = json.loads(info_json)
-        return self.url_result('http://www.dailymotion.com/video/%s' % info['id'],
-                               ie='Dailymotion')
+        webpage = self._download_webpage(url, display_id)
+        youtube_id = self._search_regex(
+            r"window\.videoURL\s*=\s*'([0-9A-Za-z_-]{11})';",
+            webpage, 'youtube id')
+        return self.url_result(youtube_id, 'Youtube', youtube_id)
 
 
 class CultureboxIE(FranceTVBaseInfoExtractor):

From fe4bfe36e148c0e1d3180a2992503536b4b4f84b Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 25 Nov 2017 15:58:28 +0100
Subject: [PATCH 1538/1696] [gamersyde] remove extractor

---
 youtube_dl/extractor/extractors.py |  3 +-
 youtube_dl/extractor/gamersyde.py  | 70 ------------------------------
 2 files changed, 1 insertion(+), 72 deletions(-)
 delete mode 100644 youtube_dl/extractor/gamersyde.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index f1a464970..bc1ca0f0a 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -374,7 +374,7 @@ from .francetv import (
     FranceTVIE,
     FranceTVEmbedIE,
     FranceTVInfoIE,
-    GenerationQuoiIE,
+    GenerationWhatIE,
     CultureboxIE,
 )
 from .freesound import FreesoundIE
@@ -390,7 +390,6 @@ from .gameone import (
     GameOneIE,
     GameOnePlaylistIE,
 )
-from .gamersyde import GamersydeIE
 from .gamespot import GameSpotIE
 from .gamestar import GameStarIE
 from .gaskrank import GaskrankIE
diff --git a/youtube_dl/extractor/gamersyde.py b/youtube_dl/extractor/gamersyde.py
deleted file mode 100644
index a218a6944..000000000
--- a/youtube_dl/extractor/gamersyde.py
+++ /dev/null
@@ -1,70 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    js_to_json,
-    parse_duration,
-    remove_start,
-)
-
-
-class GamersydeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?gamersyde\.com/hqstream_(?P<display_id>[\da-z_]+)-(?P<id>\d+)_[a-z]{2}\.html'
-    _TEST = {
-        'url': 'http://www.gamersyde.com/hqstream_bloodborne_birth_of_a_hero-34371_en.html',
-        'md5': 'f38d400d32f19724570040d5ce3a505f',
-        'info_dict': {
-            'id': '34371',
-            'ext': 'mp4',
-            'duration': 372,
-            'title': 'Bloodborne - Birth of a hero',
-            'thumbnail': r're:^https?://.*\.jpg$',
-        }
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        display_id = mobj.group('display_id')
-
-        webpage = self._download_webpage(url, display_id)
-
-        playlist = self._parse_json(
-            self._search_regex(
-                r'(?s)playlist: \[({.+?})\]\s*}\);', webpage, 'files'),
-            display_id, transform_source=js_to_json)
-
-        formats = []
-        for source in playlist['sources']:
-            video_url = source.get('file')
-            if not video_url:
-                continue
-            format_id = source.get('label')
-            f = {
-                'url': video_url,
-                'format_id': format_id,
-            }
-            m = re.search(r'^(?P<height>\d+)[pP](?P<fps>\d+)fps', format_id)
-            if m:
-                f.update({
-                    'height': int(m.group('height')),
-                    'fps': int(m.group('fps')),
-                })
-            formats.append(f)
-        self._sort_formats(formats)
-
-        title = remove_start(playlist['title'], '%s - ' % video_id)
-        thumbnail = playlist.get('image')
-        duration = parse_duration(self._search_regex(
-            r'Length:</label>([^<]+)<', webpage, 'duration', fatal=False))
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'title': title,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'formats': formats,
-        }

From dbb25af657166108e7afcd4e1ff197e2a5644724 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Nov 2017 23:08:46 +0700
Subject: [PATCH 1539/1696] [tnaflix] Don't modify download URLs (closes
 #14811)

---
 youtube_dl/extractor/tnaflix.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index 7e6ec3430..9f47ce2aa 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -91,7 +91,8 @@ class TNAFlixNetworkBaseIE(InfoExtractor):
         formats = []
 
         def extract_video_url(vl):
-            return re.sub(r'speed=\d+', 'speed=', unescapeHTML(vl.text))
+            # Any URL modification now results in HTTP Error 403: Forbidden
+            return unescapeHTML(vl.text)
 
         video_link = cfg_xml.find('./videoLink')
         if video_link is not None:

From 9105523818270cadb7ad90a3b1ea7e6f18d2dc54 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Nov 2017 23:13:17 +0700
Subject: [PATCH 1540/1696] [empflix] Fix extractrion

---
 youtube_dl/extractor/tnaflix.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index 9f47ce2aa..c4ead1fcd 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -21,6 +21,8 @@ class TNAFlixNetworkBaseIE(InfoExtractor):
         r'flashvars\.config\s*=\s*escape\("([^"]+)"',
         r'<input[^>]+name="config\d?" value="([^"]+)"',
     ]
+    _HOST = 'tna'
+    _VKEY_SUFFIX = ''
     _TITLE_REGEX = r'<input[^>]+name="title" value="([^"]+)"'
     _DESCRIPTION_REGEX = r'<input[^>]+name="description" value="([^"]+)"'
     _UPLOADER_REGEX = r'<input[^>]+name="username" value="([^"]+)"'
@@ -81,8 +83,8 @@ class TNAFlixNetworkBaseIE(InfoExtractor):
 
         if not cfg_url:
             inputs = self._hidden_inputs(webpage)
-            cfg_url = ('https://cdn-fck.tnaflix.com/tnaflix/%s.fid?key=%s&VID=%s&premium=1&vip=1&alpha'
-                       % (inputs['vkey'], inputs['nkey'], video_id))
+            cfg_url = ('https://cdn-fck.%sflix.com/%sflix/%s%s.fid?key=%s&VID=%s&premium=1&vip=1&alpha'
+                       % (self._HOST, self._HOST, inputs['vkey'], self._VKEY_SUFFIX, inputs['nkey'], video_id))
 
         cfg_xml = self._download_xml(
             cfg_url, display_id, 'Downloading metadata',
@@ -240,6 +242,8 @@ class TNAFlixIE(TNAFlixNetworkBaseIE):
 class EMPFlixIE(TNAFlixNetworkBaseIE):
     _VALID_URL = r'https?://(?:www\.)?empflix\.com/videos/(?P<display_id>.+?)-(?P<id>[0-9]+)\.html'
 
+    _HOST = 'emp'
+    _VKEY_SUFFIX = '-1'
     _UPLOADER_REGEX = r'<span[^>]+class="infoTitle"[^>]*>Uploaded By:</span>(.+?)</li>'
 
     _TESTS = [{

From b7785cf15685f7dfbe96cc6d2dd8089ff1363be3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Nov 2017 23:22:54 +0700
Subject: [PATCH 1541/1696] [empflix] Relax _VALID_URL

---
 youtube_dl/extractor/tnaflix.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index c4ead1fcd..e57b8e318 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -74,7 +74,13 @@ class TNAFlixNetworkBaseIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
-        display_id = mobj.group('display_id') if 'display_id' in mobj.groupdict() else video_id
+        for display_id_key in ('display_id', 'display_id_2'):
+            if display_id_key in mobj.groupdict():
+                display_id = mobj.group(display_id_key)
+                if display_id:
+                    break
+        else:
+            display_id = video_id
 
         webpage = self._download_webpage(url, display_id)
 
@@ -240,7 +246,7 @@ class TNAFlixIE(TNAFlixNetworkBaseIE):
 
 
 class EMPFlixIE(TNAFlixNetworkBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?empflix\.com/videos/(?P<display_id>.+?)-(?P<id>[0-9]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?empflix\.com/(?:videos/(?P<display_id>.+?)-|[^/]+/(?P<display_id_2>[^/]+)/video)(?P<id>[0-9]+)'
 
     _HOST = 'emp'
     _VKEY_SUFFIX = '-1'
@@ -264,6 +270,9 @@ class EMPFlixIE(TNAFlixNetworkBaseIE):
     }, {
         'url': 'http://www.empflix.com/videos/[AROMA][ARMD-718]-Aoi-Yoshino-Sawa-25826.html',
         'only_matching': True,
+    }, {
+        'url': 'https://www.empflix.com/amateur-porn/Amateur-Finger-Fuck/video33051',
+        'only_matching': True,
     }]
 
 

From 8cfbcfab9a94d363dbc8e1b026648c3ccf0d399e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Nov 2017 23:37:50 +0700
Subject: [PATCH 1542/1696] [tnaflix] Extract common parts of tnaflix and
 empflix

---
 youtube_dl/extractor/tnaflix.py | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index e57b8e318..0c2f8f119 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -201,18 +201,21 @@ class TNAFlixNetworkEmbedIE(TNAFlixNetworkBaseIE):
             webpage)]
 
 
-class TNAFlixIE(TNAFlixNetworkBaseIE):
+class TNAEMPFlixBaseIE(TNAFlixNetworkBaseIE):
+    _DESCRIPTION_REGEX = r'(?s)>Description:</[^>]+>(.+?)<'
+    _UPLOADER_REGEX = r'<span>by\s*<a[^>]+\bhref=["\']/profile/[^>]+>([^<]+)<'
+    _CATEGORIES_REGEX = r'(?s)<span[^>]*>Categories:</span>(.+?)</div>'
+
+
+class TNAFlixIE(TNAEMPFlixBaseIE):
     _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/[^/]+/(?P<display_id>[^/]+)/video(?P<id>\d+)'
 
     _TITLE_REGEX = r'<title>(.+?) - (?:TNAFlix Porn Videos|TNAFlix\.com)</title>'
-    _DESCRIPTION_REGEX = r'(?s)>Description:</[^>]+>(.+?)<'
-    _UPLOADER_REGEX = r'<i>\s*Verified Member\s*</i>\s*<h\d+>(.+?)<'
-    _CATEGORIES_REGEX = r'(?s)<span[^>]*>Categories:</span>(.+?)</div>'
 
     _TESTS = [{
         # anonymous uploader, no categories
         'url': 'http://www.tnaflix.com/porn-stars/Carmella-Decesare-striptease/video553878',
-        'md5': 'ecf3498417d09216374fc5907f9c6ec0',
+        'md5': '7e569419fe6d69543d01e6be22f5f7c4',
         'info_dict': {
             'id': '553878',
             'display_id': 'Carmella-Decesare-striptease',
@@ -237,7 +240,7 @@ class TNAFlixIE(TNAFlixNetworkBaseIE):
             'duration': 164,
             'age_limit': 18,
             'uploader': 'bobwhite39',
-            'categories': ['Amateur Porn', 'Squirting Videos', 'Teen Girls 18+'],
+            'categories': list,
         }
     }, {
         'url': 'https://www.tnaflix.com/amateur-porn/bunzHD-Ms.Donk/video358632',
@@ -245,16 +248,15 @@ class TNAFlixIE(TNAFlixNetworkBaseIE):
     }]
 
 
-class EMPFlixIE(TNAFlixNetworkBaseIE):
+class EMPFlixIE(TNAEMPFlixBaseIE):
     _VALID_URL = r'https?://(?:www\.)?empflix\.com/(?:videos/(?P<display_id>.+?)-|[^/]+/(?P<display_id_2>[^/]+)/video)(?P<id>[0-9]+)'
 
     _HOST = 'emp'
     _VKEY_SUFFIX = '-1'
-    _UPLOADER_REGEX = r'<span[^>]+class="infoTitle"[^>]*>Uploaded By:</span>(.+?)</li>'
 
     _TESTS = [{
         'url': 'http://www.empflix.com/videos/Amateur-Finger-Fuck-33051.html',
-        'md5': 'b1bc15b6412d33902d6e5952035fcabc',
+        'md5': 'bc30d48b91a7179448a0bda465114676',
         'info_dict': {
             'id': '33051',
             'display_id': 'Amateur-Finger-Fuck',

From 9ef909f2b2bf81e16c4fd2e428a740a523ad61cf Mon Sep 17 00:00:00 2001
From: jahudka <me@subsonic.cz>
Date: Sat, 25 Nov 2017 18:04:13 +0100
Subject: [PATCH 1543/1696] [openload] Add support for openload.link

---
 youtube_dl/extractor/openload.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index b50d6c77b..efa5d4601 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -242,7 +242,7 @@ class PhantomJSwrapper(object):
 
 
 class OpenloadIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:openload\.(?:co|io)|oload\.tv)/(?:f|embed)/(?P<id>[a-zA-Z0-9-_]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:openload\.(?:co|io|link)|oload\.tv)/(?:f|embed)/(?P<id>[a-zA-Z0-9-_]+)'
 
     _TESTS = [{
         'url': 'https://openload.co/f/kUEfGclsU9o',
@@ -286,6 +286,9 @@ class OpenloadIE(InfoExtractor):
     }, {
         'url': 'https://oload.tv/embed/KnG-kKZdcfY/',
         'only_matching': True,
+    }, {
+        'url': 'http://www.openload.link/f/KnG-kKZdcfY',
+        'only_matching': True,
     }]
 
     _USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'

From 6ff27b8d5a11f0960f53b93a1ea2423d3384af21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Nov 2017 00:05:28 +0700
Subject: [PATCH 1544/1696] [openload] Don't use bare except when removing temp
 files

---
 youtube_dl/extractor/openload.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index efa5d4601..a99af12a4 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -140,7 +140,7 @@ class PhantomJSwrapper(object):
         for name in self._TMP_FILE_NAMES:
             try:
                 os.remove(self._TMP_FILES[name].name)
-            except:
+            except (IOError, OSError):
                 pass
 
     def _save_cookies(self, url):

From c0f647a179b8d75a5f23a06f208ff8c1c0088a54 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 25 Nov 2017 18:13:17 +0100
Subject: [PATCH 1545/1696] [nexx] extract more formats

---
 youtube_dl/extractor/nexx.py | 69 ++++++++++++++++++++++++++----------
 1 file changed, 50 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/nexx.py b/youtube_dl/extractor/nexx.py
index 071879ba4..d79a1ce28 100644
--- a/youtube_dl/extractor/nexx.py
+++ b/youtube_dl/extractor/nexx.py
@@ -28,7 +28,7 @@ class NexxIE(InfoExtractor):
     _TESTS = [{
         # movie
         'url': 'https://api.nexx.cloud/v3/748/videos/byid/128907',
-        'md5': '16746bfc28c42049492385c989b26c4a',
+        'md5': '828cea195be04e66057b846288295ba1',
         'info_dict': {
             'id': '128907',
             'ext': 'mp4',
@@ -42,9 +42,6 @@ class NexxIE(InfoExtractor):
             'timestamp': 1384264416,
             'upload_date': '20131112',
         },
-        'params': {
-            'format': 'bestvideo',
-        },
     }, {
         # episode
         'url': 'https://api.nexx.cloud/v3/741/videos/byid/247858',
@@ -62,7 +59,6 @@ class NexxIE(InfoExtractor):
             'season_number': 2,
         },
         'params': {
-            'format': 'bestvideo',
             'skip_download': True,
         },
     }, {
@@ -193,35 +189,70 @@ class NexxIE(InfoExtractor):
         stream_data = video['streamdata']
         language = general.get('language_raw') or ''
 
-        # TODO: reverse more cdns and formats
+        # TODO: reverse more cdns
 
         cdn = stream_data['cdnType']
         assert cdn == 'azure'
 
         azure_locator = stream_data['azureLocator']
 
-        AZURE_URL = 'http://nx-p%02d.akamaized.net/'
+        AZURE_URL = 'http://nx%s%02d.akamaized.net/'
 
-        for secure in ('s', ''):
-            cdn_shield = stream_data.get('cdnShieldHTTP%s' % secure.upper())
-            if cdn_shield:
-                azure_base = 'http%s://%s' % (secure, cdn_shield)
-                break
-        else:
-            azure_base = AZURE_URL % int(stream_data['azureAccount'].replace('nexxplayplus', ''))
+        def get_cdn_base(cdn, prefix='-p'):
+            azure_base = None
+            for secure in ('', 's'):
+                cdn_shield = stream_data.get('cdn%sHTTP%s' % (cdn, secure.upper()))
+                if cdn_shield:
+                    azure_base = 'http%s://%s' % (secure, cdn_shield)
+                    break
+            else:
+                azure_base = AZURE_URL % (prefix, int(stream_data['azureAccount'].replace('nexxplayplus', '')))
+            return azure_base
 
+        azure_stream_base = get_cdn_base('Shield')
         is_ml = ',' in language
-        azure_m3u8_url = '%s%s/%s_src%s.ism/Manifest(format=m3u8-aapl)' % (
-            azure_base, azure_locator, video_id, ('_manifest' if is_ml else ''))
+        azure_manifest_url = '%s%s/%s_src%s.ism/Manifest' % (
+            azure_stream_base, azure_locator, video_id, ('_manifest' if is_ml else '')) + '%s'
 
         protection_token = try_get(
             video, lambda x: x['protectiondata']['token'], compat_str)
         if protection_token:
-            azure_m3u8_url += '?hdnts=%s' % protection_token
+            azure_manifest_url += '?hdnts=%s' % protection_token
 
         formats = self._extract_m3u8_formats(
-            azure_m3u8_url, video_id, 'mp4', entry_protocol='m3u8_native',
-            m3u8_id='%s-hls' % cdn)
+            azure_manifest_url % '(format=m3u8-aapl)',
+            video_id, 'mp4', 'm3u8_native',
+            m3u8_id='%s-hls' % cdn, fatal=False)
+        formats.extend(self._extract_mpd_formats(
+            azure_manifest_url % '(format=mpd-time-csf)',
+            video_id, mpd_id='%s-dash' % cdn, fatal=False))
+        formats.extend(self._extract_ism_formats(
+            azure_manifest_url % '', video_id, ism_id='%s-mss' % cdn, fatal=False))
+
+        azure_progressive_base = get_cdn_base('ShieldProg', '-d')
+        azure_file_distribution = stream_data.get('azureFileDistribution')
+        if azure_file_distribution:
+            fds = azure_file_distribution.split(',')
+            if fds:
+                for fd in fds:
+                    ss = fd.split(':')
+                    if len(ss) == 2:
+                        tbr = int_or_none(ss[0])
+                        if tbr:
+                            f = {
+                                'url': '%s%s/%s_src_%s_%d.mp4' % (
+                                    azure_progressive_base, azure_locator, video_id, ss[1], tbr),
+                                'format_id': 'http-%d' % tbr,
+                                'tbr': tbr,
+                            }
+                            width_height = ss[1].split('x')
+                            if len(width_height) == 2:
+                                f.update({
+                                    'width': int_or_none(width_height[0]),
+                                    'height': int_or_none(width_height[1]),
+                                })
+                            formats.append(f)
+
         self._sort_formats(formats)
 
         return {

From a238a868ba33b4683cea57a05893b339a302ff9f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 25 Nov 2017 18:25:00 +0100
Subject: [PATCH 1546/1696] [freespeech] fix extraction

---
 youtube_dl/extractor/freespeech.py | 31 ++++++++++++++----------------
 1 file changed, 14 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/freespeech.py b/youtube_dl/extractor/freespeech.py
index 7fa271b51..486a49c05 100644
--- a/youtube_dl/extractor/freespeech.py
+++ b/youtube_dl/extractor/freespeech.py
@@ -1,37 +1,34 @@
 from __future__ import unicode_literals
 
-import re
-import json
-
 from .common import InfoExtractor
 
 
 class FreespeechIE(InfoExtractor):
     IE_NAME = 'freespeech.org'
-    _VALID_URL = r'https?://(?:www\.)?freespeech\.org/video/(?P<title>.+)'
+    _VALID_URL = r'https?://(?:www\.)?freespeech\.org/stories/(?P<id>.+)'
     _TEST = {
         'add_ie': ['Youtube'],
-        'url': 'https://www.freespeech.org/video/obama-romney-campaign-colorado-ahead-debate-0',
+        'url': 'http://www.freespeech.org/stories/fcc-announces-net-neutrality-rollback-whats-stake/',
         'info_dict': {
-            'id': 'poKsVCZ64uU',
-            'ext': 'webm',
-            'title': 'Obama, Romney Campaign in Colorado Ahead of Debate',
-            'description': 'Obama, Romney Campaign in Colorado Ahead of Debate',
-            'uploader': 'freespeechtv',
+            'id': 'waRk6IPqyWM',
+            'ext': 'mp4',
+            'title': 'What\'s At Stake - Net Neutrality Special',
+            'description': 'Presented by MNN and FSTV',
+            'upload_date': '20170728',
             'uploader_id': 'freespeechtv',
-            'upload_date': '20121002',
+            'uploader': 'freespeechtv',
         },
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        title = mobj.group('title')
-        webpage = self._download_webpage(url, title)
-        info_json = self._search_regex(r'jQuery\.extend\(Drupal\.settings, ({.*?})\);', webpage, 'info')
-        info = json.loads(info_json)
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        youtube_url = self._search_regex(
+            r'data-video-url="([^"]+)"',
+            webpage, 'youtube url')
 
         return {
             '_type': 'url',
-            'url': info['jw_player']['basic_video_node_player']['file'],
+            'url': youtube_url,
             'ie_key': 'Youtube',
         }

From b485d5d6bf2052edf6d7218a0e4afa89884f13c6 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 25 Nov 2017 18:36:31 +0100
Subject: [PATCH 1547/1696] [nexx] make http format ids more consistent

---
 youtube_dl/extractor/nexx.py | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/nexx.py b/youtube_dl/extractor/nexx.py
index d79a1ce28..9203c0477 100644
--- a/youtube_dl/extractor/nexx.py
+++ b/youtube_dl/extractor/nexx.py
@@ -198,18 +198,15 @@ class NexxIE(InfoExtractor):
 
         AZURE_URL = 'http://nx%s%02d.akamaized.net/'
 
-        def get_cdn_base(cdn, prefix='-p'):
-            azure_base = None
+        def get_cdn_shield_base(shield_type='', prefix='-p'):
             for secure in ('', 's'):
-                cdn_shield = stream_data.get('cdn%sHTTP%s' % (cdn, secure.upper()))
+                cdn_shield = stream_data.get('cdnShield%sHTTP%s' % (shield_type, secure.upper()))
                 if cdn_shield:
-                    azure_base = 'http%s://%s' % (secure, cdn_shield)
-                    break
+                    return 'http%s://%s' % (secure, cdn_shield)
             else:
-                azure_base = AZURE_URL % (prefix, int(stream_data['azureAccount'].replace('nexxplayplus', '')))
-            return azure_base
+                return AZURE_URL % (prefix, int(stream_data['azureAccount'].replace('nexxplayplus', '')))
 
-        azure_stream_base = get_cdn_base('Shield')
+        azure_stream_base = get_cdn_shield_base()
         is_ml = ',' in language
         azure_manifest_url = '%s%s/%s_src%s.ism/Manifest' % (
             azure_stream_base, azure_locator, video_id, ('_manifest' if is_ml else '')) + '%s'
@@ -229,7 +226,7 @@ class NexxIE(InfoExtractor):
         formats.extend(self._extract_ism_formats(
             azure_manifest_url % '', video_id, ism_id='%s-mss' % cdn, fatal=False))
 
-        azure_progressive_base = get_cdn_base('ShieldProg', '-d')
+        azure_progressive_base = get_cdn_shield_base('Prog', '-d')
         azure_file_distribution = stream_data.get('azureFileDistribution')
         if azure_file_distribution:
             fds = azure_file_distribution.split(',')
@@ -242,7 +239,7 @@ class NexxIE(InfoExtractor):
                             f = {
                                 'url': '%s%s/%s_src_%s_%d.mp4' % (
                                     azure_progressive_base, azure_locator, video_id, ss[1], tbr),
-                                'format_id': 'http-%d' % tbr,
+                                'format_id': '%s-http-%d' % (cdn, tbr),
                                 'tbr': tbr,
                             }
                             width_height = ss[1].split('x')

From 87dac57cf69f7a4528c511dc7c1b9a690199cf1e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 25 Nov 2017 18:50:15 +0100
Subject: [PATCH 1548/1696] [firstpost] remove extractor

---
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/firstpost.py  | 50 ------------------------------
 2 files changed, 51 deletions(-)
 delete mode 100644 youtube_dl/extractor/firstpost.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index bc1ca0f0a..d3104f393 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -344,7 +344,6 @@ from .filmon import (
     FilmOnIE,
     FilmOnChannelIE,
 )
-from .firstpost import FirstpostIE
 from .firsttv import FirstTVIE
 from .fivemin import FiveMinIE
 from .fivetv import FiveTVIE
diff --git a/youtube_dl/extractor/firstpost.py b/youtube_dl/extractor/firstpost.py
deleted file mode 100644
index e8936cb24..000000000
--- a/youtube_dl/extractor/firstpost.py
+++ /dev/null
@@ -1,50 +0,0 @@
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-
-
-class FirstpostIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?firstpost\.com/[^/]+/.*-(?P<id>[0-9]+)\.html'
-
-    _TEST = {
-        'url': 'http://www.firstpost.com/india/india-to-launch-indigenous-aircraft-carrier-monday-1025403.html',
-        'md5': 'ee9114957692f01fb1263ed87039112a',
-        'info_dict': {
-            'id': '1025403',
-            'ext': 'mp4',
-            'title': 'India to launch indigenous aircraft carrier INS Vikrant today',
-            'description': 'md5:feef3041cb09724e0bdc02843348f5f4',
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        page = self._download_webpage(url, video_id)
-
-        title = self._html_search_meta('twitter:title', page, 'title', fatal=True)
-        description = self._html_search_meta('twitter:description', page, 'title')
-
-        data = self._download_xml(
-            'http://www.firstpost.com/getvideoxml-%s.xml' % video_id, video_id,
-            'Downloading video XML')
-
-        item = data.find('./playlist/item')
-        thumbnail = item.find('./image').text
-
-        formats = [
-            {
-                'url': details.find('./file').text,
-                'format_id': details.find('./label').text.strip(),
-                'width': int(details.find('./width').text.strip()),
-                'height': int(details.find('./height').text.strip()),
-            } for details in item.findall('./source/file_details') if details.find('./file').text
-        ]
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'formats': formats,
-        }

From 93f3f10cdcf40bdeb6c61b36bbbc0d0ab45ba66c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 25 Nov 2017 19:28:26 +0100
Subject: [PATCH 1549/1696] [fczenit] fix extraction

---
 youtube_dl/extractor/fczenit.py | 40 ++++++++++++++++++++-------------
 1 file changed, 24 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/fczenit.py b/youtube_dl/extractor/fczenit.py
index 8d1010b88..8db7c5963 100644
--- a/youtube_dl/extractor/fczenit.py
+++ b/youtube_dl/extractor/fczenit.py
@@ -2,7 +2,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
+from ..utils import (
+    int_or_none,
+    float_or_none,
+)
 
 
 class FczenitIE(InfoExtractor):
@@ -14,6 +17,8 @@ class FczenitIE(InfoExtractor):
             'id': '41044',
             'ext': 'mp4',
             'title': 'Так пишется история: казанский разгром ЦСКА на «Зенит-ТВ»',
+            'timestamp': 1462283735,
+            'upload_date': '20160503',
         },
     }
 
@@ -21,28 +26,31 @@ class FczenitIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        video_title = self._html_search_regex(
-            r'<[^>]+class=\"photoalbum__title\">([^<]+)', webpage, 'title')
+        msi_id = self._search_regex(
+            r"(?s)config\s*=\s*{.+?video_id\s*:\s*'([^']+)'", webpage, 'msi id')
 
-        video_items = self._parse_json(self._search_regex(
-            r'arrPath\s*=\s*JSON\.parse\(\'(.+)\'\)', webpage, 'video items'),
-            video_id)
-
-        def merge_dicts(*dicts):
-            ret = {}
-            for a_dict in dicts:
-                ret.update(a_dict)
-            return ret
+        msi_data = self._download_json(
+            'http://player.fc-zenit.ru/msi/video', msi_id, query={
+                'video': msi_id,
+            })['data']
+        title = msi_data['name']
 
         formats = [{
-            'url': compat_urlparse.urljoin(url, video_url),
-            'tbr': int(tbr),
-        } for tbr, video_url in merge_dicts(*video_items).items()]
+            'format_id': q.get('label'),
+            'url': q['url'],
+            'height': int_or_none(q.get('label')),
+        } for q in msi_data['qualities'] if q.get('url')]
 
         self._sort_formats(formats)
 
+        tags = [tag['label'] for tag in msi_data.get('tags', []) if tag.get('label')]
+
         return {
             'id': video_id,
-            'title': video_title,
+            'title': title,
+            'thumbnail': msi_data.get('preview'),
             'formats': formats,
+            'duration': float_or_none(msi_data.get('duration')),
+            'timestamp': int_or_none(msi_data.get('date')),
+            'tags': tags,
         }

From 7512aa986f517c301fb3272584d21c8d5cec720f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Nov 2017 20:53:10 +0700
Subject: [PATCH 1550/1696] Fix some only matching tests (closes #14855)

---
 youtube_dl/extractor/orf.py     | 6 +++---
 youtube_dl/extractor/rozhlas.py | 2 +-
 youtube_dl/extractor/vidzi.py   | 4 ++--
 youtube_dl/extractor/youku.py   | 2 +-
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index 74fe8017e..c1fb580ca 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -49,13 +49,13 @@ class ORFTVthekIE(InfoExtractor):
         'params': {
             'skip_download': True,  # rtsp downloads
         },
-        '_skip': 'Blocked outside of Austria / Germany',
+        'skip': 'Blocked outside of Austria / Germany',
     }, {
         'url': 'http://tvthek.orf.at/topic/Fluechtlingskrise/10463081/Heimat-Fremde-Heimat/13879132/Senioren-betreuen-Migrantenkinder/13879141',
-        'skip_download': True,
+        'only_matching': True,
     }, {
         'url': 'http://tvthek.orf.at/profile/Universum/35429',
-        'skip_download': True,
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
diff --git a/youtube_dl/extractor/rozhlas.py b/youtube_dl/extractor/rozhlas.py
index f8eda8dea..fccf69401 100644
--- a/youtube_dl/extractor/rozhlas.py
+++ b/youtube_dl/extractor/rozhlas.py
@@ -21,7 +21,7 @@ class RozhlasIE(InfoExtractor):
         }
     }, {
         'url': 'http://prehravac.rozhlas.cz/audio/3421320/embed',
-        'skip_download': True,
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
diff --git a/youtube_dl/extractor/vidzi.py b/youtube_dl/extractor/vidzi.py
index e64873bce..ac35d55a9 100644
--- a/youtube_dl/extractor/vidzi.py
+++ b/youtube_dl/extractor/vidzi.py
@@ -28,10 +28,10 @@ class VidziIE(InfoExtractor):
         },
     }, {
         'url': 'http://vidzi.tv/embed-4z2yb0rzphe9-600x338.html',
-        'skip_download': True,
+        'only_matching': True,
     }, {
         'url': 'http://vidzi.cc/cghql9yq6emu.html',
-        'skip_download': True,
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 6822a30bc..a2b3b4daf 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -240,7 +240,7 @@ class YoukuShowIE(InfoExtractor):
     }, {
         # Ongoing playlist. The initial page is the last one
         'url': 'http://list.youku.com/show/id_za7c275ecd7b411e1a19e.html',
-        'only_matchine': True,
+        'only_matching': True,
     }]
 
     def _extract_entries(self, playlist_data_url, show_id, note, query):

From d08dcd2dbd4c0d63ff2f4dd504aceceabfa212cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Nov 2017 21:06:14 +0700
Subject: [PATCH 1551/1696] [test_YoutubeDL] Fix typo (closes #14856)

---
 test/test_YoutubeDL.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 4af92fbd4..f0f5a8470 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -466,11 +466,11 @@ class TestFormatSelection(unittest.TestCase):
         ydl = YDL({'simulate': True})
         self.assertEqual(ydl._default_format_spec({}), 'bestvideo+bestaudio/best')
 
-        ydl = YDL({'is_live': True})
-        self.assertEqual(ydl._default_format_spec({}), 'best/bestvideo+bestaudio')
+        ydl = YDL({})
+        self.assertEqual(ydl._default_format_spec({'is_live': True}), 'best/bestvideo+bestaudio')
 
-        ydl = YDL({'simulate': True, 'is_live': True})
-        self.assertEqual(ydl._default_format_spec({}), 'bestvideo+bestaudio/best')
+        ydl = YDL({'simulate': True})
+        self.assertEqual(ydl._default_format_spec({'is_live': True}), 'bestvideo+bestaudio/best')
 
         ydl = YDL({'outtmpl': '-'})
         self.assertEqual(ydl._default_format_spec({}), 'best/bestvideo+bestaudio')

From e94d1adc360d0fbfadced82c72a7f20375a1ec87 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Nov 2017 21:10:32 +0700
Subject: [PATCH 1552/1696] Add testdata to youtube-dl.tar.gz (closes #14854)

---
 Makefile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Makefile b/Makefile
index c74eea792..141593efd 100644
--- a/Makefile
+++ b/Makefile
@@ -110,7 +110,6 @@ youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-
 		--exclude '*~' \
 		--exclude '__pycache__' \
 		--exclude '.git' \
-		--exclude 'testdata' \
 		--exclude 'docs/_build' \
 		-- \
 		bin devscripts test youtube_dl docs \

From 6c07f0b288a674c385308cccef31d034b6bd3788 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Nov 2017 21:37:27 +0700
Subject: [PATCH 1553/1696] [ChangeLog] Actualize

---
 ChangeLog | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index c4a84c597..1f1b704cc 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,8 +1,31 @@
 version <unreleased>
 
+Core
+* [extractor/common] Use final URL when dumping request (#14769)
+
 Extractors
+* [fczenit] Fix extraction
+- [firstpost] Remove extractor
+* [freespeech] Fix extraction
+* [nexx] Extract more formats
++ [openload] Add support for openload.link (#14763)
+* [empflix] Relax URL regular expression
+* [empflix] Fix extractrion
+* [tnaflix] Don't modify download URLs (#14811)
+- [gamersyde] Remove extractor
+* [francetv:generationwhat] Fix extraction
++ [massengeschmacktv] Add support for Massengeschmack TV
+* [fox9] Fix extraction
+* [faz] Fix extraction and add support for Perform Group embeds (#14714)
++ [performgroup] Add support for performgroup.com
++ [jwplatform] Add support for iframes (#14828)
+* [culturebox] Fix extraction (#14827)
 * [youku] Fix extraction; update ccode (#14815)
-+ [JWPlatform] support iframes (#14828)
+* [livestream] Make SMIL extraction non fatal (#14792)
++ [drtuber] Add support for mobile URLs (#14772)
++ [spankbang] Add support for mobile URLs (#14771)
+* [instagram] Fix description, timestamp and counters extraction (#14755)
+
 
 version 2017.11.15
 

From 5ddeb7702a2c14d51a2c76178c7abb30a4ea06e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Nov 2017 21:49:22 +0700
Subject: [PATCH 1554/1696] release 2017.11.26

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 7 +++----
 youtube_dl/version.py     | 2 +-
 4 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 4dd1a6e59..15211e5d9 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.11.15*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.11.15**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.11.26*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.11.26**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.11.15
+[debug] youtube-dl version 2017.11.26
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 1f1b704cc..a568fa233 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.11.26
 
 Core
 * [extractor/common] Use final URL when dumping request (#14769)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 6009df571..b44c9e7a3 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -266,10 +266,8 @@
  - **fc2**
  - **fc2:embed**
  - **Fczenit**
- - **fernsehkritik.tv**
  - **filmon**
  - **filmon:channel**
- - **Firstpost**
  - **FiveTV**
  - **Flickr**
  - **Flipagram**
@@ -283,7 +281,7 @@
  - **foxnews:article**
  - **foxnews:insider**
  - **FoxSports**
- - **france2.fr:generation-quoi**
+ - **france2.fr:generation-what**
  - **FranceCulture**
  - **FranceInter**
  - **FranceTV**
@@ -301,7 +299,6 @@
  - **GameInformer**
  - **GameOne**
  - **gameone:playlist**
- - **Gamersyde**
  - **GameSpot**
  - **GameStar**
  - **Gaskrank**
@@ -441,6 +438,7 @@
  - **mangomolo:live**
  - **mangomolo:video**
  - **ManyVids**
+ - **massengeschmack.tv**
  - **MatchTV**
  - **MDR**: MDR.DE and KiKA
  - **media.ccc.de**
@@ -608,6 +606,7 @@
  - **pcmag**
  - **PearVideo**
  - **People**
+ - **PerformGroup**
  - **periscope**: Periscope
  - **periscope:user**: Periscope user videos
  - **PhilharmonieDeParis**: Philharmonie de Paris
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1c3cbefeb..81e3f792d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.11.15'
+__version__ = '2017.11.26'

From f58a5060443229e017a04e79cb37d499b6347fe3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 27 Nov 2017 21:30:47 +0800
Subject: [PATCH 1555/1696] [test_InfoExtractor] Fix flake8

---
 test/test_InfoExtractor.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 686c63efa..7b95f883f 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -601,5 +601,6 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 self.ie._sort_formats(formats)
                 expect_value(self, formats, expected_formats, None)
 
+
 if __name__ == '__main__':
     unittest.main()

From 82a62de192401157a5eb44cacba16296795b4ee5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 27 Nov 2017 21:32:06 +0800
Subject: [PATCH 1556/1696] [Makefile,devscripts/run_tests.sh] Actually exclude
 network tests

Closes #14858
---
 Makefile                | 11 ++++++++++-
 devscripts/run_tests.sh |  3 ++-
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 141593efd..0c3936811 100644
--- a/Makefile
+++ b/Makefile
@@ -36,8 +36,17 @@ test:
 
 ot: offlinetest
 
+# Keep this list in sync with devscripts/run_tests.sh
 offlinetest: codetest
-	$(PYTHON) -m nose --verbose test --exclude test_download.py --exclude test_age_restriction.py --exclude test_subtitles.py --exclude test_write_annotations.py --exclude test_youtube_lists.py --exclude test_iqiyi_sdk_interpreter.py --exclude test_socks.py
+	$(PYTHON) -m nose --verbose test \
+		--exclude test_age_restriction.py \
+		--exclude test_download.py \
+		--exclude test_iqiyi_sdk_interpreter.py \
+		--exclude test_socks.py \
+		--exclude test_subtitles.py \
+		--exclude test_write_annotations.py \
+		--exclude test_youtube_lists.py \
+		--exclude test_youtube_signature.py
 
 tar: youtube-dl.tar.gz
 
diff --git a/devscripts/run_tests.sh b/devscripts/run_tests.sh
index 6ba26720d..dd37a80f5 100755
--- a/devscripts/run_tests.sh
+++ b/devscripts/run_tests.sh
@@ -1,6 +1,7 @@
 #!/bin/bash
 
-DOWNLOAD_TESTS="age_restriction|download|subtitles|write_annotations|iqiyi_sdk_interpreter|youtube_lists"
+# Keep this list in sync with the `offlinetest` target in Makefile
+DOWNLOAD_TESTS="age_restriction|download|iqiyi_sdk_interpreter|socks|subtitles|write_annotations|youtube_lists|youtube_signature"
 
 test_set=""
 multiprocess_args=""

From dafb4c66476754a46cc83fa750e9237da8fbf595 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 27 Nov 2017 22:49:35 +0800
Subject: [PATCH 1557/1696] [Makefile] Include setup.cfg in the tarball (closes
 #14857)

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 0c3936811..1c760bef8 100644
--- a/Makefile
+++ b/Makefile
@@ -124,5 +124,5 @@ youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-
 		bin devscripts test youtube_dl docs \
 		ChangeLog LICENSE README.md README.txt \
 		Makefile MANIFEST.in youtube-dl.1 youtube-dl.bash-completion \
-		youtube-dl.zsh youtube-dl.fish setup.py \
+		youtube-dl.zsh youtube-dl.fish setup.py setup.cfg \
 		youtube-dl

From ffe6979ef925cb8b2e9324c2ad4635cb01d45ab7 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 27 Nov 2017 16:53:42 +0100
Subject: [PATCH 1558/1696] [utils] add hvc1 codec code to parse_codecs

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 34866a54b..febc9d26f 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2372,7 +2372,7 @@ def parse_codecs(codecs_str):
     vcodec, acodec = None, None
     for full_codec in splited_codecs:
         codec = full_codec.split('.')[0]
-        if codec in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2', 'h263', 'h264', 'mp4v'):
+        if codec in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2', 'h263', 'h264', 'mp4v', 'hvc1'):
             if not vcodec:
                 vcodec = full_codec
         elif codec in ('mp4a', 'opus', 'vorbis', 'mp3', 'aac', 'ac-3', 'ec-3', 'eac3', 'dtsc', 'dtse', 'dtsh', 'dtsl'):

From 53f024e7c5bcea57b2446b3b5550801c011736d1 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 27 Nov 2017 16:55:08 +0100
Subject: [PATCH 1559/1696] [pbs] add another media id regex

---
 youtube_dl/extractor/pbs.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index b51dcbe10..f11d5da52 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -421,6 +421,7 @@ class PBSIE(InfoExtractor):
                 r'class="coveplayerid">([^<]+)<',                       # coveplayer
                 r'<section[^>]+data-coveid="(\d+)"',                    # coveplayer from http://www.pbs.org/wgbh/frontline/film/real-csi/
                 r'<input type="hidden" id="pbs_video_id_[0-9]+" value="([0-9]+)"/>',  # jwplayer
+                r"(?s)window\.PBS\.playerConfig\s*=\s*{.*?id\s*:\s*'([0-9]+)',",
             ]
 
             media_id = self._search_regex(

From 115afb77ec2d31764307c06866b63ba859d3aafb Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 27 Nov 2017 21:59:27 +0100
Subject: [PATCH 1560/1696] [itv] update hls formats extraction

---
 youtube_dl/extractor/itv.py | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py
index 26c48e4b8..413a219dc 100644
--- a/youtube_dl/extractor/itv.py
+++ b/youtube_dl/extractor/itv.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import uuid
 import xml.etree.ElementTree as etree
 import json
+import re
 
 from .common import InfoExtractor
 from ..compat import (
@@ -142,9 +143,9 @@ class ITVIE(InfoExtractor):
                 f['url'] = rtmp_url
             formats.append(f)
 
-        ios_playlist_url = params.get('data-video-playlist')
+        ios_playlist_url = params.get('data-video-playlist') or params.get('data-video-id')
         hmac = params.get('data-video-hmac')
-        if ios_playlist_url and hmac:
+        if ios_playlist_url and hmac and re.match(r'https?://', ios_playlist_url):
             headers = self.geo_verification_headers()
             headers.update({
                 'Accept': 'application/vnd.itv.vod.playlist.v2+json',
@@ -159,12 +160,12 @@ class ITVIE(InfoExtractor):
                         'token': ''
                     },
                     'device': {
-                        'manufacturer': 'Apple',
-                        'model': 'iPad',
+                        'manufacturer': 'Safari',
+                        'model': '5',
                         'os': {
-                            'name': 'iPhone OS',
-                            'version': '9.3',
-                            'type': 'ios'
+                            'name': 'Windows NT',
+                            'version': '6.1',
+                            'type': 'desktop'
                         }
                     },
                     'client': {
@@ -173,10 +174,10 @@ class ITVIE(InfoExtractor):
                     },
                     'variantAvailability': {
                         'featureset': {
-                            'min': ['hls', 'aes'],
-                            'max': ['hls', 'aes']
+                            'min': ['hls', 'aes', 'outband-webvtt'],
+                            'max': ['hls', 'aes', 'outband-webvtt']
                         },
-                        'platformTag': 'mobile'
+                        'platformTag': 'dotcom'
                     }
                 }).encode(), headers=headers, fatal=False)
             if ios_playlist:

From a3474aa59e44b15471f46fcac8680d79e82d5116 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 28 Nov 2017 09:04:51 +0100
Subject: [PATCH 1561/1696] [Odnoklassniki] fix api metadata request(fixes
 #14862)

---
 youtube_dl/extractor/odnoklassniki.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 854b6800c..8e13bcf1f 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -14,6 +14,7 @@ from ..utils import (
     int_or_none,
     qualities,
     unescapeHTML,
+    urlencode_postdata,
 )
 
 
@@ -56,7 +57,7 @@ class OdnoklassnikiIE(InfoExtractor):
         'url': 'http://ok.ru/video/64211978996595-1',
         'md5': '2f206894ffb5dbfcce2c5a14b909eea5',
         'info_dict': {
-            'id': '64211978996595-1',
+            'id': 'V_VztHT5BzY',
             'ext': 'mp4',
             'title': 'Космическая среда от 26 августа 2015',
             'description': 'md5:848eb8b85e5e3471a3a803dae1343ed0',
@@ -127,9 +128,14 @@ class OdnoklassnikiIE(InfoExtractor):
         if metadata:
             metadata = self._parse_json(metadata, video_id)
         else:
+            data = {}
+            st_location = flashvars.get('location')
+            if st_location:
+                data['st.location'] = st_location
             metadata = self._download_json(
                 compat_urllib_parse_unquote(flashvars['metadataUrl']),
-                video_id, 'Downloading metadata JSON')
+                video_id, 'Downloading metadata JSON',
+                data=urlencode_postdata(data))
 
         movie = metadata['movie']
 

From 5f699251e9421035cc89baf0b9dc78f3b6eacb77 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 28 Nov 2017 10:55:32 +0100
Subject: [PATCH 1562/1696] [daisuki] add support for motto.daisuki.com(fixes
 #14681)

---
 youtube_dl/extractor/daisuki.py    | 84 ++++++++++++++----------------
 youtube_dl/extractor/extractors.py |  4 +-
 2 files changed, 41 insertions(+), 47 deletions(-)

diff --git a/youtube_dl/extractor/daisuki.py b/youtube_dl/extractor/daisuki.py
index 58cc98666..5c9ac68a0 100644
--- a/youtube_dl/extractor/daisuki.py
+++ b/youtube_dl/extractor/daisuki.py
@@ -13,33 +13,30 @@ from ..aes import (
 from ..utils import (
     bytes_to_intlist,
     bytes_to_long,
-    clean_html,
+    extract_attributes,
     ExtractorError,
     intlist_to_bytes,
-    get_element_by_id,
     js_to_json,
     int_or_none,
     long_to_bytes,
     pkcs1pad,
-    remove_end,
 )
 
 
-class DaisukiIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?daisuki\.net/[^/]+/[^/]+/[^/]+/watch\.[^.]+\.(?P<id>\d+)\.html'
+class DaisukiMottoIE(InfoExtractor):
+    _VALID_URL = r'https?://motto\.daisuki\.net/framewatch/embed/[^/]+/(?P<id>[0-9a-zA-Z]{3})'
 
     _TEST = {
-        'url': 'http://www.daisuki.net/tw/en/anime/watch.TheIdolMasterCG.11213.html',
+        'url': 'http://motto.daisuki.net/framewatch/embed/embedDRAGONBALLSUPERUniverseSurvivalsaga/V2e/760/428',
         'info_dict': {
-            'id': '11213',
+            'id': 'V2e',
             'ext': 'mp4',
-            'title': '#01 Who is in the pumpkin carriage? - THE IDOLM@STER CINDERELLA GIRLS',
+            'title': '#117 SHOWDOWN OF LOVE! ANDROIDS VS UNIVERSE 2!!',
             'subtitles': {
                 'mul': [{
                     'ext': 'ttml',
                 }],
             },
-            'creator': 'BANDAI NAMCO Entertainment',
         },
         'params': {
             'skip_download': True,  # AES-encrypted HLS stream
@@ -73,15 +70,17 @@ class DaisukiIE(InfoExtractor):
 
             n, e = self._RSA_KEY
             encrypted_aeskey = long_to_bytes(pow(bytes_to_long(padded_aeskey), e, n))
-            init_data = self._download_json('http://www.daisuki.net/bin/bgn/init', video_id, query={
-                's': flashvars.get('s', ''),
-                'c': flashvars.get('ss3_prm', ''),
-                'e': url,
-                'd': base64.b64encode(intlist_to_bytes(aes_cbc_encrypt(
-                    bytes_to_intlist(json.dumps(data)),
-                    aes_key, iv))).decode('ascii'),
-                'a': base64.b64encode(encrypted_aeskey).decode('ascii'),
-            }, note='Downloading JSON metadata' + (' (try #%d)' % (idx + 1) if idx > 0 else ''))
+            init_data = self._download_json(
+                'http://motto.daisuki.net/fastAPI/bgn/init/',
+                video_id, query={
+                    's': flashvars.get('s', ''),
+                    'c': flashvars.get('ss3_prm', ''),
+                    'e': url,
+                    'd': base64.b64encode(intlist_to_bytes(aes_cbc_encrypt(
+                        bytes_to_intlist(json.dumps(data)),
+                        aes_key, iv))).decode('ascii'),
+                    'a': base64.b64encode(encrypted_aeskey).decode('ascii'),
+                }, note='Downloading JSON metadata' + (' (try #%d)' % (idx + 1) if idx > 0 else ''))
 
             if 'rtn' in init_data:
                 encrypted_rtn = init_data['rtn']
@@ -98,14 +97,11 @@ class DaisukiIE(InfoExtractor):
                 aes_key, iv)).decode('utf-8').rstrip('\0'),
             video_id)
 
+        title = rtn['title_str']
+
         formats = self._extract_m3u8_formats(
             rtn['play_url'], video_id, ext='mp4', entry_protocol='m3u8_native')
 
-        title = remove_end(self._og_search_title(webpage), ' - DAISUKI')
-
-        creator = self._html_search_regex(
-            r'Creator\s*:\s*([^<]+)', webpage, 'creator', fatal=False)
-
         subtitles = {}
         caption_url = rtn.get('caption_url')
         if caption_url:
@@ -120,21 +116,18 @@ class DaisukiIE(InfoExtractor):
             'title': title,
             'formats': formats,
             'subtitles': subtitles,
-            'creator': creator,
         }
 
 
-class DaisukiPlaylistIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)daisuki\.net/[^/]+/[^/]+/[^/]+/detail\.(?P<id>[a-zA-Z0-9]+)\.html'
+class DaisukiMottoPlaylistIE(InfoExtractor):
+    _VALID_URL = r'https?://motto\.daisuki\.net/(?P<id>information)/'
 
     _TEST = {
-        'url': 'http://www.daisuki.net/tw/en/anime/detail.TheIdolMasterCG.html',
+        'url': 'http://motto.daisuki.net/information/',
         'info_dict': {
-            'id': 'TheIdolMasterCG',
-            'title': 'THE IDOLM@STER CINDERELLA GIRLS',
-            'description': 'md5:0f2c028a9339f7a2c7fbf839edc5c5d8',
+            'title': 'DRAGON BALL SUPER',
         },
-        'playlist_count': 26,
+        'playlist_mincount': 117,
     }
 
     def _real_extract(self, url):
@@ -142,18 +135,19 @@ class DaisukiPlaylistIE(InfoExtractor):
 
         webpage = self._download_webpage(url, playlist_id)
 
-        episode_pattern = r'''(?sx)
-            <img[^>]+delay="[^"]+/(\d+)/movie\.jpg".+?
-            <p[^>]+class=".*?\bepisodeNumber\b.*?">(?:<a[^>]+>)?([^<]+)'''
-        entries = [{
-            '_type': 'url_transparent',
-            'url': url.replace('detail', 'watch').replace('.html', '.' + movie_id + '.html'),
-            'episode_id': episode_id,
-            'episode_number': int_or_none(episode_id),
-        } for movie_id, episode_id in re.findall(episode_pattern, webpage)]
+        entries = []
+        for li in re.findall(r'(<li[^>]+?data-product_id="[a-zA-Z0-9]{3}"[^>]+>)', webpage):
+            attr = extract_attributes(li)
+            ad_id = attr.get('data-ad_id')
+            product_id = attr.get('data-product_id')
+            if ad_id and product_id:
+                episode_id = attr.get('data-chapter')
+                entries.append({
+                    '_type': 'url_transparent',
+                    'url': 'http://motto.daisuki.net/framewatch/embed/%s/%s/760/428' % (ad_id, product_id),
+                    'episode_id': episode_id,
+                    'episode_number': int_or_none(episode_id),
+                    'ie_key': 'DaisukiMotto',
+                })
 
-        playlist_title = remove_end(
-            self._og_search_title(webpage, fatal=False), ' - Anime - DAISUKI')
-        playlist_description = clean_html(get_element_by_id('synopsisTxt', webpage))
-
-        return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
+        return self.playlist_result(entries, playlist_title='DRAGON BALL SUPER')
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d3104f393..6db1297f1 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -246,8 +246,8 @@ from .dailymotion import (
     DailymotionCloudIE,
 )
 from .daisuki import (
-    DaisukiIE,
-    DaisukiPlaylistIE,
+    DaisukiMottoIE,
+    DaisukiMottoPlaylistIE,
 )
 from .daum import (
     DaumIE,

From 07cf18b9c56bf8ce08404b084672f33e860a805e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 29 Nov 2017 14:21:38 +0100
Subject: [PATCH 1563/1696] [br] add support for BR Mediathek videos(fixes
 #14560)(fixes #14788)

---
 youtube_dl/extractor/br.py         | 146 ++++++++++++++++++++++++++++-
 youtube_dl/extractor/extractors.py |   5 +-
 2 files changed, 147 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/br.py b/youtube_dl/extractor/br.py
index 2c32b6ae2..9bde7f2d8 100644
--- a/youtube_dl/extractor/br.py
+++ b/youtube_dl/extractor/br.py
@@ -1,20 +1,23 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import json
 import re
 
 from .common import InfoExtractor
 from ..utils import (
+    determine_ext,
     ExtractorError,
     int_or_none,
     parse_duration,
+    parse_iso8601,
     xpath_element,
     xpath_text,
 )
 
 
 class BRIE(InfoExtractor):
-    IE_DESC = 'Bayerischer Rundfunk Mediathek'
+    IE_DESC = 'Bayerischer Rundfunk'
     _VALID_URL = r'(?P<base_url>https?://(?:www\.)?br(?:-klassik)?\.de)/(?:[a-z0-9\-_]+/)+(?P<id>[a-z0-9\-_]+)\.html'
 
     _TESTS = [
@@ -123,10 +126,10 @@ class BRIE(InfoExtractor):
         for asset in assets.findall('asset'):
             format_url = xpath_text(asset, ['downloadUrl', 'url'])
             asset_type = asset.get('type')
-            if asset_type == 'HDS':
+            if asset_type.startswith('HDS'):
                 formats.extend(self._extract_f4m_formats(
                     format_url + '?hdcore=3.2.0', media_id, f4m_id='hds', fatal=False))
-            elif asset_type == 'HLS':
+            elif asset_type.startswith('HLS'):
                 formats.extend(self._extract_m3u8_formats(
                     format_url, media_id, 'mp4', 'm3u8_native', m3u8_id='hds', fatal=False))
             else:
@@ -169,3 +172,140 @@ class BRIE(InfoExtractor):
         } for variant in variants.findall('variant') if xpath_text(variant, 'url')]
         thumbnails.sort(key=lambda x: x['width'] * x['height'], reverse=True)
         return thumbnails
+
+
+class BRMediathekIE(InfoExtractor):
+    IE_DESC = 'Bayerischer Rundfunk Mediathek'
+    _VALID_URL = r'https?://(?:www\.)?br\.de/mediathek/video/[^/?&#]*?-(?P<id>av:[0-9a-f]{24})'
+
+    _TESTS = [{
+        'url': 'https://www.br.de/mediathek/video/gesundheit-die-sendung-vom-28112017-av:5a1e6a6e8fce6d001871cc8e',
+        'md5': 'fdc3d485835966d1622587d08ba632ec',
+        'info_dict': {
+            'id': 'av:5a1e6a6e8fce6d001871cc8e',
+            'ext': 'mp4',
+            'title': 'Die Sendung vom 28.11.2017',
+            'description': 'md5:6000cdca5912ab2277e5b7339f201ccc',
+            'timestamp': 1511942766,
+            'upload_date': '20171129',
+        }
+    }]
+
+    def _real_extract(self, url):
+        clip_id = self._match_id(url)
+
+        clip = self._download_json(
+            'https://proxy-base.master.mango.express/graphql',
+            clip_id, data=json.dumps({
+                "query": """{
+  viewer {
+    clip(id: "%s") {
+      title
+      description
+      duration
+      createdAt
+      ageRestriction
+      videoFiles {
+        edges {
+          node {
+            publicLocation
+            fileSize
+            videoProfile {
+              width
+              height
+              bitrate
+              encoding
+            }
+          }
+        }
+      }
+      captionFiles {
+        edges {
+          node {
+            publicLocation
+          }
+        }
+      }
+      teaserImages {
+        edges {
+          node {
+            imageFiles {
+              edges {
+                node {
+                  publicLocation
+                  width
+                  height
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}""" % clip_id}).encode(), headers={
+                'Content-Type': 'application/json',
+            })['data']['viewer']['clip']
+        title = clip['title']
+
+        formats = []
+        for edge in clip.get('videoFiles', {}).get('edges', []):
+            node = edge.get('node', {})
+            n_url = node.get('publicLocation')
+            if not n_url:
+                continue
+            ext = determine_ext(n_url)
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    n_url, clip_id, 'mp4', 'm3u8_native',
+                    m3u8_id='hls', fatal=False))
+            else:
+                video_profile = node.get('videoProfile', {})
+                tbr = int_or_none(video_profile.get('bitrate'))
+                format_id = 'http'
+                if tbr:
+                    format_id += '-%d' % tbr
+                formats.append({
+                    'format_id': format_id,
+                    'url': n_url,
+                    'width': int_or_none(video_profile.get('width')),
+                    'height': int_or_none(video_profile.get('height')),
+                    'tbr': tbr,
+                    'filesize': int_or_none(node.get('fileSize')),
+                })
+        self._sort_formats(formats)
+
+        subtitles = {}
+        for edge in clip.get('captionFiles', {}).get('edges', []):
+            node = edge.get('node', {})
+            n_url = node.get('publicLocation')
+            if not n_url:
+                continue
+            subtitles.setdefault('de', []).append({
+                'url': n_url,
+            })
+
+        thumbnails = []
+        for edge in clip.get('teaserImages', {}).get('edges', []):
+            for image_edge in edge.get('node', {}).get('imageFiles', {}).get('edges', []):
+                node = image_edge.get('node', {})
+                n_url = node.get('publicLocation')
+                if not n_url:
+                    continue
+                thumbnails.append({
+                    'url': n_url,
+                    'width': int_or_none(node.get('width')),
+                    'height': int_or_none(node.get('height')),
+                })
+
+        return {
+            'id': clip_id,
+            'title': title,
+            'description': clip.get('description'),
+            'duration': int_or_none(clip.get('duration')),
+            'timestamp': parse_iso8601(clip.get('createdAt')),
+            'age_limit': int_or_none(clip.get('ageRestriction')),
+            'formats': formats,
+            'subtitles': subtitles,
+            'thumbnails': thumbnails,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 6db1297f1..2cc3bc463 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -127,7 +127,10 @@ from .bloomberg import BloombergIE
 from .bokecc import BokeCCIE
 from .bostonglobe import BostonGlobeIE
 from .bpb import BpbIE
-from .br import BRIE
+from .br import (
+    BRIE,
+    BRMediathekIE,
+)
 from .bravotv import BravoTVIE
 from .breakcom import BreakIE
 from .brightcove import (

From fb61b57d0f4c6422b2722f56f5740ea1b19adfcf Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 29 Nov 2017 16:09:45 +0100
Subject: [PATCH 1564/1696] [vvvvid] fix extraction for kenc videos(fixes
 #13406)

---
 youtube_dl/extractor/vvvvid.py | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vvvvid.py b/youtube_dl/extractor/vvvvid.py
index 656a4b9e5..3d0dc403b 100644
--- a/youtube_dl/extractor/vvvvid.py
+++ b/youtube_dl/extractor/vvvvid.py
@@ -22,6 +22,9 @@ class VVVVIDIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Ping Pong',
         },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         # video_type == 'video/rcs'
         'url': 'https://www.vvvvid.it/#!show/376/death-note-live-action/377/482493/episodio-01',
@@ -31,6 +34,9 @@ class VVVVIDIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Episodio 01',
         },
+        'params': {
+            'skip_download': True,
+        },
     }]
     _conn_id = None
 
@@ -116,8 +122,20 @@ class VVVVIDIE(InfoExtractor):
             embed_code = ds(embed_code)
             video_type = video_data.get('video_type')
             if video_type in ('video/rcs', 'video/kenc'):
-                formats.extend(self._extract_akamai_formats(
-                    embed_code, video_id))
+                embed_code = re.sub(r'https?://([^/]+)/z/', r'https://\1/i/', embed_code).replace('/manifest.f4m', '/master.m3u8')
+                if video_type == 'video/kenc':
+                    kenc = self._download_json(
+                        'https://www.vvvvid.it/kenc', video_id, query={
+                            'action': 'kt',
+                            'conn_id': self._conn_id,
+                            'url': embed_code,
+                        }, fatal=False) or {}
+                    kenc_message = kenc.get('message')
+                    if kenc_message:
+                        embed_code += '?' + ds(kenc_message)
+                formats.extend(self._extract_m3u8_formats(
+                    embed_code, video_id, 'mp4',
+                    m3u8_id='hls', fatal=False))
             else:
                 formats.extend(self._extract_wowza_formats(
                     'http://sb.top-ix.org/videomg/_definst_/mp4:%s/playlist.m3u8' % embed_code, video_id))

From 5ea765fb720523efb2702358e188e5157d08775b Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 29 Nov 2017 17:50:38 +0100
Subject: [PATCH 1565/1696] [mtv] correct scc extention in extracted
 subtitles(closes #13730)

---
 youtube_dl/extractor/mtv.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 1154a3536..7a3b57abd 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -115,10 +115,17 @@ class MTVServicesInfoExtractor(InfoExtractor):
             if transcript.get('kind') != 'captions':
                 continue
             lang = transcript.get('srclang')
-            subtitles[lang] = [{
-                'url': compat_str(typographic.get('src')),
-                'ext': typographic.get('format')
-            } for typographic in transcript.findall('./typographic')]
+            for typographic in transcript.findall('./typographic'):
+                sub_src = typographic.get('src')
+                if not sub_src:
+                    continue
+                ext = typographic.get('format')
+                if ext == 'cea-608':
+                    ext = 'scc'
+                subtitles.setdefault(lang, []).append({
+                    'url': compat_str(sub_src),
+                    'ext': ext
+                })
         return subtitles
 
     def _get_video_info(self, itemdoc, use_hls=True):

From 1663b329460b8b83b4eb0381ccb4f201374647db Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 29 Nov 2017 20:36:55 +0100
Subject: [PATCH 1566/1696] [xiami] add Referer header to api request

---
 youtube_dl/extractor/xiami.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/xiami.py b/youtube_dl/extractor/xiami.py
index d017e03de..7f871c8ec 100644
--- a/youtube_dl/extractor/xiami.py
+++ b/youtube_dl/extractor/xiami.py
@@ -40,9 +40,12 @@ class XiamiBaseIE(InfoExtractor):
             'subtitles': subtitles,
         }
 
-    def _extract_tracks(self, item_id, typ=None):
+    def _extract_tracks(self, item_id, referer, typ=None):
         playlist = self._download_json(
-            '%s/%s%s' % (self._API_BASE_URL, item_id, '/type/%s' % typ if typ else ''), item_id)
+            '%s/%s%s' % (self._API_BASE_URL, item_id, '/type/%s' % typ if typ else ''),
+            item_id, headers={
+                'Referer': referer,
+            })
         return [
             self._extract_track(track, item_id)
             for track in playlist['data']['trackList']]
@@ -135,13 +138,13 @@ class XiamiSongIE(XiamiBaseIE):
     }]
 
     def _real_extract(self, url):
-        return self._extract_tracks(self._match_id(url))[0]
+        return self._extract_tracks(self._match_id(url), url)[0]
 
 
 class XiamiPlaylistBaseIE(XiamiBaseIE):
     def _real_extract(self, url):
         item_id = self._match_id(url)
-        return self.playlist_result(self._extract_tracks(item_id, self._TYPE), item_id)
+        return self.playlist_result(self._extract_tracks(item_id, url, self._TYPE), item_id)
 
 
 class XiamiAlbumIE(XiamiPlaylistBaseIE):

From f5ac68d88f3f11b88213ccba5e079d7d5221e243 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 30 Nov 2017 23:37:05 +0100
Subject: [PATCH 1567/1696] [mnet] fix format extraction(fixes #14883)

---
 youtube_dl/extractor/mnet.py | 30 +++++++++++++++++++-----------
 1 file changed, 19 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/mnet.py b/youtube_dl/extractor/mnet.py
index 6a85dcbd5..0e26ca1b3 100644
--- a/youtube_dl/extractor/mnet.py
+++ b/youtube_dl/extractor/mnet.py
@@ -40,21 +40,29 @@ class MnetIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
+        # TODO: extract rtmp formats
+        # no stype -> rtmp url
+        # stype=H -> m3u8 url
+        # stype=M -> mpd url
         info = self._download_json(
-            'http://content.api.mnet.com/player/vodConfig?id=%s&ctype=CLIP' % video_id,
-            video_id, 'Downloading vod config JSON')['data']['info']
+            'http://content.api.mnet.com/player/vodConfig',
+            video_id, 'Downloading vod config JSON', query={
+                'id': video_id,
+                'ctype': 'CLIP',
+                'stype': 'H',
+            })['data']['info']
 
         title = info['title']
 
-        rtmp_info = self._download_json(
-            info['cdn'], video_id, 'Downloading vod cdn JSON')
-
-        formats = [{
-            'url': rtmp_info['serverurl'] + rtmp_info['fileurl'],
-            'ext': 'flv',
-            'page_url': url,
-            'player_url': 'http://flvfile.mnet.com/service/player/201602/cjem_player_tv.swf?v=201602191318',
-        }]
+        cdn_data = self._download_json(
+            info['cdn'], video_id, 'Downloading vod cdn JSON')['data'][0]
+        m3u8_url = cdn_data['url']
+        token = cdn_data.get('token')
+        if token and token != '-':
+            m3u8_url += '?' + token
+        formats = self._extract_wowza_formats(
+            m3u8_url, video_id, skip_protocols=['rtmp', 'rtsp', 'f4m'])
+        self._sort_formats(formats)
 
         description = info.get('ment')
         duration = parse_duration(info.get('time'))

From 0981585befc7af01331b72304ed11b9865b40401 Mon Sep 17 00:00:00 2001
From: zcanfly <zcanfly@126.com>
Date: Sat, 2 Dec 2017 19:16:22 +0800
Subject: [PATCH 1568/1696] [youku] Update ccode (closes #14872)

---
 youtube_dl/extractor/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index a2b3b4daf..f0ba01197 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -154,7 +154,7 @@ class YoukuIE(InfoExtractor):
         # request basic data
         basic_data_params = {
             'vid': video_id,
-            'ccode': '0502',
+            'ccode': '0501',
             'client_ip': '192.168.1.1',
             'utid': cna,
             'client_ts': time.time() / 1000,

From fea92aa65dbc28e3a5cd109a5b9de5048c9cdfc9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Dec 2017 19:03:24 +0700
Subject: [PATCH 1569/1696] [xhamster] Fix extraction (closes #14884)

---
 youtube_dl/extractor/xhamster.py | 87 ++++++++++++++++++++++++++++++--
 1 file changed, 83 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index be3624ef2..52f8ded2f 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -6,10 +6,12 @@ from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
     clean_html,
+    determine_ext,
     dict_get,
     ExtractorError,
     int_or_none,
     parse_duration,
+    try_get,
     unified_strdate,
 )
 
@@ -32,6 +34,7 @@ class XHamsterIE(InfoExtractor):
             'display_id': 'femaleagent_shy_beauty_takes_the_bait',
             'ext': 'mp4',
             'title': 'FemaleAgent Shy beauty takes the bait',
+            'timestamp': 1350194821,
             'upload_date': '20121014',
             'uploader': 'Ruseful2011',
             'duration': 893,
@@ -45,6 +48,7 @@ class XHamsterIE(InfoExtractor):
             'display_id': 'britney_spears_sexy_booty',
             'ext': 'mp4',
             'title': 'Britney Spears  Sexy Booty',
+            'timestamp': 1379123460,
             'upload_date': '20130914',
             'uploader': 'jojo747400',
             'duration': 200,
@@ -61,6 +65,7 @@ class XHamsterIE(InfoExtractor):
             'id': '5667973',
             'ext': 'mp4',
             'title': '....',
+            'timestamp': 1454948101,
             'upload_date': '20160208',
             'uploader': 'parejafree',
             'duration': 72,
@@ -96,6 +101,83 @@ class XHamsterIE(InfoExtractor):
         if error:
             raise ExtractorError(error, expected=True)
 
+        age_limit = self._rta_search(webpage)
+
+        def get_height(s):
+            return int_or_none(self._search_regex(
+                r'^(\d+)[pP]', s, 'height', default=None))
+
+        initials = self._parse_json(
+            self._search_regex(
+                r'window\.initials\s*=\s*({.+?})\s*;\s*\n', webpage, 'initials',
+                default='{}'),
+            video_id, fatal=False)
+        if initials:
+            video = initials['videoModel']
+            title = video['title']
+            formats = []
+            for format_id, formats_dict in video['sources'].items():
+                if not isinstance(formats_dict, dict):
+                    continue
+                for quality, format_item in formats_dict.items():
+                    if format_id == 'download':
+                        # Download link takes some time to be generated,
+                        # skipping for now
+                        continue
+                        if not isinstance(format_item, dict):
+                            continue
+                        format_url = format_item.get('link')
+                        filesize = int_or_none(
+                            format_item.get('size'), invscale=1000000)
+                    else:
+                        format_url = format_item
+                        filesize = None
+                    if not isinstance(format_url, compat_str):
+                        continue
+                    formats.append({
+                        'format_id': '%s-%s' % (format_id, quality),
+                        'url': format_url,
+                        'ext': determine_ext(format_url, 'mp4'),
+                        'height': get_height(quality),
+                        'filesize': filesize,
+                    })
+            self._sort_formats(formats)
+
+            categories_list = video.get('categories')
+            if isinstance(categories_list, list):
+                categories = []
+                for c in categories_list:
+                    if not isinstance(c, dict):
+                        continue
+                    c_name = c.get('name')
+                    if isinstance(c_name, compat_str):
+                        categories.append(c_name)
+            else:
+                categories = None
+
+            return {
+                'id': video_id,
+                'display_id': display_id,
+                'title': title,
+                'description': video.get('description'),
+                'timestamp': int_or_none(video.get('created')),
+                'uploader': try_get(
+                    video, lambda x: x['author']['name'], compat_str),
+                'thumbnail': video.get('thumbURL'),
+                'duration': int_or_none(video.get('duration')),
+                'view_count': int_or_none(video.get('views')),
+                'like_count': int_or_none(try_get(
+                    video, lambda x: x['rating']['likes'], int)),
+                'dislike_count': int_or_none(try_get(
+                    video, lambda x: x['rating']['dislikes'], int)),
+                'comment_count': int_or_none(video.get('views')),
+                'age_limit': age_limit,
+                'categories': categories,
+                'formats': formats,
+            }
+
+        # Old layout fallback
+
         title = self._html_search_regex(
             [r'<h1[^>]*>([^<]+)</h1>',
              r'<meta[^>]+itemprop=".*?caption.*?"[^>]+content="(.+?)"',
@@ -119,8 +201,7 @@ class XHamsterIE(InfoExtractor):
             formats.append({
                 'format_id': format_id,
                 'url': format_url,
-                'height': int_or_none(self._search_regex(
-                    r'^(\d+)[pP]', format_id, 'height', default=None))
+                'height': get_height(format_id),
             })
 
         video_url = self._search_regex(
@@ -167,8 +248,6 @@ class XHamsterIE(InfoExtractor):
         mobj = re.search(r'</label>Comments \((?P<commentcount>\d+)\)</div>', webpage)
         comment_count = mobj.group('commentcount') if mobj else 0
 
-        age_limit = self._rta_search(webpage)
-
         categories_html = self._search_regex(
             r'(?s)<table.+?(<span>Categories:.+?)</table>', webpage,
             'categories', default=None)

From 41bf647e895faca42cdc2565ea034ed341593f8e Mon Sep 17 00:00:00 2001
From: Petr Novak <petr.novak@cosmoboy.cz>
Date: Sat, 25 Nov 2017 02:13:23 +0100
Subject: [PATCH 1570/1696] [extractor/common] Add support for DASH manifests
 with SegmentLists with bare SegmentURLs

---
 test/test_InfoExtractor.py      |  83 +++++++++++-
 test/testdata/mpd/urls_only.mpd | 218 ++++++++++++++++++++++++++++++++
 youtube_dl/extractor/common.py  |   9 ++
 3 files changed, 309 insertions(+), 1 deletion(-)
 create mode 100644 test/testdata/mpd/urls_only.mpd

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 7b95f883f..e58452ab5 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -562,7 +562,88 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'width': 1920,
                     'height': 1080,
                 }]
-            ),
+            ), (
+                'urls_only',
+                'http://unknown/manifest.mpd',
+                [{
+                    'manifest_url': 'http://unknown/manifest.mpd',
+                    'ext': 'mp4',
+                    'format_id': 'h264_aac_144p_m4s',
+                    'format_note': 'DASH video',
+                    'protocol': 'http_dash_segments',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc3.42c01e',
+                    'tbr': 200,
+                    'width': 256,
+                    'height': 144,
+                }, {
+                    'manifest_url': 'http://unknown/manifest.mpd',
+                    'ext': 'mp4',
+                    'format_id': 'h264_aac_240p_m4s',
+                    'format_note': 'DASH video',
+                    'protocol': 'http_dash_segments',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc3.42c01e',
+                    'tbr': 400,
+                    'width': 424,
+                    'height': 240,
+                }, {
+                    'manifest_url': 'http://unknown/manifest.mpd',
+                    'ext': 'mp4',
+                    'format_id': 'h264_aac_360p_m4s',
+                    'format_note': 'DASH video',
+                    'protocol': 'http_dash_segments',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc3.42c01e',
+                    'tbr': 800,
+                    'width': 640,
+                    'height': 360,
+                }, {
+                    'manifest_url': 'http://unknown/manifest.mpd',
+                    'ext': 'mp4',
+                    'format_id': 'h264_aac_480p_m4s',
+                    'format_note': 'DASH video',
+                    'protocol': 'http_dash_segments',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc3.42c01e',
+                    'tbr': 1200,
+                    'width': 856,
+                    'height': 480,
+                }, {
+                    'manifest_url': 'http://unknown/manifest.mpd',
+                    'ext': 'mp4',
+                    'format_id': 'h264_aac_576p_m4s',
+                    'format_note': 'DASH video',
+                    'protocol': 'http_dash_segments',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc3.42c01e',
+                    'tbr': 1600,
+                    'width': 1024,
+                    'height': 576,
+                }, {
+                    'manifest_url': 'http://unknown/manifest.mpd',
+                    'ext': 'mp4',
+                    'format_id': 'h264_aac_720p_m4s',
+                    'format_note': 'DASH video',
+                    'protocol': 'http_dash_segments',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc3.42c01e',
+                    'tbr': 2400,
+                    'width': 1280,
+                    'height': 720,
+                }, {
+                    'manifest_url': 'http://unknown/manifest.mpd',
+                    'ext': 'mp4',
+                    'format_id': 'h264_aac_1080p_m4s',
+                    'format_note': 'DASH video',
+                    'protocol': 'http_dash_segments',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc3.42c01e',
+                    'tbr': 4400,
+                    'width': 1920,
+                    'height': 1080,
+                }]
+            )
         ]
 
         for mpd_file, mpd_url, expected_formats in _TEST_CASES:
diff --git a/test/testdata/mpd/urls_only.mpd b/test/testdata/mpd/urls_only.mpd
new file mode 100644
index 000000000..2b9d595d3
--- /dev/null
+++ b/test/testdata/mpd/urls_only.mpd
@@ -0,0 +1,218 @@
+<?xml version="1.0" ?>
+<MPD maxSegmentDuration="PT0H0M10.000S" mediaPresentationDuration="PT0H4M1.728S" minBufferTime="PT1.500S" profiles="urn:mpeg:dash:profile:isoff-main:2011" type="static" xmlns="urn:mpeg:dash:schema:mpd:2011">
+  <Period duration="PT0H4M1.728S">
+    <AdaptationSet bitstreamSwitching="true" lang="und" maxHeight="1080" maxWidth="1920" par="16:9" segmentAlignment="true">
+      <ContentComponent contentType="video" id="1"/>
+      <Representation audioSamplingRate="44100" bandwidth="200000" codecs="avc3.42c01e,mp4a.40.2" frameRate="25" height="144" id="h264_aac_144p_m4s" mimeType="video/mp4" sar="1:1" startWithSAP="1" width="256">
+        <SegmentList duration="10000" timescale="1000">
+          <Initialization sourceURL="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/init/432f65a0.mp4"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/0/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/1/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/2/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/3/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/4/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/5/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/6/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/7/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/8/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/9/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/10/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/11/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/12/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/13/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/14/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/15/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/16/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/17/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/18/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/19/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/20/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/21/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/22/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/23/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/24/432f65a0.m4s"/>
+        </SegmentList>
+      </Representation>
+      <Representation audioSamplingRate="44100" bandwidth="400000" codecs="avc3.42c01e,mp4a.40.2" frameRate="25" height="240" id="h264_aac_240p_m4s" mimeType="video/mp4" sar="160:159" startWithSAP="1" width="424">
+        <SegmentList duration="10000" timescale="1000">
+          <Initialization sourceURL="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/init/432f65a0.mp4"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/0/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/1/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/2/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/3/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/4/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/5/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/6/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/7/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/8/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/9/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/10/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/11/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/12/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/13/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/14/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/15/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/16/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/17/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/18/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/19/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/20/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/21/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/22/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/23/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/24/432f65a0.m4s"/>
+        </SegmentList>
+      </Representation>
+      <Representation audioSamplingRate="44100" bandwidth="800000" codecs="avc3.42c01e,mp4a.40.2" frameRate="25" height="360" id="h264_aac_360p_m4s" mimeType="video/mp4" sar="1:1" startWithSAP="1" width="640">
+        <SegmentList duration="10000" timescale="1000">
+          <Initialization sourceURL="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/init/432f65a0.mp4"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/0/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/1/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/2/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/3/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/4/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/5/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/6/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/7/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/8/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/9/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/10/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/11/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/12/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/13/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/14/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/15/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/16/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/17/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/18/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/19/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/20/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/21/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/22/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/23/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/24/432f65a0.m4s"/>
+        </SegmentList>
+      </Representation>
+      <Representation audioSamplingRate="44100" bandwidth="1200000" codecs="avc3.42c01e,mp4a.40.2" frameRate="25" height="480" id="h264_aac_480p_m4s" mimeType="video/mp4" sar="320:321" startWithSAP="1" width="856">
+        <SegmentList duration="10000" timescale="1000">
+          <Initialization sourceURL="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/init/432f65a0.mp4"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/0/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/1/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/2/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/3/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/4/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/5/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/6/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/7/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/8/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/9/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/10/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/11/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/12/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/13/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/14/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/15/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/16/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/17/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/18/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/19/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/20/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/21/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/22/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/23/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/24/432f65a0.m4s"/>
+        </SegmentList>
+      </Representation>
+      <Representation audioSamplingRate="44100" bandwidth="1600000" codecs="avc3.42c01e,mp4a.40.2" frameRate="25" height="576" id="h264_aac_576p_m4s" mimeType="video/mp4" sar="1:1" startWithSAP="1" width="1024">
+        <SegmentList duration="10000" timescale="1000">
+          <Initialization sourceURL="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/init/432f65a0.mp4"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/0/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/1/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/2/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/3/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/4/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/5/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/6/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/7/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/8/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/9/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/10/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/11/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/12/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/13/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/14/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/15/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/16/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/17/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/18/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/19/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/20/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/21/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/22/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/23/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/24/432f65a0.m4s"/>
+        </SegmentList>
+      </Representation>
+      <Representation audioSamplingRate="44100" bandwidth="2400000" codecs="avc3.42c01e,mp4a.40.2" frameRate="25" height="720" id="h264_aac_720p_m4s" mimeType="video/mp4" sar="1:1" startWithSAP="1" width="1280">
+        <SegmentList duration="10000" timescale="1000">
+          <Initialization sourceURL="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/init/432f65a0.mp4"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/0/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/1/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/2/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/3/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/4/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/5/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/6/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/7/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/8/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/9/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/10/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/11/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/12/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/13/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/14/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/15/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/16/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/17/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/18/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/19/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/20/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/21/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/22/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/23/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/24/432f65a0.m4s"/>
+        </SegmentList>
+      </Representation>
+      <Representation audioSamplingRate="44100" bandwidth="4400000" codecs="avc3.42c01e,mp4a.40.2" frameRate="25" height="1080" id="h264_aac_1080p_m4s" mimeType="video/mp4" sar="1:1" startWithSAP="1" width="1920">
+        <SegmentList duration="10000" timescale="1000">
+          <Initialization sourceURL="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/init/432f65a0.mp4"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/0/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/1/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/2/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/3/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/4/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/5/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/6/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/7/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/8/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/9/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/10/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/11/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/12/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/13/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/14/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/15/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/16/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/17/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/18/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/19/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/20/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/21/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/22/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/23/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/24/432f65a0.m4s"/>
+        </SegmentList>
+      </Representation>
+    </AdaptationSet>
+  </Period>
+</MPD>
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 8e4ee0deb..15999411b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1975,6 +1975,15 @@ class InfoExtractor(object):
                                     })
                                     segment_index += 1
                             representation_ms_info['fragments'] = fragments
+                        elif 'segment_urls' in representation_ms_info:
+                            # Segment URLs with no SegmentTimeline
+                            # Example: https://www.seznam.cz/zpravy/clanek/cesko-zasahne-vitr-o-sile-vichrice-muze-byt-i-zivotu-nebezpecny-39091
+                            fragments = []
+                            for segment_url in representation_ms_info['segment_urls']:
+                                fragments.append({
+                                    location_key(segment_url): segment_url,
+                                })
+                            representation_ms_info['fragments'] = fragments
                         # NB: MPD manifest may contain direct URLs to unfragmented media.
                         # No fragments key is present in this case.
                         if 'fragments' in representation_ms_info:

From 603fc4e0ea472c7c2a78ff201d69686a9e3fe1f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Dec 2017 21:10:35 +0700
Subject: [PATCH 1571/1696] [extractor/common] Add durations for DASH fragments
 with bare SegmentURLs

---
 youtube_dl/extractor/common.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 15999411b..3baf683d8 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1979,10 +1979,16 @@ class InfoExtractor(object):
                             # Segment URLs with no SegmentTimeline
                             # Example: https://www.seznam.cz/zpravy/clanek/cesko-zasahne-vitr-o-sile-vichrice-muze-byt-i-zivotu-nebezpecny-39091
                             fragments = []
+                            segment_duration = float_or_none(
+                                representation_ms_info['segment_duration'],
+                                representation_ms_info['timescale']) if 'segment_duration' in representation_ms_info else None
                             for segment_url in representation_ms_info['segment_urls']:
-                                fragments.append({
+                                fragment = {
                                     location_key(segment_url): segment_url,
-                                })
+                                }
+                                if segment_duration:
+                                    fragment['duration'] = segment_duration
+                                fragments.append(fragment)
                             representation_ms_info['fragments'] = fragments
                         # NB: MPD manifest may contain direct URLs to unfragmented media.
                         # No fragments key is present in this case.

From 593f2f798922d54be8d3f20f4a2048493095016d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Dec 2017 21:15:45 +0700
Subject: [PATCH 1572/1696] [downloader/fragment] Commit part file after each
 fragment In order to obtain correct resume_len on next iteration

---
 youtube_dl/downloader/fragment.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 93002e45a..7bb61a541 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -107,6 +107,7 @@ class FragmentFD(FileDownloader):
     def _append_fragment(self, ctx, frag_content):
         try:
             ctx['dest_stream'].write(frag_content)
+            ctx['dest_stream'].flush()
         finally:
             if self.__do_ytdl_file(ctx):
                 self._write_ytdl_file(ctx)

From 78593e294cc4623108d17b1b0b2d26d507953006 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Dec 2017 21:22:43 +0700
Subject: [PATCH 1573/1696] Add references for #14844

---
 test/test_InfoExtractor.py     | 1 +
 youtube_dl/extractor/common.py | 1 +
 2 files changed, 2 insertions(+)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index e58452ab5..8a372d2c9 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -563,6 +563,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 1080,
                 }]
             ), (
+                # https://github.com/rg3/youtube-dl/pull/14844
                 'urls_only',
                 'http://unknown/manifest.mpd',
                 [{
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 3baf683d8..80a9c982f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1978,6 +1978,7 @@ class InfoExtractor(object):
                         elif 'segment_urls' in representation_ms_info:
                             # Segment URLs with no SegmentTimeline
                             # Example: https://www.seznam.cz/zpravy/clanek/cesko-zasahne-vitr-o-sile-vichrice-muze-byt-i-zivotu-nebezpecny-39091
+                            # https://github.com/rg3/youtube-dl/pull/14844
                             fragments = []
                             segment_duration = float_or_none(
                                 representation_ms_info['segment_duration'],

From e25ee72657c3e27a24f62ac54d4b100c558ea662 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Dec 2017 21:29:06 +0700
Subject: [PATCH 1574/1696] [ChangeLog] Actualize

---
 ChangeLog | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index a568fa233..81eda954c 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,26 @@
+version <unreleased>
+
+Core
++ [downloader/fragment] Commit part file after each fragment
++ [extractor/common] Add durations for DASH fragments with bare SegmentURLs
++ [extractor/common] Add support for DASH manifests with SegmentLists with
+  bare SegmentURLs (#14844)
++ [utils] Add hvc1 codec code to parse_codecs
+
+Extractors
+* [xhamster] Fix extraction (#14884)
+* [youku] Update ccode (#14872)
+* [mnet] Fix format extraction (#14883)
++ [xiami] Add Referer header to API request
+* [mtv] Correct scc extention in extracted subtitles (#13730)
+* [vvvvid] Fix extraction for kenc videos (#13406)
++ [br] Add support for BR Mediathek videos (#14560, #14788)
++ [daisuki] Add support for motto.daisuki.com (#14681)
+* [odnoklassniki] Fix API metadata request (#14862)
+* [itv] Fix HLS formats extraction
++ [pbs] Add another media id regular expression
+
+
 version 2017.11.26
 
 Core

From 0d56eddc5970a9e55334add371f555227b957589 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Dec 2017 21:34:34 +0700
Subject: [PATCH 1575/1696] release 2017.12.02

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 7 ++++---
 youtube_dl/version.py     | 2 +-
 4 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 15211e5d9..513823b9b 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.11.26*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.11.26**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.12.02*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.12.02**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.11.26
+[debug] youtube-dl version 2017.12.02
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 81eda954c..63837d62b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.12.02
 
 Core
 + [downloader/fragment] Commit part file after each fragment
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index b44c9e7a3..0287a4011 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -112,11 +112,12 @@
  - **BokeCC**
  - **BostonGlobe**
  - **Bpb**: Bundeszentrale für politische Bildung
- - **BR**: Bayerischer Rundfunk Mediathek
+ - **BR**: Bayerischer Rundfunk
  - **BravoTV**
  - **Break**
  - **brightcove:legacy**
  - **brightcove:new**
+ - **BRMediathek**: Bayerischer Rundfunk Mediathek
  - **bt:article**: Bergens Tidende Articles
  - **bt:vestlendingen**: Bergens Tidende - Vestlendingen
  - **BuzzFeed**
@@ -198,8 +199,8 @@
  - **dailymotion:playlist**
  - **dailymotion:user**
  - **DailymotionCloud**
- - **Daisuki**
- - **DaisukiPlaylist**
+ - **DaisukiMotto**
+ - **DaisukiMottoPlaylist**
  - **daum.net**
  - **daum.net:clip**
  - **daum.net:playlist**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 81e3f792d..88bf1d652 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.11.26'
+__version__ = '2017.12.02'

From d7df30898115c4762514868140cfa856618cdfd5 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 2 Dec 2017 20:22:17 +0100
Subject: [PATCH 1576/1696] [toutv] fix login(closes 14614)

---
 youtube_dl/extractor/toutv.py | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/toutv.py b/youtube_dl/extractor/toutv.py
index e59ed2661..17c0adc15 100644
--- a/youtube_dl/extractor/toutv.py
+++ b/youtube_dl/extractor/toutv.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
@@ -41,7 +43,7 @@ class TouTvIE(InfoExtractor):
         email, password = self._get_login_info()
         if email is None:
             return
-        state = 'http://ici.tou.tv//'
+        state = 'http://ici.tou.tv/'
         webpage = self._download_webpage(state, None, 'Downloading homepage')
         toutvlogin = self._parse_json(self._search_regex(
             r'(?s)toutvlogin\s*=\s*({.+?});', webpage, 'toutvlogin'), None, js_to_json)
@@ -54,16 +56,30 @@ class TouTvIE(InfoExtractor):
                 'scope': 'media-drmt openid profile email id.write media-validation.read.privileged',
                 'state': state,
             })
-        login_form = self._search_regex(
-            r'(?s)(<form[^>]+(?:id|name)="Form-login".+?</form>)', login_webpage, 'login form')
-        form_data = self._hidden_inputs(login_form)
+
+        def extract_form_url_and_data(wp, default_form_url, form_spec_re=''):
+            form, form_elem = re.search(
+                r'(?s)((<form[^>]+?%s[^>]*?>).+?</form>)' % form_spec_re, wp).groups()
+            form_data = self._hidden_inputs(form)
+            form_url = extract_attributes(form_elem).get('action') or default_form_url
+            return form_url, form_data
+
+        post_url, form_data = extract_form_url_and_data(
+            login_webpage,
+            'https://services.radio-canada.ca/auth/oauth/v2/authorize/login',
+            r'(?:id|name)="Form-login"')
         form_data.update({
             'login-email': email,
             'login-password': password,
         })
-        post_url = extract_attributes(login_form).get('action') or authorize_url
-        _, urlh = self._download_webpage_handle(
+        consent_webpage = self._download_webpage(
             post_url, None, 'Logging in', data=urlencode_postdata(form_data))
+        post_url, form_data = extract_form_url_and_data(
+            consent_webpage,
+            'https://services.radio-canada.ca/auth/oauth/v2/authorize/consent')
+        _, urlh = self._download_webpage_handle(
+            post_url, None, 'Following Redirection',
+            data=urlencode_postdata(form_data))
         self._access_token = self._search_regex(
             r'access_token=([\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})',
             urlh.geturl(), 'access token')

From d4f05d473134d7bd61b054468e6ba297cef3c88f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 3 Dec 2017 00:04:43 +0100
Subject: [PATCH 1577/1696] [utils] add sami mimetype to mimetype2ext

---
 youtube_dl/utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index febc9d26f..eccbc0b1f 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2350,6 +2350,7 @@ def mimetype2ext(mt):
         'ttml+xml': 'ttml',
         'x-flv': 'flv',
         'x-mp4-fragmented': 'mp4',
+        'x-ms-sami': 'sami',
         'x-ms-wmv': 'wmv',
         'mpegurl': 'm3u8',
         'x-mpegurl': 'm3u8',

From c94427dd60f9836e3752077977cae3b4d0da47d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Dec 2017 22:34:56 +0700
Subject: [PATCH 1578/1696] [pluralsight] Detect agreement request (#14913)

---
 youtube_dl/extractor/pluralsight.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 4bf0aa786..597b11218 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -131,6 +131,13 @@ class PluralsightIE(PluralsightBaseIE):
             if BLOCKED in response:
                 raise ExtractorError(
                     'Unable to login: %s' % BLOCKED, expected=True)
+            MUST_AGREE = 'To continue using Pluralsight, you must agree to'
+            if any(p in response for p in (MUST_AGREE, '>Disagree<', '>Agree<')):
+                raise ExtractorError(
+                    'Unable to login: %s some documents. Go to pluralsight.com, '
+                    'log in and agree with what Pluralsight requires.'
+                    % MUST_AGREE, expected=True)
+
             raise ExtractorError('Unable to log in')
 
     def _get_subtitles(self, author, clip_id, lang, name, duration, video_id):

From 61d18c8a4bc91602738bfd2e506c3cbbc3a3788b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Dec 2017 22:42:02 +0700
Subject: [PATCH 1579/1696] [porncom] Fix metadata extraction (closes #14911)

---
 youtube_dl/extractor/porncom.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/porncom.py b/youtube_dl/extractor/porncom.py
index 8218c7d3b..60ade06da 100644
--- a/youtube_dl/extractor/porncom.py
+++ b/youtube_dl/extractor/porncom.py
@@ -77,12 +77,14 @@ class PornComIE(InfoExtractor):
         self._sort_formats(formats)
 
         view_count = str_to_int(self._search_regex(
-            r'class=["\']views["\'][^>]*><p>([\d,.]+)', webpage,
+            (r'Views:\s*</span>\s*<span>\s*([\d,.]+)',
+             r'class=["\']views["\'][^>]*><p>([\d,.]+)'), webpage,
             'view count', fatal=False))
 
         def extract_list(kind):
             s = self._search_regex(
-                r'(?s)<p[^>]*>%s:(.+?)</p>' % kind.capitalize(),
+                (r'(?s)%s:\s*</span>\s*<span>(.+?)</span>' % kind.capitalize(),
+                 r'(?s)<p[^>]*>%s:(.+?)</p>' % kind.capitalize()),
                 webpage, kind, fatal=False)
             return re.findall(r'<a[^>]+>([^<]+)</a>', s or '')
 

From 91328f26b05084a1ddd890866670a1133564ecd0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Dec 2017 23:01:57 +0700
Subject: [PATCH 1580/1696] [ard] Skip invalid stream URLs (closes #14906)

---
 youtube_dl/extractor/ard.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 915f8862e..ef73d5a93 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -5,6 +5,7 @@ import re
 
 from .common import InfoExtractor
 from .generic import GenericIE
+from ..compat import compat_str
 from ..utils import (
     determine_ext,
     ExtractorError,
@@ -126,6 +127,8 @@ class ARDMediathekIE(InfoExtractor):
                 quality = stream.get('_quality')
                 server = stream.get('_server')
                 for stream_url in stream_urls:
+                    if not isinstance(stream_url, compat_str) or '//' not in stream_url:
+                        continue
                     ext = determine_ext(stream_url)
                     if quality != 'auto' and ext in ('f4m', 'm3u8'):
                         continue
@@ -146,13 +149,11 @@ class ARDMediathekIE(InfoExtractor):
                                 'play_path': stream_url,
                                 'format_id': 'a%s-rtmp-%s' % (num, quality),
                             }
-                        elif stream_url.startswith('http'):
+                        else:
                             f = {
                                 'url': stream_url,
                                 'format_id': 'a%s-%s-%s' % (num, ext, quality)
                             }
-                        else:
-                            continue
                         m = re.search(r'_(?P<width>\d+)x(?P<height>\d+)\.mp4$', stream_url)
                         if m:
                             f.update({

From d3f8b76b69b9bc986354a83a8d4ee4a8617ce52d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Dec 2017 23:11:15 +0700
Subject: [PATCH 1581/1696] [extractor/generic] Fix typo (closes #14902) Don't
 pass video_id as mpd_id

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 56df2ab47..c8b7c2e63 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2195,7 +2195,7 @@ class GenericIE(InfoExtractor):
                 return self.playlist_result(self._parse_xspf(doc, video_id), video_id)
             elif re.match(r'(?i)^(?:{[^}]+})?MPD$', doc.tag):
                 info_dict['formats'] = self._parse_mpd_formats(
-                    doc, video_id,
+                    doc,
                     mpd_base_url=compat_str(full_response.geturl()).rpartition('/')[0],
                     mpd_url=url)
                 self._sort_formats(info_dict['formats'])

From b271e3352603c523bf2c6762973937fd36925aae Mon Sep 17 00:00:00 2001
From: Windom <windom@users.noreply.github.com>
Date: Tue, 5 Dec 2017 19:08:31 +0200
Subject: [PATCH 1582/1696] [xhamster] Add support for mobile URLs and fix
 thumbnail extraction

---
 youtube_dl/extractor/xhamster.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 52f8ded2f..68652a22f 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -75,6 +75,10 @@ class XHamsterIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        # mobile site
+        'url': 'https://m.xhamster.com/videos/cute-teen-jacqueline-solo-masturbation-8559111',
+        'only_matching': True,
     }, {
         'url': 'https://xhamster.com/movies/2272726/amber_slayed_by_the_knight.html',
         'only_matching': True,
@@ -93,7 +97,8 @@ class XHamsterIE(InfoExtractor):
         video_id = mobj.group('id') or mobj.group('id_2')
         display_id = mobj.group('display_id') or mobj.group('display_id_2')
 
-        webpage = self._download_webpage(url, video_id)
+        desktop_url = re.sub(r'^(https?://(?:.+?\.)?)m\.', r'\1', url)
+        webpage = self._download_webpage(desktop_url, video_id)
 
         error = self._html_search_regex(
             r'<div[^>]+id=["\']videoClosed["\'][^>]*>(.+?)</div>',
@@ -229,8 +234,8 @@ class XHamsterIE(InfoExtractor):
             webpage, 'uploader', default='anonymous')
 
         thumbnail = self._search_regex(
-            [r'''thumb\s*:\s*(?P<q>["'])(?P<thumbnail>.+?)(?P=q)''',
-             r'''<video[^>]+poster=(?P<q>["'])(?P<thumbnail>.+?)(?P=q)[^>]*>'''],
+            [r'''["']thumbUrl["']\s*:\s*(?P<q>["'])(?P<thumbnail>.+?)(?P=q)''',
+             r'''<video[^>]+"poster"=(?P<q>["'])(?P<thumbnail>.+?)(?P=q)[^>]*>'''],
             webpage, 'thumbnail', fatal=False, group='thumbnail')
 
         duration = parse_duration(self._search_regex(
@@ -274,15 +279,16 @@ class XHamsterIE(InfoExtractor):
 
 
 class XHamsterEmbedIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?xhamster\.com/xembed\.php\?video=(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:.+?\.)?xhamster\.com/xembed\.php\?video=(?P<id>\d+)'
     _TEST = {
         'url': 'http://xhamster.com/xembed.php?video=3328539',
         'info_dict': {
             'id': '3328539',
             'ext': 'mp4',
             'title': 'Pen Masturbation',
+            'timestamp': 1406581861,
             'upload_date': '20140728',
-            'uploader_id': 'anonymous',
+            'uploader': 'ManyakisArt',
             'duration': 5,
             'age_limit': 18,
         }

From 3c4fbfeca2431b120537b9eaedf5977dd0ab13b1 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 6 Dec 2017 10:54:20 +0100
Subject: [PATCH 1583/1696] [dailymotion] remove dailymotion cloud
 extractor(closes #6794)

https://web.archive.org/web/20160312110217/https://www.dmcloud.net/
---
 youtube_dl/extractor/dailymotion.py | 49 -----------------------------
 youtube_dl/extractor/extractors.py  |  1 -
 youtube_dl/extractor/francetv.py    |  9 +-----
 youtube_dl/extractor/generic.py     | 27 +---------------
 4 files changed, 2 insertions(+), 84 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 21a2d0239..0e7d587dd 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -413,52 +413,3 @@ class DailymotionUserIE(DailymotionPlaylistIE):
             'title': full_user,
             'entries': self._extract_entries(user),
         }
-
-
-class DailymotionCloudIE(DailymotionBaseInfoExtractor):
-    _VALID_URL_PREFIX = r'https?://api\.dmcloud\.net/(?:player/)?embed/'
-    _VALID_URL = r'%s[^/]+/(?P<id>[^/?]+)' % _VALID_URL_PREFIX
-    _VALID_EMBED_URL = r'%s[^/]+/[^\'"]+' % _VALID_URL_PREFIX
-
-    _TESTS = [{
-        # From http://www.francetvinfo.fr/economie/entreprises/les-entreprises-familiales-le-secret-de-la-reussite_933271.html
-        # Tested at FranceTvInfo_2
-        'url': 'http://api.dmcloud.net/embed/4e7343f894a6f677b10006b4/556e03339473995ee145930c?auth=1464865870-0-jyhsm84b-ead4c701fb750cf9367bf4447167a3db&autoplay=1',
-        'only_matching': True,
-    }, {
-        # http://www.francetvinfo.fr/societe/larguez-les-amarres-le-cobaturage-se-developpe_980101.html
-        'url': 'http://api.dmcloud.net/player/embed/4e7343f894a6f677b10006b4/559545469473996d31429f06?auth=1467430263-0-90tglw2l-a3a4b64ed41efe48d7fccad85b8b8fda&autoplay=1',
-        'only_matching': True,
-    }]
-
-    @classmethod
-    def _extract_dmcloud_url(cls, webpage):
-        mobj = re.search(r'<iframe[^>]+src=[\'"](%s)[\'"]' % cls._VALID_EMBED_URL, webpage)
-        if mobj:
-            return mobj.group(1)
-
-        mobj = re.search(
-            r'<input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=[\'"](%s)[\'"]' % cls._VALID_EMBED_URL,
-            webpage)
-        if mobj:
-            return mobj.group(1)
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage_no_ff(url, video_id)
-
-        title = self._html_search_regex(r'<title>([^>]+)</title>', webpage, 'title')
-
-        video_info = self._parse_json(self._search_regex(
-            r'var\s+info\s*=\s*([^;]+);', webpage, 'video info'), video_id)
-
-        # TODO: parse ios_url, which is in fact a manifest
-        video_url = video_info['mp4_url']
-
-        return {
-            'id': video_id,
-            'url': video_url,
-            'title': title,
-            'thumbnail': video_info.get('thumbnail_url'),
-        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 2cc3bc463..9c9739ad2 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -246,7 +246,6 @@ from .dailymotion import (
     DailymotionIE,
     DailymotionPlaylistIE,
     DailymotionUserIE,
-    DailymotionCloudIE,
 )
 from .daisuki import (
     DaisukiMottoIE,
diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 5a3abeaff..80c597015 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -13,10 +13,7 @@ from ..utils import (
     parse_duration,
     determine_ext,
 )
-from .dailymotion import (
-    DailymotionIE,
-    DailymotionCloudIE,
-)
+from .dailymotion import DailymotionIE
 
 
 class FranceTVBaseInfoExtractor(InfoExtractor):
@@ -290,10 +287,6 @@ class FranceTVInfoIE(FranceTVBaseInfoExtractor):
         page_title = mobj.group('title')
         webpage = self._download_webpage(url, page_title)
 
-        dmcloud_url = DailymotionCloudIE._extract_dmcloud_url(webpage)
-        if dmcloud_url:
-            return self.url_result(dmcloud_url, DailymotionCloudIE.ie_key())
-
         dailymotion_urls = DailymotionIE._extract_urls(webpage)
         if dailymotion_urls:
             return self.playlist_result([
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c8b7c2e63..c7b609215 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -59,10 +59,7 @@ from .tnaflix import TNAFlixNetworkEmbedIE
 from .drtuber import DrTuberIE
 from .redtube import RedTubeIE
 from .vimeo import VimeoIE
-from .dailymotion import (
-    DailymotionIE,
-    DailymotionCloudIE,
-)
+from .dailymotion import DailymotionIE
 from .dailymail import DailyMailIE
 from .onionstudios import OnionStudiosIE
 from .viewlift import ViewLiftEmbedIE
@@ -1472,23 +1469,6 @@ class GenericIE(InfoExtractor):
                 'timestamp': 1432570283,
             },
         },
-        # Dailymotion Cloud video
-        {
-            'url': 'http://replay.publicsenat.fr/vod/le-debat/florent-kolandjian,dominique-cena,axel-decourtye,laurence-abeille,bruno-parmentier/175910',
-            'md5': 'dcaf23ad0c67a256f4278bce6e0bae38',
-            'info_dict': {
-                'id': 'x2uy8t3',
-                'ext': 'mp4',
-                'title': 'Sauvons les abeilles ! - Le débat',
-                'description': 'md5:d9082128b1c5277987825d684939ca26',
-                'thumbnail': r're:^https?://.*\.jpe?g$',
-                'timestamp': 1434970506,
-                'upload_date': '20150622',
-                'uploader': 'Public Sénat',
-                'uploader_id': 'xa9gza',
-            },
-            'skip': 'File not found.',
-        },
         # OnionStudios embed
         {
             'url': 'http://www.clickhole.com/video/dont-understand-bitcoin-man-will-mumble-explanatio-2537',
@@ -2704,11 +2684,6 @@ class GenericIE(InfoExtractor):
         if senate_isvp_url:
             return self.url_result(senate_isvp_url, 'SenateISVP')
 
-        # Look for Dailymotion Cloud videos
-        dmcloud_url = DailymotionCloudIE._extract_dmcloud_url(webpage)
-        if dmcloud_url:
-            return self.url_result(dmcloud_url, 'DailymotionCloud')
-
         # Look for OnionStudios embeds
         onionstudios_url = OnionStudiosIE._extract_url(webpage)
         if onionstudios_url:

From 684ae102360dbef6d1e5fcd75a0f86266030e02d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 6 Dec 2017 22:56:14 +0100
Subject: [PATCH 1584/1696] [fox] add support for adobe pass auth and extract
 subtitles(close #14489)(closes #14205)

---
 youtube_dl/extractor/fox.py | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/fox.py b/youtube_dl/extractor/fox.py
index 5f98d017b..11d6c9c32 100644
--- a/youtube_dl/extractor/fox.py
+++ b/youtube_dl/extractor/fox.py
@@ -11,6 +11,7 @@ from ..utils import (
     parse_duration,
     try_get,
     unified_timestamp,
+    update_url_query,
 )
 
 
@@ -62,7 +63,8 @@ class FOXIE(AdobePassIE):
         duration = int_or_none(video.get('durationInSeconds')) or int_or_none(
             video.get('duration')) or parse_duration(video.get('duration'))
         timestamp = unified_timestamp(video.get('datePublished'))
-        age_limit = parse_age_limit(video.get('contentRating'))
+        rating = video.get('contentRating')
+        age_limit = parse_age_limit(rating)
 
         data = try_get(
             video, lambda x: x['trackingData']['properties'], dict) or {}
@@ -77,8 +79,24 @@ class FOXIE(AdobePassIE):
         release_year = int_or_none(video.get('releaseYear'))
 
         if data.get('authRequired'):
-            # TODO: AP
-            pass
+            resource = self._get_mvpd_resource(
+                'fbc-fox', title, video.get('guid'), rating)
+            release_url = update_url_query(
+                release_url, {
+                    'auth': self._extract_mvpd_auth(
+                        url, video_id, 'fbc-fox', resource)
+                })
+
+        subtitles = {}
+        for doc_rel in video.get('documentReleases', []):
+            rel_url = doc_rel.get('url')
+            if not url or doc_rel.get('format') != 'SCC':
+                continue
+            subtitles['en'] = [{
+                'url': rel_url,
+                'ext': 'scc',
+            }]
+            break
 
         info = {
             'id': video_id,
@@ -93,6 +111,7 @@ class FOXIE(AdobePassIE):
             'episode': episode,
             'episode_number': episode_number,
             'release_year': release_year,
+            'subtitles': subtitles,
         }
 
         urlh = self._request_webpage(HEADRequest(release_url), video_id)

From 1bd4fc96e6f8978f83e90ca913df18a979445028 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 7 Dec 2017 08:46:30 +0100
Subject: [PATCH 1585/1696] [sonyliv] extract higher quality formats and bypass
 geo restriction(closes #14922)

---
 youtube_dl/extractor/sonyliv.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/sonyliv.py b/youtube_dl/extractor/sonyliv.py
index accd112aa..c3078e285 100644
--- a/youtube_dl/extractor/sonyliv.py
+++ b/youtube_dl/extractor/sonyliv.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import smuggle_url
 
 
 class SonyLIVIE(InfoExtractor):
@@ -10,12 +11,12 @@ class SonyLIVIE(InfoExtractor):
         'url': "http://www.sonyliv.com/details/episodes/5024612095001/Ep.-1---Achaari-Cheese-Toast---Bachelor's-Delight",
         'info_dict': {
             'title': "Ep. 1 - Achaari Cheese Toast - Bachelor's Delight",
-            'id': '5024612095001',
+            'id': 'ref:5024612095001',
             'ext': 'mp4',
-            'upload_date': '20160707',
+            'upload_date': '20170923',
             'description': 'md5:7f28509a148d5be9d0782b4d5106410d',
-            'uploader_id': '4338955589001',
-            'timestamp': 1467870968,
+            'uploader_id': '5182475815001',
+            'timestamp': 1506200547,
         },
         'params': {
             'skip_download': True,
@@ -26,9 +27,11 @@ class SonyLIVIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/4338955589001/default_default/index.html?videoId=%s'
+    # BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/4338955589001/default_default/index.html?videoId=%s'
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/5182475815001/default_default/index.html?videoId=ref:%s'
 
     def _real_extract(self, url):
         brightcove_id = self._match_id(url)
         return self.url_result(
-            self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)
+            smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, {'geo_countries': ['IN']}),
+            'BrightcoveNew', brightcove_id)

From a670b1ba266533c520a436469585904321808a1b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 7 Dec 2017 22:16:41 +0700
Subject: [PATCH 1586/1696] [README.md] Add is_live, start_time and end_time to
 output template section (closes #14926)

---
 README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.md b/README.md
index ea321d536..cd30d147a 100644
--- a/README.md
+++ b/README.md
@@ -511,6 +511,9 @@ The basic usage is not to set any template arguments when downloading a single f
  - `average_rating` (numeric): Average rating give by users, the scale used depends on the webpage
  - `comment_count` (numeric): Number of comments on the video
  - `age_limit` (numeric): Age restriction for the video (years)
+ - `is_live` (boolean): Whether this video is a live stream or a fixed-length video
+ - `start_time` (numeric): Time in seconds where the reproduction should start, as specified in the URL
+ - `end_time` (numeric): Time in seconds where the reproduction should end, as specified in the URL
  - `format` (string): A human-readable description of the format 
  - `format_id` (string): Format code specified by `--format`
  - `format_note` (string): Additional info about the format

From d21d0ba6c14e8a6696130090641da4e2028e1bb3 Mon Sep 17 00:00:00 2001
From: Timendum <timedum@gmail.com>
Date: Mon, 23 Oct 2017 15:32:45 +0200
Subject: [PATCH 1587/1696] [raiplay:playlist] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/rai.py        | 26 ++++++++++++++++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 9c9739ad2..d8f9f94cc 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -857,6 +857,7 @@ from .rai import (
     RaiPlayIE,
     RaiPlayLiveIE,
     RaiIE,
+    RaiPlaylistIE,
 )
 from .rbmaradio import RBMARadioIE
 from .rds import RDSIE
diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 5bf64a56b..625458380 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -455,3 +455,29 @@ class RaiIE(RaiBaseIE):
         info.update(relinker_info)
 
         return info
+
+
+class RaiPlaylistIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?raiplay\.it/programmi/(?P<id>[^/]+)'
+    _TESTS = [{
+        'url': 'http://www.raiplay.it/programmi/nondirloalmiocapo/',
+        'info_dict': {
+            'id': 'nondirloalmiocapo',
+            'title': 'Non dirlo al mio capo',
+        },
+        'playlist_mincount': 12,
+    }]
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        webpage = self._download_webpage(url, playlist_id)
+        title = self._html_search_meta('programma', webpage, default=None)
+        video_urls = re.findall(' href="(/raiplay/video.+)"', webpage)
+        video_urls = [urljoin(url, video_url) for video_url in video_urls]
+        entries = [
+            self.url_result(
+                video_url,
+                RaiPlayIE.ie_key())
+            for video_url in video_urls if RaiPlayIE.suitable(video_url)
+        ]
+        return self.playlist_result(entries, playlist_id, title)

From 1115271ac61b89cc4ac1ca922eff8a4bed0fbf57 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Dec 2017 00:46:28 +0700
Subject: [PATCH 1588/1696] [raiplay:playlist] Fix issues and improve (closes
 #14563)

---
 youtube_dl/extractor/extractors.py |  2 +-
 youtube_dl/extractor/rai.py        | 62 +++++++++++++++++-------------
 2 files changed, 37 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d8f9f94cc..b9c97fac4 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -856,8 +856,8 @@ from .radiofrance import RadioFranceIE
 from .rai import (
     RaiPlayIE,
     RaiPlayLiveIE,
+    RaiPlayPlaylistIE,
     RaiIE,
-    RaiPlaylistIE,
 )
 from .rbmaradio import RBMARadioIE
 from .rds import RDSIE
diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 625458380..d22311031 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -17,6 +17,7 @@ from ..utils import (
     parse_duration,
     strip_or_none,
     try_get,
+    unescapeHTML,
     unified_strdate,
     unified_timestamp,
     update_url_query,
@@ -249,6 +250,41 @@ class RaiPlayLiveIE(RaiBaseIE):
         }
 
 
+class RaiPlayPlaylistIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?raiplay\.it/programmi/(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'http://www.raiplay.it/programmi/nondirloalmiocapo/',
+        'info_dict': {
+            'id': 'nondirloalmiocapo',
+            'title': 'Non dirlo al mio capo',
+            'description': 'md5:9f3d603b2947c1c7abb098f3b14fac86',
+        },
+        'playlist_mincount': 12,
+    }]
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        title = self._html_search_meta(
+            ('programma', 'nomeProgramma'), webpage, 'title')
+        description = unescapeHTML(self._html_search_meta(
+            ('description', 'og:description'), webpage, 'description'))
+        print(description)
+
+        entries = []
+        for mobj in re.finditer(
+                r'<a\b[^>]+\bhref=(["\'])(?P<path>/raiplay/video/.+?)\1',
+                webpage):
+            video_url = urljoin(url, mobj.group('path'))
+            entries.append(self.url_result(
+                video_url, ie=RaiPlayIE.ie_key(),
+                video_id=RaiPlayIE._match_id(video_url)))
+
+        return self.playlist_result(entries, playlist_id, title, description)
+
+
 class RaiIE(RaiBaseIE):
     _VALID_URL = r'https?://[^/]+\.(?:rai\.(?:it|tv)|rainews\.it)/dl/.+?-(?P<id>%s)(?:-.+?)?\.html' % RaiBaseIE._UUID_RE
     _TESTS = [{
@@ -455,29 +491,3 @@ class RaiIE(RaiBaseIE):
         info.update(relinker_info)
 
         return info
-
-
-class RaiPlaylistIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?raiplay\.it/programmi/(?P<id>[^/]+)'
-    _TESTS = [{
-        'url': 'http://www.raiplay.it/programmi/nondirloalmiocapo/',
-        'info_dict': {
-            'id': 'nondirloalmiocapo',
-            'title': 'Non dirlo al mio capo',
-        },
-        'playlist_mincount': 12,
-    }]
-
-    def _real_extract(self, url):
-        playlist_id = self._match_id(url)
-        webpage = self._download_webpage(url, playlist_id)
-        title = self._html_search_meta('programma', webpage, default=None)
-        video_urls = re.findall(' href="(/raiplay/video.+)"', webpage)
-        video_urls = [urljoin(url, video_url) for video_url in video_urls]
-        entries = [
-            self.url_result(
-                video_url,
-                RaiPlayIE.ie_key())
-            for video_url in video_urls if RaiPlayIE.suitable(video_url)
-        ]
-        return self.playlist_result(entries, playlist_id, title)

From e2707a832cd53e2cfa68b99db997890a6a5bd685 Mon Sep 17 00:00:00 2001
From: Alex Seiler <seileralex@gmail.com>
Date: Mon, 23 Oct 2017 21:15:48 +0200
Subject: [PATCH 1589/1696] [ellentube] Fix extraction (closes #14407)

---
 youtube_dl/extractor/ellentube.py  | 140 +++++++++++++++++++++++++++++
 youtube_dl/extractor/ellentv.py    | 101 ---------------------
 youtube_dl/extractor/extractors.py |   8 +-
 3 files changed, 145 insertions(+), 104 deletions(-)
 create mode 100644 youtube_dl/extractor/ellentube.py
 delete mode 100644 youtube_dl/extractor/ellentv.py

diff --git a/youtube_dl/extractor/ellentube.py b/youtube_dl/extractor/ellentube.py
new file mode 100644
index 000000000..68fe17273
--- /dev/null
+++ b/youtube_dl/extractor/ellentube.py
@@ -0,0 +1,140 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    int_or_none,
+)
+
+
+class EllenTubeIE(InfoExtractor):
+    _VALID_URL = r'''(?x)
+                    (?:
+                        https://api-prod\.ellentube\.com/ellenapi/api/item/
+                        |ellentube:
+                    )
+                    (?P<id>
+                        [\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}
+                    )'''
+
+    _TESTS = [{
+        'url': 'https://api-prod.ellentube.com/ellenapi/api/item/75c64c16-aefd-4558-b4f5-3de09b22e6fc',
+        'match_only': True,
+    }, {
+        'url': 'ellentube:734a3353-f697-4e79-9ca9-bfc3002dc1e0',
+        'match_only': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        data = self._download_json(
+            'https://api-prod.ellentube.com/ellenapi/api/item/%s' % video_id, video_id)
+        title = data['title']
+        description = data.get('description')
+        publish_time = int_or_none(data.get('publishTime'))
+        thumbnail = data.get('thumbnail')
+
+        formats = []
+        duration = None
+        for entry in data.get('media'):
+            if entry.get('id') == 'm3u8':
+                formats = self._extract_m3u8_formats(
+                    entry.get('url'), video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls')
+                duration = int_or_none(entry.get('duration'))
+                break
+        self._sort_formats(formats)
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'duration': duration,
+            'thumbnail': thumbnail,
+            'timestamp': publish_time,
+            'formats': formats,
+        }
+
+
+class EllenTubeVideoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?ellentube\.com/video/(?P<id>.+)\.html'
+
+    _TEST = {
+        'url': 'https://www.ellentube.com/video/ellen-meets-las-vegas-survivors-jesus-campos-and-stephen-schuck.html',
+        'md5': '2fabc277131bddafdd120e0fc0f974c9',
+        'info_dict': {
+            'id': '0822171c-3829-43bf-b99f-d77358ae75e3',
+            'ext': 'mp4',
+            'title': 'Ellen Meets Las Vegas Survivors Jesus Campos and Stephen Schuck',
+            'description': 'md5:76e3355e2242a78ad9e3858e5616923f',
+            'duration': 514,
+            'timestamp': 1508505120000,
+            'thumbnail': 'https://warnerbros-h.assetsadobe.com/is/image/content/dam/ellen/videos/episodes/season15/32/video--2728751654987218111',
+        }
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        video_id = self._html_search_regex(
+            r'(?s)<!--\s*CONTENT\s*-->.*data-config.+([\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})',
+            webpage, 'video id')
+        return self.url_result('ellentube:%s' % video_id, 'EllenTube')
+
+
+class EllenTubePlaylistIE(InfoExtractor):
+    def _extract_videos_from_json(self, data, display_id):
+        return [self.url_result('ellentube:%s' % elem['id'], 'EllenTube')
+                for elem in data if elem.get('type') == 'VIDEO']
+
+    def _extract_playlist(self, url, display_id, extract_description=True):
+        webpage = self._download_webpage(url, display_id)
+        playlist_data = self._html_search_regex(
+            r'<div\s+data-component\s*=\s*"Details"(.+)</div>', webpage, 'playlist data')
+        playlist_title = self._search_regex(
+            r'"title"\s*:\s*"(.+?)"', playlist_data, 'playlist title')
+        playlist_description = clean_html(self._search_regex(
+            r'"description"\s*:\s*"(.+?)"', playlist_data, 'playlist description',
+            fatal=False)) if extract_description else None
+        api_search = self._search_regex(
+            r'"filter"\s*:\s*"(.+?)"', playlist_data, 'playlist api request')
+        api_data = self._download_json(
+            'https://api-prod.ellentube.com/ellenapi/api/feed/?%s' % api_search,
+            display_id)
+        return self.playlist_result(
+            self._extract_videos_from_json(api_data, display_id),
+            display_id, playlist_title, playlist_description)
+
+
+class EllenTubeEpisodeIE(EllenTubePlaylistIE):
+    _VALID_URL = r'https?://(?:www\.)?ellentube\.com/episode/(?P<id>.+)\.html'
+
+    _TEST = {
+        'url': 'https://www.ellentube.com/episode/dax-shepard-jordan-fisher-haim.html',
+        'info_dict': {
+            'id': 'dax-shepard-jordan-fisher-haim',
+            'title': 'Dax Shepard, \'DWTS\' Team Jordan Fisher & Lindsay Arnold, HAIM',
+            'description': 'md5:aed85d42892f6126e71ec5ed2aea2a0d'
+        },
+        'playlist_count': 6,
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        return self._extract_playlist(url, display_id)
+
+
+class EllenTubeStudioIE(EllenTubePlaylistIE):
+    _VALID_URL = r'https?://(?:www\.)?ellentube\.com/studios/(?P<id>.+)\.html'
+
+    _TEST = {
+        'url': 'https://www.ellentube.com/studios/macey-goes-rving0.html',
+        'info_dict': {
+            'id': 'macey-goes-rving0',
+            'title': 'Macey Goes RVing',
+        },
+        'playlist_mincount': 3,
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        return self._extract_playlist(url, display_id, False)
diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py
deleted file mode 100644
index e0a13dd76..000000000
--- a/youtube_dl/extractor/ellentv.py
+++ /dev/null
@@ -1,101 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from .kaltura import KalturaIE
-from ..utils import NO_DEFAULT
-
-
-class EllenTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?:ellentv|ellentube)\.com/videos/(?P<id>[a-z0-9_-]+)'
-    _TESTS = [{
-        'url': 'http://www.ellentv.com/videos/0-ipq1gsai/',
-        'md5': '4294cf98bc165f218aaa0b89e0fd8042',
-        'info_dict': {
-            'id': '0_ipq1gsai',
-            'ext': 'mov',
-            'title': 'Fast Fingers of Fate',
-            'description': 'md5:3539013ddcbfa64b2a6d1b38d910868a',
-            'timestamp': 1428035648,
-            'upload_date': '20150403',
-            'uploader_id': 'batchUser',
-        },
-    }, {
-        # not available via http://widgets.ellentube.com/
-        'url': 'http://www.ellentv.com/videos/1-szkgu2m2/',
-        'info_dict': {
-            'id': '1_szkgu2m2',
-            'ext': 'flv',
-            'title': "Ellen's Amazingly Talented Audience",
-            'description': 'md5:86ff1e376ff0d717d7171590e273f0a5',
-            'timestamp': 1255140900,
-            'upload_date': '20091010',
-            'uploader_id': 'ellenkaltura@gmail.com',
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        URLS = ('http://widgets.ellentube.com/videos/%s' % video_id, url)
-
-        for num, url_ in enumerate(URLS, 1):
-            webpage = self._download_webpage(
-                url_, video_id, fatal=num == len(URLS))
-
-            default = NO_DEFAULT if num == len(URLS) else None
-
-            partner_id = self._search_regex(
-                r"var\s+partnerId\s*=\s*'([^']+)", webpage, 'partner id',
-                default=default)
-
-            kaltura_id = self._search_regex(
-                [r'id="kaltura_player_([^"]+)"',
-                 r"_wb_entry_id\s*:\s*'([^']+)",
-                 r'data-kaltura-entry-id="([^"]+)'],
-                webpage, 'kaltura id', default=default)
-
-            if partner_id and kaltura_id:
-                break
-
-        return self.url_result('kaltura:%s:%s' % (partner_id, kaltura_id), KalturaIE.ie_key())
-
-
-class EllenTVClipsIE(InfoExtractor):
-    IE_NAME = 'EllenTV:clips'
-    _VALID_URL = r'https?://(?:www\.)?ellentv\.com/episodes/(?P<id>[a-z0-9_-]+)'
-    _TEST = {
-        'url': 'http://www.ellentv.com/episodes/meryl-streep-vanessa-hudgens/',
-        'info_dict': {
-            'id': 'meryl-streep-vanessa-hudgens',
-            'title': 'Meryl Streep, Vanessa Hudgens',
-        },
-        'playlist_mincount': 5,
-    }
-
-    def _real_extract(self, url):
-        playlist_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, playlist_id)
-        playlist = self._extract_playlist(webpage, playlist_id)
-
-        return {
-            '_type': 'playlist',
-            'id': playlist_id,
-            'title': self._og_search_title(webpage),
-            'entries': self._extract_entries(playlist)
-        }
-
-    def _extract_playlist(self, webpage, playlist_id):
-        json_string = self._search_regex(r'playerView.addClips\(\[\{(.*?)\}\]\);', webpage, 'json')
-        return self._parse_json('[{' + json_string + '}]', playlist_id)
-
-    def _extract_entries(self, playlist):
-        return [
-            self.url_result(
-                'kaltura:%s:%s' % (item['kaltura_partner_id'], item['kaltura_entry_id']),
-                KalturaIE.ie_key(), video_id=item['kaltura_entry_id'])
-            for item in playlist]
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index b9c97fac4..55d2dd1fe 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -311,9 +311,11 @@ from .ehow import EHowIE
 from .eighttracks import EightTracksIE
 from .einthusan import EinthusanIE
 from .eitb import EitbIE
-from .ellentv import (
-    EllenTVIE,
-    EllenTVClipsIE,
+from .ellentube import (
+    EllenTubeIE,
+    EllenTubeEpisodeIE,
+    EllenTubeStudioIE,
+    EllenTubeVideoIE,
 )
 from .elpais import ElPaisIE
 from .embedly import EmbedlyIE

From 2a57b62b8007973b5b4974a1d9f5ab06ae78c86e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Dec 2017 02:16:23 +0700
Subject: [PATCH 1590/1696] [ellentube] Fix issues, improve and simplify
 (closes #14570)

---
 youtube_dl/extractor/ellentube.py  | 167 ++++++++++++++---------------
 youtube_dl/extractor/extractors.py |   3 +-
 2 files changed, 81 insertions(+), 89 deletions(-)

diff --git a/youtube_dl/extractor/ellentube.py b/youtube_dl/extractor/ellentube.py
index 68fe17273..544473274 100644
--- a/youtube_dl/extractor/ellentube.py
+++ b/youtube_dl/extractor/ellentube.py
@@ -4,137 +4,130 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..utils import (
     clean_html,
+    extract_attributes,
+    float_or_none,
     int_or_none,
+    try_get,
 )
 
 
-class EllenTubeIE(InfoExtractor):
-    _VALID_URL = r'''(?x)
-                    (?:
-                        https://api-prod\.ellentube\.com/ellenapi/api/item/
-                        |ellentube:
-                    )
-                    (?P<id>
-                        [\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}
-                    )'''
+class EllenTubeBaseIE(InfoExtractor):
+    def _extract_data_config(self, webpage, video_id):
+        details = self._search_regex(
+            r'(<[^>]+\bdata-component=(["\'])[Dd]etails.+?></div>)', webpage,
+            'details')
+        return self._parse_json(
+            extract_attributes(details)['data-config'], video_id)
 
-    _TESTS = [{
-        'url': 'https://api-prod.ellentube.com/ellenapi/api/item/75c64c16-aefd-4558-b4f5-3de09b22e6fc',
-        'match_only': True,
-    }, {
-        'url': 'ellentube:734a3353-f697-4e79-9ca9-bfc3002dc1e0',
-        'match_only': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        data = self._download_json(
-            'https://api-prod.ellentube.com/ellenapi/api/item/%s' % video_id, video_id)
+    def _extract_video(self, data, video_id):
         title = data['title']
-        description = data.get('description')
-        publish_time = int_or_none(data.get('publishTime'))
-        thumbnail = data.get('thumbnail')
 
         formats = []
         duration = None
         for entry in data.get('media'):
             if entry.get('id') == 'm3u8':
                 formats = self._extract_m3u8_formats(
-                    entry.get('url'), video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls')
+                    entry['url'], video_id, 'mp4',
+                    entry_protocol='m3u8_native', m3u8_id='hls')
                 duration = int_or_none(entry.get('duration'))
                 break
         self._sort_formats(formats)
+
+        def get_insight(kind):
+            return int_or_none(try_get(
+                data, lambda x: x['insight']['%ss' % kind]))
+
         return {
+            'extractor_key': EllenTubeIE.ie_key(),
             'id': video_id,
             'title': title,
-            'description': description,
+            'description': data.get('description'),
             'duration': duration,
-            'thumbnail': thumbnail,
-            'timestamp': publish_time,
+            'thumbnail': data.get('thumbnail'),
+            'timestamp': float_or_none(data.get('publishTime'), scale=1000),
+            'view_count': get_insight('view'),
+            'like_count': get_insight('like'),
             'formats': formats,
         }
 
 
-class EllenTubeVideoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ellentube\.com/video/(?P<id>.+)\.html'
-
-    _TEST = {
-        'url': 'https://www.ellentube.com/video/ellen-meets-las-vegas-survivors-jesus-campos-and-stephen-schuck.html',
+class EllenTubeIE(EllenTubeBaseIE):
+    _VALID_URL = r'''(?x)
+                        (?:
+                            ellentube:|
+                            https://api-prod\.ellentube\.com/ellenapi/api/item/
+                        )
+                        (?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})
+                    '''
+    _TESTS = [{
+        'url': 'https://api-prod.ellentube.com/ellenapi/api/item/0822171c-3829-43bf-b99f-d77358ae75e3',
         'md5': '2fabc277131bddafdd120e0fc0f974c9',
         'info_dict': {
             'id': '0822171c-3829-43bf-b99f-d77358ae75e3',
             'ext': 'mp4',
             'title': 'Ellen Meets Las Vegas Survivors Jesus Campos and Stephen Schuck',
             'description': 'md5:76e3355e2242a78ad9e3858e5616923f',
+            'thumbnail': r're:^https?://.+?',
             'duration': 514,
-            'timestamp': 1508505120000,
-            'thumbnail': 'https://warnerbros-h.assetsadobe.com/is/image/content/dam/ellen/videos/episodes/season15/32/video--2728751654987218111',
+            'timestamp': 1508505120,
+            'upload_date': '20171020',
+            'view_count': int,
+            'like_count': int,
         }
+    }, {
+        'url': 'ellentube:734a3353-f697-4e79-9ca9-bfc3002dc1e0',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        data = self._download_json(
+            'https://api-prod.ellentube.com/ellenapi/api/item/%s' % video_id,
+            video_id)
+        return self._extract_video(data, video_id)
+
+
+class EllenTubeVideoIE(EllenTubeBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?ellentube\.com/video/(?P<id>.+?)\.html'
+    _TEST = {
+        'url': 'https://www.ellentube.com/video/ellen-meets-las-vegas-survivors-jesus-campos-and-stephen-schuck.html',
+        'only_matching': True,
     }
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
-        video_id = self._html_search_regex(
-            r'(?s)<!--\s*CONTENT\s*-->.*data-config.+([\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})',
-            webpage, 'video id')
-        return self.url_result('ellentube:%s' % video_id, 'EllenTube')
+        video_id = self._extract_data_config(webpage, display_id)['id']
+        return self.url_result(
+            'ellentube:%s' % video_id, ie=EllenTubeIE.ie_key(),
+            video_id=video_id)
 
 
-class EllenTubePlaylistIE(InfoExtractor):
-    def _extract_videos_from_json(self, data, display_id):
-        return [self.url_result('ellentube:%s' % elem['id'], 'EllenTube')
-                for elem in data if elem.get('type') == 'VIDEO']
-
-    def _extract_playlist(self, url, display_id, extract_description=True):
-        webpage = self._download_webpage(url, display_id)
-        playlist_data = self._html_search_regex(
-            r'<div\s+data-component\s*=\s*"Details"(.+)</div>', webpage, 'playlist data')
-        playlist_title = self._search_regex(
-            r'"title"\s*:\s*"(.+?)"', playlist_data, 'playlist title')
-        playlist_description = clean_html(self._search_regex(
-            r'"description"\s*:\s*"(.+?)"', playlist_data, 'playlist description',
-            fatal=False)) if extract_description else None
-        api_search = self._search_regex(
-            r'"filter"\s*:\s*"(.+?)"', playlist_data, 'playlist api request')
-        api_data = self._download_json(
-            'https://api-prod.ellentube.com/ellenapi/api/feed/?%s' % api_search,
-            display_id)
-        return self.playlist_result(
-            self._extract_videos_from_json(api_data, display_id),
-            display_id, playlist_title, playlist_description)
-
-
-class EllenTubeEpisodeIE(EllenTubePlaylistIE):
-    _VALID_URL = r'https?://(?:www\.)?ellentube\.com/episode/(?P<id>.+)\.html'
-
-    _TEST = {
+class EllenTubePlaylistIE(EllenTubeBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?ellentube\.com/(?:episode|studios)/(?P<id>.+?)\.html'
+    _TESTS = [{
         'url': 'https://www.ellentube.com/episode/dax-shepard-jordan-fisher-haim.html',
         'info_dict': {
             'id': 'dax-shepard-jordan-fisher-haim',
-            'title': 'Dax Shepard, \'DWTS\' Team Jordan Fisher & Lindsay Arnold, HAIM',
-            'description': 'md5:aed85d42892f6126e71ec5ed2aea2a0d'
+            'title': "Dax Shepard, 'DWTS' Team Jordan Fisher & Lindsay Arnold, HAIM",
+            'description': 'md5:bfc982194dabb3f4e325e43aa6b2e21c',
         },
         'playlist_count': 6,
-    }
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        return self._extract_playlist(url, display_id)
-
-
-class EllenTubeStudioIE(EllenTubePlaylistIE):
-    _VALID_URL = r'https?://(?:www\.)?ellentube\.com/studios/(?P<id>.+)\.html'
-
-    _TEST = {
+    }, {
         'url': 'https://www.ellentube.com/studios/macey-goes-rving0.html',
-        'info_dict': {
-            'id': 'macey-goes-rving0',
-            'title': 'Macey Goes RVing',
-        },
-        'playlist_mincount': 3,
-    }
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
-        return self._extract_playlist(url, display_id, False)
+        webpage = self._download_webpage(url, display_id)
+        data = self._extract_data_config(webpage, display_id)['data']
+        feed = self._download_json(
+            'https://api-prod.ellentube.com/ellenapi/api/feed/?%s'
+            % data['filter'], display_id)
+        entries = [
+            self._extract_video(elem, elem['id'])
+            for elem in feed if elem.get('type') == 'VIDEO' and elem.get('id')]
+        return self.playlist_result(
+            entries, display_id, data.get('title'),
+            clean_html(data.get('description')))
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 55d2dd1fe..0177a2cff 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -313,9 +313,8 @@ from .einthusan import EinthusanIE
 from .eitb import EitbIE
 from .ellentube import (
     EllenTubeIE,
-    EllenTubeEpisodeIE,
-    EllenTubeStudioIE,
     EllenTubeVideoIE,
+    EllenTubePlaylistIE,
 )
 from .elpais import ElPaisIE
 from .embedly import EmbedlyIE

From f4cc03d60b5dd713fb8964cd9ecf8ca2b1a8a556 Mon Sep 17 00:00:00 2001
From: Andrew Bottom <abottom@wennsoft.com>
Date: Tue, 24 Oct 2017 11:50:02 -0500
Subject: [PATCH 1591/1696] [stretchinternet] Add extractor

---
 youtube_dl/extractor/extractors.py      |  1 +
 youtube_dl/extractor/stretchinternet.py | 28 +++++++++++++++++++++++++
 2 files changed, 29 insertions(+)
 create mode 100644 youtube_dl/extractor/stretchinternet.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 0177a2cff..612fd35d4 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1001,6 +1001,7 @@ from .streamango import StreamangoIE
 from .streamcloud import StreamcloudIE
 from .streamcz import StreamCZIE
 from .streetvoice import StreetVoiceIE
+from .stretchinternet import StretchInternetIE
 from .sunporno import SunPornoIE
 from .svt import (
     SVTIE,
diff --git a/youtube_dl/extractor/stretchinternet.py b/youtube_dl/extractor/stretchinternet.py
new file mode 100644
index 000000000..9a0ec0e65
--- /dev/null
+++ b/youtube_dl/extractor/stretchinternet.py
@@ -0,0 +1,28 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class StretchInternetIE(InfoExtractor):
+    IE_DESC = 'StretchInternet'
+    _VALID_URL = r'https?://.*?stretchinternet\.com/[^/_?].*(?<=eventId=)(?P<id>.*)(?=&).*'
+    _TEST = {
+        'url': 'https://portal.stretchinternet.com/umary/portal.htm?eventId=313900&streamType=video',
+        'info_dict': {
+            'id': '313900',
+            'ext': 'mp4',
+            'title': 'StretchInternet'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        stream = self._download_json('https://neo-client.stretchinternet.com/streamservice/v1/media/stream/v%s' % video_id, video_id)
+        stream_url = stream.get('source')
+        return {
+            'ie_key': 'Generic',
+            'id': video_id,
+            'url': 'http://%s' % stream_url,
+            'title': 'StretchInternet'
+        }

From a3de5e6c0e0efef4e8ff0cd37961c594b13c7fb9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Dec 2017 17:58:08 +0700
Subject: [PATCH 1592/1696] [stretchinternet] Fix issues and improve (closes
 #14576)

---
 youtube_dl/extractor/stretchinternet.py | 38 +++++++++++++++++++------
 1 file changed, 29 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/stretchinternet.py b/youtube_dl/extractor/stretchinternet.py
index 9a0ec0e65..ae2ac1b42 100644
--- a/youtube_dl/extractor/stretchinternet.py
+++ b/youtube_dl/extractor/stretchinternet.py
@@ -1,28 +1,48 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import int_or_none
 
 
 class StretchInternetIE(InfoExtractor):
-    IE_DESC = 'StretchInternet'
-    _VALID_URL = r'https?://.*?stretchinternet\.com/[^/_?].*(?<=eventId=)(?P<id>.*)(?=&).*'
+    _VALID_URL = r'https?://portal\.stretchinternet\.com/[^/]+/portal\.htm\?.*?\beventId=(?P<id>\d+)'
     _TEST = {
         'url': 'https://portal.stretchinternet.com/umary/portal.htm?eventId=313900&streamType=video',
         'info_dict': {
             'id': '313900',
             'ext': 'mp4',
-            'title': 'StretchInternet'
+            'title': 'Augustana (S.D.) Baseball vs University of Mary',
+            'description': 'md5:7578478614aae3bdd4a90f578f787438',
+            'timestamp': 1490468400,
+            'upload_date': '20170325',
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        stream = self._download_json('https://neo-client.stretchinternet.com/streamservice/v1/media/stream/v%s' % video_id, video_id)
-        stream_url = stream.get('source')
+
+        stream = self._download_json(
+            'https://neo-client.stretchinternet.com/streamservice/v1/media/stream/v%s'
+            % video_id, video_id)
+
+        video_url = 'https://%s' % stream['source']
+
+        event = self._download_json(
+            'https://neo-client.stretchinternet.com/portal-ws/getEvent.json',
+            video_id, query={
+                'clientID': 99997,
+                'eventID': video_id,
+                'token': 'asdf',
+            })['event']
+
+        title = event.get('title') or event['mobileTitle']
+        description = event.get('customText')
+        timestamp = int_or_none(event.get('longtime'))
+
         return {
-            'ie_key': 'Generic',
             'id': video_id,
-            'url': 'http://%s' % stream_url,
-            'title': 'StretchInternet'
+            'title': title,
+            'description': description,
+            'timestamp': timestamp,
+            'url': video_url,
         }

From 6f1ec339a0332041d6469887a91cf4e3ff557477 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Dec 2017 19:52:31 +0700
Subject: [PATCH 1593/1696] [udemy] Improve course id extraction (closes
 #14938)

---
 youtube_dl/extractor/udemy.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index c248ea727..b66033923 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -62,11 +62,11 @@ class UdemyIE(InfoExtractor):
     def _extract_course_info(self, webpage, video_id):
         course = self._parse_json(
             unescapeHTML(self._search_regex(
-                r'ng-init=["\'].*\bcourse=({.+?});', webpage, 'course', default='{}')),
+                r'ng-init=["\'].*\bcourse=({.+?})[;"\']',
+                webpage, 'course', default='{}')),
             video_id, fatal=False) or {}
         course_id = course.get('id') or self._search_regex(
-            (r'&quot;id&quot;\s*:\s*(\d+)', r'data-course-id=["\'](\d+)'),
-            webpage, 'course id')
+            r'data-course-id=["\'](\d+)', webpage, 'course id')
         return course_id, course.get('title')
 
     def _enroll_course(self, base_url, webpage, course_id):

From 913b61eeee9436a3ddf5675ec935d00c3a05b7a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Dec 2017 20:02:19 +0700
Subject: [PATCH 1594/1696] [udemy] Extract more HLS formats

---
 youtube_dl/extractor/udemy.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index b66033923..195f5ce78 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -257,6 +257,11 @@ class UdemyIE(InfoExtractor):
                 video_url = source.get('file') or source.get('src')
                 if not video_url or not isinstance(video_url, compat_str):
                     continue
+                if source.get('type') == 'application/x-mpegURL' or determine_ext(video_url) == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        video_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                        m3u8_id='hls', fatal=False))
+                    continue
                 format_id = source.get('label')
                 f = {
                     'url': video_url,

From 51f2863357af55dfe026bfe47808bf104e7eebf9 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 10 Dec 2017 14:10:52 +0100
Subject: [PATCH 1595/1696] [twitter] improve extraction(closes #14197)

---
 youtube_dl/extractor/twitter.py | 141 +++++++++++++++-----------------
 1 file changed, 64 insertions(+), 77 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 1b0b96371..d7e425041 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -43,7 +43,7 @@ class TwitterBaseIE(InfoExtractor):
 
 class TwitterCardIE(TwitterBaseIE):
     IE_NAME = 'twitter:card'
-    _VALID_URL = r'https?://(?:www\.)?twitter\.com/i/(?:cards/tfw/v1|videos(?:/tweet)?)/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?twitter\.com/i/(?P<path>cards/tfw/v1|videos(?:/tweet)?)/(?P<id>\d+)'
     _TESTS = [
         {
             'url': 'https://twitter.com/i/cards/tfw/v1/560070183650213889',
@@ -51,11 +51,10 @@ class TwitterCardIE(TwitterBaseIE):
             'info_dict': {
                 'id': '560070183650213889',
                 'ext': 'mp4',
-                'title': 'Twitter Card',
+                'title': 'Twitter web player',
                 'thumbnail': r're:^https?://.*\.jpg$',
                 'duration': 30.033,
             },
-            'skip': 'Video gone',
         },
         {
             'url': 'https://twitter.com/i/cards/tfw/v1/623160978427936768',
@@ -63,11 +62,9 @@ class TwitterCardIE(TwitterBaseIE):
             'info_dict': {
                 'id': '623160978427936768',
                 'ext': 'mp4',
-                'title': 'Twitter Card',
-                'thumbnail': r're:^https?://.*\.jpg',
-                'duration': 80.155,
+                'title': 'Twitter web player',
+                'thumbnail': r're:^https?://.*(?:\bformat=|\.)jpg',
             },
-            'skip': 'Video gone',
         },
         {
             'url': 'https://twitter.com/i/cards/tfw/v1/654001591733886977',
@@ -120,15 +117,15 @@ class TwitterCardIE(TwitterBaseIE):
             elif media_url.endswith('.mpd'):
                 formats.extend(self._extract_mpd_formats(media_url, video_id, mpd_id='dash'))
             else:
-                vbr = int_or_none(dict_get(media_variant, ('bitRate', 'bitrate')), scale=1000)
+                tbr = int_or_none(dict_get(media_variant, ('bitRate', 'bitrate')), scale=1000)
                 a_format = {
                     'url': media_url,
-                    'format_id': 'http-%d' % vbr if vbr else 'http',
-                    'vbr': vbr,
+                    'format_id': 'http-%d' % tbr if tbr else 'http',
+                    'tbr': tbr,
                 }
                 # Reported bitRate may be zero
-                if not a_format['vbr']:
-                    del a_format['vbr']
+                if not a_format['tbr']:
+                    del a_format['tbr']
 
                 self._search_dimensions_in_video_url(a_format, media_url)
 
@@ -150,79 +147,83 @@ class TwitterCardIE(TwitterBaseIE):
         bearer_token = self._search_regex(
             r'BEARER_TOKEN\s*:\s*"([^"]+)"',
             main_script, 'bearer token')
-        guest_token = self._search_regex(
-            r'document\.cookie\s*=\s*decodeURIComponent\("gt=(\d+)',
-            webpage, 'guest token')
+        # https://developer.twitter.com/en/docs/tweets/post-and-engage/api-reference/get-statuses-show-id
         api_data = self._download_json(
-            'https://api.twitter.com/2/timeline/conversation/%s.json' % video_id,
-            video_id, 'Downloading mobile API data',
+            'https://api.twitter.com/1.1/statuses/show/%s.json' % video_id,
+            video_id, 'Downloading API data',
             headers={
                 'Authorization': 'Bearer ' + bearer_token,
-                'x-guest-token': guest_token,
             })
-        media_info = try_get(api_data, lambda o: o['globalObjects']['tweets'][video_id]
-                                                  ['extended_entities']['media'][0]['video_info']) or {}
+        media_info = try_get(api_data, lambda o: o['extended_entities']['media'][0]['video_info']) or {}
         return self._parse_media_info(media_info, video_id)
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        path, video_id = re.search(self._VALID_URL, url).groups()
 
         config = None
         formats = []
         duration = None
 
-        webpage = self._download_webpage(url, video_id)
+        urls = [url]
+        if path.startswith('cards/'):
+            urls.append('https://twitter.com/i/videos/' + video_id)
 
-        iframe_url = self._html_search_regex(
-            r'<iframe[^>]+src="((?:https?:)?//(?:www\.youtube\.com/embed/[^"]+|(?:www\.)?vine\.co/v/\w+/card))"',
-            webpage, 'video iframe', default=None)
-        if iframe_url:
-            return self.url_result(iframe_url)
+        for u in urls:
+            webpage = self._download_webpage(u, video_id)
 
-        config = self._parse_json(self._html_search_regex(
-            r'data-(?:player-)?config="([^"]+)"', webpage,
-            'data player config', default='{}'),
-            video_id)
+            iframe_url = self._html_search_regex(
+                r'<iframe[^>]+src="((?:https?:)?//(?:www\.youtube\.com/embed/[^"]+|(?:www\.)?vine\.co/v/\w+/card))"',
+                webpage, 'video iframe', default=None)
+            if iframe_url:
+                return self.url_result(iframe_url)
 
-        if config.get('source_type') == 'vine':
-            return self.url_result(config['player_url'], 'Vine')
+            config = self._parse_json(self._html_search_regex(
+                r'data-(?:player-)?config="([^"]+)"', webpage,
+                'data player config', default='{}'),
+                video_id)
 
-        periscope_url = PeriscopeIE._extract_url(webpage)
-        if periscope_url:
-            return self.url_result(periscope_url, PeriscopeIE.ie_key())
+            if config.get('source_type') == 'vine':
+                return self.url_result(config['player_url'], 'Vine')
 
-        video_url = config.get('video_url') or config.get('playlist', [{}])[0].get('source')
+            periscope_url = PeriscopeIE._extract_url(webpage)
+            if periscope_url:
+                return self.url_result(periscope_url, PeriscopeIE.ie_key())
 
-        if video_url:
-            if determine_ext(video_url) == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(video_url, video_id, ext='mp4', m3u8_id='hls'))
-            else:
-                f = {
-                    'url': video_url,
-                }
+            video_url = config.get('video_url') or config.get('playlist', [{}])[0].get('source')
 
-                self._search_dimensions_in_video_url(f, video_url)
+            if video_url:
+                if determine_ext(video_url) == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(video_url, video_id, ext='mp4', m3u8_id='hls'))
+                else:
+                    f = {
+                        'url': video_url,
+                    }
 
-                formats.append(f)
+                    self._search_dimensions_in_video_url(f, video_url)
 
-        vmap_url = config.get('vmapUrl') or config.get('vmap_url')
-        if vmap_url:
-            formats.extend(
-                self._extract_formats_from_vmap_url(vmap_url, video_id))
+                    formats.append(f)
 
-        media_info = None
+            vmap_url = config.get('vmapUrl') or config.get('vmap_url')
+            if vmap_url:
+                formats.extend(
+                    self._extract_formats_from_vmap_url(vmap_url, video_id))
 
-        for entity in config.get('status', {}).get('entities', []):
-            if 'mediaInfo' in entity:
-                media_info = entity['mediaInfo']
+            media_info = None
 
-        if media_info:
-            formats.extend(self._parse_media_info(media_info, video_id))
-            duration = float_or_none(media_info.get('duration', {}).get('nanos'), scale=1e9)
+            for entity in config.get('status', {}).get('entities', []):
+                if 'mediaInfo' in entity:
+                    media_info = entity['mediaInfo']
 
-        username = config.get('user', {}).get('screen_name')
-        if username:
-            formats.extend(self._extract_mobile_formats(username, video_id))
+            if media_info:
+                formats.extend(self._parse_media_info(media_info, video_id))
+                duration = float_or_none(media_info.get('duration', {}).get('nanos'), scale=1e9)
+
+            username = config.get('user', {}).get('screen_name')
+            if username:
+                formats.extend(self._extract_mobile_formats(username, video_id))
+
+            if formats:
+                break
 
         self._remove_duplicate_formats(formats)
         self._sort_formats(formats)
@@ -258,9 +259,6 @@ class TwitterIE(InfoExtractor):
             'uploader_id': 'freethenipple',
             'duration': 12.922,
         },
-        'params': {
-            'skip_download': True,  # requires ffmpeg
-        },
     }, {
         'url': 'https://twitter.com/giphz/status/657991469417025536/photo/1',
         'md5': 'f36dcd5fb92bf7057f155e7d927eeb42',
@@ -277,7 +275,6 @@ class TwitterIE(InfoExtractor):
         'skip': 'Account suspended',
     }, {
         'url': 'https://twitter.com/starwars/status/665052190608723968',
-        'md5': '39b7199856dee6cd4432e72c74bc69d4',
         'info_dict': {
             'id': '665052190608723968',
             'ext': 'mp4',
@@ -303,20 +300,16 @@ class TwitterIE(InfoExtractor):
         },
     }, {
         'url': 'https://twitter.com/jaydingeer/status/700207533655363584',
-        'md5': '',
         'info_dict': {
             'id': '700207533655363584',
             'ext': 'mp4',
-            'title': 'あかさ - BEAT PROD: @suhmeduh #Damndaniel',
-            'description': 'あかさ on Twitter: "BEAT PROD: @suhmeduh  https://t.co/HBrQ4AfpvZ #Damndaniel https://t.co/byBooq2ejZ"',
+            'title': 'JG - BEAT PROD: @suhmeduh #Damndaniel',
+            'description': 'JG on Twitter: "BEAT PROD: @suhmeduh  https://t.co/HBrQ4AfpvZ #Damndaniel https://t.co/byBooq2ejZ"',
             'thumbnail': r're:^https?://.*\.jpg',
-            'uploader': 'あかさ',
+            'uploader': 'JG',
             'uploader_id': 'jaydingeer',
             'duration': 30.0,
         },
-        'params': {
-            'skip_download': True,  # requires ffmpeg
-        },
     }, {
         'url': 'https://twitter.com/Filmdrunk/status/713801302971588609',
         'md5': '89a15ed345d13b86e9a5a5e051fa308a',
@@ -342,9 +335,6 @@ class TwitterIE(InfoExtractor):
             'uploader': 'Captain America',
             'duration': 3.17,
         },
-        'params': {
-            'skip_download': True,  # requires ffmpeg
-        },
     }, {
         'url': 'https://twitter.com/OPP_HSD/status/779210622571536384',
         'info_dict': {
@@ -370,9 +360,6 @@ class TwitterIE(InfoExtractor):
             'uploader_id': 'news_al3alm',
             'duration': 277.4,
         },
-        'params': {
-            'format': 'best[format_id^=http-]',
-        },
     }, {
         'url': 'https://twitter.com/i/web/status/910031516746514432',
         'info_dict': {

From c38970ca10fc8c8e3ba05019d5f73ff235de9fc8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Dec 2017 22:46:21 +0700
Subject: [PATCH 1596/1696] [culturebox] Improve video id extraction (closes
 #14947)

---
 youtube_dl/extractor/francetv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 80c597015..095bb3954 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -356,6 +356,7 @@ class CultureboxIE(FranceTVBaseInfoExtractor):
             raise ExtractorError('Video %s is not available' % name, expected=True)
 
         video_id, catalogue = self._search_regex(
-            r'"https?://videos\.francetv\.fr/video/([^@]+@[^"]+)"', webpage, 'video id').split('@')
+            r'["\'>]https?://videos\.francetv\.fr/video/([^@]+@.+?)["\'<]',
+            webpage, 'video id').split('@')
 
         return self._extract_video(video_id, catalogue)

From fa1dd6d2cdaa41b67ee2fc47dc5184a040004d41 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Dec 2017 23:15:24 +0700
Subject: [PATCH 1597/1696] [ChangeLog] Actualize

---
 ChangeLog | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 63837d62b..fd3f3ebf8 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,30 @@
+version <unreleased>
+
+Core
++ [utils] Add sami mimetype to mimetype2ext
+
+Extractors
+* [culturebox] Improve video id extraction (#14947)
+* [twitter] Improve extraction (#14197)
++ [udemy] Extract more HLS formats
+* [udemy] Improve course id extraction (#14938)
++ [stretchinternet] Add support for portal.stretchinternet.com (#14576)
+* [ellentube] Fix extraction (#14407, #14570)
++ [raiplay:playlist] Add support for playlists (#14563)
+* [sonyliv] Bypass geo restriction
+* [sonyliv] Extract higher quality formats (#14922)
+* [fox] Extract subtitles
++ [fox] Add support for Adobe Pass authentication (#14205, #14489)
+- [dailymotion:cloud] Remove extractor (#6794)
+* [xhamster] Fix thumbnail extraction (#14780)
++ [xhamster] Add support for mobile URLs (#14780)
+* [generic] Don't pass video id as mpd id while extracting DASH (#14902)
+* [ard] Skip invalid stream URLs (#14906)
+* [porncom] Fix metadata extraction (#14911)
+* [pluralsight] Detect agreement request (#14913)
+* [toutv] Fix login (#14614)
+
+
 version 2017.12.02
 
 Core

From 1fa0dce2c044826eec87b712f90964b90198aaa4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Dec 2017 23:18:53 +0700
Subject: [PATCH 1598/1696] release 2017.12.10

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 8 +++++---
 youtube_dl/version.py     | 2 +-
 4 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 513823b9b..736869bf0 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.12.02*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.12.02**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.12.10*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.12.10**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.12.02
+[debug] youtube-dl version 2017.12.10
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index fd3f3ebf8..a1fdcab99 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.12.10
 
 Core
 + [utils] Add sami mimetype to mimetype2ext
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 0287a4011..d5e3a8a67 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -198,7 +198,6 @@
  - **dailymotion**
  - **dailymotion:playlist**
  - **dailymotion:user**
- - **DailymotionCloud**
  - **DaisukiMotto**
  - **DaisukiMottoPlaylist**
  - **daum.net**
@@ -243,8 +242,9 @@
  - **eHow**
  - **Einthusan**
  - **eitb.tv**
- - **EllenTV**
- - **EllenTV:clips**
+ - **EllenTube**
+ - **EllenTubePlaylist**
+ - **EllenTubeVideo**
  - **ElPais**: El País
  - **Embedly**
  - **EMPFlix**
@@ -662,6 +662,7 @@
  - **Rai**
  - **RaiPlay**
  - **RaiPlayLive**
+ - **RaiPlayPlaylist**
  - **RBMARadio**
  - **RDS**: RDS.ca
  - **RedBullTV**
@@ -781,6 +782,7 @@
  - **streamcloud.eu**
  - **StreamCZ**
  - **StreetVoice**
+ - **StretchInternet**
  - **SunPorno**
  - **SVT**
  - **SVTPlay**: SVT Play and Öppet arkiv
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 88bf1d652..d0c438317 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.12.02'
+__version__ = '2017.12.10'

From b6f78d76c14a1787606aad23c2df0d5158d1fde1 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 11 Dec 2017 13:38:55 +0100
Subject: [PATCH 1599/1696] [tbs] fix extraction(fixes #13658)

---
 youtube_dl/extractor/tbs.py    | 134 +++++++++++++++++++++++----------
 youtube_dl/extractor/turner.py |  48 ++++++------
 2 files changed, 120 insertions(+), 62 deletions(-)

diff --git a/youtube_dl/extractor/tbs.py b/youtube_dl/extractor/tbs.py
index e9474533f..460bc5d74 100644
--- a/youtube_dl/extractor/tbs.py
+++ b/youtube_dl/extractor/tbs.py
@@ -4,58 +4,110 @@ from __future__ import unicode_literals
 import re
 
 from .turner import TurnerBaseIE
-from ..utils import extract_attributes
+from ..utils import (
+    float_or_none,
+    int_or_none,
+    strip_or_none,
+)
 
 
 class TBSIE(TurnerBaseIE):
-    # https://github.com/rg3/youtube-dl/issues/13658
-    _WORKING = False
-
-    _VALID_URL = r'https?://(?:www\.)?(?P<site>tbs|tntdrama)\.com/videos/(?:[^/]+/)+(?P<id>[^/?#]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?(?P<site>tbs|tntdrama)\.com/(?:movies|shows/[^/]+/(?:clips|season-\d+/episode-\d+))/(?P<id>[^/?#]+)'
     _TESTS = [{
-        'url': 'http://www.tbs.com/videos/people-of-earth/season-1/extras/2007318/theatrical-trailer.html',
-        'md5': '9e61d680e2285066ade7199e6408b2ee',
+        'url': 'http://www.tntdrama.com/shows/the-alienist/clips/monster',
         'info_dict': {
-            'id': '2007318',
+            'id': '8d384cde33b89f3a43ce5329de42903ed5099887',
             'ext': 'mp4',
-            'title': 'Theatrical Trailer',
-            'description': 'Catch the latest comedy from TBS, People of Earth, premiering Halloween night--Monday, October 31, at 9/8c.',
+            'title': 'Monster',
+            'description': 'Get a first look at the theatrical trailer for TNT’s highly anticipated new psychological thriller The Alienist, which premieres January 22 on TNT.',
+            'timestamp': 1508175329,
+            'upload_date': '20171016',
         },
-        'skip': 'TBS videos are deleted after a while',
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
     }, {
-        'url': 'http://www.tntdrama.com/videos/good-behavior/season-1/extras/1538823/you-better-run.html',
-        'md5': 'ce53c6ead5e9f3280b4ad2031a6fab56',
-        'info_dict': {
-            'id': '1538823',
-            'ext': 'mp4',
-            'title': 'You Better Run',
-            'description': 'Letty Raines must figure out what she\'s running toward while running away from her past. Good Behavior premieres November 15 at 9/8c.',
-        },
-        'skip': 'TBS videos are deleted after a while',
+        'url': 'http://www.tbs.com/shows/search-party/season-1/episode-1/explicit-the-mysterious-disappearance-of-the-girl-no-one-knew',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.tntdrama.com/movies/star-wars-a-new-hope',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         domain, display_id = re.match(self._VALID_URL, url).groups()
         site = domain[:3]
         webpage = self._download_webpage(url, display_id)
-        video_params = extract_attributes(self._search_regex(r'(<[^>]+id="page-video"[^>]*>)', webpage, 'video params'))
-        query = None
-        clip_id = video_params.get('clipid')
-        if clip_id:
-            query = 'id=' + clip_id
-        else:
-            query = 'titleId=' + video_params['titleid']
-        return self._extract_cvp_info(
-            'http://www.%s.com/service/cvpXml?%s' % (domain, query), display_id, {
-                'default': {
-                    'media_src': 'http://ht.cdn.turner.com/%s/big' % site,
-                },
-                'secure': {
-                    'media_src': 'http://androidhls-secure.cdn.turner.com/%s/big' % site,
-                    'tokenizer_src': 'http://www.%s.com/video/processors/services/token_ipadAdobe.do' % domain,
-                },
-            }, {
-                'url': url,
-                'site_name': site.upper(),
-                'auth_required': video_params.get('isAuthRequired') != 'false',
-            })
+        video_data = self._parse_json(self._search_regex(
+            r'<script[^>]+?data-drupal-selector="drupal-settings-json"[^>]*?>({.+?})</script>',
+            webpage, 'drupal setting'), display_id)['turner_playlist'][0]
+
+        media_id = video_data['mediaID']
+        title = video_data['title']
+
+        streams_data = self._download_json(
+            'http://medium.ngtv.io/media/%s/tv' % media_id,
+            media_id)['media']['tv']
+        duration = None
+        chapters = []
+        formats = []
+        for supported_type in ('unprotected', 'bulkaes'):
+            stream_data = streams_data.get(supported_type, {})
+            m3u8_url = stream_data.get('secureUrl') or stream_data.get('url')
+            if not m3u8_url:
+                continue
+            if stream_data.get('playlistProtection') == 'spe':
+                m3u8_url = self._add_akamai_spe_token(
+                    'http://www.%s.com/service/token_spe' % site,
+                    m3u8_url, media_id, {
+                        'url': url,
+                        'site_name': site.upper(),
+                        'auth_required': video_data.get('authRequired') == '1',
+                    })
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, media_id, 'mp4', m3u8_id='hls', fatal=False))
+
+            duration = float_or_none(stream_data.get('totalRuntime') or video_data.get('duration'))
+
+            if not chapters:
+                for chapter in stream_data.get('contentSegments', []):
+                    start_time = float_or_none(chapter.get('start'))
+                    duration = float_or_none(chapter.get('duration'))
+                    if start_time is None or duration is None:
+                        continue
+                    chapters.append({
+                        'start_time': start_time,
+                        'end_time': start_time + duration,
+                    })
+        self._sort_formats(formats)
+
+        thumbnails = []
+        for image_id, image in video_data.get('images', {}).items():
+            image_url = image.get('url')
+            if not image_url or image.get('type') != 'video':
+                continue
+            i = {
+                'id': image_id,
+                'url': image_url,
+            }
+            mobj = re.search(r'(\d+)x(\d+)', image_url)
+            if mobj:
+                i.update({
+                    'width': int(mobj.group(1)),
+                    'height': int(mobj.group(2)),
+                })
+            thumbnails.append(i)
+
+        return {
+            'id': media_id,
+            'title': title,
+            'description': strip_or_none(video_data.get('descriptionNoTags') or video_data.get('shortDescriptionNoTags')),
+            'duration': duration,
+            'timestamp': int_or_none(video_data.get('created')),
+            'season_number': int_or_none(video_data.get('season')),
+            'episode_number': int_or_none(video_data.get('episode')),
+            'cahpters': chapters,
+            'thumbnails': thumbnails,
+            'formats': formats,
+        }
diff --git a/youtube_dl/extractor/turner.py b/youtube_dl/extractor/turner.py
index efeb677ee..e73b64aeb 100644
--- a/youtube_dl/extractor/turner.py
+++ b/youtube_dl/extractor/turner.py
@@ -18,9 +18,32 @@ from ..utils import (
 
 
 class TurnerBaseIE(AdobePassIE):
+    _AKAMAI_SPE_TOKEN_CACHE = {}
+
     def _extract_timestamp(self, video_data):
         return int_or_none(xpath_attr(video_data, 'dateCreated', 'uts'))
 
+    def _add_akamai_spe_token(self, tokenizer_src, video_url, content_id, ap_data):
+        secure_path = self._search_regex(r'https?://[^/]+(.+/)', video_url, 'secure path') + '*'
+        token = self._AKAMAI_SPE_TOKEN_CACHE.get(secure_path)
+        if not token:
+            query = {
+                'path': secure_path,
+                'videoId': content_id,
+            }
+            if ap_data.get('auth_required'):
+                query['accessToken'] = self._extract_mvpd_auth(ap_data['url'], content_id, ap_data['site_name'], ap_data['site_name'])
+            auth = self._download_xml(
+                tokenizer_src, content_id, query=query)
+            error_msg = xpath_text(auth, 'error/msg')
+            if error_msg:
+                raise ExtractorError(error_msg, expected=True)
+            token = xpath_text(auth, 'token')
+            if not token:
+                return video_url
+            self._AKAMAI_SPE_TOKEN_CACHE[secure_path] = token
+        return video_url + '?hdnea=' + token
+
     def _extract_cvp_info(self, data_src, video_id, path_data={}, ap_data={}):
         video_data = self._download_xml(data_src, video_id)
         video_id = video_data.attrib['id']
@@ -33,7 +56,6 @@ class TurnerBaseIE(AdobePassIE):
         #         rtmp_src = splited_rtmp_src[1]
         # aifp = xpath_text(video_data, 'akamai/aifp', default='')
 
-        tokens = {}
         urls = []
         formats = []
         rex = re.compile(
@@ -67,26 +89,10 @@ class TurnerBaseIE(AdobePassIE):
                 secure_path_data = path_data.get('secure')
                 if not secure_path_data:
                     continue
-                video_url = secure_path_data['media_src'] + video_url
-                secure_path = self._search_regex(r'https?://[^/]+(.+/)', video_url, 'secure path') + '*'
-                token = tokens.get(secure_path)
-                if not token:
-                    query = {
-                        'path': secure_path,
-                        'videoId': content_id,
-                    }
-                    if ap_data.get('auth_required'):
-                        query['accessToken'] = self._extract_mvpd_auth(ap_data['url'], video_id, ap_data['site_name'], ap_data['site_name'])
-                    auth = self._download_xml(
-                        secure_path_data['tokenizer_src'], video_id, query=query)
-                    error_msg = xpath_text(auth, 'error/msg')
-                    if error_msg:
-                        raise ExtractorError(error_msg, expected=True)
-                    token = xpath_text(auth, 'token')
-                    if not token:
-                        continue
-                    tokens[secure_path] = token
-                video_url = video_url + '?hdnea=' + token
+                video_url = self._add_akamai_spe_token(
+                    secure_path_data['tokenizer_src'],
+                    secure_path_data['media_src'] + video_url,
+                    content_id, ap_data)
             elif not re.match('https?://', video_url):
                 base_path_data = path_data.get(ext, path_data.get('default', {}))
                 media_src = base_path_data.get('media_src')

From 5868079e9921bfb03c64439d2188b765fc30dcfc Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 11 Dec 2017 14:37:03 +0100
Subject: [PATCH 1600/1696] [nick.com] improve extraction(fixes #14876)

---
 youtube_dl/extractor/nick.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
index 310eea2cf..cae8faf3e 100644
--- a/youtube_dl/extractor/nick.py
+++ b/youtube_dl/extractor/nick.py
@@ -10,7 +10,7 @@ from ..utils import update_url_query
 class NickIE(MTVServicesInfoExtractor):
     # None of videos on the website are still alive?
     IE_NAME = 'nick.com'
-    _VALID_URL = r'https?://(?:(?:www|beta)\.)?nick(?:jr)?\.com/(?:[^/]+/)?(?:videos/clip|[^/]+/videos)/(?P<id>[^/?#.]+)'
+    _VALID_URL = r'https?://(?P<domain>(?:(?:www|beta)\.)?nick(?:jr)?\.com)/(?:[^/]+/)?(?:videos/clip|[^/]+/videos)/(?P<id>[^/?#.]+)'
     _FEED_URL = 'http://udat.mtvnservices.com/service1/dispatch.htm'
     _GEO_COUNTRIES = ['US']
     _TESTS = [{
@@ -69,8 +69,14 @@ class NickIE(MTVServicesInfoExtractor):
             'mgid': uri,
         }
 
-    def _extract_mgid(self, webpage):
-        return self._search_regex(r'data-contenturi="([^"]+)', webpage, 'mgid')
+    def _real_extract(self, url):
+        domain, display_id = re.match(self._VALID_URL, url).groups()
+        video_data = self._download_json(
+            'http://%s/data/video.endLevel.json' % domain,
+            display_id, query={
+                'urlKey': display_id,
+            })
+        return self._get_videos_info(video_data['player'] + video_data['id'])
 
 
 class NickDeIE(MTVServicesInfoExtractor):

From 08d77a95c98a1d1dcaa5df99b709f119e7a64f37 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 11 Dec 2017 17:18:05 +0100
Subject: [PATCH 1601/1696] [nickelodeon:br] add support for
 Nickelodeon(Brazil) websites(closes #14893)

---
 youtube_dl/extractor/nick.py | 45 ++++++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
index cae8faf3e..f38c2c725 100644
--- a/youtube_dl/extractor/nick.py
+++ b/youtube_dl/extractor/nick.py
@@ -79,6 +79,51 @@ class NickIE(MTVServicesInfoExtractor):
         return self._get_videos_info(video_data['player'] + video_data['id'])
 
 
+class NickBeIE(MTVServicesInfoExtractor):
+    IE_NAME = 'nickelodeon:br'
+    _VALID_URL = r'https?://(?P<domain>(?:www\.)?nickjr|mundonick\.uol)\.com\.br/(?:programas/)?[^/]+/videos/(?:episodios/)?(?P<id>[^/?#.]+)'
+    _TESTS = [{
+        'url': 'http://www.nickjr.com.br/patrulha-canina/videos/210-labirinto-de-pipoca/',
+        'only_matching': True,
+    }, {
+        'url': 'http://mundonick.uol.com.br/programas/the-loud-house/videos/muitas-irmas/7ljo9j',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        domain, display_id = re.match(self._VALID_URL, url).groups()
+        webpage = self._download_webpage(url, display_id)
+        uri = self._search_regex(
+            r'data-(?:contenturi|mgid)="([^"]+)', webpage, 'mgid')
+        video_id = self._id_from_uri(uri)
+        config = self._download_json(
+            'http://media.mtvnservices.com/pmt/e1/access/index.html',
+            video_id, query={
+                'uri': uri,
+                'configtype': 'edge',
+            }, headers={
+                'Referer': url,
+            })
+        info_url = self._remove_template_parameter(config['feedWithQueryParams'])
+        if info_url == 'None':
+            if domain.startswith('www.'):
+                domain = domain[4:]
+            content_domain = {
+                'mundonick.uol': 'mundonick.com.br',
+                'nickjr': 'br.nickelodeonjunior.tv',
+            }[domain]
+            query = {
+                'mgid': uri,
+                'imageEp': content_domain,
+                'arcEp': content_domain,
+            }
+            if domain == 'nickjr.com.br':
+                query['ep'] = 'c4b16088'
+            info_url = update_url_query(
+                'http://feeds.mtvnservices.com/od/feed/intl-mrss-player-feed', query)
+        return self._get_videos_info_from_url(info_url, video_id)
+
+
 class NickDeIE(MTVServicesInfoExtractor):
     IE_NAME = 'nick.de'
     _VALID_URL = r'https?://(?:www\.)?(?P<host>nick\.(?:de|com\.pl|ch)|nickelodeon\.(?:nl|be|at|dk|no|se))/[^/]+/(?:[^/]+/)*(?P<id>[^/?#&]+)'

From e4f201bc1b4ffbf6423278e8d744727aa7d6f415 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 11 Dec 2017 17:19:16 +0100
Subject: [PATCH 1602/1696] [extractors] add import for NickBrIE

---
 youtube_dl/extractor/extractors.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 612fd35d4..25c4ce0de 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -689,6 +689,7 @@ from .nhl import (
 )
 from .nick import (
     NickIE,
+    NickBeIE,
     NickDeIE,
     NickNightIE,
     NickRuIE,

From 127e98d31dd96b135f0f511c4ca171a300e089cc Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 11 Dec 2017 17:21:31 +0100
Subject: [PATCH 1603/1696] [nickelodeon:br] correct extractor name

---
 youtube_dl/extractor/extractors.py | 2 +-
 youtube_dl/extractor/nick.py       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 25c4ce0de..99887ee89 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -689,7 +689,7 @@ from .nhl import (
 )
 from .nick import (
     NickIE,
-    NickBeIE,
+    NickBrIE,
     NickDeIE,
     NickNightIE,
     NickRuIE,
diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
index f38c2c725..7edd68472 100644
--- a/youtube_dl/extractor/nick.py
+++ b/youtube_dl/extractor/nick.py
@@ -79,7 +79,7 @@ class NickIE(MTVServicesInfoExtractor):
         return self._get_videos_info(video_data['player'] + video_data['id'])
 
 
-class NickBeIE(MTVServicesInfoExtractor):
+class NickBrIE(MTVServicesInfoExtractor):
     IE_NAME = 'nickelodeon:br'
     _VALID_URL = r'https?://(?P<domain>(?:www\.)?nickjr|mundonick\.uol)\.com\.br/(?:programas/)?[^/]+/videos/(?:episodios/)?(?P<id>[^/?#.]+)'
     _TESTS = [{

From 23b6e23002d8bd3af3b2ccb2e1b833c1dd1b3cbf Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 11 Dec 2017 21:36:12 +0100
Subject: [PATCH 1604/1696] [tvnow] fix extraction(closes #7831)

---
 youtube_dl/extractor/extractors.py |   8 +-
 youtube_dl/extractor/nowtv.py      | 261 -----------------------------
 youtube_dl/extractor/tvnow.py      | 175 +++++++++++++++++++
 3 files changed, 179 insertions(+), 265 deletions(-)
 delete mode 100644 youtube_dl/extractor/nowtv.py
 create mode 100644 youtube_dl/extractor/tvnow.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 99887ee89..fb9f365f2 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -722,10 +722,6 @@ from .nowness import (
     NownessPlaylistIE,
     NownessSeriesIE,
 )
-from .nowtv import (
-    NowTVIE,
-    NowTVListIE,
-)
 from .noz import NozIE
 from .npo import (
     AndereTijdenIE,
@@ -1105,6 +1101,10 @@ from .tvigle import TvigleIE
 from .tvland import TVLandIE
 from .tvn24 import TVN24IE
 from .tvnoe import TVNoeIE
+from .tvnow import (
+    TVNowIE,
+    TVNowListIE,
+)
 from .tvp import (
     TVPEmbedIE,
     TVPIE,
diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
deleted file mode 100644
index e43b37136..000000000
--- a/youtube_dl/extractor/nowtv.py
+++ /dev/null
@@ -1,261 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..compat import compat_str
-from ..utils import (
-    ExtractorError,
-    determine_ext,
-    int_or_none,
-    parse_iso8601,
-    parse_duration,
-    remove_start,
-)
-
-
-class NowTVBaseIE(InfoExtractor):
-    _VIDEO_FIELDS = (
-        'id', 'title', 'free', 'geoblocked', 'articleLong', 'articleShort',
-        'broadcastStartDate', 'seoUrl', 'duration', 'files',
-        'format.defaultImage169Format', 'format.defaultImage169Logo')
-
-    def _extract_video(self, info, display_id=None):
-        video_id = compat_str(info['id'])
-
-        files = info['files']
-        if not files:
-            if info.get('geoblocked', False):
-                raise ExtractorError(
-                    'Video %s is not available from your location due to geo restriction' % video_id,
-                    expected=True)
-            if not info.get('free', True):
-                raise ExtractorError(
-                    'Video %s is not available for free' % video_id, expected=True)
-
-        formats = []
-        for item in files['items']:
-            if determine_ext(item['path']) != 'f4v':
-                continue
-            app, play_path = remove_start(item['path'], '/').split('/', 1)
-            formats.append({
-                'url': 'rtmpe://fms.rtl.de',
-                'app': app,
-                'play_path': 'mp4:%s' % play_path,
-                'ext': 'flv',
-                'page_url': 'http://rtlnow.rtl.de',
-                'player_url': 'http://cdn.static-fra.de/now/vodplayer.swf',
-                'tbr': int_or_none(item.get('bitrate')),
-            })
-        self._sort_formats(formats)
-
-        title = info['title']
-        description = info.get('articleLong') or info.get('articleShort')
-        timestamp = parse_iso8601(info.get('broadcastStartDate'), ' ')
-        duration = parse_duration(info.get('duration'))
-
-        f = info.get('format', {})
-        thumbnail = f.get('defaultImage169Format') or f.get('defaultImage169Logo')
-
-        return {
-            'id': video_id,
-            'display_id': display_id or info.get('seoUrl'),
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'timestamp': timestamp,
-            'duration': duration,
-            'formats': formats,
-        }
-
-
-class NowTVIE(NowTVBaseIE):
-    _WORKING = False
-    _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at|ch)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<show_id>[^/]+)/(?:(?:list/[^/]+|jahr/\d{4}/\d{1,2})/)?(?P<id>[^/]+)/(?:player|preview)'
-
-    _TESTS = [{
-        # rtl
-        'url': 'http://www.nowtv.de/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit/player',
-        'info_dict': {
-            'id': '203519',
-            'display_id': 'bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit',
-            'ext': 'flv',
-            'title': 'Inka Bause stellt die neuen Bauern vor',
-            'description': 'md5:e234e1ed6d63cf06be5c070442612e7e',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'timestamp': 1432580700,
-            'upload_date': '20150525',
-            'duration': 2786,
-        },
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        },
-    }, {
-        # rtl2
-        'url': 'http://www.nowtv.de/rtl2/berlin-tag-nacht/berlin-tag-nacht-folge-934/player',
-        'info_dict': {
-            'id': '203481',
-            'display_id': 'berlin-tag-nacht/berlin-tag-nacht-folge-934',
-            'ext': 'flv',
-            'title': 'Berlin - Tag & Nacht (Folge 934)',
-            'description': 'md5:c85e88c2e36c552dfe63433bc9506dd0',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'timestamp': 1432666800,
-            'upload_date': '20150526',
-            'duration': 2641,
-        },
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        },
-    }, {
-        # rtlnitro
-        'url': 'http://www.nowtv.de/rtlnitro/alarm-fuer-cobra-11-die-autobahnpolizei/hals-und-beinbruch-2014-08-23-21-10-00/player',
-        'info_dict': {
-            'id': '165780',
-            'display_id': 'alarm-fuer-cobra-11-die-autobahnpolizei/hals-und-beinbruch-2014-08-23-21-10-00',
-            'ext': 'flv',
-            'title': 'Hals- und Beinbruch',
-            'description': 'md5:b50d248efffe244e6f56737f0911ca57',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'timestamp': 1432415400,
-            'upload_date': '20150523',
-            'duration': 2742,
-        },
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        },
-    }, {
-        # superrtl
-        'url': 'http://www.nowtv.de/superrtl/medicopter-117/angst/player',
-        'info_dict': {
-            'id': '99205',
-            'display_id': 'medicopter-117/angst',
-            'ext': 'flv',
-            'title': 'Angst!',
-            'description': 'md5:30cbc4c0b73ec98bcd73c9f2a8c17c4e',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'timestamp': 1222632900,
-            'upload_date': '20080928',
-            'duration': 3025,
-        },
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        },
-    }, {
-        # ntv
-        'url': 'http://www.nowtv.de/ntv/ratgeber-geld/thema-ua-der-erste-blick-die-apple-watch/player',
-        'info_dict': {
-            'id': '203521',
-            'display_id': 'ratgeber-geld/thema-ua-der-erste-blick-die-apple-watch',
-            'ext': 'flv',
-            'title': 'Thema u.a.: Der erste Blick: Die Apple Watch',
-            'description': 'md5:4312b6c9d839ffe7d8caf03865a531af',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'timestamp': 1432751700,
-            'upload_date': '20150527',
-            'duration': 1083,
-        },
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        },
-    }, {
-        # vox
-        'url': 'http://www.nowtv.de/vox/der-hundeprofi/buero-fall-chihuahua-joel/player',
-        'info_dict': {
-            'id': '128953',
-            'display_id': 'der-hundeprofi/buero-fall-chihuahua-joel',
-            'ext': 'flv',
-            'title': "Büro-Fall / Chihuahua 'Joel'",
-            'description': 'md5:e62cb6bf7c3cc669179d4f1eb279ad8d',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'timestamp': 1432408200,
-            'upload_date': '20150523',
-            'duration': 3092,
-        },
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        },
-    }, {
-        'url': 'http://www.nowtv.de/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit/preview',
-        'only_matching': True,
-    }, {
-        'url': 'http://www.nowtv.at/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit/preview?return=/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit',
-        'only_matching': True,
-    }, {
-        'url': 'http://www.nowtv.de/rtl2/echtzeit/list/aktuell/schnelles-geld-am-ende-der-welt/player',
-        'only_matching': True,
-    }, {
-        'url': 'http://www.nowtv.de/rtl2/zuhause-im-glueck/jahr/2015/11/eine-erschuetternde-diagnose/player',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = '%s/%s' % (mobj.group('show_id'), mobj.group('id'))
-
-        info = self._download_json(
-            'https://api.nowtv.de/v3/movies/%s?fields=%s'
-            % (display_id, ','.join(self._VIDEO_FIELDS)), display_id)
-
-        return self._extract_video(info, display_id)
-
-
-class NowTVListIE(NowTVBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at|ch)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<show_id>[^/]+)/list/(?P<id>[^?/#&]+)$'
-
-    _SHOW_FIELDS = ('title', )
-    _SEASON_FIELDS = ('id', 'headline', 'seoheadline', )
-
-    _TESTS = [{
-        'url': 'http://www.nowtv.at/rtl/stern-tv/list/aktuell',
-        'info_dict': {
-            'id': '17006',
-            'title': 'stern TV - Aktuell',
-        },
-        'playlist_count': 1,
-    }, {
-        'url': 'http://www.nowtv.at/rtl/das-supertalent/list/free-staffel-8',
-        'info_dict': {
-            'id': '20716',
-            'title': 'Das Supertalent - FREE Staffel 8',
-        },
-        'playlist_count': 14,
-    }]
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        show_id = mobj.group('show_id')
-        season_id = mobj.group('id')
-
-        fields = []
-        fields.extend(self._SHOW_FIELDS)
-        fields.extend('formatTabs.%s' % field for field in self._SEASON_FIELDS)
-        fields.extend(
-            'formatTabs.formatTabPages.container.movies.%s' % field
-            for field in self._VIDEO_FIELDS)
-
-        list_info = self._download_json(
-            'https://api.nowtv.de/v3/formats/seo?fields=%s&name=%s.php'
-            % (','.join(fields), show_id),
-            season_id)
-
-        season = next(
-            season for season in list_info['formatTabs']['items']
-            if season.get('seoheadline') == season_id)
-
-        title = '%s - %s' % (list_info['title'], season['headline'])
-
-        entries = []
-        for container in season['formatTabPages']['items']:
-            for info in ((container.get('container') or {}).get('movies') or {}).get('items') or []:
-                entries.append(self._extract_video(info))
-
-        return self.playlist_result(
-            entries, compat_str(season.get('id') or season_id), title)
diff --git a/youtube_dl/extractor/tvnow.py b/youtube_dl/extractor/tvnow.py
new file mode 100644
index 000000000..e2169f2bc
--- /dev/null
+++ b/youtube_dl/extractor/tvnow.py
@@ -0,0 +1,175 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    ExtractorError,
+    parse_iso8601,
+    parse_duration,
+    update_url_query,
+)
+
+
+class TVNowBaseIE(InfoExtractor):
+    _VIDEO_FIELDS = (
+        'id', 'title', 'free', 'geoblocked', 'articleLong', 'articleShort',
+        'broadcastStartDate', 'isDrm', 'duration', 'manifest.dashclear',
+        'format.defaultImage169Format', 'format.defaultImage169Logo')
+
+    def _call_api(self, path, video_id, query):
+        return self._download_json(
+            'https://api.tvnow.de/v3/' + path,
+            video_id, query=query)
+
+    def _extract_video(self, info, display_id):
+        video_id = compat_str(info['id'])
+        title = info['title']
+
+        mpd_url = info['manifest']['dashclear']
+        if not mpd_url:
+            if info.get('isDrm'):
+                raise ExtractorError(
+                    'Video %s is DRM protected' % video_id, expected=True)
+            if info.get('geoblocked'):
+                raise ExtractorError(
+                    'Video %s is not available from your location due to geo restriction' % video_id,
+                    expected=True)
+            if not info.get('free', True):
+                raise ExtractorError(
+                    'Video %s is not available for free' % video_id, expected=True)
+
+        mpd_url = update_url_query(mpd_url, {'filter': ''})
+        formats = self._extract_mpd_formats(mpd_url, video_id, mpd_id='dash', fatal=False)
+        formats.extend(self._extract_ism_formats(
+            mpd_url.replace('dash.', 'hss.').replace('/.mpd', '/Manifest'),
+            video_id, ism_id='mss', fatal=False))
+        formats.extend(self._extract_m3u8_formats(
+            mpd_url.replace('dash.', 'hls.').replace('/.mpd', '/.m3u8'),
+            video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+        self._sort_formats(formats)
+
+        description = info.get('articleLong') or info.get('articleShort')
+        timestamp = parse_iso8601(info.get('broadcastStartDate'), ' ')
+        duration = parse_duration(info.get('duration'))
+
+        f = info.get('format', {})
+        thumbnail = f.get('defaultImage169Format') or f.get('defaultImage169Logo')
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'duration': duration,
+            'formats': formats,
+        }
+
+
+class TVNowIE(TVNowBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?tvnow\.(?:de|at|ch)/(?:rtl(?:2|plus)?|nitro|superrtl|ntv|vox)/(?P<show_id>[^/]+)/(?:(?:list/[^/]+|jahr/\d{4}/\d{1,2})/)?(?P<id>[^/]+)/(?:player|preview)'
+
+    _TESTS = [{
+        # rtl
+        'url': 'https://www.tvnow.de/rtl/alarm-fuer-cobra-11/freier-fall/player?return=/rtl',
+        'info_dict': {
+            'id': '385314',
+            'display_id': 'alarm-fuer-cobra-11/freier-fall',
+            'ext': 'mp4',
+            'title': 'Freier Fall',
+            'description': 'md5:8c2d8f727261adf7e0dc18366124ca02',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'timestamp': 1512677700,
+            'upload_date': '20171207',
+            'duration': 2862.0,
+        },
+    }, {
+        # rtl2
+        'url': 'https://www.tvnow.de/rtl2/armes-deutschland/episode-0008/player',
+        'only_matching': 'True',
+    }, {
+        # rtlnitro
+        'url': 'https://www.tvnow.de/nitro/alarm-fuer-cobra-11-die-autobahnpolizei/auf-eigene-faust-pilot/player',
+        'only_matching': 'True',
+    }, {
+        # superrtl
+        'url': 'https://www.tvnow.de/superrtl/die-lustigsten-schlamassel-der-welt/u-a-ketchup-effekt/player',
+        'only_matching': 'True',
+    }, {
+        # ntv
+        'url': 'https://www.tvnow.de/ntv/startup-news/goetter-in-weiss/player',
+        'only_matching': 'True',
+    }, {
+        # vox
+        'url': 'https://www.tvnow.de/vox/auto-mobil/neues-vom-automobilmarkt-2017-11-19-17-00-00/player',
+        'only_matching': 'True',
+    }, {
+        # rtlplus
+        'url': 'https://www.tvnow.de/rtlplus/op-ruft-dr-bruckner/die-vernaehte-frau/player',
+        'only_matching': 'True',
+    }]
+
+    def _real_extract(self, url):
+        display_id = '%s/%s' % re.match(self._VALID_URL, url).groups()
+
+        info = self._call_api(
+            'movies/' + display_id, display_id, query={
+                'fields': ','.join(self._VIDEO_FIELDS),
+            })
+
+        return self._extract_video(info, display_id)
+
+
+class TVNowListIE(TVNowBaseIE):
+    _VALID_URL = r'(?P<base_url>https?://(?:www\.)?tvnow\.(?:de|at|ch)/(?:rtl(?:2|plus)?|nitro|superrtl|ntv|vox)/(?P<show_id>[^/]+)/)list/(?P<id>[^?/#&]+)$'
+
+    _SHOW_FIELDS = ('title', )
+    _SEASON_FIELDS = ('id', 'headline', 'seoheadline', )
+    _VIDEO_FIELDS = ('id', 'headline', 'seoUrl', )
+
+    _TESTS = [{
+        'url': 'https://www.tvnow.de/rtl/30-minuten-deutschland/list/aktuell',
+        'info_dict': {
+            'id': '28296',
+            'title': '30 Minuten Deutschland - Aktuell',
+        },
+        'playlist_mincount': 1,
+    }]
+
+    def _real_extract(self, url):
+        base_url, show_id, season_id = re.match(self._VALID_URL, url).groups()
+
+        fields = []
+        fields.extend(self._SHOW_FIELDS)
+        fields.extend('formatTabs.%s' % field for field in self._SEASON_FIELDS)
+        fields.extend(
+            'formatTabs.formatTabPages.container.movies.%s' % field
+            for field in self._VIDEO_FIELDS)
+
+        list_info = self._call_api(
+            'formats/seo', season_id, query={
+                'fields': ','.join(fields),
+                'name': show_id + '.php'
+            })
+
+        season = next(
+            season for season in list_info['formatTabs']['items']
+            if season.get('seoheadline') == season_id)
+
+        title = '%s - %s' % (list_info['title'], season['headline'])
+
+        entries = []
+        for container in season['formatTabPages']['items']:
+            for info in ((container.get('container') or {}).get('movies') or {}).get('items') or []:
+                seo_url = info.get('seoUrl')
+                if not seo_url:
+                    continue
+                entries.append(self.url_result(
+                    base_url + seo_url + '/player', 'TVNow', info.get('id')))
+
+        return self.playlist_result(
+            entries, compat_str(season.get('id') or season_id), title)

From cb0c2310fbf232e09ae41013be3400034171d6d2 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 12 Dec 2017 10:33:03 +0100
Subject: [PATCH 1605/1696] [discovery] fix free videos
 extraction(#14157)(#14954)

---
 youtube_dl/extractor/discovery.py   | 122 +++++++++++++---------------
 youtube_dl/extractor/discoverygo.py |  72 ++++++++--------
 2 files changed, 94 insertions(+), 100 deletions(-)

diff --git a/youtube_dl/extractor/discovery.py b/youtube_dl/extractor/discovery.py
index 55853f76f..f9cec1d23 100644
--- a/youtube_dl/extractor/discovery.py
+++ b/youtube_dl/extractor/discovery.py
@@ -1,14 +1,18 @@
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
+import random
+import re
+import string
+
+from .discoverygo import DiscoveryGoBaseIE
 from ..utils import (
-    parse_duration,
-    parse_iso8601,
+    ExtractorError,
+    update_url_query,
 )
-from ..compat import compat_str
+from ..compat import compat_HTTPError
 
 
-class DiscoveryIE(InfoExtractor):
+class DiscoveryIE(DiscoveryGoBaseIE):
     _VALID_URL = r'''(?x)https?://(?:www\.)?(?:
             discovery|
             investigationdiscovery|
@@ -19,79 +23,65 @@ class DiscoveryIE(InfoExtractor):
             sciencechannel|
             tlc|
             velocity
-        )\.com/(?:[^/]+/)*(?P<id>[^./?#]+)'''
+        )\.com(?P<path>/tv-shows/[^/]+/(?:video|full-episode)s/(?P<id>[^./?#]+))'''
     _TESTS = [{
-        'url': 'http://www.discovery.com/tv-shows/mythbusters/videos/mission-impossible-outtakes.htm',
+        'url': 'https://www.discovery.com/tv-shows/cash-cab/videos/dave-foley',
         'info_dict': {
-            'id': '20769',
+            'id': '5a2d9b4d6b66d17a5026e1fd',
             'ext': 'mp4',
-            'title': 'Mission Impossible Outtakes',
-            'description': ('Watch Jamie Hyneman and Adam Savage practice being'
-                            ' each other -- to the point of confusing Jamie\'s dog -- and '
-                            'don\'t miss Adam moon-walking as Jamie ... behind Jamie\'s'
-                            ' back.'),
-            'duration': 156,
-            'timestamp': 1302032462,
-            'upload_date': '20110405',
-            'uploader_id': '103207',
+            'title': 'Dave Foley',
+            'description': 'md5:4b39bcafccf9167ca42810eb5f28b01f',
+            'duration': 608,
         },
         'params': {
             'skip_download': True,  # requires ffmpeg
         }
     }, {
-        'url': 'http://www.discovery.com/tv-shows/mythbusters/videos/mythbusters-the-simpsons',
-        'info_dict': {
-            'id': 'mythbusters-the-simpsons',
-            'title': 'MythBusters: The Simpsons',
-        },
-        'playlist_mincount': 10,
-    }, {
-        'url': 'http://www.animalplanet.com/longfin-eels-maneaters/',
-        'info_dict': {
-            'id': '78326',
-            'ext': 'mp4',
-            'title': 'Longfin Eels: Maneaters?',
-            'description': 'Jeremy Wade tests whether or not New Zealand\'s longfin eels are man-eaters by covering himself in fish guts and getting in the water with them.',
-            'upload_date': '20140725',
-            'timestamp': 1406246400,
-            'duration': 116,
-            'uploader_id': '103207',
-        },
-        'params': {
-            'skip_download': True,  # requires ffmpeg
-        }
+        'url': 'https://www.investigationdiscovery.com/tv-shows/final-vision/full-episodes/final-vision',
+        'only_matching': True,
     }]
+    _GEO_COUNTRIES = ['US']
+    _GEO_BYPASS = False
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
-        info = self._download_json(url + '?flat=1', display_id)
+        path, display_id = re.match(self._VALID_URL, url).groups()
+        webpage = self._download_webpage(url, display_id)
 
-        video_title = info.get('playlist_title') or info.get('video_title')
+        react_data = self._parse_json(self._search_regex(
+            r'window\.__reactTransmitPacket\s*=\s*({.+?});',
+            webpage, 'react data'), display_id)
+        content_blocks = react_data['layout'][path]['contentBlocks']
+        video = next(cb for cb in content_blocks if cb.get('type') == 'video')['content']['items'][0]
+        video_id = video['id']
 
-        entries = []
+        access_token = self._download_json(
+            'https://www.discovery.com/anonymous', display_id, query={
+                'authLink': update_url_query(
+                    'https://login.discovery.com/v1/oauth2/authorize', {
+                        'client_id': react_data['application']['apiClientId'],
+                        'redirect_uri': 'https://fusion.ddmcdn.com/app/mercury-sdk/180/redirectHandler.html',
+                        'response_type': 'anonymous',
+                        'state': 'nonce,' + ''.join([random.choice(string.ascii_letters) for _ in range(32)]),
+                    })
+            })['access_token']
 
-        for idx, video_info in enumerate(info['playlist']):
-            subtitles = {}
-            caption_url = video_info.get('captionsUrl')
-            if caption_url:
-                subtitles = {
-                    'en': [{
-                        'url': caption_url,
-                    }]
-                }
+        try:
+            stream = self._download_json(
+                'https://api.discovery.com/v1/streaming/video/' + video_id,
+                display_id, headers={
+                    'Authorization': 'Bearer ' + access_token,
+                })
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+                e_description = self._parse_json(
+                    e.cause.read().decode(), display_id)['description']
+                if 'resource not available for country' in e_description:
+                    self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
+                if 'Authorized Networks' in e_description:
+                    raise ExtractorError(
+                        'This video is only available via cable service provider subscription that'
+                        ' is not currently supported. You may want to use --cookies.', expected=True)
+                raise ExtractorError(e_description)
+            raise
 
-            entries.append({
-                '_type': 'url_transparent',
-                'url': 'http://players.brightcove.net/103207/default_default/index.html?videoId=ref:%s' % video_info['referenceId'],
-                'id': compat_str(video_info['id']),
-                'title': video_info['title'],
-                'description': video_info.get('description'),
-                'duration': parse_duration(video_info.get('video_length')),
-                'webpage_url': video_info.get('href') or video_info.get('url'),
-                'thumbnail': video_info.get('thumbnailURL'),
-                'alt_title': video_info.get('secondary_title'),
-                'timestamp': parse_iso8601(video_info.get('publishedDate')),
-                'subtitles': subtitles,
-            })
-
-        return self.playlist_result(entries, display_id, video_title)
+        return self._extract_video_info(video, stream, display_id)
diff --git a/youtube_dl/extractor/discoverygo.py b/youtube_dl/extractor/discoverygo.py
index 7cd5d4291..99376454b 100644
--- a/youtube_dl/extractor/discoverygo.py
+++ b/youtube_dl/extractor/discoverygo.py
@@ -27,42 +27,9 @@ class DiscoveryGoBaseIE(InfoExtractor):
             velocitychannel
         )go\.com/%s(?P<id>[^/?#&]+)'''
 
-
-class DiscoveryGoIE(DiscoveryGoBaseIE):
-    _VALID_URL = DiscoveryGoBaseIE._VALID_URL_TEMPLATE % r'(?:[^/]+/)+'
-    _GEO_COUNTRIES = ['US']
-    _TEST = {
-        'url': 'https://www.discoverygo.com/bering-sea-gold/reaper-madness/',
-        'info_dict': {
-            'id': '58c167d86b66d12f2addeb01',
-            'ext': 'mp4',
-            'title': 'Reaper Madness',
-            'description': 'md5:09f2c625c99afb8946ed4fb7865f6e78',
-            'duration': 2519,
-            'series': 'Bering Sea Gold',
-            'season_number': 8,
-            'episode_number': 6,
-            'age_limit': 14,
-        },
-    }
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, display_id)
-
-        container = extract_attributes(
-            self._search_regex(
-                r'(<div[^>]+class=["\']video-player-container[^>]+>)',
-                webpage, 'video container'))
-
-        video = self._parse_json(
-            container.get('data-video') or container.get('data-json'),
-            display_id)
-
+    def _extract_video_info(self, video, stream, display_id):
         title = video['name']
 
-        stream = video.get('stream')
         if not stream:
             if video.get('authenticated') is True:
                 raise ExtractorError(
@@ -124,6 +91,43 @@ class DiscoveryGoIE(DiscoveryGoBaseIE):
         }
 
 
+class DiscoveryGoIE(DiscoveryGoBaseIE):
+    _VALID_URL = DiscoveryGoBaseIE._VALID_URL_TEMPLATE % r'(?:[^/]+/)+'
+    _GEO_COUNTRIES = ['US']
+    _TEST = {
+        'url': 'https://www.discoverygo.com/bering-sea-gold/reaper-madness/',
+        'info_dict': {
+            'id': '58c167d86b66d12f2addeb01',
+            'ext': 'mp4',
+            'title': 'Reaper Madness',
+            'description': 'md5:09f2c625c99afb8946ed4fb7865f6e78',
+            'duration': 2519,
+            'series': 'Bering Sea Gold',
+            'season_number': 8,
+            'episode_number': 6,
+            'age_limit': 14,
+        },
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        container = extract_attributes(
+            self._search_regex(
+                r'(<div[^>]+class=["\']video-player-container[^>]+>)',
+                webpage, 'video container'))
+
+        video = self._parse_json(
+            container.get('data-video') or container.get('data-json'),
+            display_id)
+
+        stream = video.get('stream')
+
+        return self._extract_video_info(video, stream, display_id)
+
+
 class DiscoveryGoPlaylistIE(DiscoveryGoBaseIE):
     _VALID_URL = DiscoveryGoBaseIE._VALID_URL_TEMPLATE % ''
     _TEST = {

From e6b8803d599df4de3c115e4de4ef192a1e10c749 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 12 Dec 2017 11:11:44 +0100
Subject: [PATCH 1606/1696] [toutv] add support special video urls(closes
 #14179)

---
 youtube_dl/extractor/toutv.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/toutv.py b/youtube_dl/extractor/toutv.py
index 17c0adc15..2e7876cc5 100644
--- a/youtube_dl/extractor/toutv.py
+++ b/youtube_dl/extractor/toutv.py
@@ -16,7 +16,7 @@ from ..utils import (
 class TouTvIE(InfoExtractor):
     _NETRC_MACHINE = 'toutv'
     IE_NAME = 'tou.tv'
-    _VALID_URL = r'https?://ici\.tou\.tv/(?P<id>[a-zA-Z0-9_-]+(?:/S[0-9]+E[0-9]+)?)'
+    _VALID_URL = r'https?://ici\.tou\.tv/(?P<id>[a-zA-Z0-9_-]+(?:/S[0-9]+[EC][0-9]+)?)'
     _access_token = None
     _claims = None
 
@@ -37,6 +37,9 @@ class TouTvIE(InfoExtractor):
     }, {
         'url': 'http://ici.tou.tv/hackers',
         'only_matching': True,
+    }, {
+        'url': 'https://ici.tou.tv/l-age-adulte/S01C501',
+        'only_matching': True,
     }]
 
     def _real_initialize(self):

From 6b2d8c91823c8f4c5dcda42cf174ab5468c44661 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 12 Dec 2017 18:04:05 +0100
Subject: [PATCH 1607/1696] [bbc.co.uk] fix extraction for 320k m3u8 streams
 broken since 197224b7a4e37a6581bf1a0da18d0f67ea61a476

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 5525f7c9b..8b20c03d6 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -386,7 +386,7 @@ class BBCCoUkIE(InfoExtractor):
                             m3u8_id=format_id, fatal=False))
                         if re.search(self._USP_RE, href):
                             usp_formats = self._extract_m3u8_formats(
-                                re.sub(self._USP_RE, r'/\1\.ism/\1\.m3u8', href),
+                                re.sub(self._USP_RE, r'/\1.ism/\1.m3u8', href),
                                 programme_id, ext='mp4', entry_protocol='m3u8_native',
                                 m3u8_id=format_id, fatal=False)
                             for f in usp_formats:

From 15960255fe3caaae58132116d772d1c1bcf415fa Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 12 Dec 2017 18:16:45 +0100
Subject: [PATCH 1608/1696] [tbs] fix typo

---
 youtube_dl/extractor/tbs.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tbs.py b/youtube_dl/extractor/tbs.py
index 460bc5d74..eab22c38f 100644
--- a/youtube_dl/extractor/tbs.py
+++ b/youtube_dl/extractor/tbs.py
@@ -36,8 +36,7 @@ class TBSIE(TurnerBaseIE):
     }]
 
     def _real_extract(self, url):
-        domain, display_id = re.match(self._VALID_URL, url).groups()
-        site = domain[:3]
+        site, display_id = re.match(self._VALID_URL, url).groups()
         webpage = self._download_webpage(url, display_id)
         video_data = self._parse_json(self._search_regex(
             r'<script[^>]+?data-drupal-selector="drupal-settings-json"[^>]*?>({.+?})</script>',
@@ -62,7 +61,7 @@ class TBSIE(TurnerBaseIE):
                     'http://www.%s.com/service/token_spe' % site,
                     m3u8_url, media_id, {
                         'url': url,
-                        'site_name': site.upper(),
+                        'site_name': site[:3].upper(),
                         'auth_required': video_data.get('authRequired') == '1',
                     })
             formats.extend(self._extract_m3u8_formats(

From c8be7d5f7417f5779cda2c02653f38401dd7a6b7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 13 Dec 2017 23:14:30 +0700
Subject: [PATCH 1609/1696] [byutv] Fix extraction (closes #14966, closes
 #14967)

---
 youtube_dl/extractor/byutv.py      | 71 ++++++++----------------------
 youtube_dl/extractor/extractors.py |  5 +--
 2 files changed, 19 insertions(+), 57 deletions(-)

diff --git a/youtube_dl/extractor/byutv.py b/youtube_dl/extractor/byutv.py
index 8ef089653..dd7d2f858 100644
--- a/youtube_dl/extractor/byutv.py
+++ b/youtube_dl/extractor/byutv.py
@@ -3,20 +3,19 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import ExtractorError
 
 
 class BYUtvIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?byutv\.org/watch/(?!event/)(?P<id>[0-9a-f-]+)(?:/(?P<display_id>[^/?#&]+))?'
+    _VALID_URL = r'https?://(?:www\.)?byutv\.org/(?:watch|player)/(?!event/)(?P<id>[0-9a-f-]+)(?:/(?P<display_id>[^/?#&]+))?'
     _TESTS = [{
         'url': 'http://www.byutv.org/watch/6587b9a3-89d2-42a6-a7f7-fd2f81840a7d/studio-c-season-5-episode-5',
         'info_dict': {
-            'id': '6587b9a3-89d2-42a6-a7f7-fd2f81840a7d',
+            'id': 'ZvanRocTpW-G5_yZFeltTAMv6jxOU9KH',
             'display_id': 'studio-c-season-5-episode-5',
             'ext': 'mp4',
             'title': 'Season 5 Episode 5',
-            'description': 'md5:e07269172baff037f8e8bf9956bc9747',
-            'thumbnail': r're:^https?://.*\.jpg$',
+            'description': 'md5:1d31dc18ef4f075b28f6a65937d22c65',
+            'thumbnail': r're:^https?://.*',
             'duration': 1486.486,
         },
         'params': {
@@ -26,6 +25,9 @@ class BYUtvIE(InfoExtractor):
     }, {
         'url': 'http://www.byutv.org/watch/6587b9a3-89d2-42a6-a7f7-fd2f81840a7d',
         'only_matching': True,
+    }, {
+        'url': 'https://www.byutv.org/player/27741493-dc83-40b0-8420-e7ae38a2ae98/byu-football-toledo-vs-byu-93016?listid=4fe0fee5-0d3c-4a29-b725-e4948627f472&listindex=0&q=toledo',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -33,16 +35,16 @@ class BYUtvIE(InfoExtractor):
         video_id = mobj.group('id')
         display_id = mobj.group('display_id') or video_id
 
-        webpage = self._download_webpage(url, display_id)
-        episode_code = self._search_regex(
-            r'(?s)episode:(.*?\}),\s*\n', webpage, 'episode information')
-
-        ep = self._parse_json(
-            episode_code, display_id, transform_source=lambda s:
-            re.sub(r'(\n\s+)([a-zA-Z]+):\s+\'(.*?)\'', r'\1"\2": "\3"', s))
-
-        if ep['providerType'] != 'Ooyala':
-            raise ExtractorError('Unsupported provider %s' % ep['provider'])
+        ep = self._download_json(
+            'https://api.byutv.org/api3/catalog/getvideosforcontent', video_id,
+            query={
+                'contentid': video_id,
+                'channel': 'byutv_global',
+                'x-byutv-context': 'web$Global',
+            }, headers={
+                'x-byutv-context': 'web$Global',
+                'x-byutv-platformkey': 'xsaaw9c7y5',
+            })['ooyalaVOD']
 
         return {
             '_type': 'url_transparent',
@@ -50,44 +52,7 @@ class BYUtvIE(InfoExtractor):
             'url': 'ooyala:%s' % ep['providerId'],
             'id': video_id,
             'display_id': display_id,
-            'title': ep['title'],
+            'title': ep.get('title'),
             'description': ep.get('description'),
             'thumbnail': ep.get('imageThumbnail'),
         }
-
-
-class BYUtvEventIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?byutv\.org/watch/event/(?P<id>[0-9a-f-]+)'
-    _TEST = {
-        'url': 'http://www.byutv.org/watch/event/29941b9b-8bf6-48d2-aebf-7a87add9e34b',
-        'info_dict': {
-            'id': '29941b9b-8bf6-48d2-aebf-7a87add9e34b',
-            'ext': 'mp4',
-            'title': 'Toledo vs. BYU (9/30/16)',
-        },
-        'params': {
-            'skip_download': True,
-        },
-        'add_ie': ['Ooyala'],
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        ooyala_id = self._search_regex(
-            r'providerId\s*:\s*(["\'])(?P<id>(?:(?!\1).)+)\1',
-            webpage, 'ooyala id', group='id')
-
-        title = self._search_regex(
-            r'class=["\']description["\'][^>]*>\s*<h1>([^<]+)</h1>', webpage,
-            'title').strip()
-
-        return {
-            '_type': 'url_transparent',
-            'ie_key': 'Ooyala',
-            'url': 'ooyala:%s' % ooyala_id,
-            'id': video_id,
-            'title': title,
-        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index fb9f365f2..407245513 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -138,10 +138,7 @@ from .brightcove import (
     BrightcoveNewIE,
 )
 from .buzzfeed import BuzzFeedIE
-from .byutv import (
-    BYUtvIE,
-    BYUtvEventIE,
-)
+from .byutv import BYUtvIE
 from .c56 import C56IE
 from .camdemy import (
     CamdemyIE,

From bec49996c68fe73b4e17e7418290072a4c4b7cdb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 13 Dec 2017 23:49:05 +0700
Subject: [PATCH 1610/1696] [downloader/http] Return actual download result
 (closes #14971)

---
 youtube_dl/downloader/http.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 8a6638cc2..3ff26ff70 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -284,8 +284,7 @@ class HttpFD(FileDownloader):
         while count <= retries:
             try:
                 establish_connection()
-                download()
-                return True
+                return download()
             except RetryDownload as e:
                 count += 1
                 if count <= retries:

From 6bf9c28b0ac5d44381008cebe4741f8f1d78c53d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 13 Dec 2017 17:51:24 +0100
Subject: [PATCH 1611/1696] [byutv] add support for geo restricted videos

---
 youtube_dl/extractor/byutv.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/byutv.py b/youtube_dl/extractor/byutv.py
index dd7d2f858..4bf4efe1f 100644
--- a/youtube_dl/extractor/byutv.py
+++ b/youtube_dl/extractor/byutv.py
@@ -39,10 +39,10 @@ class BYUtvIE(InfoExtractor):
             'https://api.byutv.org/api3/catalog/getvideosforcontent', video_id,
             query={
                 'contentid': video_id,
-                'channel': 'byutv_global',
-                'x-byutv-context': 'web$Global',
+                'channel': 'byutv',
+                'x-byutv-context': 'web$US',
             }, headers={
-                'x-byutv-context': 'web$Global',
+                'x-byutv-context': 'web$US',
                 'x-byutv-platformkey': 'xsaaw9c7y5',
             })['ooyalaVOD']
 

From 7974e289a10f87ae52fdc32b5f5cc7e14a0fafda Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Dec 2017 01:00:14 +0700
Subject: [PATCH 1612/1696] [postprocessor/xattr] Clarify NO_SPACE message
 (#14970)

---
 youtube_dl/postprocessor/xattrpp.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dl/postprocessor/xattrpp.py
index fbdfa02ac..b0aed9ca7 100644
--- a/youtube_dl/postprocessor/xattrpp.py
+++ b/youtube_dl/postprocessor/xattrpp.py
@@ -42,6 +42,7 @@ class XAttrMetadataPP(PostProcessor):
                 'user.dublincore.format': 'format',
             }
 
+            num_written = 0
             for xattrname, infoname in xattr_mapping.items():
 
                 value = info.get(infoname)
@@ -52,6 +53,7 @@ class XAttrMetadataPP(PostProcessor):
 
                     byte_value = value.encode('utf-8')
                     write_xattr(filename, xattrname, byte_value)
+                    num_written += 1
 
             return [], info
 
@@ -62,8 +64,8 @@ class XAttrMetadataPP(PostProcessor):
         except XAttrMetadataError as e:
             if e.reason == 'NO_SPACE':
                 self._downloader.report_warning(
-                    'There\'s no disk space left or disk quota exceeded. ' +
-                    'Extended attributes are not written.')
+                    'There\'s no disk space left, disk quota exceeded or filesystem xattr limit exceeded. ' +
+                    (('Some ' if num_written else '') + 'extended attributes are not written.').capitalize())
             elif e.reason == 'VALUE_TOO_LONG':
                 self._downloader.report_warning(
                     'Unable to write extended attributes due to too long values.')

From 3fae11ac006567bfbfe9368bc11f56da090e267a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Dec 2017 04:49:07 +0700
Subject: [PATCH 1613/1696] [itv] Improve extraction, extract more subtitles
 and duration (closes #14944)

---
 youtube_dl/extractor/itv.py | 125 ++++++++++++++++++++++--------------
 1 file changed, 77 insertions(+), 48 deletions(-)

diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py
index 413a219dc..18a7d7f8c 100644
--- a/youtube_dl/extractor/itv.py
+++ b/youtube_dl/extractor/itv.py
@@ -26,7 +26,7 @@ from ..utils import (
 class ITVIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?itv\.com/hub/[^/]+/(?P<id>[0-9a-zA-Z]+)'
     _GEO_COUNTRIES = ['GB']
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.itv.com/hub/mr-bean-animated-series/2a2936a0053',
         'info_dict': {
             'id': '2a2936a0053',
@@ -37,7 +37,11 @@ class ITVIE(InfoExtractor):
             # rtmp download
             'skip_download': True,
         },
-    }
+    }, {
+        # unavailable via data-playlist-url
+        'url': 'https://www.itv.com/hub/through-the-keyhole/2a2271a0033',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -101,6 +105,18 @@ class ITVIE(InfoExtractor):
             'Content-Type': 'text/xml; charset=utf-8',
             'SOAPAction': 'http://tempuri.org/PlaylistService/GetPlaylist',
         })
+
+        info = self._search_json_ld(webpage, video_id, default={})
+        formats = []
+        subtitles = {}
+
+        def extract_subtitle(sub_url):
+            ext = determine_ext(sub_url, 'ttml')
+            subtitles.setdefault('en', []).append({
+                'url': sub_url,
+                'ext': 'ttml' if ext == 'xml' else ext,
+            })
+
         resp_env = self._download_xml(
             params['data-playlist-url'], video_id,
             headers=headers, data=etree.tostring(req_env))
@@ -111,37 +127,55 @@ class ITVIE(InfoExtractor):
             if fault_code == 'InvalidGeoRegion':
                 self.raise_geo_restricted(
                     msg=fault_string, countries=self._GEO_COUNTRIES)
-            raise ExtractorError('%s said: %s' % (self.IE_NAME, fault_string))
-        title = xpath_text(playlist, 'EpisodeTitle', fatal=True)
-        video_element = xpath_element(playlist, 'VideoEntries/Video', fatal=True)
-        media_files = xpath_element(video_element, 'MediaFiles', fatal=True)
-        rtmp_url = media_files.attrib['base']
+            elif fault_code != 'InvalidEntity':
+                raise ExtractorError(
+                    '%s said: %s' % (self.IE_NAME, fault_string), expected=True)
+            info.update({
+                'title': self._og_search_title(webpage),
+                'episode_title': params.get('data-video-episode'),
+                'series': params.get('data-video-title'),
+            })
+        else:
+            title = xpath_text(playlist, 'EpisodeTitle', default=None)
+            info.update({
+                'title': title,
+                'episode_title': title,
+                'episode_number': int_or_none(xpath_text(playlist, 'EpisodeNumber')),
+                'series': xpath_text(playlist, 'ProgrammeTitle'),
+                'duration': parse_duration(xpath_text(playlist, 'Duration')),
+            })
+            video_element = xpath_element(playlist, 'VideoEntries/Video', fatal=True)
+            media_files = xpath_element(video_element, 'MediaFiles', fatal=True)
+            rtmp_url = media_files.attrib['base']
 
-        formats = []
-        for media_file in media_files.findall('MediaFile'):
-            play_path = xpath_text(media_file, 'URL')
-            if not play_path:
-                continue
-            tbr = int_or_none(media_file.get('bitrate'), 1000)
-            f = {
-                'format_id': 'rtmp' + ('-%d' % tbr if tbr else ''),
-                'play_path': play_path,
-                # Providing this swfVfy allows to avoid truncated downloads
-                'player_url': 'http://www.itv.com/mercury/Mercury_VideoPlayer.swf',
-                'page_url': url,
-                'tbr': tbr,
-                'ext': 'flv',
-            }
-            app = self._search_regex(
-                'rtmpe?://[^/]+/(.+)$', rtmp_url, 'app', default=None)
-            if app:
-                f.update({
-                    'url': rtmp_url.split('?', 1)[0],
-                    'app': app,
-                })
-            else:
-                f['url'] = rtmp_url
-            formats.append(f)
+            for media_file in media_files.findall('MediaFile'):
+                play_path = xpath_text(media_file, 'URL')
+                if not play_path:
+                    continue
+                tbr = int_or_none(media_file.get('bitrate'), 1000)
+                f = {
+                    'format_id': 'rtmp' + ('-%d' % tbr if tbr else ''),
+                    'play_path': play_path,
+                    # Providing this swfVfy allows to avoid truncated downloads
+                    'player_url': 'http://www.itv.com/mercury/Mercury_VideoPlayer.swf',
+                    'page_url': url,
+                    'tbr': tbr,
+                    'ext': 'flv',
+                }
+                app = self._search_regex(
+                    'rtmpe?://[^/]+/(.+)$', rtmp_url, 'app', default=None)
+                if app:
+                    f.update({
+                        'url': rtmp_url.split('?', 1)[0],
+                        'app': app,
+                    })
+                else:
+                    f['url'] = rtmp_url
+                formats.append(f)
+
+            for caption_url in video_element.findall('ClosedCaptioningURIs/URL'):
+                if caption_url.text:
+                    extract_subtitle(caption_url.text)
 
         ios_playlist_url = params.get('data-video-playlist') or params.get('data-video-id')
         hmac = params.get('data-video-hmac')
@@ -198,27 +232,22 @@ class ITVIE(InfoExtractor):
                         formats.append({
                             'url': href,
                         })
+                subs = video_data.get('Subtitles')
+                if isinstance(subs, list):
+                    for sub in subs:
+                        if not isinstance(sub, dict):
+                            continue
+                        href = sub.get('Href')
+                        if isinstance(href, compat_str):
+                            extract_subtitle(href)
+                if not info.get('duration'):
+                    info['duration'] = parse_duration(video_data.get('Duration'))
+
         self._sort_formats(formats)
 
-        subtitles = {}
-        for caption_url in video_element.findall('ClosedCaptioningURIs/URL'):
-            if not caption_url.text:
-                continue
-            ext = determine_ext(caption_url.text, 'ttml')
-            subtitles.setdefault('en', []).append({
-                'url': caption_url.text,
-                'ext': 'ttml' if ext == 'xml' else ext,
-            })
-
-        info = self._search_json_ld(webpage, video_id, default={})
         info.update({
             'id': video_id,
-            'title': title,
             'formats': formats,
             'subtitles': subtitles,
-            'episode_title': title,
-            'episode_number': int_or_none(xpath_text(playlist, 'EpisodeNumber')),
-            'series': xpath_text(playlist, 'ProgrammeTitle'),
-            'duartion': parse_duration(xpath_text(playlist, 'Duration')),
         })
         return info

From c6a5a811a197e1c6520764df1f5f93473692dd18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Dec 2017 05:15:51 +0700
Subject: [PATCH 1614/1696] [ChangeLog] Actualize

---
 ChangeLog | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index a1fdcab99..254264c4a 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,23 @@
+version <unreleased>
+
+Core
+* [postprocessor/xattr] Clarify NO_SPACE message (#14970)
+* [downloader/http] Return actual download result from real_download (#14971)
+
+Extractors
++ [itv] Extract more subtitles and duration
+* [itv] Improve extraction (#14944)
++ [byutv] Add support for geo restricted videos
+* [byutv] Fix extraction (#14966, #14967)
++ [bbccouk] Fix extraction for 320k HLS streams
++ [toutv] Add support for special video URLs (#14179)
+* [discovery] Fix free videos extraction (#14157, #14954)
+* [tvnow] Fix extraction (#7831)
++ [nickelodeon:br] Add support for nickelodeon brazil websites (#14893)
+* [nick] Improve extraction (#14876)
+* [tbs] Fix extraction (#13658)
+
+
 version 2017.12.10
 
 Core

From 8ff2b16435cdf53c3e684532afe699dfdf34403c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Dec 2017 05:19:21 +0700
Subject: [PATCH 1615/1696] release 2017.12.14

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 8 ++++----
 youtube_dl/version.py     | 2 +-
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 736869bf0..f37d8aa42 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.12.10*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.12.10**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.12.14*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.12.14**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.12.10
+[debug] youtube-dl version 2017.12.14
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 254264c4a..03d2defb7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.12.14
 
 Core
 * [postprocessor/xattr] Clarify NO_SPACE message (#14970)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index d5e3a8a67..ebddd5b9d 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -122,7 +122,6 @@
  - **bt:vestlendingen**: Bergens Tidende - Vestlendingen
  - **BuzzFeed**
  - **BYUtv**
- - **BYUtvEvent**
  - **Camdemy**
  - **CamdemyFolder**
  - **CamWithHer**
@@ -538,6 +537,7 @@
  - **nhl.com:videocenter:category**: NHL videocenter category
  - **nick.com**
  - **nick.de**
+ - **nickelodeon:br**
  - **nickelodeonru**
  - **nicknight**
  - **niconico**: ニコニコ動画
@@ -556,8 +556,6 @@
  - **nowness**
  - **nowness:playlist**
  - **nowness:series**
- - **NowTV** (Currently broken)
- - **NowTVList**
  - **nowvideo**: NowVideo
  - **Noz**
  - **npo**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
@@ -794,7 +792,7 @@
  - **tagesschau:player**
  - **Tass**
  - **TastyTrade**
- - **TBS** (Currently broken)
+ - **TBS**
  - **TDSLifeway**
  - **teachertube**: teachertube.com videos
  - **teachertube:user:collection**: teachertube.com user and collection videos
@@ -865,6 +863,8 @@
  - **tvland.com**
  - **TVN24**
  - **TVNoe**
+ - **TVNow**
+ - **TVNowList**
  - **tvp**: Telewizja Polska
  - **tvp:embed**: Telewizja Polska
  - **tvp:series**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d0c438317..2b5a63464 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.12.10'
+__version__ = '2017.12.14'

From 1c4804ef9b03e92ef6b1d6720f9a93bb54d98324 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 14 Dec 2017 23:05:07 +0100
Subject: [PATCH 1616/1696] [voot] fix format extraction(closes #14758)

---
 youtube_dl/extractor/voot.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/voot.py b/youtube_dl/extractor/voot.py
index 5de3deb8c..5fd1c3d80 100644
--- a/youtube_dl/extractor/voot.py
+++ b/youtube_dl/extractor/voot.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from .kaltura import KalturaIE
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -17,11 +16,10 @@ class VootIE(InfoExtractor):
     _TESTS = [{
         'url': 'https://www.voot.com/shows/ishq-ka-rang-safed/1/360558/is-this-the-end-of-kamini-/441353',
         'info_dict': {
-            'id': '0_8ledb18o',
+            'id': '441353',
             'ext': 'mp4',
             'title': 'Ishq Ka Rang Safed - Season 01 - Episode 340',
             'description': 'md5:06291fbbbc4dcbe21235c40c262507c1',
-            'uploader_id': 'batchUser',
             'timestamp': 1472162937,
             'upload_date': '20160825',
             'duration': 1146,
@@ -61,8 +59,10 @@ class VootIE(InfoExtractor):
 
         media = media_info['assets']
 
-        entry_id = media['EntryId']
         title = media['MediaName']
+        formats = self._extract_m3u8_formats(
+            'https://cdnapisec.kaltura.com/p/1982551/playManifest/pt/https/f/applehttp/t/web/e/' + media['EntryId'],
+            video_id, 'mp4', m3u8_id='hls', fatal=False)
 
         description, series, season_number, episode, episode_number = [None] * 5
 
@@ -82,9 +82,7 @@ class VootIE(InfoExtractor):
                 episode_number = int_or_none(value)
 
         return {
-            '_type': 'url_transparent',
-            'url': 'kaltura:1982551:%s' % entry_id,
-            'ie_key': KalturaIE.ie_key(),
+            'id': video_id,
             'title': title,
             'description': description,
             'series': series,
@@ -95,4 +93,5 @@ class VootIE(InfoExtractor):
             'duration': int_or_none(media.get('Duration')),
             'view_count': int_or_none(media.get('ViewCounter')),
             'like_count': int_or_none(media.get('like_counter')),
+            'formats': formats,
         }

From 23f511f5c74d85222e69259996666b7ef97b9421 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 15 Dec 2017 09:05:59 +0100
Subject: [PATCH 1617/1696] [voot] sort formats

---
 youtube_dl/extractor/voot.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/voot.py b/youtube_dl/extractor/voot.py
index 5fd1c3d80..426754489 100644
--- a/youtube_dl/extractor/voot.py
+++ b/youtube_dl/extractor/voot.py
@@ -62,7 +62,8 @@ class VootIE(InfoExtractor):
         title = media['MediaName']
         formats = self._extract_m3u8_formats(
             'https://cdnapisec.kaltura.com/p/1982551/playManifest/pt/https/f/applehttp/t/web/e/' + media['EntryId'],
-            video_id, 'mp4', m3u8_id='hls', fatal=False)
+            video_id, 'mp4', m3u8_id='hls')
+        self._sort_formats(formats)
 
         description, series, season_number, episode, episode_number = [None] * 5
 

From d05ba4b89e195ed18e8d51561649712f945769b9 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 15 Dec 2017 09:27:56 +0100
Subject: [PATCH 1618/1696] [disney] skip Apple FairPlay formats(#14982)

---
 youtube_dl/extractor/disney.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/disney.py b/youtube_dl/extractor/disney.py
index 968c4c7fd..0eee82fd6 100644
--- a/youtube_dl/extractor/disney.py
+++ b/youtube_dl/extractor/disney.py
@@ -10,6 +10,7 @@ from ..utils import (
     compat_str,
     determine_ext,
     ExtractorError,
+    update_url_query,
 )
 
 
@@ -108,9 +109,16 @@ class DisneyIE(InfoExtractor):
                 continue
             tbr = int_or_none(flavor.get('bitrate'))
             if tbr == 99999:
-                formats.extend(self._extract_m3u8_formats(
+                # wrong ks(Kaltura Signature) causes 404 Error
+                flavor_url = update_url_query(flavor_url, {'ks': ''})
+                m3u8_formats = self._extract_m3u8_formats(
                     flavor_url, video_id, 'mp4',
-                    m3u8_id=flavor_format, fatal=False))
+                    m3u8_id=flavor_format, fatal=False)
+                for f in m3u8_formats:
+                    # Apple FairPlay
+                    if '/fpshls/' in f['url']:
+                        continue
+                    formats.append(f)
                 continue
             format_id = []
             if flavor_format:

From 498a8a4ca56e66adb84f8f1488c0239d048c7adc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Dec 2017 22:53:56 +0700
Subject: [PATCH 1619/1696] [vk] Make view count optional (closes #14979)

---
 youtube_dl/extractor/vk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 0d8376522..d4838b3e5 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -414,7 +414,7 @@ class VKIE(VKBaseIE):
 
         view_count = str_to_int(self._search_regex(
             r'class=["\']mv_views_count[^>]+>\s*([\d,.]+)',
-            info_page, 'view count', fatal=False))
+            info_page, 'view count', default=None))
 
         formats = []
         for format_id, format_url in data.items():

From c402e7f3a03aceeb80dd032831ceafb6d0aaa935 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 16 Dec 2017 12:55:20 +0100
Subject: [PATCH 1620/1696] [discoverygo] correct ttml subtitle extension

---
 youtube_dl/extractor/discoverygo.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/discoverygo.py b/youtube_dl/extractor/discoverygo.py
index 99376454b..3368c4c07 100644
--- a/youtube_dl/extractor/discoverygo.py
+++ b/youtube_dl/extractor/discoverygo.py
@@ -5,6 +5,7 @@ import re
 from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
+    determine_ext,
     extract_attributes,
     ExtractorError,
     int_or_none,
@@ -73,7 +74,11 @@ class DiscoveryGoBaseIE(InfoExtractor):
                         not subtitle_url.startswith('http')):
                     continue
                 lang = caption.get('fileLang', 'en')
-                subtitles.setdefault(lang, []).append({'url': subtitle_url})
+                ext = determine_ext(subtitle_url)
+                subtitles.setdefault(lang, []).append({
+                    'url': subtitle_url,
+                    'ext': 'ttml' if ext == 'xml' else ext,
+                })
 
         return {
             'id': video_id,

From b555ae9bf146ce9e2bf81327746847f0bc4d63e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Dec 2017 21:56:16 +0700
Subject: [PATCH 1621/1696] [utils] Add another date format pattern (#14999)

---
 test/test_utils.py  | 1 +
 youtube_dl/utils.py | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index cc13f795c..0857c0fc0 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -343,6 +343,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(unified_timestamp('Feb 7, 2016 at 6:35 pm'), 1454870100)
         self.assertEqual(unified_timestamp('2017-03-30T17:52:41Q'), 1490896361)
         self.assertEqual(unified_timestamp('Sep 11, 2013 | 5:49 AM'), 1378878540)
+        self.assertEqual(unified_timestamp('December 15, 2017 at 7:49 am'), 1513324140)
 
     def test_determine_ext(self):
         self.assertEqual(determine_ext('http://example.com/foo/bar.mp4/?download'), 'mp4')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index eccbc0b1f..2843a3dc0 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -159,6 +159,8 @@ DATE_FORMATS = (
     '%Y-%m-%dT%H:%M',
     '%b %d %Y at %H:%M',
     '%b %d %Y at %H:%M:%S',
+    '%B %d %Y at %H:%M',
+    '%B %d %Y at %H:%M:%S',
 )
 
 DATE_FORMATS_DAY_FIRST = list(DATE_FORMATS)

From 06dbcd7be41f71346e75f78f3ad77d9eca0219ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Dec 2017 21:57:30 +0700
Subject: [PATCH 1622/1696] [cbslocal] Fix timestamp extraction (closes #14999,
 closes #15000)

---
 youtube_dl/extractor/cbslocal.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/cbslocal.py b/youtube_dl/extractor/cbslocal.py
index 7d78e3aae..90852a9ef 100644
--- a/youtube_dl/extractor/cbslocal.py
+++ b/youtube_dl/extractor/cbslocal.py
@@ -91,12 +91,10 @@ class CBSLocalIE(AnvatoIE):
 
         info_dict = self._extract_anvato_videos(webpage, display_id)
 
-        time_str = self._html_search_regex(
-            r'class="entry-date">([^<]+)<', webpage, 'released date', default=None)
-        if time_str:
-            timestamp = unified_timestamp(time_str)
-        else:
-            timestamp = parse_iso8601(self._html_search_meta('uploadDate', webpage))
+        timestamp = unified_timestamp(self._html_search_regex(
+            r'class="(?:entry|post)-date"[^>]*>([^<]+)', webpage,
+            'released date', default=None)) or parse_iso8601(
+            self._html_search_meta('uploadDate', webpage))
 
         info_dict.update({
             'display_id': display_id,

From 3dfa9ec21362f8781d0a2d2eef1c9474b525af79 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 17 Dec 2017 09:14:52 +0100
Subject: [PATCH 1623/1696] [crunchyroll] Future-proof XML element
 checks(closes #15013)

---
 youtube_dl/extractor/crunchyroll.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index b53f2d705..b92f25447 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -392,7 +392,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
                 'Downloading subtitles for ' + sub_name, data={
                     'subtitle_script_id': sub_id,
                 })
-            if not sub_doc:
+            if sub_doc is None:
                 continue
             sid = sub_doc.get('id')
             iv = xpath_text(sub_doc, 'iv', 'subtitle iv')
@@ -479,9 +479,9 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
                     'video_quality': stream_quality,
                     'current_page': url,
                 })
-            if streamdata:
+            if streamdata is not None:
                 stream_info = streamdata.find('./{default}preload/stream_info')
-                if stream_info:
+                if stream_info is not None:
                     stream_infos.append(stream_info)
             stream_info = self._call_rpc_api(
                 'VideoEncode_GetStreamInfo', video_id,
@@ -490,7 +490,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
                     'video_format': stream_format,
                     'video_encode_quality': stream_quality,
                 })
-            if stream_info:
+            if stream_info is not None:
                 stream_infos.append(stream_info)
             for stream_info in stream_infos:
                 video_encode_id = xpath_text(stream_info, './video_encode_id')

From 25475dfab356e625ee480e1631601b86403afb5a Mon Sep 17 00:00:00 2001
From: Hongjie Dong <hongjieuf@gmail.com>
Date: Mon, 4 Dec 2017 00:26:09 -0800
Subject: [PATCH 1624/1696] [mailru] Add support for embed URLs

---
 youtube_dl/extractor/mailru.py | 36 +++++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/mailru.py b/youtube_dl/extractor/mailru.py
index f7cc3c832..f1865fd11 100644
--- a/youtube_dl/extractor/mailru.py
+++ b/youtube_dl/extractor/mailru.py
@@ -13,8 +13,7 @@ from ..utils import (
 class MailRuIE(InfoExtractor):
     IE_NAME = 'mailru'
     IE_DESC = 'Видео@Mail.Ru'
-    _VALID_URL = r'https?://(?:(?:www|m)\.)?my\.mail\.ru/(?:video/.*#video=/?(?P<idv1>(?:[^/]+/){3}\d+)|(?:(?P<idv2prefix>(?:[^/]+/){2})video/(?P<idv2suffix>[^/]+/\d+))\.html)'
-
+    _VALID_URL = r'https?://(?:(?:www|m)\.)?my\.mail\.ru/(?:video/.*#video=/?(?P<idv1>(?:[^/]+/){3}\d+)|(?:(?P<idv2prefix>(?:[^/]+/){2})video/(?P<idv2suffix>[^/]+/\d+))\.html|video/embed/(?P<meta_id>\d+))'
     _TESTS = [
         {
             'url': 'http://my.mail.ru/video/top#video=/mail/sonypicturesrus/75/76',
@@ -65,25 +64,34 @@ class MailRuIE(InfoExtractor):
         {
             'url': 'http://m.my.mail.ru/mail/3sktvtr/video/_myvideo/138.html',
             'only_matching': True,
+        },
+        {
+            'url': 'https://my.mail.ru/video/embed/7949340477499637815',
+            'only_matching': True,
         }
     ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('idv1')
-
-        if not video_id:
-            video_id = mobj.group('idv2prefix') + mobj.group('idv2suffix')
-
-        webpage = self._download_webpage(url, video_id)
-
+        meta_id = mobj.group('meta_id')
+        page_config = None
         video_data = None
+        video_id = None
+        if not meta_id:
+            video_id = mobj.group('idv1')
+            if not video_id:
+                video_id = mobj.group('idv2prefix') + mobj.group('idv2suffix')
 
-        page_config = self._parse_json(self._search_regex(
-            r'(?s)<script[^>]+class="sp-video__page-config"[^>]*>(.+?)</script>',
-            webpage, 'page config', default='{}'), video_id, fatal=False)
-        if page_config:
-            meta_url = page_config.get('metaUrl') or page_config.get('video', {}).get('metaUrl')
+            webpage = self._download_webpage(url, video_id)
+
+            page_config = self._parse_json(self._search_regex(
+                r'(?s)<script[^>]+class="sp-video__page-config"[^>]*>(.+?)</script>',
+                webpage, 'page config', default='{}'), video_id, fatal=False)
+        if page_config or meta_id:
+            if page_config:
+                meta_url = page_config.get('metaUrl') or page_config.get('video', {}).get('metaUrl')
+            elif meta_id:
+                meta_url = 'https://my.mail.ru/+/video/meta/' + meta_id
             if meta_url:
                 video_data = self._download_json(
                     meta_url, video_id, 'Downloading video meta JSON', fatal=False)

From 549bb416f5a9d15c03749a98abd582d0e40418ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Dec 2017 18:36:19 +0700
Subject: [PATCH 1625/1696] [mailru] Fix issues and improve (closes #14904)

---
 youtube_dl/extractor/mailru.py | 43 ++++++++++++++++++++++------------
 1 file changed, 28 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/mailru.py b/youtube_dl/extractor/mailru.py
index f1865fd11..6b7c5e3e0 100644
--- a/youtube_dl/extractor/mailru.py
+++ b/youtube_dl/extractor/mailru.py
@@ -13,7 +13,15 @@ from ..utils import (
 class MailRuIE(InfoExtractor):
     IE_NAME = 'mailru'
     IE_DESC = 'Видео@Mail.Ru'
-    _VALID_URL = r'https?://(?:(?:www|m)\.)?my\.mail\.ru/(?:video/.*#video=/?(?P<idv1>(?:[^/]+/){3}\d+)|(?:(?P<idv2prefix>(?:[^/]+/){2})video/(?P<idv2suffix>[^/]+/\d+))\.html|video/embed/(?P<meta_id>\d+))'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:(?:www|m)\.)?my\.mail\.ru/
+                        (?:
+                            video/.*\#video=/?(?P<idv1>(?:[^/]+/){3}\d+)|
+                            (?:(?P<idv2prefix>(?:[^/]+/){2})video/(?P<idv2suffix>[^/]+/\d+))\.html|
+                            (?:video/embed|\+/video/meta)/(?P<metaid>\d+)
+                        )
+                    '''
     _TESTS = [
         {
             'url': 'http://my.mail.ru/video/top#video=/mail/sonypicturesrus/75/76',
@@ -22,7 +30,7 @@ class MailRuIE(InfoExtractor):
                 'id': '46301138_76',
                 'ext': 'mp4',
                 'title': 'Новый Человек-Паук. Высокое напряжение. Восстание Электро',
-                'timestamp': 1393232740,
+                'timestamp': 1393235077,
                 'upload_date': '20140224',
                 'uploader': 'sonypicturesrus',
                 'uploader_id': 'sonypicturesrus@mail.ru',
@@ -39,7 +47,7 @@ class MailRuIE(InfoExtractor):
                 'title': 'Samsung Galaxy S5 Hammer Smash Fail Battery Explosion',
                 'timestamp': 1397039888,
                 'upload_date': '20140409',
-                'uploader': 'hitech@corp.mail.ru',
+                'uploader': 'hitech',
                 'uploader_id': 'hitech@corp.mail.ru',
                 'duration': 245,
             },
@@ -68,33 +76,38 @@ class MailRuIE(InfoExtractor):
         {
             'url': 'https://my.mail.ru/video/embed/7949340477499637815',
             'only_matching': True,
+        },
+        {
+            'url': 'http://my.mail.ru/+/video/meta/7949340477499637815',
+            'only_matching': True,
         }
     ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        meta_id = mobj.group('meta_id')
-        page_config = None
-        video_data = None
+        meta_id = mobj.group('metaid')
+
         video_id = None
-        if not meta_id:
+        if meta_id:
+            meta_url = 'https://my.mail.ru/+/video/meta/%s' % meta_id
+        else:
             video_id = mobj.group('idv1')
             if not video_id:
                 video_id = mobj.group('idv2prefix') + mobj.group('idv2suffix')
-
             webpage = self._download_webpage(url, video_id)
-
             page_config = self._parse_json(self._search_regex(
                 r'(?s)<script[^>]+class="sp-video__page-config"[^>]*>(.+?)</script>',
                 webpage, 'page config', default='{}'), video_id, fatal=False)
-        if page_config or meta_id:
             if page_config:
                 meta_url = page_config.get('metaUrl') or page_config.get('video', {}).get('metaUrl')
-            elif meta_id:
-                meta_url = 'https://my.mail.ru/+/video/meta/' + meta_id
-            if meta_url:
-                video_data = self._download_json(
-                    meta_url, video_id, 'Downloading video meta JSON', fatal=False)
+            else:
+                meta_url = None
+
+        video_data = None
+        if meta_url:
+            video_data = self._download_json(
+                meta_url, video_id or meta_id, 'Downloading video meta JSON',
+                fatal=not video_id)
 
         # Fallback old approach
         if not video_data:

From 7e810109877bb59d213574b115e46d9621db9a18 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 17 Dec 2017 19:13:25 +0100
Subject: [PATCH 1626/1696] [cspan] add support for audio only pages and catch
 page errors(closes #14995)

---
 youtube_dl/extractor/cspan.py | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index 171820e27..67d6df4b0 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -4,13 +4,14 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
-    int_or_none,
-    unescapeHTML,
-    find_xpath_attr,
-    smuggle_url,
     determine_ext,
     ExtractorError,
     extract_attributes,
+    find_xpath_attr,
+    get_element_by_class,
+    int_or_none,
+    smuggle_url,
+    unescapeHTML,
 )
 from .senateisvp import SenateISVPIE
 from .ustream import UstreamIE
@@ -68,6 +69,10 @@ class CSpanIE(InfoExtractor):
             'uploader': 'HouseCommittee',
             'uploader_id': '12987475',
         },
+    }, {
+        # Audio Only
+        'url': 'https://www.c-span.org/video/?437336-1/judiciary-antitrust-competition-policy-consumer-rights',
+        'only_matching': True,
     }]
     BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
 
@@ -111,7 +116,15 @@ class CSpanIE(InfoExtractor):
                     title = self._og_search_title(webpage)
                     surl = smuggle_url(senate_isvp_url, {'force_title': title})
                     return self.url_result(surl, 'SenateISVP', video_id, title)
+                video_id = self._search_regex(
+                    r'jwsetup\.clipprog\s*=\s*(\d+);',
+                    webpage, 'jwsetup program id', default=None)
+                if video_id:
+                    video_type = 'program'
         if video_type is None or video_id is None:
+            error_message = get_element_by_class('VLplayer-error-message', webpage)
+            if error_message:
+                raise ExtractorError(error_message)
             raise ExtractorError('unable to find video id and type')
 
         def get_text_attr(d, attr):
@@ -138,7 +151,7 @@ class CSpanIE(InfoExtractor):
         entries = []
         for partnum, f in enumerate(files):
             formats = []
-            for quality in f['qualities']:
+            for quality in f.get('qualities', []):
                 formats.append({
                     'format_id': '%s-%sp' % (get_text_attr(quality, 'bitrate'), get_text_attr(quality, 'height')),
                     'url': unescapeHTML(get_text_attr(quality, 'file')),

From 99081da90c9b8ce4ee7fe9452787507fed4251a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 18 Dec 2017 03:31:53 +0700
Subject: [PATCH 1627/1696] [downloader/fragment] Encode filename of fragment
 being removed (closes #15020)

---
 youtube_dl/downloader/fragment.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 7bb61a541..ea5e3a4b5 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -112,7 +112,7 @@ class FragmentFD(FileDownloader):
             if self.__do_ytdl_file(ctx):
                 self._write_ytdl_file(ctx)
             if not self.params.get('keep_fragments', False):
-                os.remove(ctx['fragment_filename_sanitized'])
+                os.remove(encodeFilename(ctx['fragment_filename_sanitized']))
             del ctx['fragment_filename_sanitized']
 
     def _prepare_frag_download(self, ctx):

From 4a109f81bc39ed09931fd6f9d21d20f7abdc9742 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 Dec 2017 00:38:39 +0700
Subject: [PATCH 1628/1696] [afreecatv] Improve format extraction (closes
 #15019)

---
 youtube_dl/extractor/afreecatv.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py
index e6513c7a4..513dd81df 100644
--- a/youtube_dl/extractor/afreecatv.py
+++ b/youtube_dl/extractor/afreecatv.py
@@ -228,10 +228,19 @@ class AfreecaTVIE(InfoExtractor):
                     r'^(\d{8})_', key, 'upload date', default=None)
                 file_duration = int_or_none(file_element.get('duration'))
                 format_id = key if key else '%s_%s' % (video_id, file_num)
-                formats = self._extract_m3u8_formats(
-                    file_url, video_id, 'mp4', entry_protocol='m3u8_native',
-                    m3u8_id='hls',
-                    note='Downloading part %d m3u8 information' % file_num)
+                if determine_ext(file_url) == 'm3u8':
+                    formats = self._extract_m3u8_formats(
+                        file_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                        m3u8_id='hls',
+                        note='Downloading part %d m3u8 information' % file_num)
+                else:
+                    formats = [{
+                        'url': file_url,
+                        'format_id': 'http',
+                    }]
+                if not formats:
+                    continue
+                self._sort_formats(formats)
                 file_info = common_entry.copy()
                 file_info.update({
                     'id': format_id,

From c10c93238e6db0df8746fc185ca316b9d8ccece5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 Dec 2017 03:51:03 +0700
Subject: [PATCH 1629/1696] [extractor/common] Introduce uploader, uploader_id
 and uploader_url meta fields for playlists (#11427, #15018)

---
 youtube_dl/extractor/common.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 80a9c982f..e5ef5e490 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -301,8 +301,9 @@ class InfoExtractor(object):
     There must be a key "entries", which is a list, an iterable, or a PagedList
     object, each element of which is a valid dictionary by this specification.
 
-    Additionally, playlists can have "title", "description" and "id" attributes
-    with the same semantics as videos (see above).
+    Additionally, playlists can have "id", "title", "description", "uploader",
+    "uploader_id", "uploader_url" attributes with the same semantics as videos
+    (see above).
 
 
     _type "multi_video" indicates that there are multiple videos that

From 07aeced68e5aa24f3e2562aa7cb9ddd2f11b59ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 Dec 2017 03:51:28 +0700
Subject: [PATCH 1630/1696] [youtube] Extract uploader, uploader_id and
 uploader_url for playlists (#11427, #15018)

---
 youtube_dl/extractor/youtube.py | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 9943dddc1..0919bef0e 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2270,6 +2270,19 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
             r'(?s)<h1 class="pl-header-title[^"]*"[^>]*>\s*(.*?)\s*</h1>',
             page, 'title', default=None)
 
+        _UPLOADER_BASE = r'class=["\']pl-header-details[^>]+>\s*<li>\s*<a[^>]+\bhref='
+        uploader = self._search_regex(
+            r'%s["\']/(?:user|channel)/[^>]+>([^<]+)' % _UPLOADER_BASE,
+            page, 'uploader', default=None)
+        mobj = re.search(
+            r'%s(["\'])(?P<path>/(?:user|channel)/(?P<uploader_id>.+?))\1' % _UPLOADER_BASE,
+            page)
+        if mobj:
+            uploader_id = mobj.group('uploader_id')
+            uploader_url = compat_urlparse.urljoin(url, mobj.group('path'))
+        else:
+            uploader_id = uploader_url = None
+
         has_videos = True
 
         if not playlist_title:
@@ -2280,8 +2293,15 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
             except StopIteration:
                 has_videos = False
 
-        return has_videos, self.playlist_result(
+        playlist = self.playlist_result(
             self._entries(page, playlist_id), playlist_id, playlist_title)
+        playlist.update({
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'uploader_url': uploader_url,
+        })
+
+        return has_videos, playlist
 
     def _check_download_just_video(self, url, playlist_id):
         # Check if it's a video-specific URL

From 3961c6cb9d3a1c30fe31db774b0809095952f1bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 Dec 2017 03:53:44 +0700
Subject: [PATCH 1631/1696] [YoutubeDL] Add support for playlist_uploader and
 playlist_uploader_id in output template (closes #11427, #15018)

---
 README.md               | 2 ++
 youtube_dl/YoutubeDL.py | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/README.md b/README.md
index cd30d147a..47b0640ab 100644
--- a/README.md
+++ b/README.md
@@ -539,6 +539,8 @@ The basic usage is not to set any template arguments when downloading a single f
  - `playlist_index` (numeric): Index of the video in the playlist padded with leading zeros according to the total length of the playlist
  - `playlist_id` (string): Playlist identifier
  - `playlist_title` (string): Playlist title
+ - `playlist_uploader` (string): Full name of the playlist uploader
+ - `playlist_uploader_id` (string): Nickname or id of the playlist uploader
 
 Available for the video that belongs to some logical chapter or section:
 
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 68721e9ab..ace80f14b 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -975,6 +975,8 @@ class YoutubeDL(object):
                     'playlist': playlist,
                     'playlist_id': ie_result.get('id'),
                     'playlist_title': ie_result.get('title'),
+                    'playlist_uploader': ie_result.get('uploader'),
+                    'playlist_uploader_id': ie_result.get('uploader_id'),
                     'playlist_index': i + playliststart,
                     'extractor': ie_result['extractor'],
                     'webpage_url': ie_result['webpage_url'],

From 78466fcab519d1b92fd9846bc8073885308a7e22 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 19 Dec 2017 02:00:13 +0100
Subject: [PATCH 1632/1696] [shahid] add support for show pages(closes #7401)

---
 youtube_dl/extractor/aws.py             |  78 +++++++++++
 youtube_dl/extractor/extractors.py      |   5 +-
 youtube_dl/extractor/scrippsnetworks.py | 103 +++------------
 youtube_dl/extractor/shahid.py          | 164 +++++++++++++++++-------
 4 files changed, 219 insertions(+), 131 deletions(-)
 create mode 100644 youtube_dl/extractor/aws.py

diff --git a/youtube_dl/extractor/aws.py b/youtube_dl/extractor/aws.py
new file mode 100644
index 000000000..670abce0c
--- /dev/null
+++ b/youtube_dl/extractor/aws.py
@@ -0,0 +1,78 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import datetime
+import hashlib
+import hmac
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse_urlencode
+
+
+class AWSIE(InfoExtractor):
+    _AWS_ALGORITHM = 'AWS4-HMAC-SHA256'
+    _AWS_REGION = 'us-east-1'
+
+    def _aws_execute_api(self, aws_dict, video_id, query=None):
+        query = query or {}
+        amz_date = datetime.datetime.utcnow().strftime('%Y%m%dT%H%M%SZ')
+        date = amz_date[:8]
+        headers = {
+            'Accept': 'application/json',
+            'Host': self._AWS_PROXY_HOST,
+            'X-Amz-Date': amz_date,
+        }
+        session_token = aws_dict.get('session_token')
+        if session_token:
+            headers['X-Amz-Security-Token'] = session_token
+        headers['X-Api-Key'] = self._AWS_API_KEY
+
+        def aws_hash(s):
+            return hashlib.sha256(s.encode('utf-8')).hexdigest()
+
+        # Task 1: http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+        canonical_querystring = compat_urllib_parse_urlencode(query)
+        canonical_headers = ''
+        for header_name, header_value in headers.items():
+            canonical_headers += '%s:%s\n' % (header_name.lower(), header_value)
+        signed_headers = ';'.join([header.lower() for header in headers.keys()])
+        canonical_request = '\n'.join([
+            'GET',
+            aws_dict['uri'],
+            canonical_querystring,
+            canonical_headers,
+            signed_headers,
+            aws_hash('')
+        ])
+
+        # Task 2: http://docs.aws.amazon.com/general/latest/gr/sigv4-create-string-to-sign.html
+        credential_scope_list = [date, self._AWS_REGION, 'execute-api', 'aws4_request']
+        credential_scope = '/'.join(credential_scope_list)
+        string_to_sign = '\n'.join([self._AWS_ALGORITHM, amz_date, credential_scope, aws_hash(canonical_request)])
+
+        # Task 3: http://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
+        def aws_hmac(key, msg):
+            return hmac.new(key, msg.encode('utf-8'), hashlib.sha256)
+
+        def aws_hmac_digest(key, msg):
+            return aws_hmac(key, msg).digest()
+
+        def aws_hmac_hexdigest(key, msg):
+            return aws_hmac(key, msg).hexdigest()
+
+        k_signing = ('AWS4' + aws_dict['secret_key']).encode('utf-8')
+        for value in credential_scope_list:
+            k_signing = aws_hmac_digest(k_signing, value)
+
+        signature = aws_hmac_hexdigest(k_signing, string_to_sign)
+
+        # Task 4: http://docs.aws.amazon.com/general/latest/gr/sigv4-add-signature-to-request.html
+        headers['Authorization'] = ', '.join([
+            '%s Credential=%s/%s' % (self._AWS_ALGORITHM, aws_dict['access_key'], credential_scope),
+            'SignedHeaders=%s' % signed_headers,
+            'Signature=%s' % signature,
+        ])
+
+        return self._download_json(
+            'https://%s%s%s' % (self._AWS_PROXY_HOST, aws_dict['uri'], '?' + canonical_querystring if canonical_querystring else ''),
+            video_id, headers=headers)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 407245513..513074801 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -927,7 +927,10 @@ from .sendtonews import SendtoNewsIE
 from .servingsys import ServingSysIE
 from .servus import ServusIE
 from .sexu import SexuIE
-from .shahid import ShahidIE
+from .shahid import (
+    ShahidIE,
+    ShahidShowIE,
+)
 from .shared import (
     SharedIE,
     VivoIE,
diff --git a/youtube_dl/extractor/scrippsnetworks.py b/youtube_dl/extractor/scrippsnetworks.py
index b446a02ba..4023aeef8 100644
--- a/youtube_dl/extractor/scrippsnetworks.py
+++ b/youtube_dl/extractor/scrippsnetworks.py
@@ -1,13 +1,11 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import datetime
 import json
 import hashlib
-import hmac
 import re
 
-from .common import InfoExtractor
+from .aws import AWSIE
 from .anvato import AnvatoIE
 from ..utils import (
     smuggle_url,
@@ -16,7 +14,7 @@ from ..utils import (
 )
 
 
-class ScrippsNetworksWatchIE(InfoExtractor):
+class ScrippsNetworksWatchIE(AWSIE):
     IE_NAME = 'scrippsnetworks:watch'
     _VALID_URL = r'''(?x)
                     https?://
@@ -64,44 +62,27 @@ class ScrippsNetworksWatchIE(InfoExtractor):
         'travelchannel': 'trav',
         'geniuskitchen': 'genius',
     }
-    _SNI_HOST = 'web.api.video.snidigital.com'
 
-    _AWS_REGION = 'us-east-1'
-    _AWS_IDENTITY_ID_JSON = json.dumps({
-        'IdentityId': '%s:7655847c-0ae7-4d9b-80d6-56c062927eb3' % _AWS_REGION
-    })
-    _AWS_USER_AGENT = 'aws-sdk-js/2.80.0 callback'
     _AWS_API_KEY = 'E7wSQmq0qK6xPrF13WmzKiHo4BQ7tip4pQcSXVl1'
-    _AWS_SERVICE = 'execute-api'
-    _AWS_REQUEST = 'aws4_request'
-    _AWS_SIGNED_HEADERS = ';'.join([
-        'host', 'x-amz-date', 'x-amz-security-token', 'x-api-key'])
-    _AWS_CANONICAL_REQUEST_TEMPLATE = '''GET
-%(uri)s
+    _AWS_PROXY_HOST = 'web.api.video.snidigital.com'
 
-host:%(host)s
-x-amz-date:%(date)s
-x-amz-security-token:%(token)s
-x-api-key:%(key)s
-
-%(signed_headers)s
-%(payload_hash)s'''
+    _AWS_USER_AGENT = 'aws-sdk-js/2.80.0 callback'
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         site_id, video_id = mobj.group('site', 'id')
 
-        def aws_hash(s):
-            return hashlib.sha256(s.encode('utf-8')).hexdigest()
-
+        aws_identity_id_json = json.dumps({
+            'IdentityId': '%s:7655847c-0ae7-4d9b-80d6-56c062927eb3' % self._AWS_REGION
+        }).encode('utf-8')
         token = self._download_json(
-            'https://cognito-identity.us-east-1.amazonaws.com/', video_id,
-            data=self._AWS_IDENTITY_ID_JSON.encode('utf-8'),
+            'https://cognito-identity.%s.amazonaws.com/' % self._AWS_REGION, video_id,
+            data=aws_identity_id_json,
             headers={
                 'Accept': '*/*',
                 'Content-Type': 'application/x-amz-json-1.1',
                 'Referer': url,
-                'X-Amz-Content-Sha256': aws_hash(self._AWS_IDENTITY_ID_JSON),
+                'X-Amz-Content-Sha256': hashlib.sha256(aws_identity_id_json).hexdigest(),
                 'X-Amz-Target': 'AWSCognitoIdentityService.GetOpenIdToken',
                 'X-Amz-User-Agent': self._AWS_USER_AGENT,
             })['Token']
@@ -124,64 +105,12 @@ x-api-key:%(key)s
                 sts, './/{https://sts.amazonaws.com/doc/2011-06-15/}%s' % key,
                 fatal=True)
 
-        access_key_id = get('AccessKeyId')
-        secret_access_key = get('SecretAccessKey')
-        session_token = get('SessionToken')
-
-        # Task 1: http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
-        uri = '/1/web/brands/%s/episodes/scrid/%s' % (self._SNI_TABLE[site_id], video_id)
-        datetime_now = datetime.datetime.utcnow().strftime('%Y%m%dT%H%M%SZ')
-        date = datetime_now[:8]
-        canonical_string = self._AWS_CANONICAL_REQUEST_TEMPLATE % {
-            'uri': uri,
-            'host': self._SNI_HOST,
-            'date': datetime_now,
-            'token': session_token,
-            'key': self._AWS_API_KEY,
-            'signed_headers': self._AWS_SIGNED_HEADERS,
-            'payload_hash': aws_hash(''),
-        }
-
-        # Task 2: http://docs.aws.amazon.com/general/latest/gr/sigv4-create-string-to-sign.html
-        credential_string = '/'.join([date, self._AWS_REGION, self._AWS_SERVICE, self._AWS_REQUEST])
-        string_to_sign = '\n'.join([
-            'AWS4-HMAC-SHA256', datetime_now, credential_string,
-            aws_hash(canonical_string)])
-
-        # Task 3: http://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
-        def aws_hmac(key, msg):
-            return hmac.new(key, msg.encode('utf-8'), hashlib.sha256)
-
-        def aws_hmac_digest(key, msg):
-            return aws_hmac(key, msg).digest()
-
-        def aws_hmac_hexdigest(key, msg):
-            return aws_hmac(key, msg).hexdigest()
-
-        k_secret = 'AWS4' + secret_access_key
-        k_date = aws_hmac_digest(k_secret.encode('utf-8'), date)
-        k_region = aws_hmac_digest(k_date, self._AWS_REGION)
-        k_service = aws_hmac_digest(k_region, self._AWS_SERVICE)
-        k_signing = aws_hmac_digest(k_service, self._AWS_REQUEST)
-
-        signature = aws_hmac_hexdigest(k_signing, string_to_sign)
-
-        auth_header = ', '.join([
-            'AWS4-HMAC-SHA256 Credential=%s' % '/'.join(
-                [access_key_id, date, self._AWS_REGION, self._AWS_SERVICE, self._AWS_REQUEST]),
-            'SignedHeaders=%s' % self._AWS_SIGNED_HEADERS,
-            'Signature=%s' % signature,
-        ])
-
-        mcp_id = self._download_json(
-            'https://%s%s' % (self._SNI_HOST, uri), video_id, headers={
-                'Accept': '*/*',
-                'Referer': url,
-                'Authorization': auth_header,
-                'X-Amz-Date': datetime_now,
-                'X-Amz-Security-Token': session_token,
-                'X-Api-Key': self._AWS_API_KEY,
-            })['results'][0]['mcpId']
+        mcp_id = self._aws_execute_api({
+            'uri': '/1/web/brands/%s/episodes/scrid/%s' % (self._SNI_TABLE[site_id], video_id),
+            'access_key': get('AccessKeyId'),
+            'secret_key': get('SecretAccessKey'),
+            'session_token': get('SessionToken'),
+        }, video_id)['results'][0]['mcpId']
 
         return self.url_result(
             smuggle_url(
diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index 374f7faf9..5c2a6206b 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -1,22 +1,53 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
 import json
+import math
+import re
 
-from .common import InfoExtractor
+from .aws import AWSIE
 from ..compat import compat_HTTPError
 from ..utils import (
+    clean_html,
     ExtractorError,
+    InAdvancePagedList,
     int_or_none,
     parse_iso8601,
     str_or_none,
     urlencode_postdata,
-    clean_html,
 )
 
 
-class ShahidIE(InfoExtractor):
+class ShahidBaseIE(AWSIE):
+    _AWS_PROXY_HOST = 'api2.shahid.net'
+    _AWS_API_KEY = '2RRtuMHx95aNI1Kvtn2rChEuwsCogUd4samGPjLh'
+
+    def _handle_error(self, e):
+        fail_data = self._parse_json(
+            e.cause.read().decode('utf-8'), None, fatal=False)
+        if fail_data:
+            faults = fail_data.get('faults', [])
+            faults_message = ', '.join([clean_html(fault['userMessage']) for fault in faults if fault.get('userMessage')])
+            if faults_message:
+                raise ExtractorError(faults_message, expected=True)
+
+    def _call_api(self, path, video_id, request=None):
+        query = {}
+        if request:
+            query['request'] = json.dumps(request)
+        try:
+            return self._aws_execute_api({
+                'uri': '/proxy/v2/' + path,
+                'access_key': 'AKIAI6X4TYCIXM2B7MUQ',
+                'secret_key': '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',
+            }, video_id, query)
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError):
+                self._handle_error(e)
+            raise
+
+
+class ShahidIE(ShahidBaseIE):
     _NETRC_MACHINE = 'shahid'
     _VALID_URL = r'https?://shahid\.mbc\.net/ar/(?:serie|show|movie)s/[^/]+/(?P<type>episode|clip|movie)-(?P<id>\d+)'
     _TESTS = [{
@@ -41,34 +72,25 @@ class ShahidIE(InfoExtractor):
         'only_matching': True
     }]
 
-    def _api2_request(self, *args, **kwargs):
-        try:
-            return self._download_json(*args, **kwargs)
-        except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError):
-                fail_data = self._parse_json(
-                    e.cause.read().decode('utf-8'), None, fatal=False)
-                if fail_data:
-                    faults = fail_data.get('faults', [])
-                    faults_message = ', '.join([clean_html(fault['userMessage']) for fault in faults if fault.get('userMessage')])
-                    if faults_message:
-                        raise ExtractorError(faults_message, expected=True)
-            raise
-
     def _real_initialize(self):
         email, password = self._get_login_info()
         if email is None:
             return
 
-        user_data = self._api2_request(
-            'https://shahid.mbc.net/wd/service/users/login',
-            None, 'Logging in', data=json.dumps({
-                'email': email,
-                'password': password,
-                'basic': 'false',
-            }).encode('utf-8'), headers={
-                'Content-Type': 'application/json; charset=UTF-8',
-            })['user']
+        try:
+            user_data = self._download_json(
+                'https://shahid.mbc.net/wd/service/users/login',
+                None, 'Logging in', data=json.dumps({
+                    'email': email,
+                    'password': password,
+                    'basic': 'false',
+                }).encode('utf-8'), headers={
+                    'Content-Type': 'application/json; charset=UTF-8',
+                })['user']
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError):
+                self._handle_error(e)
+            raise
 
         self._download_webpage(
             'https://shahid.mbc.net/populateContext',
@@ -81,25 +103,13 @@ class ShahidIE(InfoExtractor):
                 'sessionId': user_data['sessionId'],
             }))
 
-    def _get_api_data(self, response):
-        data = response.get('data', {})
-
-        error = data.get('error')
-        if error:
-            raise ExtractorError(
-                '%s returned error: %s' % (self.IE_NAME, '\n'.join(error.values())),
-                expected=True)
-
-        return data
-
     def _real_extract(self, url):
         page_type, video_id = re.match(self._VALID_URL, url).groups()
         if page_type == 'clip':
             page_type = 'episode'
 
-        playout = self._api2_request(
-            'https://api2.shahid.net/proxy/v2/playout/url/' + video_id,
-            video_id, 'Downloading player JSON')['playout']
+        playout = self._call_api(
+            'playout/url/' + video_id, video_id)['playout']
 
         if playout.get('drm'):
             raise ExtractorError('This video is DRM protected.', expected=True)
@@ -107,13 +117,27 @@ class ShahidIE(InfoExtractor):
         formats = self._extract_m3u8_formats(playout['url'], video_id, 'mp4')
         self._sort_formats(formats)
 
-        video = self._get_api_data(self._download_json(
+        # video = self._call_api(
+        #     'product/id', video_id, {
+        #         'id': video_id,
+        #         'productType': 'ASSET',
+        #         'productSubType': page_type.upper()
+        #     })['productModel']
+
+        response = self._download_json(
             'http://api.shahid.net/api/v1_1/%s/%s' % (page_type, video_id),
             video_id, 'Downloading video JSON', query={
                 'apiKey': 'sh@hid0nlin3',
                 'hash': 'b2wMCTHpSmyxGqQjJFOycRmLSex+BpTK/ooxy6vHaqs=',
-            }))[page_type]
+            })
+        data = response.get('data', {})
+        error = data.get('error')
+        if error:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, '\n'.join(error.values())),
+                expected=True)
 
+        video = data[page_type]
         title = video['title']
         categories = [
             category['name']
@@ -135,3 +159,57 @@ class ShahidIE(InfoExtractor):
             'episode_id': video_id,
             'formats': formats,
         }
+
+
+class ShahidShowIE(ShahidBaseIE):
+    _VALID_URL = r'https?://shahid\.mbc\.net/ar/(?:show|serie)s/[^/]+/(?:show|series)-(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://shahid.mbc.net/ar/shows/%D8%B1%D8%A7%D9%85%D8%B2-%D9%82%D8%B1%D8%B4-%D8%A7%D9%84%D8%A8%D8%AD%D8%B1/show-79187',
+        'info_dict': {
+            'id': '79187',
+            'title': 'رامز قرش البحر',
+            'description': 'md5:c88fa7e0f02b0abd39d417aee0d046ff',
+        },
+        'playlist_mincount': 32,
+    }, {
+        'url': 'https://shahid.mbc.net/ar/series/How-to-live-Longer-(The-Big-Think)/series-291861',
+        'only_matching': True
+    }]
+    _PAGE_SIZE = 30
+
+    def _real_extract(self, url):
+        show_id = self._match_id(url)
+
+        product = self._call_api(
+            'playableAsset', show_id, {'showId': show_id})['productModel']
+        playlist = product['playlist']
+        playlist_id = playlist['id']
+        show = product.get('show', {})
+
+        def page_func(page_num):
+            playlist = self._call_api(
+                'product/playlist', show_id, {
+                    'playListId': playlist_id,
+                    'pageNumber': page_num,
+                    'pageSize': 30,
+                    'sorts': [{
+                        'order': 'DESC',
+                        'type': 'SORTDATE'
+                    }],
+                })
+            for product in playlist.get('productList', {}).get('products', []):
+                product_url = product.get('productUrl', []).get('url')
+                if not product_url:
+                    continue
+                yield self.url_result(
+                    product_url, 'Shahid',
+                    str_or_none(product.get('id')),
+                    product.get('title'))
+
+        entries = InAdvancePagedList(
+            page_func,
+            math.ceil(playlist['count'] / self._PAGE_SIZE),
+            self._PAGE_SIZE)
+
+        return self.playlist_result(
+            entries, show_id, show.get('title'), show.get('description'))

From 17c3aced5d0d2cf7df41e9978500260756ee8ad9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 Dec 2017 22:53:04 +0700
Subject: [PATCH 1633/1696] [animeondemand] Relax login error regex

---
 youtube_dl/extractor/animeondemand.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index 34c2b363e..be032d5b4 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -85,8 +85,8 @@ class AnimeOnDemandIE(InfoExtractor):
 
         if all(p not in response for p in ('>Logout<', 'href="/users/sign_out"')):
             error = self._search_regex(
-                r'<p class="alert alert-danger">(.+?)</p>',
-                response, 'error', default=None)
+                r'<p[^>]+\bclass=(["\'])(?:(?!\1).)*\balert\s(?:(?!\1).)*\1[^>]*>(?P<error>.+?)</p>',
+                response, 'error', default=None, group='error')
             if error:
                 raise ExtractorError('Unable to login: %s' % error, expected=True)
             raise ExtractorError('Unable to log in')

From d2d766bc6d6f976c28fad8b69a1de060b55f5b17 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 20 Dec 2017 23:17:36 +0700
Subject: [PATCH 1634/1696] [animeondemand] Fix typo

---
 youtube_dl/extractor/animeondemand.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index be032d5b4..e4fa72f46 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -85,7 +85,7 @@ class AnimeOnDemandIE(InfoExtractor):
 
         if all(p not in response for p in ('>Logout<', 'href="/users/sign_out"')):
             error = self._search_regex(
-                r'<p[^>]+\bclass=(["\'])(?:(?!\1).)*\balert\s(?:(?!\1).)*\1[^>]*>(?P<error>.+?)</p>',
+                r'<p[^>]+\bclass=(["\'])(?:(?!\1).)*\balert\b(?:(?!\1).)*\1[^>]*>(?P<error>.+?)</p>',
                 response, 'error', default=None, group='error')
             if error:
                 raise ExtractorError('Unable to login: %s' % error, expected=True)

From 963d237d26c7e6da7b6f514c1d240a7046501b05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 Dec 2017 23:38:16 +0700
Subject: [PATCH 1635/1696] Add LICENSE, AUTHORS and ChangeLog to PyPI package
 (closes #15054)

---
 MANIFEST.in | 3 +++
 setup.py    | 1 +
 2 files changed, 4 insertions(+)

diff --git a/MANIFEST.in b/MANIFEST.in
index 5743f605a..af7518e0d 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,4 +1,7 @@
 include README.md
+include LICENSE
+include AUTHORS
+include ChangeLog
 include test/*.py
 include test/*.json
 include youtube-dl.bash-completion
diff --git a/setup.py b/setup.py
index 67d6633ed..7dbb5805f 100644
--- a/setup.py
+++ b/setup.py
@@ -109,6 +109,7 @@ setup(
     author_email='ytdl@yt-dl.org',
     maintainer='Sergey M.',
     maintainer_email='dstftw@gmail.com',
+    license='Unlicense',
     packages=[
         'youtube_dl',
         'youtube_dl.extractor', 'youtube_dl.downloader',

From 3e191da6d9d1cbe62d8f638ed68a93a46348b38b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 Dec 2017 23:46:08 +0700
Subject: [PATCH 1636/1696] [Makefile] Add AUTHORS to youtube-dl.tar.gz

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 1c760bef8..fe247810f 100644
--- a/Makefile
+++ b/Makefile
@@ -110,7 +110,7 @@ _EXTRACTOR_FILES = $(shell find youtube_dl/extractor -iname '*.py' -and -not -in
 youtube_dl/extractor/lazy_extractors.py: devscripts/make_lazy_extractors.py devscripts/lazy_load_template.py $(_EXTRACTOR_FILES)
 	$(PYTHON) devscripts/make_lazy_extractors.py $@
 
-youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish ChangeLog
+youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish ChangeLog AUTHORS
 	@tar -czf youtube-dl.tar.gz --transform "s|^|youtube-dl/|" --owner 0 --group 0 \
 		--exclude '*.DS_Store' \
 		--exclude '*.kate-swp' \
@@ -122,7 +122,7 @@ youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-
 		--exclude 'docs/_build' \
 		-- \
 		bin devscripts test youtube_dl docs \
-		ChangeLog LICENSE README.md README.txt \
+		ChangeLog AUTHORS LICENSE README.md README.txt \
 		Makefile MANIFEST.in youtube-dl.1 youtube-dl.bash-completion \
 		youtube-dl.zsh youtube-dl.fish setup.py setup.cfg \
 		youtube-dl

From 9e3682d555d431514d9583170ae8be1b6fc12839 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 Dec 2017 23:53:27 +0700
Subject: [PATCH 1637/1696] [MANIFEST.in] Include all test data in PyPI package

---
 MANIFEST.in | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/MANIFEST.in b/MANIFEST.in
index af7518e0d..4e43e99f3 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -2,9 +2,8 @@ include README.md
 include LICENSE
 include AUTHORS
 include ChangeLog
-include test/*.py
-include test/*.json
 include youtube-dl.bash-completion
 include youtube-dl.fish
 include youtube-dl.1
 recursive-include docs Makefile conf.py *.rst
+recursive-include test *

From 4b7dd1705a7c16c1426ed7ed39e51e275124b4f3 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 23 Dec 2017 13:21:33 +0100
Subject: [PATCH 1638/1696] [7plus] Add new extractor(closes #15043)

---
 youtube_dl/extractor/brightcove.py | 116 +++++++++++++++--------------
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/sevenplus.py  |  67 +++++++++++++++++
 3 files changed, 128 insertions(+), 56 deletions(-)
 create mode 100644 youtube_dl/extractor/sevenplus.py

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 0ed59bcbc..f04505011 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -464,7 +464,7 @@ class BrightcoveNewIE(AdobePassIE):
             'timestamp': 1441391203,
             'upload_date': '20150904',
             'uploader_id': '929656772001',
-            'formats': 'mincount:22',
+            'formats': 'mincount:20',
         },
     }, {
         # with rtmp streams
@@ -478,7 +478,7 @@ class BrightcoveNewIE(AdobePassIE):
             'timestamp': 1433556729,
             'upload_date': '20150606',
             'uploader_id': '4036320279001',
-            'formats': 'mincount:41',
+            'formats': 'mincount:39',
         },
         'params': {
             # m3u8 download
@@ -564,59 +564,7 @@ class BrightcoveNewIE(AdobePassIE):
 
         return entries
 
-    def _real_extract(self, url):
-        url, smuggled_data = unsmuggle_url(url, {})
-        self._initialize_geo_bypass(smuggled_data.get('geo_countries'))
-
-        account_id, player_id, embed, video_id = re.match(self._VALID_URL, url).groups()
-
-        webpage = self._download_webpage(
-            'http://players.brightcove.net/%s/%s_%s/index.min.js'
-            % (account_id, player_id, embed), video_id)
-
-        policy_key = None
-
-        catalog = self._search_regex(
-            r'catalog\(({.+?})\);', webpage, 'catalog', default=None)
-        if catalog:
-            catalog = self._parse_json(
-                js_to_json(catalog), video_id, fatal=False)
-            if catalog:
-                policy_key = catalog.get('policyKey')
-
-        if not policy_key:
-            policy_key = self._search_regex(
-                r'policyKey\s*:\s*(["\'])(?P<pk>.+?)\1',
-                webpage, 'policy key', group='pk')
-
-        api_url = 'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s' % (account_id, video_id)
-        try:
-            json_data = self._download_json(api_url, video_id, headers={
-                'Accept': 'application/json;pk=%s' % policy_key
-            })
-        except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
-                json_data = self._parse_json(e.cause.read().decode(), video_id)[0]
-                message = json_data.get('message') or json_data['error_code']
-                if json_data.get('error_subcode') == 'CLIENT_GEO':
-                    self.raise_geo_restricted(msg=message)
-                raise ExtractorError(message, expected=True)
-            raise
-
-        errors = json_data.get('errors')
-        if errors and errors[0].get('error_subcode') == 'TVE_AUTH':
-            custom_fields = json_data['custom_fields']
-            tve_token = self._extract_mvpd_auth(
-                smuggled_data['source_url'], video_id,
-                custom_fields['bcadobepassrequestorid'],
-                custom_fields['bcadobepassresourceid'])
-            json_data = self._download_json(
-                api_url, video_id, headers={
-                    'Accept': 'application/json;pk=%s' % policy_key
-                }, query={
-                    'tveToken': tve_token,
-                })
-
+    def _parse_brightcove_metadata(self, json_data, video_id):
         title = json_data['name'].strip()
 
         formats = []
@@ -682,6 +630,7 @@ class BrightcoveNewIE(AdobePassIE):
                     })
                 formats.append(f)
 
+        errors = json_data.get('errors')
         if not formats and errors:
             error = errors[0]
             raise ExtractorError(
@@ -708,9 +657,64 @@ class BrightcoveNewIE(AdobePassIE):
             'thumbnail': json_data.get('thumbnail') or json_data.get('poster'),
             'duration': duration,
             'timestamp': parse_iso8601(json_data.get('published_at')),
-            'uploader_id': account_id,
+            'uploader_id': json_data.get('account_id'),
             'formats': formats,
             'subtitles': subtitles,
             'tags': json_data.get('tags', []),
             'is_live': is_live,
         }
+
+    def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
+        self._initialize_geo_bypass(smuggled_data.get('geo_countries'))
+
+        account_id, player_id, embed, video_id = re.match(self._VALID_URL, url).groups()
+
+        webpage = self._download_webpage(
+            'http://players.brightcove.net/%s/%s_%s/index.min.js'
+            % (account_id, player_id, embed), video_id)
+
+        policy_key = None
+
+        catalog = self._search_regex(
+            r'catalog\(({.+?})\);', webpage, 'catalog', default=None)
+        if catalog:
+            catalog = self._parse_json(
+                js_to_json(catalog), video_id, fatal=False)
+            if catalog:
+                policy_key = catalog.get('policyKey')
+
+        if not policy_key:
+            policy_key = self._search_regex(
+                r'policyKey\s*:\s*(["\'])(?P<pk>.+?)\1',
+                webpage, 'policy key', group='pk')
+
+        api_url = 'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s' % (account_id, video_id)
+        try:
+            json_data = self._download_json(api_url, video_id, headers={
+                'Accept': 'application/json;pk=%s' % policy_key
+            })
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+                json_data = self._parse_json(e.cause.read().decode(), video_id)[0]
+                message = json_data.get('message') or json_data['error_code']
+                if json_data.get('error_subcode') == 'CLIENT_GEO':
+                    self.raise_geo_restricted(msg=message)
+                raise ExtractorError(message, expected=True)
+            raise
+
+        errors = json_data.get('errors')
+        if errors and errors[0].get('error_subcode') == 'TVE_AUTH':
+            custom_fields = json_data['custom_fields']
+            tve_token = self._extract_mvpd_auth(
+                smuggled_data['source_url'], video_id,
+                custom_fields['bcadobepassrequestorid'],
+                custom_fields['bcadobepassresourceid'])
+            json_data = self._download_json(
+                api_url, video_id, headers={
+                    'Accept': 'application/json;pk=%s' % policy_key
+                }, query={
+                    'tveToken': tve_token,
+                })
+
+        return self._parse_brightcove_metadata(json_data, video_id)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 513074801..9ba1be2cd 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -926,6 +926,7 @@ from .senateisvp import SenateISVPIE
 from .sendtonews import SendtoNewsIE
 from .servingsys import ServingSysIE
 from .servus import ServusIE
+from .sevenplus import SevenPlusIE
 from .sexu import SexuIE
 from .shahid import (
     ShahidIE,
diff --git a/youtube_dl/extractor/sevenplus.py b/youtube_dl/extractor/sevenplus.py
new file mode 100644
index 000000000..9792f820a
--- /dev/null
+++ b/youtube_dl/extractor/sevenplus.py
@@ -0,0 +1,67 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .brightcove import BrightcoveNewIE
+from ..utils import update_url_query
+
+
+class SevenPlusIE(BrightcoveNewIE):
+    IE_NAME = '7plus'
+    _VALID_URL = r'https?://(?:www\.)?7plus\.com\.au/(?P<path>[^?]+\?.*?\bepisode-id=(?P<id>[^&#]+))'
+    _TESTS = [{
+        'url': 'https://7plus.com.au/BEAT?episode-id=BEAT-001',
+        'info_dict': {
+            'id': 'BEAT-001',
+            'ext': 'mp4',
+            'title': 'S1 E1 - Help / Lucy In The Sky With Diamonds',
+            'description': 'md5:37718bea20a8eedaca7f7361af566131',
+            'uploader_id': '5303576322001',
+            'upload_date': '20171031',
+            'timestamp': 1509440068,
+        },
+        'params': {
+            'format': 'bestvideo',
+            'skip_download': True,
+        }
+    }, {
+        'url': 'https://7plus.com.au/UUUU?episode-id=AUMS43-001',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        path, episode_id = re.match(self._VALID_URL, url).groups()
+
+        media = self._download_json(
+            'https://videoservice.swm.digital/playback', episode_id, query={
+                'appId': '7plus',
+                'deviceType': 'web',
+                'platformType': 'web',
+                'accountId': 5303576322001,
+                'referenceId': 'ref:' + episode_id,
+                'deliveryId': 'csai',
+                'videoType': 'vod',
+            })['media']
+
+        for source in media.get('sources', {}):
+            src = source.get('src')
+            if not src:
+                continue
+            source['src'] = update_url_query(src, {'rule': ''})
+
+        info = self._parse_brightcove_metadata(media, episode_id)
+
+        content = self._download_json(
+            'https://component-cdn.swm.digital/content/' + path,
+            episode_id, headers={
+                'market-id': 4,
+            }, fatal=False) or {}
+        for item in content.get('items', {}):
+            if item.get('componentData', {}).get('componentType') == 'infoPanel':
+                for src_key, dst_key in [('title', 'title'), ('shortSynopsis', 'description')]:
+                    value = item.get(src_key)
+                    if value:
+                        info[dst_key] = value
+
+        return info

From 2132edaa03857085821b6a1214ce1410e0c2e463 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Dec 2017 20:57:35 +0700
Subject: [PATCH 1639/1696] [extractor/common] Move X-Forwarded-For setup code
 into _request_webpage

---
 youtube_dl/extractor/common.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e5ef5e490..3b79b8cb4 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -495,6 +495,16 @@ class InfoExtractor(object):
                 self.to_screen('%s' % (note,))
             else:
                 self.to_screen('%s: %s' % (video_id, note))
+
+        # Some sites check X-Forwarded-For HTTP header in order to figure out
+        # the origin of the client behind proxy. This allows bypassing geo
+        # restriction by faking this header's value to IP that belongs to some
+        # geo unrestricted country. We will do so once we encounter any
+        # geo restriction error.
+        if self._x_forwarded_for_ip:
+            if 'X-Forwarded-For' not in headers:
+                headers['X-Forwarded-For'] = self._x_forwarded_for_ip
+
         if isinstance(url_or_request, compat_urllib_request.Request):
             url_or_request = update_Request(
                 url_or_request, data=data, headers=headers, query=query)
@@ -524,15 +534,6 @@ class InfoExtractor(object):
         if isinstance(url_or_request, (compat_str, str)):
             url_or_request = url_or_request.partition('#')[0]
 
-        # Some sites check X-Forwarded-For HTTP header in order to figure out
-        # the origin of the client behind proxy. This allows bypassing geo
-        # restriction by faking this header's value to IP that belongs to some
-        # geo unrestricted country. We will do so once we encounter any
-        # geo restriction error.
-        if self._x_forwarded_for_ip:
-            if 'X-Forwarded-For' not in headers:
-                headers['X-Forwarded-For'] = self._x_forwarded_for_ip
-
         urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal, data=data, headers=headers, query=query)
         if urlh is False:
             assert not fatal

From 5c5e60cff894e5372f89e6ba45d7ab6575c0a0b6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Dec 2017 20:59:14 +0700
Subject: [PATCH 1640/1696] [voot] Fix video identification

---
 youtube_dl/extractor/voot.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/voot.py b/youtube_dl/extractor/voot.py
index 426754489..751b21ee5 100644
--- a/youtube_dl/extractor/voot.py
+++ b/youtube_dl/extractor/voot.py
@@ -16,7 +16,7 @@ class VootIE(InfoExtractor):
     _TESTS = [{
         'url': 'https://www.voot.com/shows/ishq-ka-rang-safed/1/360558/is-this-the-end-of-kamini-/441353',
         'info_dict': {
-            'id': '441353',
+            'id': '0_8ledb18o',
             'ext': 'mp4',
             'title': 'Ishq Ka Rang Safed - Season 01 - Episode 340',
             'description': 'md5:06291fbbbc4dcbe21235c40c262507c1',
@@ -59,9 +59,10 @@ class VootIE(InfoExtractor):
 
         media = media_info['assets']
 
+        entry_id = media['EntryId']
         title = media['MediaName']
         formats = self._extract_m3u8_formats(
-            'https://cdnapisec.kaltura.com/p/1982551/playManifest/pt/https/f/applehttp/t/web/e/' + media['EntryId'],
+            'https://cdnapisec.kaltura.com/p/1982551/playManifest/pt/https/f/applehttp/t/web/e/' + entry_id,
             video_id, 'mp4', m3u8_id='hls')
         self._sort_formats(formats)
 
@@ -83,7 +84,8 @@ class VootIE(InfoExtractor):
                 episode_number = int_or_none(value)
 
         return {
-            'id': video_id,
+            'extractor_key': 'Kaltura',
+            'id': entry_id,
             'title': title,
             'description': description,
             'series': series,

From 69d69da98aa093c05776371beac1f2ffb4f5eea6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Dec 2017 21:16:32 +0700
Subject: [PATCH 1641/1696] [kaltura] Add another embed pattern for entry_id
 For cases when player configuration map is setup via indexing operator, e.g.
 kalturaPlayerConfiguration_1_lre6rg3i_10[entry_id] = 1_lre6rg3i (see
 https://www.heise.de/video/artikel/odcast-c-t-uplink-20-1-Apple-CarPlay-vs-Android-Auto-Galileo-3D-Sound-erklaert-3919694.html)

---
 youtube_dl/extractor/kaltura.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index bdac2df3e..e369959e3 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -125,9 +125,12 @@ class KalturaIE(InfoExtractor):
                         (?:https?:)?//cdnapi(?:sec)?\.kaltura\.com(?::\d+)?/(?:(?!(?P=q1)).)*\b(?:p|partner_id)/(?P<partner_id>\d+)(?:(?!(?P=q1)).)*
                     (?P=q1).*?
                     (?:
-                        entry_?[Ii]d|
-                        (?P<q2>["'])entry_?[Ii]d(?P=q2)
-                    )\s*:\s*
+                        (?:
+                            entry_?[Ii]d|
+                            (?P<q2>["'])entry_?[Ii]d(?P=q2)
+                        )\s*:\s*|
+                        \[\s*(?P<q2_1>["'])entry_?[Ii]d(?P=q2_1)\s*\]?\s*=\s*
+                    )
                     (?P<q3>["'])(?P<id>(?:(?!(?P=q3)).)+)(?P=q3)
                 ''', webpage) or
             re.search(

From f5a6321107db17ec8efaccaa2a4febc64b5aa5ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Dec 2017 21:17:28 +0700
Subject: [PATCH 1642/1696] [ChangeLog] Actualize

---
 ChangeLog | 33 +++++++++++++++++++++++++++++++--
 1 file changed, 31 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 03d2defb7..2d62f2fde 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,32 @@
+version <unreleased>
+
+Core
+* [extractor/common] Move X-Forwarded-For setup code into _request_webpage
++ [YoutubeDL] Add support for playlist_uploader and playlist_uploader_id in
+  output template (#11427, #15018)
++ [extractor/common] Introduce uploader, uploader_id and uploader_url
+  meta fields for playlists (#11427, #15018)
+* [downloader/fragment] Encode filename of fragment being removed (#15020)
++ [utils] Add another date format pattern (#14999)
+
+Extractors
++ [kaltura] Add another embed pattern for entry_id
++ [7plus] Add support for 7plus.com.au (#15043)
+* [animeondemand] Relax login error regular expression
++ [shahid] Add support for show pages (#7401)
++ [youtube] Extract uploader, uploader_id and uploader_url for playlists
+  (#11427, #15018)
+* [afreecatv] Improve format extraction (#15019)
++ [cspan] Add support for audio only pages and catch page errors (#14995)
++ [mailru] Add support for embed URLs (#14904)
+* [crunchyroll] Future-proof XML element checks (#15013)
+* [cbslocal] Fix timestamp extraction (#14999, #15000)
+* [discoverygo] Correct TTML subtitle extension
+* [vk] Make view count optional (#14979)
+* [disney] Skip Apple FairPlay formats (#14982)
+* [voot] Fix format extraction (#14758)
+
+
 version 2017.12.14
 
 Core
@@ -148,8 +177,8 @@ Extractors
 + [fxnetworks] Extract series metadata (#14603)
 + [younow] Add support for younow.com (#9255, #9432, #12436)
 * [dctptv] Fix extraction (#14599)
-* [youtube] Restrict embed regex (#14600)
-* [vimeo] Restrict iframe embed regex (#14600)
+* [youtube] Restrict embed regular expression (#14600)
+* [vimeo] Restrict iframe embed regular expression (#14600)
 * [soundgasm] Improve extraction (#14588)
 - [myvideo] Remove extractor (#8557)
 + [nbc] Add support for classic-tv videos (#14575)

From c2f2f8b120628f7e0e4b0a6f7184884fa976d9c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Dec 2017 21:22:41 +0700
Subject: [PATCH 1643/1696] [kaltura] Fix typo

---
 youtube_dl/extractor/kaltura.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index e369959e3..562e25f6d 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -129,7 +129,7 @@ class KalturaIE(InfoExtractor):
                             entry_?[Ii]d|
                             (?P<q2>["'])entry_?[Ii]d(?P=q2)
                         )\s*:\s*|
-                        \[\s*(?P<q2_1>["'])entry_?[Ii]d(?P=q2_1)\s*\]?\s*=\s*
+                        \[\s*(?P<q2_1>["'])entry_?[Ii]d(?P=q2_1)\s*\]\s*=\s*
                     )
                     (?P<q3>["'])(?P<id>(?:(?!(?P=q3)).)+)(?P=q3)
                 ''', webpage) or

From 307a7588b0a9205688e8ebc2539c1b0e19f68a6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Dec 2017 21:24:18 +0700
Subject: [PATCH 1644/1696] release 2017.12.23

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 ++
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index f37d8aa42..d7a91239f 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.12.14*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.12.14**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.12.23*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.12.23**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.12.14
+[debug] youtube-dl version 2017.12.23
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 2d62f2fde..ba64f3e02 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.12.23
 
 Core
 * [extractor/common] Move X-Forwarded-For setup code into _request_webpage
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index ebddd5b9d..eac35e390 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -10,6 +10,7 @@
  - **56.com**
  - **5min**
  - **6play**
+ - **7plus**
  - **8tracks**
  - **91porn**
  - **9c9media**
@@ -728,6 +729,7 @@
  - **Servus**
  - **Sexu**
  - **Shahid**
+ - **ShahidShow**
  - **Shared**: shared.sx
  - **ShowRoomLive**
  - **Sina**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 2b5a63464..f999584d7 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.12.14'
+__version__ = '2017.12.23'

From 0e25a1a27875369a4fdf11b6a7fcfc969b1f482d Mon Sep 17 00:00:00 2001
From: JianxinLi <justcoxier@gmail.com>
Date: Mon, 4 Dec 2017 10:33:56 +0800
Subject: [PATCH 1645/1696] [youku] Update ccode

Change-Id: Id397e814e81ff560506d68563b7409eebbe5943d
---
 youtube_dl/extractor/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index f0ba01197..9d0caee93 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -154,7 +154,7 @@ class YoukuIE(InfoExtractor):
         # request basic data
         basic_data_params = {
             'vid': video_id,
-            'ccode': '0501',
+            'ccode': '0507',
             'client_ip': '192.168.1.1',
             'utid': cna,
             'client_ts': time.time() / 1000,

From 116561697d605f22f749e3d092e8e4795ca0573d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 23 Dec 2017 23:41:24 +0800
Subject: [PATCH 1646/1696] [ChangeLog] Update after #14903

---
 ChangeLog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index ba64f3e02..658c00c5b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unrequested>
+
+Extractors
+* [youku] Update ccode
+
+
 version 2017.12.23
 
 Core

From b954e72c8731e65b9b0548a537cd0e3275b54e4d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 23 Dec 2017 23:42:02 +0800
Subject: [PATCH 1647/1696] [ChangeLog] typo

---
 ChangeLog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 658c00c5b..cb750e270 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unrequested>
+version <unreleased>
 
 Extractors
 * [youku] Update ccode

From 273c23d960cbd2da18fadaef002473db41b5f56b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Dec 2017 13:53:27 +0700
Subject: [PATCH 1648/1696] [openload] Add support for oload.stream (closes
 #15070)

---
 youtube_dl/extractor/openload.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index a99af12a4..aed579f36 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -242,7 +242,7 @@ class PhantomJSwrapper(object):
 
 
 class OpenloadIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?:openload\.(?:co|io|link)|oload\.tv)/(?:f|embed)/(?P<id>[a-zA-Z0-9-_]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:openload\.(?:co|io|link)|oload\.(?:tv|stream))/(?:f|embed)/(?P<id>[a-zA-Z0-9-_]+)'
 
     _TESTS = [{
         'url': 'https://openload.co/f/kUEfGclsU9o',
@@ -289,6 +289,9 @@ class OpenloadIE(InfoExtractor):
     }, {
         'url': 'http://www.openload.link/f/KnG-kKZdcfY',
         'only_matching': True,
+    }, {
+        'url': 'https://oload.stream/f/KnG-kKZdcfY',
+        'only_matching': True,
     }]
 
     _USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'

From a75419586bb900df711de49adf5047afa9f083ef Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 24 Dec 2017 20:47:42 +0800
Subject: [PATCH 1649/1696] [openload] Remove a confusing exception

If phantomjs is not installed, there's an error besides the missing
phantomjs exception:

Exception ignored in: <bound method PhantomJSwrapper.__del__ of <youtube_dl.extractor.openload.PhantomJSwrapper object at 0x7f8ad5e78278>>
Traceback (most recent call last):
  File "/home/yen/Projects/youtube-dl/youtube_dl/extractor/openload.py", line 142, in __del__
    os.remove(self._TMP_FILES[name].name)
AttributeError: 'PhantomJSwrapper' object has no attribute '_TMP_FILES'
---
 youtube_dl/extractor/openload.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index aed579f36..d1eb3be25 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -112,6 +112,8 @@ class PhantomJSwrapper(object):
         return get_exe_version('phantomjs', version_re=r'([0-9.]+)')
 
     def __init__(self, extractor, required_version=None, timeout=10000):
+        self._TMP_FILES = {}
+
         self.exe = check_executable('phantomjs', ['-v'])
         if not self.exe:
             raise ExtractorError('PhantomJS executable not found in PATH, '
@@ -130,7 +132,6 @@ class PhantomJSwrapper(object):
         self.options = {
             'timeout': timeout,
         }
-        self._TMP_FILES = {}
         for name in self._TMP_FILE_NAMES:
             tmp = tempfile.NamedTemporaryFile(delete=False)
             tmp.close()
@@ -140,7 +141,7 @@ class PhantomJSwrapper(object):
         for name in self._TMP_FILE_NAMES:
             try:
                 os.remove(self._TMP_FILES[name].name)
-            except (IOError, OSError):
+            except (IOError, OSError, KeyError):
                 pass
 
     def _save_cookies(self, url):

From d99a1000c7522cb37910afe772d7317687521eb0 Mon Sep 17 00:00:00 2001
From: JianxinLi <justcoxier@gmail.com>
Date: Sun, 24 Dec 2017 00:30:27 +0800
Subject: [PATCH 1650/1696] [youku] Fix list extraction.(close #15065)

Change-Id: I578fdc5b69509bdcd8d3191e3917afe47c234ff6
---
 youtube_dl/extractor/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 9d0caee93..3e64cce38 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -276,9 +276,9 @@ class YoukuShowIE(InfoExtractor):
             r'<div[^>]+id="(reload_\d+)', first_page, 'first page reload id')
         # The first reload_id has the same items as first_page
         reload_ids = re.findall('<li[^>]+data-id="([^"]+)">', first_page)
+        entries.extend(initial_entries)
         for idx, reload_id in enumerate(reload_ids):
             if reload_id == first_page_reload_id:
-                entries.extend(initial_entries)
                 continue
             _, new_entries = self._extract_entries(
                 'http://list.youku.com/show/episode', show_id,

From d3ca28323545a36819d9c32797907bc190095b5c Mon Sep 17 00:00:00 2001
From: JianxinLi <justcoxier@gmail.com>
Date: Mon, 25 Dec 2017 21:39:10 +0800
Subject: [PATCH 1651/1696] [youku] Add test case.

Some playlist has no data-id value.

Change-Id: I97455f2907f08bda03b538cdc13ec827e2f8ce26
---
 youtube_dl/extractor/youku.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 3e64cce38..c7947d4a1 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -241,6 +241,10 @@ class YoukuShowIE(InfoExtractor):
         # Ongoing playlist. The initial page is the last one
         'url': 'http://list.youku.com/show/id_za7c275ecd7b411e1a19e.html',
         'only_matching': True,
+    }, {
+        #  No data-id value.
+        'url': 'http://list.youku.com/show/id_zefbfbd61237fefbfbdef.html',
+        'only_matching': True,
     }]
 
     def _extract_entries(self, playlist_data_url, show_id, note, query):

From 173558ce9620bf1b22ba2d4c67288e2a45c715fc Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 25 Dec 2017 22:06:18 +0800
Subject: [PATCH 1652/1696] [ChangeLog] Update after #15065

---
 ChangeLog | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index cb750e270..420a1bd11 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,8 @@
 version <unreleased>
 
 Extractors
-* [youku] Update ccode
+* [youku] Update ccode (#14880)
+* [youku] Fix list extraction (#15065)
 
 
 version 2017.12.23

From 0f897e0929b2a3ebcae616f8b1bbdac8cd9c6f75 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 25 Dec 2017 23:28:51 +0100
Subject: [PATCH 1653/1696] [espn] add support for espnfc and extract more
 formats(closes #8053)

---
 youtube_dl/extractor/espn.py | 70 +++++++++++++++++++++++++-----------
 1 file changed, 50 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/espn.py b/youtube_dl/extractor/espn.py
index 7a7436068..0e135b8bc 100644
--- a/youtube_dl/extractor/espn.py
+++ b/youtube_dl/extractor/espn.py
@@ -1,6 +1,9 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from .once import OnceIE
 from ..compat import compat_str
 from ..utils import (
     determine_ext,
@@ -9,22 +12,27 @@ from ..utils import (
 )
 
 
-class ESPNIE(InfoExtractor):
+class ESPNIE(OnceIE):
     _VALID_URL = r'''(?x)
                     https?://
-                        (?:
-                            (?:(?:\w+\.)+)?espn\.go|
-                            (?:www\.)?espn
-                        )\.com/
                         (?:
                             (?:
-                                video/clip|
-                                watch/player
-                            )
-                            (?:
-                                \?.*?\bid=|
-                                /_/id/
-                            )
+                                (?:
+                                    (?:(?:\w+\.)+)?espn\.go|
+                                    (?:www\.)?espn
+                                )\.com/
+                                (?:
+                                    (?:
+                                        video/(?:clip|iframe/twitter)|
+                                        watch/player
+                                    )
+                                    (?:
+                                        .*?\?.*?\bid=|
+                                        /_/id/
+                                    )
+                                )
+                            )|
+                            (?:www\.)espnfc\.(?:com|us)/(?:video/)?[^/]+/\d+/video/
                         )
                         (?P<id>\d+)
                     '''
@@ -77,6 +85,15 @@ class ESPNIE(InfoExtractor):
     }, {
         'url': 'http://www.espn.com/video/clip/_/id/17989860',
         'only_matching': True,
+    }, {
+        'url': 'https://espn.go.com/video/iframe/twitter/?cms=espn&id=10365079',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.espnfc.us/video/espn-fc-tv/86/video/3319154/nashville-unveiled-as-the-newest-club-in-mls',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.espnfc.com/english-premier-league/23/video/3324163/premier-league-in-90-seconds-golden-tweets',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -93,7 +110,9 @@ class ESPNIE(InfoExtractor):
 
         def traverse_source(source, base_source_id=None):
             for source_id, source in source.items():
-                if isinstance(source, compat_str):
+                if source_id == 'alert':
+                    continue
+                elif isinstance(source, compat_str):
                     extract_source(source, base_source_id)
                 elif isinstance(source, dict):
                     traverse_source(
@@ -106,7 +125,9 @@ class ESPNIE(InfoExtractor):
                 return
             format_urls.add(source_url)
             ext = determine_ext(source_url)
-            if ext == 'smil':
+            if OnceIE.suitable(source_url):
+                formats.extend(self._extract_once_formats(source_url))
+            elif ext == 'smil':
                 formats.extend(self._extract_smil_formats(
                     source_url, video_id, fatal=False))
             elif ext == 'f4m':
@@ -117,12 +138,24 @@ class ESPNIE(InfoExtractor):
                     source_url, video_id, 'mp4', entry_protocol='m3u8_native',
                     m3u8_id=source_id, fatal=False))
             else:
-                formats.append({
+                f = {
                     'url': source_url,
                     'format_id': source_id,
-                })
+                }
+                mobj = re.search(r'(\d+)p(\d+)_(\d+)k\.', source_url)
+                if mobj:
+                    f.update({
+                        'height': int(mobj.group(1)),
+                        'fps': int(mobj.group(2)),
+                        'tbr': int(mobj.group(3)),
+                    })
+                if source_id == 'mezzanine':
+                    f['preference'] = 1
+                formats.append(f)
 
-        traverse_source(clip['links']['source'])
+        links = clip.get('links', {})
+        traverse_source(links.get('source', {}))
+        traverse_source(links.get('mobile', {}))
         self._sort_formats(formats)
 
         description = clip.get('caption') or clip.get('description')
@@ -144,9 +177,6 @@ class ESPNIE(InfoExtractor):
 class ESPNArticleIE(InfoExtractor):
     _VALID_URL = r'https?://(?:espn\.go|(?:www\.)?espn)\.com/(?:[^/]+/)*(?P<id>[^/]+)'
     _TESTS = [{
-        'url': 'https://espn.go.com/video/iframe/twitter/?cms=espn&id=10365079',
-        'only_matching': True,
-    }, {
         'url': 'http://espn.go.com/nba/recap?gameId=400793786',
         'only_matching': True,
     }, {

From 45d20488f188b680daa39c5b9fa88d0bba102ab5 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 26 Dec 2017 12:32:04 +0100
Subject: [PATCH 1654/1696] [umg:de] Add new extractor(closes #11582)(closes
 #11584)

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/umg.py        | 103 +++++++++++++++++++++++++++++
 2 files changed, 104 insertions(+)
 create mode 100644 youtube_dl/extractor/umg.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 9ba1be2cd..3269ed743 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1143,6 +1143,7 @@ from .udemy import (
 from .udn import UDNEmbedIE
 from .uktvplay import UKTVPlayIE
 from .digiteka import DigitekaIE
+from .umg import UMGDeIE
 from .unistra import UnistraIE
 from .unity import UnityIE
 from .uol import UOLIE
diff --git a/youtube_dl/extractor/umg.py b/youtube_dl/extractor/umg.py
new file mode 100644
index 000000000..d815cd9a6
--- /dev/null
+++ b/youtube_dl/extractor/umg.py
@@ -0,0 +1,103 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_filesize,
+    parse_iso8601,
+)
+
+
+class UMGDeIE(InfoExtractor):
+    IE_NAME = 'umg:de'
+    IE_DESC = 'Universal Music Deutschland'
+    _VALID_URL = r'https?://(?:www\.)?universal-music\.de/[^/]+/videos/[^/?#]+-(?P<id>\d+)'
+    _TEST = {
+        'url': 'https://www.universal-music.de/sido/videos/jedes-wort-ist-gold-wert-457803',
+        'md5': 'ebd90f48c80dcc82f77251eb1902634f',
+        'info_dict': {
+            'id': '457803',
+            'ext': 'mp4',
+            'title': 'Jedes Wort ist Gold wert',
+            'timestamp': 1513591800,
+            'upload_date': '20171218',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_data = self._download_json(
+            'https://api.universal-music.de/graphql',
+            video_id, query={
+                'query': '''{
+  universalMusic(channel:16) {
+    video(id:%s) {
+      headline
+      formats {
+        formatId
+        url
+        type
+        width
+        height
+        mimeType
+        fileSize
+      }
+      duration
+      createdDate
+    }
+  }
+}''' % video_id})['data']['universalMusic']['video']
+
+        title = video_data['headline']
+        hls_url_template = 'http://mediadelivery.universal-music-services.de/vod/mp4:autofill/storage/' + '/'.join(list(video_id)) + '/content/%s/file/playlist.m3u8'
+
+        thumbnails = []
+        formats = []
+
+        def add_m3u8_format(format_id):
+            m3u8_formats = self._extract_m3u8_formats(
+                hls_url_template % format_id, video_id, 'mp4',
+                'm3u8_native', m3u8_id='hls', fatal='False')
+            if m3u8_formats and m3u8_formats[0].get('height'):
+                formats.extend(m3u8_formats)
+
+        for f in video_data.get('formats', []):
+            f_url = f.get('url')
+            mime_type = f.get('mimeType')
+            if not f_url or mime_type == 'application/mxf':
+                continue
+            fmt = {
+                'url': f_url,
+                'width': int_or_none(f.get('width')),
+                'height': int_or_none(f.get('height')),
+                'filesize': parse_filesize(f.get('fileSize')),
+            }
+            f_type = f.get('type')
+            if f_type == 'Image':
+                thumbnails.append(fmt)
+            elif f_type == 'Video':
+                format_id = f.get('formatId')
+                if format_id:
+                    fmt['format_id'] = format_id
+                    if mime_type == 'video/mp4':
+                        add_m3u8_format(format_id)
+                urlh = self._request_webpage(f_url, video_id, fatal=False)
+                if urlh:
+                    first_byte = urlh.read(1)
+                    if first_byte not in (b'F', b'\x00'):
+                        continue
+                    formats.append(fmt)
+        if not formats:
+            for format_id in (867, 836, 940):
+                add_m3u8_format(format_id)
+        self._sort_formats(formats, ('width', 'height', 'filesize', 'tbr'))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'duration': int_or_none(video_data.get('duration')),
+            'timestamp': parse_iso8601(video_data.get('createdDate'), ' '),
+            'thumbnails': thumbnails,
+            'formats': formats,
+        }

From db145ee54a57f5ccc89639de8c589eb111a91b19 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 26 Dec 2017 14:20:21 +0100
Subject: [PATCH 1655/1696] [espn] Add new extractor for
 http://fivethirtyeight.com(closes #6864)

---
 youtube_dl/extractor/espn.py       | 31 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  1 +
 2 files changed, 32 insertions(+)

diff --git a/youtube_dl/extractor/espn.py b/youtube_dl/extractor/espn.py
index 0e135b8bc..127c69b2e 100644
--- a/youtube_dl/extractor/espn.py
+++ b/youtube_dl/extractor/espn.py
@@ -205,3 +205,34 @@ class ESPNArticleIE(InfoExtractor):
 
         return self.url_result(
             'http://espn.go.com/video/clip?id=%s' % video_id, ESPNIE.ie_key())
+
+
+class FiveThirtyEightIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?fivethirtyeight\.com/features/(?P<id>[^/?#]+)'
+    _TEST = {
+        'url': 'http://fivethirtyeight.com/features/how-the-6-8-raiders-can-still-make-the-playoffs/',
+        'info_dict': {
+            'id': '21846851',
+            'ext': 'mp4',
+            'title': 'FiveThirtyEight: The Raiders can still make the playoffs',
+            'description': 'Neil Paine breaks down the simplest scenario that will put the Raiders into the playoffs at 8-8.',
+            'timestamp': 1513960621,
+            'upload_date': '20171222',
+        },
+        'params': {
+            'skip_download': True,
+        },
+        'expected_warnings': ['Unable to download f4m manifest'],
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        video_id = self._search_regex(
+            r'data-video-id=["\'](?P<id>\d+)',
+            webpage, 'video id', group='id')
+
+        return self.url_result(
+            'http://espn.go.com/video/clip?id=%s' % video_id, ESPNIE.ie_key())
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 3269ed743..91bd3287c 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -322,6 +322,7 @@ from .escapist import EscapistIE
 from .espn import (
     ESPNIE,
     ESPNArticleIE,
+    FiveThirtyEightIE,
 )
 from .esri import EsriVideoIE
 from .etonline import ETOnlineIE

From a14001a5a13b1639dc98b75b0775d251487aad1d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?D=C3=A9stin=20Reed?= <trox1972@users.noreply.github.com>
Date: Wed, 17 Aug 2016 16:02:59 +0200
Subject: [PATCH 1656/1696] [Filmweb] Add extractor

---
 youtube_dl/extractor/extractors.py       |  2 ++
 youtube_dl/extractor/filmweb.py          | 45 ++++++++++++++++++++++++
 youtube_dl/extractor/twentythreevideo.py | 45 ++++++++++++++++++++++++
 3 files changed, 92 insertions(+)
 create mode 100644 youtube_dl/extractor/filmweb.py
 create mode 100644 youtube_dl/extractor/twentythreevideo.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 91bd3287c..04125d9f2 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -345,6 +345,7 @@ from .filmon import (
     FilmOnIE,
     FilmOnChannelIE,
 )
+from .filmweb import FilmwebIE
 from .firsttv import FirstTVIE
 from .fivemin import FiveMinIE
 from .fivetv import FiveTVIE
@@ -1120,6 +1121,7 @@ from .tvplayer import TVPlayerIE
 from .tweakers import TweakersIE
 from .twentyfourvideo import TwentyFourVideoIE
 from .twentymin import TwentyMinutenIE
+from .twentythreevideo import TwentyThreeVideoIE
 from .twitch import (
     TwitchVideoIE,
     TwitchChapterIE,
diff --git a/youtube_dl/extractor/filmweb.py b/youtube_dl/extractor/filmweb.py
new file mode 100644
index 000000000..a3d9f872e
--- /dev/null
+++ b/youtube_dl/extractor/filmweb.py
@@ -0,0 +1,45 @@
+from __future__ import unicode_literals
+
+from .twentythreevideo import TwentyThreeVideoIE
+
+
+class FilmwebIE(TwentyThreeVideoIE):
+    IE_NAME = 'Filmweb'
+    _VALID_URL = r'https?://(?:www\.)?filmweb\.no/trailere/article(?P<id>\d+).ece'
+    _TEST = {
+        'url': 'http://www.filmweb.no/trailere/article1264921.ece',
+        'md5': 'e353f47df98e557d67edaceda9dece89',
+        'info_dict': {
+            'id': '1264921',
+            'title': 'Det som en gang var',
+            'ext': 'mp4',
+            'description': 'Trailer: Scener fra et vennskap',
+        }
+    }
+
+    _CLIENT_NAME = 'filmweb'
+    _CLIENT_ID = '12732917'
+    _EMBED_BASE_URL = 'http://www.filmweb.no/template/ajax/json_trailerEmbed.jsp?articleId=%s&autoplay=true'
+
+    def _real_extract(self, url):
+        article_id = self._match_id(url)
+        webpage = self._download_webpage(url, article_id)
+
+        title = self._search_regex(r'var\s+jsTitle\s*=\s*escape\("([^"]+)"\);',
+            webpage, 'title', fatal=True)
+
+        format_url = self._proto_relative_url(
+            self._html_search_regex(r'"(//filmweb\.23video\.com/[^"]+)"',
+                self._download_json(self._EMBED_BASE_URL % article_id,
+                    article_id)['embedCode'], 'format url'))
+
+        formats = self._extract_formats(format_url, self._CLIENT_ID)
+        self._sort_formats(formats)
+
+        return {
+            'id': article_id,
+            'title': title,
+            'alt_title': self._og_search_title(webpage),
+            'formats': formats,
+            'description': self._og_search_description(webpage),
+        }
diff --git a/youtube_dl/extractor/twentythreevideo.py b/youtube_dl/extractor/twentythreevideo.py
new file mode 100644
index 000000000..2bad2dbd6
--- /dev/null
+++ b/youtube_dl/extractor/twentythreevideo.py
@@ -0,0 +1,45 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class TwentyThreeVideoIE(InfoExtractor):
+    IE_NAME = '23video'
+    _VALID_URL = r'https?://(?:www\.)?(?P<client>[\w-]+)\.23video\.com/v.ihtml/player.html.*photo_id=(?P<id>\d+)'
+    _TEST = {}
+
+    _URL_TEMPLATE = 'https://%s.23video.com/%s/%s/%s/%s/download-video.mp4'
+    _FORMATS = {
+        'video_hd': {
+            'width': 1280,
+            'height': 720,
+        },
+        'video_medium': {
+            'width': 640,
+            'height': 360,
+        },
+        'video_mobile_high': {
+            'width': 320,
+            'height': 180,
+        }
+    }
+
+    def _extract_formats(self, url, client_id):
+        client_name = self._search_regex(r'([a-z]+)\.23video\.com', url, 'client name')
+        video_id = self._search_regex(r'photo%5fid=([^?&]+)', url, 'video id')
+        token = self._search_regex(r'token=([^?&]+)', url, 'token')
+
+        formats = []
+        for format_key in self._FORMATS.keys():
+            formats.append({
+                'url': self._URL_TEMPLATE % (client_name, client_id, video_id,
+                    token, format_key),
+                'width': self._FORMATS.get(format_key, {}).get('width'),
+                'height': self._FORMATS.get(format_key, {}).get('height'),
+            })
+
+        return formats
+
+    def _real_extract(self, url):
+        # TODO: Find out how to extract client_id
+        raise NotImplementedError('Not able to extract the `client_id`')

From be069839b4acb645799f7b216d14c046fb4a3400 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 26 Dec 2017 19:41:08 +0100
Subject: [PATCH 1657/1696] [filmweb] improve extraction

---
 youtube_dl/extractor/filmweb.py          | 53 +++++++-------
 youtube_dl/extractor/twentythreevideo.py | 90 ++++++++++++++++--------
 2 files changed, 86 insertions(+), 57 deletions(-)

diff --git a/youtube_dl/extractor/filmweb.py b/youtube_dl/extractor/filmweb.py
index a3d9f872e..56000bc5b 100644
--- a/youtube_dl/extractor/filmweb.py
+++ b/youtube_dl/extractor/filmweb.py
@@ -1,45 +1,42 @@
 from __future__ import unicode_literals
 
-from .twentythreevideo import TwentyThreeVideoIE
+import re
+
+from .common import InfoExtractor
 
 
-class FilmwebIE(TwentyThreeVideoIE):
-    IE_NAME = 'Filmweb'
-    _VALID_URL = r'https?://(?:www\.)?filmweb\.no/trailere/article(?P<id>\d+).ece'
+class FilmwebIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?filmweb\.no/(?P<type>trailere|filmnytt)/article(?P<id>\d+)\.ece'
     _TEST = {
         'url': 'http://www.filmweb.no/trailere/article1264921.ece',
         'md5': 'e353f47df98e557d67edaceda9dece89',
         'info_dict': {
-            'id': '1264921',
-            'title': 'Det som en gang var',
+            'id': '13033574',
             'ext': 'mp4',
-            'description': 'Trailer: Scener fra et vennskap',
+            'title': 'Det som en gang var',
+            'upload_date': '20160316',
+            'timestamp': 1458140101,
+            'uploader_id': '12639966',
+            'uploader': 'Live Roaldset',
         }
     }
 
-    _CLIENT_NAME = 'filmweb'
-    _CLIENT_ID = '12732917'
-    _EMBED_BASE_URL = 'http://www.filmweb.no/template/ajax/json_trailerEmbed.jsp?articleId=%s&autoplay=true'
-
     def _real_extract(self, url):
-        article_id = self._match_id(url)
-        webpage = self._download_webpage(url, article_id)
-
-        title = self._search_regex(r'var\s+jsTitle\s*=\s*escape\("([^"]+)"\);',
-            webpage, 'title', fatal=True)
-
-        format_url = self._proto_relative_url(
-            self._html_search_regex(r'"(//filmweb\.23video\.com/[^"]+)"',
-                self._download_json(self._EMBED_BASE_URL % article_id,
-                    article_id)['embedCode'], 'format url'))
-
-        formats = self._extract_formats(format_url, self._CLIENT_ID)
-        self._sort_formats(formats)
+        article_type, article_id = re.match(self._VALID_URL, url).groups()
+        if article_type == 'filmnytt':
+            webpage = self._download_webpage(url, article_id)
+            article_id = self._search_regex(r'data-videoid="(\d+)"', webpage, 'article id')
+        embed_code = self._download_json(
+            'https://www.filmweb.no/template_v2/ajax/json_trailerEmbed.jsp',
+            article_id, query={
+                'articleId': article_id,
+            })['embedCode']
+        iframe_url = self._proto_relative_url(self._search_regex(
+            r'<iframe[^>]+src="([^"]+)', embed_code, 'iframe url'))
 
         return {
+            '_type': 'url_transparent',
             'id': article_id,
-            'title': title,
-            'alt_title': self._og_search_title(webpage),
-            'formats': formats,
-            'description': self._og_search_description(webpage),
+            'url': iframe_url,
+            'ie_key': 'TwentyThreeVideo',
         }
diff --git a/youtube_dl/extractor/twentythreevideo.py b/youtube_dl/extractor/twentythreevideo.py
index 2bad2dbd6..aa0c6e90f 100644
--- a/youtube_dl/extractor/twentythreevideo.py
+++ b/youtube_dl/extractor/twentythreevideo.py
@@ -1,45 +1,77 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..utils import int_or_none
 
 
 class TwentyThreeVideoIE(InfoExtractor):
     IE_NAME = '23video'
-    _VALID_URL = r'https?://(?:www\.)?(?P<client>[\w-]+)\.23video\.com/v.ihtml/player.html.*photo_id=(?P<id>\d+)'
-    _TEST = {}
-
-    _URL_TEMPLATE = 'https://%s.23video.com/%s/%s/%s/%s/download-video.mp4'
-    _FORMATS = {
-        'video_hd': {
-            'width': 1280,
-            'height': 720,
-        },
-        'video_medium': {
-            'width': 640,
-            'height': 360,
-        },
-        'video_mobile_high': {
-            'width': 320,
-            'height': 180,
+    _VALID_URL = r'https?://video\.(?P<domain>twentythree\.net|23video\.com|filmweb\.no)/v\.ihtml/player\.html\?(?P<query>.*?\bphoto(?:_|%5f)id=(?P<id>\d+).*)'
+    _TEST = {
+        'url': 'https://video.twentythree.net/v.ihtml/player.html?showDescriptions=0&source=site&photo%5fid=20448876&autoPlay=1',
+        'md5': '75fcf216303eb1dae9920d651f85ced4',
+        'info_dict': {
+            'id': '20448876',
+            'ext': 'mp4',
+            'title': 'Video Marketing Minute: Personalized Video',
+            'timestamp': 1513855354,
+            'upload_date': '20171221',
+            'uploader_id': '12258964',
+            'uploader': 'Rasmus Bysted',
         }
     }
 
-    def _extract_formats(self, url, client_id):
-        client_name = self._search_regex(r'([a-z]+)\.23video\.com', url, 'client name')
-        video_id = self._search_regex(r'photo%5fid=([^?&]+)', url, 'video id')
-        token = self._search_regex(r'token=([^?&]+)', url, 'token')
+    def _real_extract(self, url):
+        domain, query, photo_id = re.match(self._VALID_URL, url).groups()
+        base_url = 'https://video.%s' % domain
+        photo_data = self._download_json(
+            base_url + '/api/photo/list?' + query, photo_id, query={
+                'format': 'json',
+            }, transform_source=lambda s: self._search_regex(r'(?s)({.+})', s, 'photo data'))['photo']
+        title = photo_data['title']
 
         formats = []
-        for format_key in self._FORMATS.keys():
+
+        audio_path = photo_data.get('audio_download')
+        if audio_path:
             formats.append({
-                'url': self._URL_TEMPLATE % (client_name, client_id, video_id,
-                    token, format_key),
-                'width': self._FORMATS.get(format_key, {}).get('width'),
-                'height': self._FORMATS.get(format_key, {}).get('height'),
+                'format_id': 'audio',
+                'url': base_url + audio_path,
+                'filesize': int_or_none(photo_data.get('audio_size')),
+                'vcodec': 'none',
             })
 
-        return formats
+        def add_common_info_to_list(l, template, id_field, id_value):
+            f_base = template % id_value
+            f_path = photo_data.get(f_base + 'download')
+            if not f_path:
+                return
+            l.append({
+                id_field: id_value,
+                'url': base_url + f_path,
+                'width': int_or_none(photo_data.get(f_base + 'width')),
+                'height': int_or_none(photo_data.get(f_base + 'height')),
+                'filesize': int_or_none(photo_data.get(f_base + 'size')),
+            })
 
-    def _real_extract(self, url):
-        # TODO: Find out how to extract client_id
-        raise NotImplementedError('Not able to extract the `client_id`')
+        for f in ('mobile_high', 'medium', 'hd', '1080p', '4k'):
+            add_common_info_to_list(formats, 'video_%s_', 'format_id', f)
+
+        thumbnails = []
+        for t in ('quad16', 'quad50', 'quad75', 'quad100', 'small', 'portrait', 'standard', 'medium', 'large', 'original'):
+            add_common_info_to_list(thumbnails, '%s_', 'id', t)
+
+        return {
+            'id': photo_id,
+            'title': title,
+            'timestamp': int_or_none(photo_data.get('creation_date_epoch')),
+            'duration': int_or_none(photo_data.get('video_length')),
+            'view_count': int_or_none(photo_data.get('view_count')),
+            'comment_count': int_or_none(photo_data.get('number_of_comments')),
+            'uploader_id': photo_data.get('user_id'),
+            'uploader': photo_data.get('display_name'),
+            'thumbnails': thumbnails,
+            'formats': formats,
+        }

From 616bb95b280e74b1a5048e95128259e2a85ca962 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ond=C5=99ej=20Caletka?= <ondrej@caletka.cz>
Date: Wed, 27 Dec 2017 16:57:26 +0100
Subject: [PATCH 1658/1696] [playtvak] Relax video regex and make description
 optional

---
 youtube_dl/extractor/playtvak.py | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/playtvak.py b/youtube_dl/extractor/playtvak.py
index 391e1bd09..4c5f57919 100644
--- a/youtube_dl/extractor/playtvak.py
+++ b/youtube_dl/extractor/playtvak.py
@@ -24,7 +24,7 @@ class PlaytvakIE(InfoExtractor):
             'id': 'A150730_150323_hodinovy-manzel_kuko',
             'ext': 'mp4',
             'title': 'Vyžeňte vosy a sršně ze zahrady',
-            'description': 'md5:f93d398691044d303bc4a3de62f3e976',
+            'description': 'md5:4436e61b7df227a093778efb7e373571',
             'thumbnail': r're:(?i)^https?://.*\.(?:jpg|png)$',
             'duration': 279,
             'timestamp': 1438732860,
@@ -36,9 +36,19 @@ class PlaytvakIE(InfoExtractor):
         'info_dict': {
             'id': 'A150624_164934_planespotting_cat',
             'ext': 'flv',
-            'title': 're:^Přímý přenos iDNES.cz [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'title': 're:^Planespotting [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
             'description': 'Sledujte provoz na ranveji Letiště Václava Havla v Praze',
-            'thumbnail': r're:(?i)^https?://.*\.(?:jpg|png)$',
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,  # requires rtmpdump
+        },
+    }, {  # another live stream, this one without Misc.videoFLV
+        'url': 'https://slowtv.playtvak.cz/zive-sledujte-vlaky-v-primem-prenosu-dwi-/hlavni-nadrazi.aspx?c=A151218_145728_hlavni-nadrazi_plap',
+        'info_dict': {
+            'id': 'A151218_145728_hlavni-nadrazi_plap',
+            'ext': 'flv',
+            'title': 're:^Hlavní nádraží [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
             'is_live': True,
         },
         'params': {
@@ -95,7 +105,7 @@ class PlaytvakIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         info_url = self._html_search_regex(
-            r'Misc\.videoFLV\(\s*{\s*data\s*:\s*"([^"]+)"', webpage, 'info url')
+            r'Misc\.video(?:FLV)?\(\s*{\s*data\s*:\s*"([^"]+)"', webpage, 'info url')
 
         parsed_url = compat_urlparse.urlparse(info_url)
 
@@ -160,7 +170,7 @@ class PlaytvakIE(InfoExtractor):
         if is_live:
             title = self._live_title(title)
         description = self._og_search_description(webpage, default=None) or self._html_search_meta(
-            'description', webpage, 'description')
+            'description', webpage, 'description', default=None)
         timestamp = None
         duration = None
         if not is_live:

From 1ae0f0a21dec0cfce85b6f8285cc64cd702846b5 Mon Sep 17 00:00:00 2001
From: Leonardo Taccari <iamleot@gmail.com>
Date: Wed, 13 Dec 2017 12:24:07 +0100
Subject: [PATCH 1659/1696] [internazionale] Add extractor

---
 youtube_dl/extractor/extractors.py     |  1 +
 youtube_dl/extractor/internazionale.py | 46 ++++++++++++++++++++++++++
 2 files changed, 47 insertions(+)
 create mode 100644 youtube_dl/extractor/internazionale.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 04125d9f2..e7b93a699 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -466,6 +466,7 @@ from .indavideo import (
 )
 from .infoq import InfoQIE
 from .instagram import InstagramIE, InstagramUserIE
+from .internazionale import InternazionaleIE
 from .internetvideoarchive import InternetVideoArchiveIE
 from .iprima import IPrimaIE
 from .iqiyi import IqiyiIE
diff --git a/youtube_dl/extractor/internazionale.py b/youtube_dl/extractor/internazionale.py
new file mode 100644
index 000000000..2ebf05ded
--- /dev/null
+++ b/youtube_dl/extractor/internazionale.py
@@ -0,0 +1,46 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class InternazionaleIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?internazionale\.it/video/(?P<id>.*)'
+    _TESTS = [{
+        'url': 'https://www.internazionale.it/video/2015/02/19/richard-linklater-racconta-una-scena-di-boyhood',
+        'md5': '11b54a3d3333e455c00684e50a65c58e',
+        'info_dict': {
+            'id': '265968',
+            'ext': 'mp4',
+            'description': 'md5:efb7e5bbfb1a54ae2ed5a4a015f0e665',
+            'title': 'Richard Linklater racconta una scena di Boyhood',
+            'thumbnail': r're:^https?://.*\.jpg$',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        data_job_id = self._html_search_regex(r'data-job-id="([^"]+)"', webpage, 'data-job-id')
+        data_video_path = self._html_search_regex(r'data-video-path="([^"]+)"', webpage, 'data-video-path')
+
+        formats = []
+
+        formats.extend(self._extract_m3u8_formats(
+            'https://video.internazionale.it/%s/%s.m3u8' % (data_video_path, data_job_id),
+            video_id))
+
+        formats.extend(self._extract_mpd_formats(
+            'https://video.internazionale.it/%s/%s.mpd' % (data_video_path, data_job_id),
+            video_id))
+
+        self._sort_formats(formats)
+
+        return {
+            'id': data_job_id,
+            'title': self._og_search_title(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'description': self._og_search_description(webpage),
+            'formats': formats,
+        }

From 640788f6f476f4344c13ed7c369c73575d3e705c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 27 Dec 2017 23:26:48 +0700
Subject: [PATCH 1660/1696] [internazionale] Improve extraction (closes #14973)

---
 youtube_dl/extractor/internazionale.py | 56 +++++++++++++++++---------
 1 file changed, 37 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/internazionale.py b/youtube_dl/extractor/internazionale.py
index 2ebf05ded..10ba1f6cf 100644
--- a/youtube_dl/extractor/internazionale.py
+++ b/youtube_dl/extractor/internazionale.py
@@ -2,45 +2,63 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import unified_timestamp
 
 
 class InternazionaleIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?internazionale\.it/video/(?P<id>.*)'
-    _TESTS = [{
+    _VALID_URL = r'https?://(?:www\.)?internazionale\.it/video/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _TEST = {
         'url': 'https://www.internazionale.it/video/2015/02/19/richard-linklater-racconta-una-scena-di-boyhood',
-        'md5': '11b54a3d3333e455c00684e50a65c58e',
+        'md5': '3e39d32b66882c1218e305acbf8348ca',
         'info_dict': {
             'id': '265968',
+            'display_id': 'richard-linklater-racconta-una-scena-di-boyhood',
             'ext': 'mp4',
-            'description': 'md5:efb7e5bbfb1a54ae2ed5a4a015f0e665',
             'title': 'Richard Linklater racconta una scena di Boyhood',
+            'description': 'md5:efb7e5bbfb1a54ae2ed5a4a015f0e665',
+            'timestamp': 1424354635,
+            'upload_date': '20150219',
             'thumbnail': r're:^https?://.*\.jpg$',
-        }
-    }]
+        },
+        'params': {
+            'format': 'bestvideo',
+        },
+    }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        display_id = self._match_id(url)
 
-        data_job_id = self._html_search_regex(r'data-job-id="([^"]+)"', webpage, 'data-job-id')
-        data_video_path = self._html_search_regex(r'data-video-path="([^"]+)"', webpage, 'data-video-path')
+        webpage = self._download_webpage(url, display_id)
 
-        formats = []
+        DATA_RE = r'data-%s=(["\'])(?P<value>(?:(?!\1).)+)\1'
 
-        formats.extend(self._extract_m3u8_formats(
-            'https://video.internazionale.it/%s/%s.m3u8' % (data_video_path, data_job_id),
-            video_id))
+        title = self._search_regex(
+            DATA_RE % 'video-title', webpage, 'title', default=None,
+            group='value') or self._og_search_title(webpage)
 
+        video_id = self._search_regex(
+            DATA_RE % 'job-id', webpage, 'video id', group='value')
+        video_path = self._search_regex(
+            DATA_RE % 'video-path', webpage, 'video path', group='value')
+
+        video_base = 'https://video.internazionale.it/%s/%s.' % (video_path, video_id)
+
+        formats = self._extract_m3u8_formats(
+            video_base + 'm3u8', display_id, 'mp4',
+            entry_protocol='m3u8_native', m3u8_id='hls', fatal=False)
         formats.extend(self._extract_mpd_formats(
-            'https://video.internazionale.it/%s/%s.mpd' % (data_video_path, data_job_id),
-            video_id))
-
+            video_base + 'mpd', display_id, mpd_id='dash', fatal=False))
         self._sort_formats(formats)
 
+        timestamp = unified_timestamp(self._html_search_meta(
+            'article:published_time', webpage, 'timestamp'))
+
         return {
-            'id': data_job_id,
-            'title': self._og_search_title(webpage),
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
             'thumbnail': self._og_search_thumbnail(webpage),
             'description': self._og_search_description(webpage),
+            'timestamp': timestamp,
             'formats': formats,
         }

From 99277daaac5b2cb6990f1d592b78ca8839635c72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Dec 2017 23:10:42 +0700
Subject: [PATCH 1661/1696] [ChangeLog] Actualize

---
 ChangeLog | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 420a1bd11..794b00223 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,16 @@
 version <unreleased>
 
 Extractors
++ [internazionale] Add support for internazionale.it (#14973)
+* [playtvak] Relax video regular expression and make description optional
+  (#15037)
++ [filmweb] Add support for filmweb.no (#8773, #10368)
++ [23video] Add support for 23video.com
++ [espn] Add support for fivethirtyeight.com (#6864)
++ [umg:de] Add support for universal-music.de (#11582, #11584)
++ [espn] Add support for espnfc and extract more formats (#8053)
 * [youku] Update ccode (#14880)
++ [openload] Add support for oload.stream (#15070)
 * [youku] Fix list extraction (#15065)
 
 

From a491fd0c6f5abd68bae128cacbcc9926a017cee3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Dec 2017 23:12:56 +0700
Subject: [PATCH 1662/1696] release 2017.12.28

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 5 +++++
 youtube_dl/version.py     | 2 +-
 4 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index d7a91239f..f40927d36 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.12.23*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.12.23**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.12.28*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.12.28**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.12.23
+[debug] youtube-dl version 2017.12.28
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 794b00223..4323cfc5c 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.12.28
 
 Extractors
 + [internazionale] Add support for internazionale.it (#14973)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index eac35e390..6b935e572 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -3,6 +3,7 @@
  - **1up.com**
  - **20min**
  - **220.ro**
+ - **23video**
  - **24video**
  - **3qsdn**: 3Q SDN
  - **3sat**
@@ -269,6 +270,8 @@
  - **Fczenit**
  - **filmon**
  - **filmon:channel**
+ - **Filmweb**
+ - **FiveThirtyEight**
  - **FiveTV**
  - **Flickr**
  - **Flipagram**
@@ -359,6 +362,7 @@
  - **InfoQ**
  - **Instagram**
  - **instagram:user**: Instagram user profile
+ - **Internazionale**
  - **InternetVideoArchive**
  - **IPrima**
  - **iqiyi**: 爱奇艺
@@ -889,6 +893,7 @@
  - **udemy:course**
  - **UDNEmbed**: 聯合影音
  - **UKTVPlay**
+ - **umg:de**: Universal Music Deutschland
  - **Unistra**
  - **Unity**
  - **uol.com.br**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index f999584d7..f4a1b67a2 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.12.23'
+__version__ = '2017.12.28'

From 84f085d4bdb66ee025fb337bcd571eab7469da97 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 29 Dec 2017 00:13:23 +0100
Subject: [PATCH 1663/1696] [aws] fix canonical/signed headers generation in
 python 2(closes #15102)

---
 youtube_dl/extractor/aws.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/aws.py b/youtube_dl/extractor/aws.py
index 670abce0c..dccfeaf73 100644
--- a/youtube_dl/extractor/aws.py
+++ b/youtube_dl/extractor/aws.py
@@ -21,11 +21,11 @@ class AWSIE(InfoExtractor):
             'Accept': 'application/json',
             'Host': self._AWS_PROXY_HOST,
             'X-Amz-Date': amz_date,
+            'X-Api-Key': self._AWS_API_KEY
         }
         session_token = aws_dict.get('session_token')
         if session_token:
             headers['X-Amz-Security-Token'] = session_token
-        headers['X-Api-Key'] = self._AWS_API_KEY
 
         def aws_hash(s):
             return hashlib.sha256(s.encode('utf-8')).hexdigest()
@@ -33,9 +33,9 @@ class AWSIE(InfoExtractor):
         # Task 1: http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
         canonical_querystring = compat_urllib_parse_urlencode(query)
         canonical_headers = ''
-        for header_name, header_value in headers.items():
+        for header_name, header_value in sorted(headers.items()):
             canonical_headers += '%s:%s\n' % (header_name.lower(), header_value)
-        signed_headers = ';'.join([header.lower() for header in headers.keys()])
+        signed_headers = ';'.join([header.lower() for header in sorted(headers.keys())])
         canonical_request = '\n'.join([
             'GET',
             aws_dict['uri'],

From 9d6ac71c27b1dfb662c795ef598dbfd0286682da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 29 Dec 2017 23:14:15 +0700
Subject: [PATCH 1664/1696] [extractor/common] Fix extraction of DASH formats
 with the same representation id (closes #15111)

---
 test/test_InfoExtractor.py     | 11 +++++++++++
 youtube_dl/extractor/common.py | 18 ++++++++----------
 2 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 8a372d2c9..7b31d5198 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -493,9 +493,20 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
         _TEST_CASES = [
             (
                 # https://github.com/rg3/youtube-dl/issues/13919
+                # Also tests duplicate representation ids, see
+                # https://github.com/rg3/youtube-dl/issues/15111
                 'float_duration',
                 'http://unknown/manifest.mpd',
                 [{
+                    'manifest_url': 'http://unknown/manifest.mpd',
+                    'ext': 'm4a',
+                    'format_id': '318597',
+                    'format_note': 'DASH audio',
+                    'protocol': 'http_dash_segments',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'none',
+                    'tbr': 61.587,
+                }, {
                     'manifest_url': 'http://unknown/manifest.mpd',
                     'ext': 'mp4',
                     'format_id': '318597',
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 3b79b8cb4..35d427eec 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2007,16 +2007,14 @@ class InfoExtractor(object):
                                     f['url'] = initialization_url
                                 f['fragments'].append({location_key(initialization_url): initialization_url})
                             f['fragments'].extend(representation_ms_info['fragments'])
-                        try:
-                            existing_format = next(
-                                fo for fo in formats
-                                if fo['format_id'] == representation_id)
-                        except StopIteration:
-                            full_info = formats_dict.get(representation_id, {}).copy()
-                            full_info.update(f)
-                            formats.append(full_info)
-                        else:
-                            existing_format.update(f)
+                        # According to [1, 5.3.5.2, Table 7, page 35] @id of Representation
+                        # is not necessarily unique within a Period thus formats with
+                        # the same `format_id` are quite possible. There are numerous examples
+                        # of such manifests (see https://github.com/rg3/youtube-dl/issues/15111,
+                        # https://github.com/rg3/youtube-dl/issues/13919)
+                        full_info = formats_dict.get(representation_id, {}).copy()
+                        full_info.update(f)
+                        formats.append(full_info)
                     else:
                         self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
         return formats

From 580f3c79d52ae3f880b4ca64e22665bf094ae65f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Dec 2017 03:54:14 +0700
Subject: [PATCH 1665/1696] [vimeo] Improve password protected videos
 extraction (closes #15114)

---
 youtube_dl/extractor/vimeo.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index cedb54876..6af705657 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -468,11 +468,12 @@ class VimeoIE(VimeoBaseInfoExtractor):
         request = sanitized_Request(url, headers=headers)
         try:
             webpage, urlh = self._download_webpage_handle(request, video_id)
+            redirect_url = compat_str(urlh.geturl())
             # Some URLs redirect to ondemand can't be extracted with
             # this extractor right away thus should be passed through
             # ondemand extractor (e.g. https://vimeo.com/73445910)
-            if VimeoOndemandIE.suitable(urlh.geturl()):
-                return self.url_result(urlh.geturl(), VimeoOndemandIE.ie_key())
+            if VimeoOndemandIE.suitable(redirect_url):
+                return self.url_result(redirect_url, VimeoOndemandIE.ie_key())
         except ExtractorError as ee:
             if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
                 errmsg = ee.cause.read()
@@ -541,15 +542,15 @@ class VimeoIE(VimeoBaseInfoExtractor):
             if re.search(r'<form[^>]+?id="pw_form"', webpage) is not None:
                 if '_video_password_verified' in data:
                     raise ExtractorError('video password verification failed!')
-                self._verify_video_password(url, video_id, webpage)
+                self._verify_video_password(redirect_url, video_id, webpage)
                 return self._real_extract(
-                    smuggle_url(url, {'_video_password_verified': 'verified'}))
+                    smuggle_url(redirect_url, {'_video_password_verified': 'verified'}))
             else:
                 raise ExtractorError('Unable to extract info section',
                                      cause=e)
         else:
             if config.get('view') == 4:
-                config = self._verify_player_video_password(url, video_id)
+                config = self._verify_player_video_password(redirect_url, video_id)
 
         def is_rented():
             if '>You rented this title.<' in webpage:

From d2c5b5a951868ae974bc3af6659ab39b8abd2157 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Dec 2017 05:52:35 +0700
Subject: [PATCH 1666/1696] [openload] Fallback on f-page extraction (closes
 #14665, closes #14879)

---
 youtube_dl/extractor/openload.py | 28 +++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index d1eb3be25..81c1317b6 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -284,6 +284,11 @@ class OpenloadIE(InfoExtractor):
         # for title and ext
         'url': 'https://openload.co/embed/Sxz5sADo82g/',
         'only_matching': True,
+    }, {
+        # unavailable via https://openload.co/embed/e-Ixz9ZR5L0/ but available
+        # via https://openload.co/f/e-Ixz9ZR5L0/
+        'url': 'https://openload.co/f/e-Ixz9ZR5L0/',
+        'only_matching': True,
     }, {
         'url': 'https://oload.tv/embed/KnG-kKZdcfY/',
         'only_matching': True,
@@ -305,18 +310,27 @@ class OpenloadIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        url = 'https://openload.co/embed/%s/' % video_id
+        url_pattern = 'https://openload.co/%%s/%s/' % video_id
         headers = {
             'User-Agent': self._USER_AGENT,
         }
 
-        webpage = self._download_webpage(url, video_id, headers=headers)
-
-        if 'File not found' in webpage or 'deleted by the owner' in webpage:
-            raise ExtractorError('File not found', expected=True, video_id=video_id)
+        for path in ('embed', 'f'):
+            page_url = url_pattern % path
+            last = path == 'f'
+            webpage = self._download_webpage(
+                page_url, video_id, 'Downloading %s webpage' % path,
+                headers=headers, fatal=last)
+            if not webpage:
+                continue
+            if 'File not found' in webpage or 'deleted by the owner' in webpage:
+                if not last:
+                    continue
+                raise ExtractorError('File not found', expected=True, video_id=video_id)
+            break
 
         phantom = PhantomJSwrapper(self, required_version='2.0')
-        webpage, _ = phantom.get(url, html=webpage, video_id=video_id, headers=headers)
+        webpage, _ = phantom.get(page_url, html=webpage, video_id=video_id, headers=headers)
 
         decoded_id = get_element_by_id('streamurl', webpage)
 
@@ -327,7 +341,7 @@ class OpenloadIE(InfoExtractor):
             'title', default=None) or self._html_search_meta(
             'description', webpage, 'title', fatal=True)
 
-        entries = self._parse_html5_media_entries(url, webpage, video_id)
+        entries = self._parse_html5_media_entries(page_url, webpage, video_id)
         entry = entries[0] if entries else {}
         subtitles = entry.get('subtitles')
 

From 2c8e11b4af253fec9096a800ccd26b4daef521a1 Mon Sep 17 00:00:00 2001
From: 50csent <32077344+50csent@users.noreply.github.com>
Date: Fri, 29 Dec 2017 22:59:49 +0200
Subject: [PATCH 1667/1696] [pluralsight] Fix missing first line of subtitles
 (closes #11118)

---
 youtube_dl/extractor/pluralsight.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 597b11218..aacc5d4bb 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -171,12 +171,12 @@ class PluralsightIE(PluralsightBaseIE):
         for num, current in enumerate(subs):
             current = subs[num]
             start, text = (
-                float_or_none(dict_get(current, TIME_OFFSET_KEYS)),
+                float_or_none(dict_get(current, TIME_OFFSET_KEYS, skip_false_values=False)),
                 dict_get(current, TEXT_KEYS))
             if start is None or text is None:
                 continue
             end = duration if num == len(subs) - 1 else float_or_none(
-                dict_get(subs[num + 1], TIME_OFFSET_KEYS))
+                dict_get(subs[num + 1], TIME_OFFSET_KEYS, skip_false_values=False))
             if end is None:
                 continue
             srt += os.linesep.join(

From d97cb84b313f94da76bff0520392334e753e2238 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 30 Dec 2017 00:30:15 +0100
Subject: [PATCH 1668/1696] [ufctv] Add new extractor(closes #14520)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/ufctv.py      | 55 ++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 youtube_dl/extractor/ufctv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index e7b93a699..5ed031833 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1145,6 +1145,7 @@ from .udemy import (
     UdemyCourseIE
 )
 from .udn import UDNEmbedIE
+from .ufctv import UFCTVIE
 from .uktvplay import UKTVPlayIE
 from .digiteka import DigitekaIE
 from .umg import UMGDeIE
diff --git a/youtube_dl/extractor/ufctv.py b/youtube_dl/extractor/ufctv.py
new file mode 100644
index 000000000..ab823814b
--- /dev/null
+++ b/youtube_dl/extractor/ufctv.py
@@ -0,0 +1,55 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    parse_duration,
+    parse_iso8601,
+)
+
+
+class UFCTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?ufc\.tv/video/(?P<id>[^/]+)'
+    _TEST = {
+        'url': 'https://www.ufc.tv/video/ufc-219-countdown-full-episode',
+        'info_dict': {
+            'id': '34167',
+            'ext': 'mp4',
+            'title': 'UFC 219 Countdown: Full Episode',
+            'description': 'md5:26d4e8bf4665ae5878842d7050c3c646',
+            'timestamp': 1513962360,
+            'upload_date': '20171222',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        video_data = self._download_json(url, display_id, query={
+            'format': 'json',
+        })
+        video_id = str(video_data['id'])
+        title = video_data['name']
+        m3u8_url = self._download_json(
+            'https://www.ufc.tv/service/publishpoint', video_id, query={
+                'type': 'video',
+                'format': 'json',
+                'id': video_id,
+            }, headers={
+                'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0_1 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A402 Safari/604.1',
+            })['path']
+        m3u8_url = m3u8_url.replace('_iphone.', '.')
+        formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4')
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': video_data.get('description'),
+            'duration': parse_duration(video_data.get('runtime')),
+            'timestamp': parse_iso8601(video_data.get('releaseDate')),
+            'formats': formats,
+        }

From 2501d41ef4b9ed0349cf4f9838e12873350e60d5 Mon Sep 17 00:00:00 2001
From: felix <felix.von.s@posteo.de>
Date: Sat, 12 Nov 2016 22:15:51 +0100
Subject: [PATCH 1669/1696] [common] use AACL as the default fourcc when
 AudioTag is 255

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 35d427eec..5e7e7a3f7 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2054,7 +2054,7 @@ class InfoExtractor(object):
             stream_timescale = int_or_none(stream.get('TimeScale')) or timescale
             stream_name = stream.get('Name')
             for track in stream.findall('QualityLevel'):
-                fourcc = track.get('FourCC')
+                fourcc = track.get('FourCC', 'AACL' if track.get('AudioTag') == '255' else None)
                 # TODO: add support for WVC1 and WMAP
                 if fourcc not in ('H264', 'AVC1', 'AACL'):
                     self.report_warning('%s is not a supported codec' % fourcc)

From 8056c8542d008ac2f103b4fcb862dc1ad78dc8fc Mon Sep 17 00:00:00 2001
From: felix <felix.von.s@posteo.de>
Date: Sat, 12 Nov 2016 21:52:02 +0100
Subject: [PATCH 1670/1696] [mediasite] Add extractor, subsume sandia and
 collegerama extractors

---
 youtube_dl/extractor/collegerama.py |  93 --------------
 youtube_dl/extractor/extractors.py  |   3 +-
 youtube_dl/extractor/generic.py     |  10 ++
 youtube_dl/extractor/mediasite.py   | 184 ++++++++++++++++++++++++++++
 youtube_dl/extractor/sandia.py      |  65 ----------
 5 files changed, 195 insertions(+), 160 deletions(-)
 delete mode 100644 youtube_dl/extractor/collegerama.py
 create mode 100644 youtube_dl/extractor/mediasite.py
 delete mode 100644 youtube_dl/extractor/sandia.py

diff --git a/youtube_dl/extractor/collegerama.py b/youtube_dl/extractor/collegerama.py
deleted file mode 100644
index 6a41db87c..000000000
--- a/youtube_dl/extractor/collegerama.py
+++ /dev/null
@@ -1,93 +0,0 @@
-from __future__ import unicode_literals
-
-import json
-
-from .common import InfoExtractor
-from ..utils import (
-    float_or_none,
-    int_or_none,
-    sanitized_Request,
-)
-
-
-class CollegeRamaIE(InfoExtractor):
-    _VALID_URL = r'https?://collegerama\.tudelft\.nl/Mediasite/Play/(?P<id>[\da-f]+)'
-    _TESTS = [
-        {
-            'url': 'https://collegerama.tudelft.nl/Mediasite/Play/585a43626e544bdd97aeb71a0ec907a01d',
-            'md5': '481fda1c11f67588c0d9d8fbdced4e39',
-            'info_dict': {
-                'id': '585a43626e544bdd97aeb71a0ec907a01d',
-                'ext': 'mp4',
-                'title': 'Een nieuwe wereld: waarden, bewustzijn en techniek van de mensheid 2.0.',
-                'description': '',
-                'thumbnail': r're:^https?://.*\.jpg(?:\?.*?)?$',
-                'duration': 7713.088,
-                'timestamp': 1413309600,
-                'upload_date': '20141014',
-            },
-        },
-        {
-            'url': 'https://collegerama.tudelft.nl/Mediasite/Play/86a9ea9f53e149079fbdb4202b521ed21d?catalog=fd32fd35-6c99-466c-89d4-cd3c431bc8a4',
-            'md5': 'ef1fdded95bdf19b12c5999949419c92',
-            'info_dict': {
-                'id': '86a9ea9f53e149079fbdb4202b521ed21d',
-                'ext': 'wmv',
-                'title': '64ste Vakantiecursus: Afvalwater',
-                'description': 'md5:7fd774865cc69d972f542b157c328305',
-                'thumbnail': r're:^https?://.*\.jpg(?:\?.*?)?$',
-                'duration': 10853,
-                'timestamp': 1326446400,
-                'upload_date': '20120113',
-            },
-        },
-    ]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        player_options_request = {
-            'getPlayerOptionsRequest': {
-                'ResourceId': video_id,
-                'QueryString': '',
-            }
-        }
-
-        request = sanitized_Request(
-            'http://collegerama.tudelft.nl/Mediasite/PlayerService/PlayerService.svc/json/GetPlayerOptions',
-            json.dumps(player_options_request))
-        request.add_header('Content-Type', 'application/json')
-
-        player_options = self._download_json(request, video_id)
-
-        presentation = player_options['d']['Presentation']
-        title = presentation['Title']
-        description = presentation.get('Description')
-        thumbnail = None
-        duration = float_or_none(presentation.get('Duration'), 1000)
-        timestamp = int_or_none(presentation.get('UnixTime'), 1000)
-
-        formats = []
-        for stream in presentation['Streams']:
-            for video in stream['VideoUrls']:
-                thumbnail_url = stream.get('ThumbnailUrl')
-                if thumbnail_url:
-                    thumbnail = 'http://collegerama.tudelft.nl' + thumbnail_url
-                format_id = video['MediaType']
-                if format_id == 'SS':
-                    continue
-                formats.append({
-                    'url': video['Location'],
-                    'format_id': format_id,
-                })
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'timestamp': timestamp,
-            'formats': formats,
-        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 5ed031833..e64defe62 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -205,7 +205,6 @@ from .cnn import (
     CNNArticleIE,
 )
 from .coub import CoubIE
-from .collegerama import CollegeRamaIE
 from .comedycentral import (
     ComedyCentralFullEpisodesIE,
     ComedyCentralIE,
@@ -576,6 +575,7 @@ from .massengeschmacktv import MassengeschmackTVIE
 from .matchtv import MatchTVIE
 from .mdr import MDRIE
 from .mediaset import MediasetIE
+from .mediasite import MediasiteIE
 from .medici import MediciIE
 from .megaphone import MegaphoneIE
 from .meipai import MeipaiIE
@@ -912,7 +912,6 @@ from .rutube import (
 from .rutv import RUTVIE
 from .ruutu import RuutuIE
 from .ruv import RuvIE
-from .sandia import SandiaIE
 from .safari import (
     SafariIE,
     SafariApiIE,
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c7b609215..d5622c823 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2883,6 +2883,16 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 vshare_urls, video_id, video_title, ie=VShareIE.ie_key())
 
+        # Look for Mediasite embeds
+        mobj = re.search(r'''(?xi)
+                <iframe[^>]+src="((?:https?://[a-z0-9\-\.:\[\]]+)?
+                    /Mediasite/Play/[0-9a-f]{32,34}(?:\?.*?)?)"
+            ''', webpage)
+        if mobj is not None:
+            return self.url_result(smuggle_url(
+                compat_urlparse.urljoin(url, unescapeHTML(mobj.group(1))),
+                { 'UrlReferrer': url }), 'Livestream')
+
         def merge_dicts(dict1, dict2):
             merged = {}
             for k, v in dict1.items():
diff --git a/youtube_dl/extractor/mediasite.py b/youtube_dl/extractor/mediasite.py
new file mode 100644
index 000000000..5d281684e
--- /dev/null
+++ b/youtube_dl/extractor/mediasite.py
@@ -0,0 +1,184 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import (
+    ExtractorError,
+    unsmuggle_url,
+    mimetype2ext,
+    float_or_none,
+)
+
+
+class MediasiteIE(InfoExtractor):
+    _VALID_URL = r'''(?xi)
+        https?://[a-z0-9\-\.:\[\]]+/Mediasite/Play/
+        (?P<id>[0-9a-f]{32,34})
+        (?P<QueryString>\?[^#]+|)
+    '''
+    _TESTS = [
+        {
+            'url': 'https://hitsmediaweb.h-its.org/mediasite/Play/2db6c271681e4f199af3c60d1f82869b1d',
+            'info_dict': {
+                'id': '2db6c271681e4f199af3c60d1f82869b1d',
+                'ext': 'mp4',
+                'title': 'Lecture: Tuesday, September 20, 2016 - Sir Andrew Wiles',
+                'description': 'Sir Andrew Wiles: “Equations in arithmetic”\\n\\nI will describe some of the interactions between modern number theory and the problem of solving equations in rational numbers or integers\\u0027.',
+                'timestamp': 1474268400.0,
+                'upload_date': '20160919',
+            },
+        },
+        {
+            'url': 'http://mediasite.uib.no/Mediasite/Play/90bb363295d945d6b548c867d01181361d?catalog=a452b7df-9ae1-46b7-a3ba-aceeb285f3eb',
+            'info_dict': {
+                'id': '90bb363295d945d6b548c867d01181361d',
+                'ext': 'mp4',
+                'upload_date': '20150429',
+                'title': '5) IT-forum 2015-Dag 1  - Dungbeetle -  How and why Rain created a tiny bug tracker for Unity',
+                'timestamp': 1430311380.0,
+            },
+        },
+        {
+            'url': 'https://collegerama.tudelft.nl/Mediasite/Play/585a43626e544bdd97aeb71a0ec907a01d',
+            'md5': '481fda1c11f67588c0d9d8fbdced4e39',
+            'info_dict': {
+                'id': '585a43626e544bdd97aeb71a0ec907a01d',
+                'ext': 'mp4',
+                'title': 'Een nieuwe wereld: waarden, bewustzijn en techniek van de mensheid 2.0.',
+                'description': '',
+                'thumbnail': r're:^https?://.*\.jpg(?:\?.*)?$',
+                'duration': 7713.088,
+                'timestamp': 1413309600,
+                'upload_date': '20141014',
+            },
+        },
+        {
+            'url': 'https://collegerama.tudelft.nl/Mediasite/Play/86a9ea9f53e149079fbdb4202b521ed21d?catalog=fd32fd35-6c99-466c-89d4-cd3c431bc8a4',
+            'md5': 'ef1fdded95bdf19b12c5999949419c92',
+            'info_dict': {
+                'id': '86a9ea9f53e149079fbdb4202b521ed21d',
+                'ext': 'wmv',
+                'title': '64ste Vakantiecursus: Afvalwater',
+                'description': 'md5:7fd774865cc69d972f542b157c328305',
+                'thumbnail': r're:^https?://.*\.jpg(?:\?.*?)?$',
+                'duration': 10853,
+                'timestamp': 1326446400,
+                'upload_date': '20120113',
+            },
+        },
+        {
+            'url': 'http://digitalops.sandia.gov/Mediasite/Play/24aace4429fc450fb5b38cdbf424a66e1d',
+            'md5': '9422edc9b9a60151727e4b6d8bef393d',
+            'info_dict': {
+                'id': '24aace4429fc450fb5b38cdbf424a66e1d',
+                'ext': 'mp4',
+                'title': 'Xyce Software Training - Section 1',
+                'description': r're:(?s)SAND Number: SAND 2013-7800.{200,}',
+                'upload_date': '20120409',
+                'timestamp': 1333983600,
+                'duration': 7794,
+            }
+        }
+    ]
+
+    # look in Mediasite.Core.js (Mediasite.ContentStreamType[*])
+    _STREAM_TYPES = {
+        0: 'video1', # the main video
+        2: 'slide',
+        3: 'presentation',
+        4: 'video2', # screencast?
+        5: 'video3',
+    }
+
+    def _real_extract(self, url):
+        url, data = unsmuggle_url(url, {})
+        mobj = re.match(self._VALID_URL, url)
+        ResourceId = mobj.group('id')
+        QueryString = mobj.group('QueryString')
+
+        webpage = self._download_webpage(url, ResourceId) # XXX: add UrlReferrer?
+
+        # XXX: might have also extracted UrlReferrer and QueryString from the html
+        ServicePath = compat_urlparse.urljoin(url, self._html_search_regex(
+            r'<div id="ServicePath">(.+?)</div>', webpage, ResourceId,
+            default='/Mediasite/PlayerService/PlayerService.svc/json'))
+
+        PlayerOptions = self._download_json(
+            '%s/GetPlayerOptions' % (ServicePath), ResourceId,
+            headers={
+                'Content-type': 'application/json; charset=utf-8',
+                'X-Requested-With': 'XMLHttpRequest',
+            },
+            data=json.dumps({
+                'getPlayerOptionsRequest': {
+                    'ResourceId': ResourceId,
+                    'QueryString': QueryString,
+                    'UrlReferrer': data.get('UrlReferrer', ''),
+                    'UseScreenReader': False,
+                }
+            }).encode('utf-8'))
+        Presentation = PlayerOptions['d']['Presentation']
+        if Presentation is None:
+            raise ExtractorError('Mediasite says: %s' %
+                (PlayerOptions['d']['PlayerPresentationStatusMessage'],),
+                expected=True)
+
+        thumbnails = []
+        formats = []
+        for snum, Stream in enumerate(Presentation['Streams']):
+            stream_type = self._STREAM_TYPES.get(
+                Stream['StreamType'], 'type%u' % Stream['StreamType'])
+
+            stream_formats = []
+            for unum, VideoUrl in enumerate(Stream['VideoUrls']):
+                url = VideoUrl['Location']
+                # XXX: if Stream.get('CanChangeScheme', False), switch scheme to HTTP/HTTPS
+
+                if VideoUrl['MediaType'] == 'SS':
+                    stream_formats.extend(self._extract_ism_formats(
+                        url, ResourceId, ism_id='%s-%u.%u' % (stream_type, snum, unum)))
+                    continue
+
+                stream_formats.append({
+                    'format_id': '%s-%u.%u' % (stream_type, snum, unum),
+                    'url': url,
+                    'ext': mimetype2ext(VideoUrl['MimeType']),
+                })
+
+            # TODO: if Stream['HasSlideContent']:
+            # synthesise an MJPEG video stream '%s-%u.slides' % (stream_type, snum)
+            # from Stream['Slides']
+            # this will require writing a custom downloader...
+
+            # disprefer 'secondary' streams
+            if Stream['StreamType'] != 0:
+                for fmt in stream_formats:
+                    fmt['preference'] = -1
+
+            ThumbnailUrl = Stream.get('ThumbnailUrl')
+            if ThumbnailUrl:
+                thumbnails.append({
+                    'id': '%s-%u' % (stream_type, snum),
+                    'url': compat_urlparse.urljoin(url, ThumbnailUrl),
+                    'preference': -1 if Stream['StreamType'] != 0 else 0,
+                })
+            formats.extend(stream_formats)
+
+        self._sort_formats(formats)
+
+        # XXX: Presentation['Presenters']
+        # XXX: Presentation['Transcript']
+
+        return {
+            'id': ResourceId,
+            'title': Presentation['Title'],
+            'description': Presentation.get('Description'),
+            'duration': float_or_none(Presentation.get('Duration'), 1000),
+            'timestamp': float_or_none(Presentation.get('UnixTime'), 1000),
+            'formats': formats,
+            'thumbnails': thumbnails,
+        }
diff --git a/youtube_dl/extractor/sandia.py b/youtube_dl/extractor/sandia.py
deleted file mode 100644
index 96e43af84..000000000
--- a/youtube_dl/extractor/sandia.py
+++ /dev/null
@@ -1,65 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import json
-
-from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    mimetype2ext,
-)
-
-
-class SandiaIE(InfoExtractor):
-    IE_DESC = 'Sandia National Laboratories'
-    _VALID_URL = r'https?://digitalops\.sandia\.gov/Mediasite/Play/(?P<id>[0-9a-f]+)'
-    _TEST = {
-        'url': 'http://digitalops.sandia.gov/Mediasite/Play/24aace4429fc450fb5b38cdbf424a66e1d',
-        'md5': '9422edc9b9a60151727e4b6d8bef393d',
-        'info_dict': {
-            'id': '24aace4429fc450fb5b38cdbf424a66e1d',
-            'ext': 'mp4',
-            'title': 'Xyce Software Training - Section 1',
-            'description': 're:(?s)SAND Number: SAND 2013-7800.{200,}',
-            'upload_date': '20120409',
-            'timestamp': 1333983600,
-            'duration': 7794,
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        presentation_data = self._download_json(
-            'http://digitalops.sandia.gov/Mediasite/PlayerService/PlayerService.svc/json/GetPlayerOptions',
-            video_id, data=json.dumps({
-                'getPlayerOptionsRequest': {
-                    'ResourceId': video_id,
-                    'QueryString': '',
-                }
-            }), headers={
-                'Content-Type': 'application/json; charset=utf-8',
-            })['d']['Presentation']
-
-        title = presentation_data['Title']
-
-        formats = []
-        for stream in presentation_data.get('Streams', []):
-            for fd in stream.get('VideoUrls', []):
-                formats.append({
-                    'format_id': fd['MediaType'],
-                    'format_note': fd['MimeType'].partition('/')[2],
-                    'ext': mimetype2ext(fd['MimeType']),
-                    'url': fd['Location'],
-                    'protocol': 'f4m' if fd['MimeType'] == 'video/x-mp4-fragmented' else None,
-                })
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': presentation_data.get('Description'),
-            'formats': formats,
-            'timestamp': int_or_none(presentation_data.get('UnixTime'), 1000),
-            'duration': int_or_none(presentation_data.get('Duration'), 1000),
-        }

From 2ca7ed41fed73cf37581b07d0c67d3bad8a6acc3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Dec 2017 07:28:18 +0700
Subject: [PATCH 1671/1696] [mediasite] Improve extraction and code style, add
 support for DASH (closes #11185, closes #14343, refs #5428)

---
 youtube_dl/extractor/generic.py   |  29 +++++--
 youtube_dl/extractor/mediasite.py | 128 ++++++++++++++++++------------
 2 files changed, 100 insertions(+), 57 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index d5622c823..cc4c90b8c 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -100,6 +100,7 @@ from .megaphone import MegaphoneIE
 from .vzaar import VzaarIE
 from .channel9 import Channel9IE
 from .vshare import VShareIE
+from .mediasite import MediasiteIE
 
 
 class GenericIE(InfoExtractor):
@@ -1925,6 +1926,18 @@ class GenericIE(InfoExtractor):
                 'title': 'vl14062007715967',
                 'ext': 'mp4',
             }
+        },
+        {
+            'url': 'http://www.heidelberg-laureate-forum.org/blog/video/lecture-friday-september-23-2016-sir-c-antony-r-hoare/',
+            'md5': 'aecd089f55b1cb5a59032cb049d3a356',
+            'info_dict': {
+                'id': '90227f51a80c4d8f86c345a7fa62bd9a1d',
+                'ext': 'mp4',
+                'title': 'Lecture: Friday, September 23, 2016 - Sir Tony Hoare',
+                'description': 'md5:5a51db84a62def7b7054df2ade403c6c',
+                'timestamp': 1474354800,
+                'upload_date': '20160920',
+            }
         }
         # {
         #     # TODO: find another test
@@ -2884,14 +2897,14 @@ class GenericIE(InfoExtractor):
                 vshare_urls, video_id, video_title, ie=VShareIE.ie_key())
 
         # Look for Mediasite embeds
-        mobj = re.search(r'''(?xi)
-                <iframe[^>]+src="((?:https?://[a-z0-9\-\.:\[\]]+)?
-                    /Mediasite/Play/[0-9a-f]{32,34}(?:\?.*?)?)"
-            ''', webpage)
-        if mobj is not None:
-            return self.url_result(smuggle_url(
-                compat_urlparse.urljoin(url, unescapeHTML(mobj.group(1))),
-                { 'UrlReferrer': url }), 'Livestream')
+        mediasite_urls = MediasiteIE._extract_urls(webpage)
+        if mediasite_urls:
+            entries = [
+                self.url_result(smuggle_url(
+                    compat_urlparse.urljoin(url, mediasite_url),
+                    {'UrlReferrer': url}), ie=MediasiteIE.ie_key())
+                for mediasite_url in mediasite_urls]
+            return self.playlist_result(entries, video_id, video_title)
 
         def merge_dicts(dict1, dict2):
             merged = {}
diff --git a/youtube_dl/extractor/mediasite.py b/youtube_dl/extractor/mediasite.py
index 5d281684e..0e2645c55 100644
--- a/youtube_dl/extractor/mediasite.py
+++ b/youtube_dl/extractor/mediasite.py
@@ -5,21 +5,22 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
+from ..compat import (
+    compat_str,
+    compat_urlparse,
+)
 from ..utils import (
     ExtractorError,
-    unsmuggle_url,
-    mimetype2ext,
     float_or_none,
+    mimetype2ext,
+    unescapeHTML,
+    unsmuggle_url,
+    urljoin,
 )
 
 
 class MediasiteIE(InfoExtractor):
-    _VALID_URL = r'''(?xi)
-        https?://[a-z0-9\-\.:\[\]]+/Mediasite/Play/
-        (?P<id>[0-9a-f]{32,34})
-        (?P<QueryString>\?[^#]+|)
-    '''
+    _VALID_URL = r'(?xi)https?://[^/]+/Mediasite/Play/(?P<id>[0-9a-f]{32,34})(?P<query>\?[^#]+|)'
     _TESTS = [
         {
             'url': 'https://hitsmediaweb.h-its.org/mediasite/Play/2db6c271681e4f199af3c60d1f82869b1d',
@@ -87,67 +88,96 @@ class MediasiteIE(InfoExtractor):
 
     # look in Mediasite.Core.js (Mediasite.ContentStreamType[*])
     _STREAM_TYPES = {
-        0: 'video1', # the main video
+        0: 'video1',  # the main video
         2: 'slide',
         3: 'presentation',
-        4: 'video2', # screencast?
+        4: 'video2',  # screencast?
         5: 'video3',
     }
 
+    @staticmethod
+    def _extract_urls(webpage):
+        return [
+            unescapeHTML(mobj.group('url'))
+            for mobj in re.finditer(
+                r'(?xi)<iframe\b[^>]+\bsrc=(["\'])(?P<url>(?:(?:https?:)?//[^/]+)?/Mediasite/Play/[0-9a-f]{32,34}(?:\?.*?)?)\1',
+                webpage)]
+
     def _real_extract(self, url):
         url, data = unsmuggle_url(url, {})
         mobj = re.match(self._VALID_URL, url)
-        ResourceId = mobj.group('id')
-        QueryString = mobj.group('QueryString')
+        resource_id = mobj.group('id')
+        query = mobj.group('query')
 
-        webpage = self._download_webpage(url, ResourceId) # XXX: add UrlReferrer?
+        webpage, urlh = self._download_webpage_handle(url, resource_id)  # XXX: add UrlReferrer?
+        redirect_url = compat_str(urlh.geturl())
 
         # XXX: might have also extracted UrlReferrer and QueryString from the html
-        ServicePath = compat_urlparse.urljoin(url, self._html_search_regex(
-            r'<div id="ServicePath">(.+?)</div>', webpage, ResourceId,
+        service_path = compat_urlparse.urljoin(redirect_url, self._html_search_regex(
+            r'<div[^>]+\bid=["\']ServicePath[^>]+>(.+?)</div>', webpage, resource_id,
             default='/Mediasite/PlayerService/PlayerService.svc/json'))
 
-        PlayerOptions = self._download_json(
-            '%s/GetPlayerOptions' % (ServicePath), ResourceId,
+        player_options = self._download_json(
+            '%s/GetPlayerOptions' % service_path, resource_id,
             headers={
                 'Content-type': 'application/json; charset=utf-8',
                 'X-Requested-With': 'XMLHttpRequest',
             },
             data=json.dumps({
                 'getPlayerOptionsRequest': {
-                    'ResourceId': ResourceId,
-                    'QueryString': QueryString,
+                    'ResourceId': resource_id,
+                    'QueryString': query,
                     'UrlReferrer': data.get('UrlReferrer', ''),
                     'UseScreenReader': False,
                 }
-            }).encode('utf-8'))
-        Presentation = PlayerOptions['d']['Presentation']
-        if Presentation is None:
-            raise ExtractorError('Mediasite says: %s' %
-                (PlayerOptions['d']['PlayerPresentationStatusMessage'],),
+            }).encode('utf-8'))['d']
+
+        presentation = player_options['Presentation']
+        title = presentation['Title']
+
+        if presentation is None:
+            raise ExtractorError(
+                'Mediasite says: %s' % player_options['PlayerPresentationStatusMessage'],
                 expected=True)
 
         thumbnails = []
         formats = []
-        for snum, Stream in enumerate(Presentation['Streams']):
-            stream_type = self._STREAM_TYPES.get(
-                Stream['StreamType'], 'type%u' % Stream['StreamType'])
+        for snum, Stream in enumerate(presentation['Streams']):
+            stream_type = Stream.get('StreamType')
+            if stream_type is None:
+                continue
+
+            video_urls = Stream.get('VideoUrls')
+            if not isinstance(video_urls, list):
+                video_urls = []
+
+            stream_id = self._STREAM_TYPES.get(
+                stream_type, 'type%u' % stream_type)
 
             stream_formats = []
-            for unum, VideoUrl in enumerate(Stream['VideoUrls']):
-                url = VideoUrl['Location']
+            for unum, VideoUrl in enumerate(video_urls):
+                video_url = VideoUrl.get('Location')
+                if not video_url or not isinstance(video_url, compat_str):
+                    continue
                 # XXX: if Stream.get('CanChangeScheme', False), switch scheme to HTTP/HTTPS
 
-                if VideoUrl['MediaType'] == 'SS':
+                media_type = VideoUrl.get('MediaType')
+                if media_type == 'SS':
                     stream_formats.extend(self._extract_ism_formats(
-                        url, ResourceId, ism_id='%s-%u.%u' % (stream_type, snum, unum)))
-                    continue
-
-                stream_formats.append({
-                    'format_id': '%s-%u.%u' % (stream_type, snum, unum),
-                    'url': url,
-                    'ext': mimetype2ext(VideoUrl['MimeType']),
-                })
+                        video_url, resource_id,
+                        ism_id='%s-%u.%u' % (stream_id, snum, unum),
+                        fatal=False))
+                elif media_type == 'Dash':
+                    stream_formats.extend(self._extract_mpd_formats(
+                        video_url, resource_id,
+                        mpd_id='%s-%u.%u' % (stream_id, snum, unum),
+                        fatal=False))
+                else:
+                    stream_formats.append({
+                        'format_id': '%s-%u.%u' % (stream_id, snum, unum),
+                        'url': video_url,
+                        'ext': mimetype2ext(VideoUrl.get('MimeType')),
+                    })
 
             # TODO: if Stream['HasSlideContent']:
             # synthesise an MJPEG video stream '%s-%u.slides' % (stream_type, snum)
@@ -155,16 +185,16 @@ class MediasiteIE(InfoExtractor):
             # this will require writing a custom downloader...
 
             # disprefer 'secondary' streams
-            if Stream['StreamType'] != 0:
+            if stream_type != 0:
                 for fmt in stream_formats:
                     fmt['preference'] = -1
 
-            ThumbnailUrl = Stream.get('ThumbnailUrl')
-            if ThumbnailUrl:
+            thumbnail_url = Stream.get('ThumbnailUrl')
+            if thumbnail_url:
                 thumbnails.append({
-                    'id': '%s-%u' % (stream_type, snum),
-                    'url': compat_urlparse.urljoin(url, ThumbnailUrl),
-                    'preference': -1 if Stream['StreamType'] != 0 else 0,
+                    'id': '%s-%u' % (stream_id, snum),
+                    'url': urljoin(redirect_url, thumbnail_url),
+                    'preference': -1 if stream_type != 0 else 0,
                 })
             formats.extend(stream_formats)
 
@@ -174,11 +204,11 @@ class MediasiteIE(InfoExtractor):
         # XXX: Presentation['Transcript']
 
         return {
-            'id': ResourceId,
-            'title': Presentation['Title'],
-            'description': Presentation.get('Description'),
-            'duration': float_or_none(Presentation.get('Duration'), 1000),
-            'timestamp': float_or_none(Presentation.get('UnixTime'), 1000),
+            'id': resource_id,
+            'title': title,
+            'description': presentation.get('Description'),
+            'duration': float_or_none(presentation.get('Duration'), 1000),
+            'timestamp': float_or_none(presentation.get('UnixTime'), 1000),
             'formats': formats,
             'thumbnails': thumbnails,
         }

From 620ee8712e264c71b444748c86a51b258818b4a1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Dec 2017 15:03:13 +0800
Subject: [PATCH 1672/1696] [openload] Fix extraction (closes #15118)

---
 ChangeLog                        | 6 ++++++
 youtube_dl/extractor/openload.py | 3 ++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 4323cfc5c..08aed1ba7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [openload] Fix extraction (#15118)
+
+
 version 2017.12.28
 
 Extractors
diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 81c1317b6..b282bcfd9 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -332,7 +332,8 @@ class OpenloadIE(InfoExtractor):
         phantom = PhantomJSwrapper(self, required_version='2.0')
         webpage, _ = phantom.get(page_url, html=webpage, video_id=video_id, headers=headers)
 
-        decoded_id = get_element_by_id('streamurl', webpage)
+        decoded_id = (get_element_by_id('streamurl', webpage) or
+                      get_element_by_id('streamuri', webpage))
 
         video_url = 'https://openload.co/stream/%s?mime=true' % decoded_id
 

From 538d4f8681535507838d19942d2581478ebdc28b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 Dec 2017 01:15:35 +0700
Subject: [PATCH 1673/1696] [downloader/hls] Use HTTP headers for key request

---
 youtube_dl/downloader/hls.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 1a6e226c8..4dc3ab46a 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -163,7 +163,8 @@ class HlsFD(FragmentFD):
                         return False
                     if decrypt_info['METHOD'] == 'AES-128':
                         iv = decrypt_info.get('IV') or compat_struct_pack('>8xq', media_sequence)
-                        decrypt_info['KEY'] = decrypt_info.get('KEY') or self.ydl.urlopen(decrypt_info['URI']).read()
+                        decrypt_info['KEY'] = decrypt_info.get('KEY') or self.ydl.urlopen(
+                            self._prepare_url(info_dict, decrypt_info['URI'])).read()
                         frag_content = AES.new(
                             decrypt_info['KEY'], AES.MODE_CBC, iv).decrypt(frag_content)
                     self._append_fragment(ctx, frag_content)

From 2e65e7db9e39f22a8d20ea950906ee39fbf2618c Mon Sep 17 00:00:00 2001
From: d2au <d2au.dev@gmail.com>
Date: Sat, 18 Nov 2017 05:28:58 +1030
Subject: [PATCH 1674/1696] [abc:iview] Fix extraction (closes #14711)

ABC dropped unmetering, so change to metered hls urls which
require auth.
---
 youtube_dl/extractor/abc.py | 28 +++++++++++++++++++++++-----
 1 file changed, 23 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py
index 60f753b95..9610d0cb7 100644
--- a/youtube_dl/extractor/abc.py
+++ b/youtube_dl/extractor/abc.py
@@ -1,6 +1,9 @@
 from __future__ import unicode_literals
 
+import hashlib
+import hmac
 import re
+import time
 
 from .common import InfoExtractor
 from ..compat import compat_str
@@ -126,20 +129,35 @@ class ABCIViewIE(InfoExtractor):
         title = video_params.get('title') or video_params['seriesTitle']
         stream = next(s for s in video_params['playlist'] if s.get('type') == 'program')
 
-        format_urls = [
-            try_get(stream, lambda x: x['hds-unmetered'], compat_str)]
+        key = 'android.content.res.Resources'.encode('utf-8')
+        time_str = str(int(time.time()))
+        house_number = video_params.get('episodeHouseNumber')
+        path = '/auth/hls/sign?ts={0}&hn={1}&d=android-mobile'.format(
+            time_str, house_number)
+        sig = hmac.new(key, path.encode('utf-8'), hashlib.sha256).hexdigest()
+        auth_url = 'http://iview.abc.net.au{0}&sig={1}'.format(path, sig)
+        token = self._download_webpage(auth_url, video_id)
+
+        format_urls = []
+
+        def tokenize_url(url, token):
+            return ''.join([url, '?hdnea=', token])
 
         # May have higher quality video
         sd_url = try_get(
-            stream, lambda x: x['streams']['hds']['sd'], compat_str)
+            stream, lambda x: x['streams']['hls']['sd'], compat_str)
         if sd_url:
-            format_urls.append(sd_url.replace('metered', 'um'))
+            format_urls.append(tokenize_url(sd_url, token))
+        else:
+            sd_low_url = try_get(
+                stream, lambda x: x['streams']['hls']['sd-low'], compat_str)
+            format_urls.append(tokenize_url(sd_low_url, token))
 
         formats = []
         for format_url in format_urls:
             if format_url:
                 formats.extend(
-                    self._extract_akamai_formats(format_url, video_id))
+                    self._extract_m3u8_formats(format_url, video_id, 'mp4'))
         self._sort_formats(formats)
 
         subtitles = {}

From 77341dae1418fa1442bdc7689e428450ff83e5d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 Dec 2017 01:22:15 +0700
Subject: [PATCH 1675/1696] [abc:iview] Improve extraction and bypass geo
 restriction (closes #14782)

---
 youtube_dl/extractor/abc.py | 63 ++++++++++++++++++-------------------
 1 file changed, 31 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py
index 9610d0cb7..87017ed39 100644
--- a/youtube_dl/extractor/abc.py
+++ b/youtube_dl/extractor/abc.py
@@ -13,6 +13,7 @@ from ..utils import (
     int_or_none,
     parse_iso8601,
     try_get,
+    update_url_query,
 )
 
 
@@ -104,21 +105,24 @@ class ABCIE(InfoExtractor):
 class ABCIViewIE(InfoExtractor):
     IE_NAME = 'abc.net.au:iview'
     _VALID_URL = r'https?://iview\.abc\.net\.au/programs/[^/]+/(?P<id>[^/?#]+)'
+    _GEO_COUNTRIES = ['AU']
 
     # ABC iview programs are normally available for 14 days only.
     _TESTS = [{
-        'url': 'http://iview.abc.net.au/programs/diaries-of-a-broken-mind/ZX9735A001S00',
+        'url': 'http://iview.abc.net.au/programs/call-the-midwife/ZW0898A003S00',
         'md5': 'cde42d728b3b7c2b32b1b94b4a548afc',
         'info_dict': {
-            'id': 'ZX9735A001S00',
+            'id': 'ZW0898A003S00',
             'ext': 'mp4',
-            'title': 'Diaries Of A Broken Mind',
-            'description': 'md5:7de3903874b7a1be279fe6b68718fc9e',
-            'upload_date': '20161010',
-            'uploader_id': 'abc2',
-            'timestamp': 1476064920,
+            'title': 'Series 5 Ep 3',
+            'description': 'md5:e0ef7d4f92055b86c4f33611f180ed79',
+            'upload_date': '20171228',
+            'uploader_id': 'abc1',
+            'timestamp': 1514499187,
+        },
+        'params': {
+            'skip_download': True,
         },
-        'skip': 'Video gone',
     }]
 
     def _real_extract(self, url):
@@ -129,35 +133,30 @@ class ABCIViewIE(InfoExtractor):
         title = video_params.get('title') or video_params['seriesTitle']
         stream = next(s for s in video_params['playlist'] if s.get('type') == 'program')
 
-        key = 'android.content.res.Resources'.encode('utf-8')
-        time_str = str(int(time.time()))
         house_number = video_params.get('episodeHouseNumber')
         path = '/auth/hls/sign?ts={0}&hn={1}&d=android-mobile'.format(
-            time_str, house_number)
-        sig = hmac.new(key, path.encode('utf-8'), hashlib.sha256).hexdigest()
-        auth_url = 'http://iview.abc.net.au{0}&sig={1}'.format(path, sig)
-        token = self._download_webpage(auth_url, video_id)
-
-        format_urls = []
+            int(time.time()), house_number)
+        sig = hmac.new(
+            'android.content.res.Resources'.encode('utf-8'),
+            path.encode('utf-8'), hashlib.sha256).hexdigest()
+        token = self._download_webpage(
+            'http://iview.abc.net.au{0}&sig={1}'.format(path, sig), video_id)
 
         def tokenize_url(url, token):
-            return ''.join([url, '?hdnea=', token])
+            return update_url_query(url, {
+                'hdnea': token,
+            })
 
-        # May have higher quality video
-        sd_url = try_get(
-            stream, lambda x: x['streams']['hls']['sd'], compat_str)
-        if sd_url:
-            format_urls.append(tokenize_url(sd_url, token))
-        else:
-            sd_low_url = try_get(
-                stream, lambda x: x['streams']['hls']['sd-low'], compat_str)
-            format_urls.append(tokenize_url(sd_low_url, token))
-
-        formats = []
-        for format_url in format_urls:
-            if format_url:
-                formats.extend(
-                    self._extract_m3u8_formats(format_url, video_id, 'mp4'))
+        for sd in ('sd', 'sd-low'):
+            sd_url = try_get(
+                stream, lambda x: x['streams']['hls'][sd], compat_str)
+            if not sd_url:
+                continue
+            formats = self._extract_m3u8_formats(
+                tokenize_url(sd_url, token), video_id, 'mp4',
+                entry_protocol='m3u8_native', m3u8_id='hls', fatal=False)
+            if formats:
+                break
         self._sort_formats(formats)
 
         subtitles = {}

From 4f5cf319772b693c7781de47dbc5451c3c18a81c Mon Sep 17 00:00:00 2001
From: Windom <windom@users.noreply.github.com>
Date: Sat, 30 Dec 2017 20:41:07 +0200
Subject: [PATCH 1676/1696] [slutload] Add support for mobile URLs

---
 youtube_dl/extractor/slutload.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/slutload.py b/youtube_dl/extractor/slutload.py
index 7145d285a..6fc2ff60d 100644
--- a/youtube_dl/extractor/slutload.py
+++ b/youtube_dl/extractor/slutload.py
@@ -1,11 +1,13 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 
 
 class SlutloadIE(InfoExtractor):
     _VALID_URL = r'^https?://(?:\w+\.)?slutload\.com/video/[^/]+/(?P<id>[^/]+)/?$'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.slutload.com/video/virginie-baisee-en-cam/TD73btpBqSxc/',
         'md5': '868309628ba00fd488cf516a113fd717',
         'info_dict': {
@@ -15,11 +17,17 @@ class SlutloadIE(InfoExtractor):
             'age_limit': 18,
             'thumbnail': r're:https?://.*?\.jpg'
         }
-    }
+    }, {
+        # mobile site
+        'url': 'http://mobile.slutload.com/video/masturbation-solo/fviFLmc6kzJ/',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+
+        desktop_url = re.sub(r'^(https?://)mobile\.', r'\1', url)
+        webpage = self._download_webpage(desktop_url, video_id)
 
         video_title = self._html_search_regex(r'<h1><strong>([^<]+)</strong>',
                                               webpage, 'title').strip()

From 126f225bcffbdc55725fe2272daf22489abacf54 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ond=C5=99ej=20Caletka?= <ondrej@caletka.cz>
Date: Sat, 30 Dec 2017 22:02:46 +0100
Subject: [PATCH 1677/1696] [extractor/common] Add container meta field for
 formats extracted in _parse_mpd_formats

---
 youtube_dl/extractor/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5e7e7a3f7..5b6a09c0b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1880,6 +1880,7 @@ class InfoExtractor(object):
                             'language': lang if lang not in ('mul', 'und', 'zxx', 'mis') else None,
                             'format_note': 'DASH %s' % content_type,
                             'filesize': filesize,
+                            'container': mimetype2ext(mime_type) + '_dash',
                         }
                         f.update(parse_codecs(representation_attrib.get('codecs')))
                         representation_ms_info = extract_multisegment_info(representation, adaption_set_ms_info)

From c95c08a856af2499729e2141f5b683b266324fd0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 Dec 2017 04:28:01 +0700
Subject: [PATCH 1678/1696] [ChangeLog] Actualize

---
 ChangeLog | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 08aed1ba7..eff1fe8f7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,28 @@
 version <unreleased>
 
+Core
++ [extractor/common] Add container meta field for formats extracted
+  in _parse_mpd_formats (#13616)
++ [downloader/hls] Use HTTP headers for key request
+* [common] Use AACL as the default fourcc when AudioTag is 255
+* [extractor/common] Fix extraction of DASH formats with the same
+  representation id (#15111)
+
 Extractors
++ [slutload] Add support for mobile URLs (#14806)
+* [abc:iview] Bypass geo restriction
+* [abc:iview] Fix extraction (#14711, #14782, #14838, #14917, #14963, #14985,
+  #15035, #15057, #15061, #15071, #15095, #15106)
 * [openload] Fix extraction (#15118)
+- [sandia] Remove extractor
+- [collegerama] Remove extractor
++ [mediasite] Add support for sites based on Mediasite Video Platform (#5428,
+  #11185, #14343)
++ [ufctv] Add support for ufc.tv (#14520)
+* [pluralsight] Fix missing first line of subtitles (#11118)
+* [openload] Fallback on f-page extraction (#14665, #14879)
+* [vimeo] Improve password protected videos extraction (#15114)
+* [aws] Fix canonical/signed headers generation on python 2 (#15102)
 
 
 version 2017.12.28

From 04cf1a191a6b47bac93364ed8eb237a25058ee5e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 Dec 2017 04:30:49 +0700
Subject: [PATCH 1679/1696] release 2017.12.31

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 4 ++--
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index f40927d36..3f8984943 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.12.28*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.12.28**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.12.31*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.12.31**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.12.28
+[debug] youtube-dl version 2017.12.31
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index eff1fe8f7..bfffb1f5f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.12.31
 
 Core
 + [extractor/common] Add container meta field for formats extracted
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 6b935e572..75bd5c922 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -171,7 +171,6 @@
  - **CNN**
  - **CNNArticle**
  - **CNNBlogs**
- - **CollegeRama**
  - **ComCarCoff**
  - **ComedyCentral**
  - **ComedyCentralFullEpisodes**
@@ -449,6 +448,7 @@
  - **media.ccc.de**
  - **Medialaan**
  - **Mediaset**
+ - **Mediasite**
  - **Medici**
  - **megaphone.fm**: megaphone.fm embedded players
  - **Meipai**: 美拍
@@ -717,7 +717,6 @@
  - **safari**: safaribooksonline.com online video
  - **safari:api**
  - **safari:course**: safaribooksonline.com online courses
- - **Sandia**: Sandia National Laboratories
  - **Sapo**: SAPO Vídeos
  - **savefrom.net**
  - **SBS**: sbs.com.au
@@ -892,6 +891,7 @@
  - **udemy**
  - **udemy:course**
  - **UDNEmbed**: 聯合影音
+ - **UFCTV**
  - **UKTVPlay**
  - **umg:de**: Universal Music Deutschland
  - **Unistra**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index f4a1b67a2..a3f84b9ea 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.12.28'
+__version__ = '2017.12.31'

From b300cda4769a987c501db298f3f107326521d45c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 1 Jan 2018 21:52:24 +0700
Subject: [PATCH 1680/1696] [YoutubeDL] Output python implementation in debug
 header

---
 youtube_dl/YoutubeDL.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index ace80f14b..97bd9c526 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -2233,8 +2233,16 @@ class YoutubeDL(object):
                 sys.exc_clear()
             except Exception:
                 pass
-        self._write_string('[debug] Python version %s - %s\n' % (
-            platform.python_version(), platform_name()))
+
+        def python_implementation():
+            impl_name = platform.python_implementation()
+            if impl_name == 'PyPy' and hasattr(sys, 'pypy_version_info'):
+                return impl_name + ' version %d.%d.%d' % sys.pypy_version_info[:3]
+            return impl_name
+
+        self._write_string('[debug] Python version %s (%s) - %s\n' % (
+            platform.python_version(), python_implementation(),
+            platform_name()))
 
         exe_versions = FFmpegPostProcessor.get_versions(self)
         exe_versions['rtmpdump'] = rtmpdump_version()

From 54009c246e8eed38cfa8dc3eecb5619c1c81a1f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 1 Jan 2018 21:54:28 +0700
Subject: [PATCH 1681/1696] [travis] Add PyPy builds

---
 .travis.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 5f4f3922b..7b175e9c9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,6 +7,8 @@ python:
   - "3.4"
   - "3.5"
   - "3.6"
+  - "pypy"
+  - "pypy3"
 sudo: false
 env:
   - YTDL_TEST_SET=core

From d7cd9a9e847fab3ac3f0fb5b4ad2e4788aeea775 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 04:01:08 +0100
Subject: [PATCH 1682/1696] [utils] Fix youtube-dl under PyPy3 on Windows

---
 youtube_dl/compat.py | 21 +++++++++++++++++++++
 youtube_dl/utils.py  | 15 ++++++++-------
 2 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 2a62248ef..41ca9adf1 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -3,12 +3,14 @@ from __future__ import unicode_literals
 
 import binascii
 import collections
+import ctypes
 import email
 import getpass
 import io
 import itertools
 import optparse
 import os
+import platform
 import re
 import shlex
 import shutil
@@ -2906,6 +2908,24 @@ except ImportError:  # not 2.6+ or is 3.x
     except ImportError:
         compat_zip = zip
 
+if platform.python_implementation() == 'PyPy' and sys.pypy_version_info < (5, 4, 0):
+    # PyPy2 prior to version 5.4.0 expects byte strings as Windows function
+    # names, see the original PyPy issue [1] and the youtube-dl one [2].
+    # 1. https://bitbucket.org/pypy/pypy/issues/2360/windows-ctypescdll-typeerror-function-name
+    # 2. https://github.com/rg3/youtube-dl/pull/4392
+    def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
+        real = ctypes.WINFUNCTYPE(*args, **kwargs)
+
+        def resf(tpl, *args, **kwargs):
+            funcname, dll = tpl
+            return real((str(funcname), dll), *args, **kwargs)
+
+        return resf
+else:
+    def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
+        return ctypes.WINFUNCTYPE(*args, **kwargs)
+
+
 __all__ = [
     'compat_HTMLParseError',
     'compat_HTMLParser',
@@ -2914,6 +2934,7 @@ __all__ = [
     'compat_chr',
     'compat_cookiejar',
     'compat_cookies',
+    'compat_ctypes_WINFUNCTYPE',
     'compat_etree_fromstring',
     'compat_etree_register_namespace',
     'compat_expanduser',
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 2843a3dc0..386897a85 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -39,6 +39,7 @@ from .compat import (
     compat_HTMLParser,
     compat_basestring,
     compat_chr,
+    compat_ctypes_WINFUNCTYPE,
     compat_etree_fromstring,
     compat_expanduser,
     compat_html_entities,
@@ -1330,24 +1331,24 @@ def _windows_write_string(s, out):
     if fileno not in WIN_OUTPUT_IDS:
         return False
 
-    GetStdHandle = ctypes.WINFUNCTYPE(
+    GetStdHandle = compat_ctypes_WINFUNCTYPE(
         ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
-        (b'GetStdHandle', ctypes.windll.kernel32))
+        ('GetStdHandle', ctypes.windll.kernel32))
     h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
 
-    WriteConsoleW = ctypes.WINFUNCTYPE(
+    WriteConsoleW = compat_ctypes_WINFUNCTYPE(
         ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
         ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
-        ctypes.wintypes.LPVOID)((b'WriteConsoleW', ctypes.windll.kernel32))
+        ctypes.wintypes.LPVOID)(('WriteConsoleW', ctypes.windll.kernel32))
     written = ctypes.wintypes.DWORD(0)
 
-    GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)((b'GetFileType', ctypes.windll.kernel32))
+    GetFileType = compat_ctypes_WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)(('GetFileType', ctypes.windll.kernel32))
     FILE_TYPE_CHAR = 0x0002
     FILE_TYPE_REMOTE = 0x8000
-    GetConsoleMode = ctypes.WINFUNCTYPE(
+    GetConsoleMode = compat_ctypes_WINFUNCTYPE(
         ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
         ctypes.POINTER(ctypes.wintypes.DWORD))(
-        (b'GetConsoleMode', ctypes.windll.kernel32))
+        ('GetConsoleMode', ctypes.windll.kernel32))
     INVALID_HANDLE_VALUE = ctypes.wintypes.DWORD(-1).value
 
     def not_a_console(handle):

From 7a6c204fcb6ba5a1a5149ea7a3c186eab87fc7e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 1 Jan 2018 23:21:39 +0700
Subject: [PATCH 1683/1696] [travis] Add Jython build

---
 .travis.yml                  | 11 +++++++----
 devscripts/install_jython.sh |  5 +++++
 2 files changed, 12 insertions(+), 4 deletions(-)
 create mode 100755 devscripts/install_jython.sh

diff --git a/.travis.yml b/.travis.yml
index 7b175e9c9..92f326860 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -14,11 +14,14 @@ env:
   - YTDL_TEST_SET=core
   - YTDL_TEST_SET=download
 matrix:
+  include:
+    - env: JYTHON=true; YTDL_TEST_SET=core
+    - env: JYTHON=true; YTDL_TEST_SET=download
   fast_finish: true
   allow_failures:
     - env: YTDL_TEST_SET=download
+    - env: JYTHON=true; YTDL_TEST_SET=core
+    - env: JYTHON=true; YTDL_TEST_SET=download
+before_install:
+  - if [ "$JYTHON" == "true" ]; then ./devscripts/install_jython.sh; export PATH="$HOME/jython/bin:$PATH"; fi
 script: ./devscripts/run_tests.sh
-notifications:
-  email:
-    - filippo.valsorda@gmail.com
-    - yasoob.khld@gmail.com
diff --git a/devscripts/install_jython.sh b/devscripts/install_jython.sh
new file mode 100755
index 000000000..bafca4da4
--- /dev/null
+++ b/devscripts/install_jython.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+wget http://central.maven.org/maven2/org/python/jython-installer/2.7.1/jython-installer-2.7.1.jar
+java -jar jython-installer-2.7.1.jar -s -d "$HOME/jython"
+$HOME/jython/bin/jython -m pip install nose

From b5e531f31a57d2cdc9f0edd77e0cbef426e016b9 Mon Sep 17 00:00:00 2001
From: Mattias Wadman <mattias.wadman@gmail.com>
Date: Tue, 2 Jan 2018 17:32:17 +0100
Subject: [PATCH 1684/1696] [acast] Fix extraction

---
 youtube_dl/extractor/acast.py | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/acast.py b/youtube_dl/extractor/acast.py
index 6dace3051..5871e72dc 100644
--- a/youtube_dl/extractor/acast.py
+++ b/youtube_dl/extractor/acast.py
@@ -8,7 +8,7 @@ from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
     int_or_none,
-    parse_iso8601,
+    unified_timestamp,
     OnDemandPagedList,
 )
 
@@ -32,7 +32,7 @@ class ACastIE(InfoExtractor):
     }, {
         # test with multiple blings
         'url': 'https://www.acast.com/sparpodcast/2.raggarmordet-rosterurdetforflutna',
-        'md5': '55c0097badd7095f494c99a172f86501',
+        'md5': 'e87d5b8516cd04c0d81b6ee1caca28d0',
         'info_dict': {
             'id': '2a92b283-1a75-4ad8-8396-499c641de0d9',
             'ext': 'mp3',
@@ -40,23 +40,24 @@ class ACastIE(InfoExtractor):
             'timestamp': 1477346700,
             'upload_date': '20161024',
             'description': 'md5:4f81f6d8cf2e12ee21a321d8bca32db4',
-            'duration': 2797,
+            'duration': 2766,
         }
     }]
 
     def _real_extract(self, url):
         channel, display_id = re.match(self._VALID_URL, url).groups()
         cast_data = self._download_json(
-            'https://embed.acast.com/api/acasts/%s/%s' % (channel, display_id), display_id)
+            'https://play-api.acast.com/splash/%s/%s' % (channel, display_id), display_id)
+        e = cast_data['result']['episode']
         return {
-            'id': compat_str(cast_data['id']),
+            'id': compat_str(e['id']),
             'display_id': display_id,
-            'url': [b['audio'] for b in cast_data['blings'] if b['type'] == 'BlingAudio'][0],
-            'title': cast_data['name'],
-            'description': cast_data.get('description'),
-            'thumbnail': cast_data.get('image'),
-            'timestamp': parse_iso8601(cast_data.get('publishingDate')),
-            'duration': int_or_none(cast_data.get('duration')),
+            'url': e['mediaUrl'],
+            'title': e['name'],
+            'description': e.get('description'),
+            'thumbnail': e.get('image'),
+            'timestamp': unified_timestamp(e.get('publishingDate')),
+            'duration': int_or_none(e.get('duration')),
         }
 
 

From 9650c3e91d96fc054e37485d2bfd86a6a12417e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@fastmail.net>
Date: Tue, 2 Jan 2018 21:12:39 +0100
Subject: [PATCH 1685/1696] [rtve.es:alacarta] Fix extraction of some new URLs

---
 ChangeLog                    | 6 ++++++
 youtube_dl/extractor/rtve.py | 3 +++
 2 files changed, 9 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index bfffb1f5f..3e6afca92 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [rtve.es:alacarta] Fix extraction of some new URLs
+
+
 version 2017.12.31
 
 Core
diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index d9edf9da2..fa60ffd5e 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -31,6 +31,9 @@ def _decrypt_url(png):
     hash_index = data.index('#')
     alphabet_data = data[:hash_index]
     url_data = data[hash_index + 1:]
+    if url_data[0] == 'H' and url_data[3] == '%':
+        # remove useless HQ%% at the start
+        url_data = url_data[4:]
 
     alphabet = []
     e = 0

From f0c6c2bce29281d37d2bbd589143b35323e38e3d Mon Sep 17 00:00:00 2001
From: Luc Ritchie <luc.ritchie@gmail.com>
Date: Wed, 3 Jan 2018 04:22:55 -0500
Subject: [PATCH 1686/1696] [twitch] Pass video id to url_result when
 extracting playlist

---
 youtube_dl/extractor/twitch.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index bf57eac01..f9164af09 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -358,9 +358,16 @@ class TwitchPlaylistBaseIE(TwitchBaseIE):
                 break
             offset += limit
         return self.playlist_result(
-            [self.url_result(entry) for entry in orderedSet(entries)],
+            [self._make_url_result(entry) for entry in orderedSet(entries)],
             channel_id, channel_name)
 
+    def _make_url_result(self, url):
+        try:
+            video_id = 'v%s' % TwitchVodIE._match_id(url)
+            return self.url_result(url, TwitchVodIE.ie_key(), video_id=video_id)
+        except AssertionError:
+            return self.url_result(url)
+
     def _extract_playlist_page(self, response):
         videos = response.get('videos')
         return [video['url'] for video in videos] if videos else []

From 75ba0efb52c898601ef9febcba3e7a84ff2f311e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 3 Jan 2018 16:41:28 +0700
Subject: [PATCH 1687/1696] [lynda] Skip invalid subtitles (closes #15159)

---
 youtube_dl/extractor/lynda.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 1b6f5091d..0d6026aad 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -244,8 +244,9 @@ class LyndaIE(LyndaBaseIE):
     def _get_subtitles(self, video_id):
         url = 'https://www.lynda.com/ajax/player?videoId=%s&type=transcript' % video_id
         subs = self._download_json(url, None, False)
-        if subs:
-            return {'en': [{'ext': 'srt', 'data': self._fix_subtitles(subs)}]}
+        fixed_subs = self._fix_subtitles(subs)
+        if fixed_subs:
+            return {'en': [{'ext': 'srt', 'data': fixed_subs}]}
         else:
             return {}
 

From de329f64abd920e148701436d6a20bfc5b2d3ef3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 4 Jan 2018 13:26:08 +0800
Subject: [PATCH 1688/1696] [openload] Fix extraction (closes #15166)

---
 ChangeLog                        | 1 +
 youtube_dl/extractor/openload.py | 6 +++++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 3e6afca92..94b27b6a1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [openload] Fix extraction (#15166)
 * [rtve.es:alacarta] Fix extraction of some new URLs
 
 
diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index b282bcfd9..eaaaf8a08 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -333,7 +333,11 @@ class OpenloadIE(InfoExtractor):
         webpage, _ = phantom.get(page_url, html=webpage, video_id=video_id, headers=headers)
 
         decoded_id = (get_element_by_id('streamurl', webpage) or
-                      get_element_by_id('streamuri', webpage))
+                      get_element_by_id('streamuri', webpage) or
+                      get_element_by_id('streamurj', webpage))
+
+        if not decoded_id:
+            raise ExtractorError('Can\'t find stream URL', video_id=video_id)
 
         video_url = 'https://openload.co/stream/%s?mime=true' % decoded_id
 

From da35331c6c23f90fc29ca3315f7e00481aff1d5c Mon Sep 17 00:00:00 2001
From: JianxinLi <justcoxier@gmail.com>
Date: Thu, 4 Jan 2018 22:25:28 +0800
Subject: [PATCH 1689/1696] [youku] Fix list extraction.(close #15135) (#15137)

* [youku] Fix list extraction.(close #15135)

Change-Id: I2e9c920143f4f16012252625943a8f18b8ff40eb

* [youku] Remove KeyError try-except

Change-Id: Ic46327905cbef1356b7b12d5eb3db5d9746ca338
---
 youtube_dl/extractor/youku.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index c7947d4a1..5b0b248cd 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -245,13 +245,19 @@ class YoukuShowIE(InfoExtractor):
         #  No data-id value.
         'url': 'http://list.youku.com/show/id_zefbfbd61237fefbfbdef.html',
         'only_matching': True,
+    }, {
+        #  Wrong number of reload_id.
+        'url': 'http://list.youku.com/show/id_z20eb4acaf5c211e3b2ad.html',
+        'only_matching': True,
     }]
 
     def _extract_entries(self, playlist_data_url, show_id, note, query):
         query['callback'] = 'cb'
         playlist_data = self._download_json(
             playlist_data_url, show_id, query=query, note=note,
-            transform_source=lambda s: js_to_json(strip_jsonp(s)))['html']
+            transform_source=lambda s: js_to_json(strip_jsonp(s))).get('html')
+        if playlist_data is None:
+            return [None, None]
         drama_list = (get_element_by_class('p-drama-grid', playlist_data) or
                       get_element_by_class('p-drama-half-row', playlist_data))
         if drama_list is None:
@@ -291,8 +297,8 @@ class YoukuShowIE(InfoExtractor):
                     'id': page_config['showid'],
                     'stage': reload_id,
                 })
-            entries.extend(new_entries)
-
+            if new_entries is not None:
+                entries.extend(new_entries)
         desc = self._html_search_meta('description', webpage, fatal=False)
         playlist_title = desc.split(',')[0] if desc else None
         detail_li = get_element_by_class('p-intro', webpage)

From c2f18e1c4915d43b37c12337dc4fbae68d43a304 Mon Sep 17 00:00:00 2001
From: Chih-Hsuan Yen <yan12125@gmail.com>
Date: Thu, 4 Jan 2018 22:28:00 +0800
Subject: [PATCH 1690/1696] [ChangeLog] Update after #15137

[skip ci]
---
 ChangeLog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChangeLog b/ChangeLog
index 94b27b6a1..96bc471f3 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [youku] Fix list extraction (#15135)
 * [openload] Fix extraction (#15166)
 * [rtve.es:alacarta] Fix extraction of some new URLs
 

From 0b0870f9d0dd3e72be3ff6be6bfa9fa43b693b50 Mon Sep 17 00:00:00 2001
From: Parmjit Virk <pvirk@mts.net>
Date: Thu, 4 Jan 2018 19:25:42 -0600
Subject: [PATCH 1691/1696] [soundcloud] Fallback to avatar picture for
 thumbnail (closes #12878)

---
 youtube_dl/extractor/soundcloud.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 8894f4b0c..6c9816eef 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -136,6 +136,25 @@ class SoundcloudIE(InfoExtractor):
                 'license': 'all-rights-reserved',
             },
         },
+        # no album art, use avatar pic for thumbnail
+        {
+            'url': 'https://soundcloud.com/garyvee/sideways-prod-mad-real',
+            'md5': '59c7872bc44e5d99b7211891664760c2',
+            'info_dict': {
+                'id': '309699954',
+                'ext': 'mp3',
+                'title': 'Sideways (Prod. Mad Real)',
+                'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
+                'uploader': 'garyvee',
+                'upload_date': '20170226',
+                'duration': 207,
+                'thumbnail': r're:https?://.*\.jpg',
+                'license': 'all-rights-reserved',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
     ]
 
     _CLIENT_ID = 'c6CU49JDMapyrQo06UxU9xouB9ZVzqCn'
@@ -160,7 +179,7 @@ class SoundcloudIE(InfoExtractor):
         name = full_title or track_id
         if quiet:
             self.report_extraction(name)
-        thumbnail = info.get('artwork_url')
+        thumbnail = info.get('artwork_url') or info.get('user', {}).get('avatar_url')
         if isinstance(thumbnail, compat_str):
             thumbnail = thumbnail.replace('-large', '-t500x500')
         ext = 'mp3'

From b7c74c04036c07f8a81d3048b482afd6ef384b40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 Jan 2018 23:12:30 +0700
Subject: [PATCH 1692/1696] [lynda] Relax _VALID_URL (closes #15185)

---
 youtube_dl/extractor/lynda.py | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 0d6026aad..f5c7abc13 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -94,7 +94,15 @@ class LyndaBaseIE(InfoExtractor):
 class LyndaIE(LyndaBaseIE):
     IE_NAME = 'lynda'
     IE_DESC = 'lynda.com videos'
-    _VALID_URL = r'https?://(?:www\.)?(?:lynda\.com|educourse\.ga)/(?:[^/]+/[^/]+/(?P<course_id>\d+)|player/embed)/(?P<id>\d+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:www\.)?(?:lynda\.com|educourse\.ga)/
+                        (?:
+                            (?:[^/]+/){2,3}(?P<course_id>\d+)|
+                            player/embed
+                        )/
+                        (?P<id>\d+)
+                    '''
 
     _TIMECODE_REGEX = r'\[(?P<timecode>\d+:\d+:\d+[\.,]\d+)\]'
 
@@ -113,6 +121,9 @@ class LyndaIE(LyndaBaseIE):
     }, {
         'url': 'https://educourse.ga/Bootstrap-tutorials/Using-exercise-files/110885/114408-4.html',
         'only_matching': True,
+    }, {
+        'url': 'https://www.lynda.com/de/Graphic-Design-tutorials/Willkommen-Grundlagen-guten-Gestaltung/393570/393572-4.html',
+        'only_matching': True,
     }]
 
     def _raise_unavailable(self, video_id):
@@ -257,7 +268,15 @@ class LyndaCourseIE(LyndaBaseIE):
 
     # Course link equals to welcome/introduction video link of same course
     # We will recognize it as course link
-    _VALID_URL = r'https?://(?:www|m)\.(?:lynda\.com|educourse\.ga)/(?P<coursepath>[^/]+/[^/]+/(?P<courseid>\d+))-\d\.html'
+    _VALID_URL = r'https?://(?:www|m)\.(?:lynda\.com|educourse\.ga)/(?P<coursepath>(?:[^/]+/){2,3}(?P<courseid>\d+))-2\.html'
+
+    _TESTS = [{
+        'url': 'https://www.lynda.com/Graphic-Design-tutorials/Grundlagen-guten-Gestaltung/393570-2.html',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.lynda.com/de/Graphic-Design-tutorials/Grundlagen-guten-Gestaltung/393570-2.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 45283afdec81af21ba50ff3aca3d86fb6d2584b0 Mon Sep 17 00:00:00 2001
From: Martin Weinelt <mweinelt@users.noreply.github.com>
Date: Sat, 6 Jan 2018 17:33:40 +0100
Subject: [PATCH 1693/1696] [motherless] Add support for groups

---
 youtube_dl/extractor/extractors.py |  5 +-
 youtube_dl/extractor/motherless.py | 73 ++++++++++++++++++++++++++++++
 2 files changed, 77 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index e64defe62..fb0997d39 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -609,7 +609,10 @@ from .mofosex import MofosexIE
 from .mojvideo import MojvideoIE
 from .moniker import MonikerIE
 from .morningstar import MorningstarIE
-from .motherless import MotherlessIE
+from .motherless import (
+    MotherlessIE,
+    MotherlessGroupIE
+)
 from .motorsport import MotorsportIE
 from .movieclips import MovieClipsIE
 from .moviezine import MoviezineIE
diff --git a/youtube_dl/extractor/motherless.py b/youtube_dl/extractor/motherless.py
index 6fe3b6049..90ed91ba6 100644
--- a/youtube_dl/extractor/motherless.py
+++ b/youtube_dl/extractor/motherless.py
@@ -4,8 +4,11 @@ import datetime
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
     ExtractorError,
+    InAdvancePagedList,
+    orderedSet,
     str_to_int,
     unified_strdate,
 )
@@ -114,3 +117,73 @@ class MotherlessIE(InfoExtractor):
             'age_limit': age_limit,
             'url': video_url,
         }
+
+
+class MotherlessGroupIE(InfoExtractor):
+    _VALID_URL = 'https?://(?:www\.)?motherless\.com/gv?/(?P<id>[a-z0-9_]+)'
+    _TESTS = [{
+        'url': 'http://motherless.com/g/movie_scenes',
+        'info_dict': {
+            'id': 'movie_scenes',
+            'title': 'Movie Scenes',
+            'description': 'Hot and sexy scenes from "regular" movies... '
+                           'Beautiful actresses fully nude... A looot of '
+                           'skin! :)Enjoy!',
+        },
+        'playlist_mincount': 662,
+    }, {
+        'url': 'http://motherless.com/gv/sex_must_be_funny',
+        'info_dict': {
+            'id': 'sex_must_be_funny',
+            'title': 'Sex must be funny',
+            'description': 'Sex can be funny. Wide smiles,laugh, games, fun of '
+                           'any kind!'
+        },
+        'playlist_mincount': 9,
+    }]
+
+    @classmethod
+    def suitable(cls, url):
+        return (False if MotherlessIE.suitable(url)
+                else super(MotherlessGroupIE, cls).suitable(url))
+
+    def _extract_entries(self, webpage, base):
+        return [
+            self.url_result(
+                compat_urlparse.urljoin(base, video_path),
+                MotherlessIE.ie_key(), video_title=title)
+            for video_path, title in orderedSet(re.findall(
+                r'href="/([^"]+)"[^>]+>\s+<img[^>]+alt="[^-]+-\s([^"]+)"',
+                webpage))
+        ]
+
+    def _real_extract(self, url):
+        group_id = self._match_id(url)
+        page_url = compat_urlparse.urljoin(url, '/gv/%s' % group_id)
+        webpage = self._download_webpage(page_url, group_id)
+        title = self._search_regex(
+            r'<title>([\w\s]+\w)\s+-', webpage, 'title', fatal=False)
+        description = self._html_search_meta(
+            'description', webpage, fatal=False)
+        page_count = self._int(self._search_regex(
+            r'(\d+)</(?:a|span)><(?:a|span)[^>]+>\s*NEXT',
+            webpage, 'page_count'), 'page_count')
+        PAGE_SIZE = 80
+
+        def _get_page(idx):
+            webpage = self._download_webpage(
+                page_url, group_id, query={'page': idx + 1},
+                note='Downloading page %d/%d' % (idx + 1, page_count)
+            )
+            for entry in self._extract_entries(webpage, url):
+                yield entry
+
+        playlist = InAdvancePagedList(_get_page, page_count, PAGE_SIZE)
+
+        return {
+            '_type': 'playlist',
+            'id': group_id,
+            'title': title,
+            'description': description,
+            'entries': playlist
+        }

From f12628f934ff50cc8e6441c4e64fe61019ebae5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 Jan 2018 23:58:00 +0700
Subject: [PATCH 1694/1696] [mitele] Fix extraction (closes #15186)

---
 youtube_dl/extractor/mitele.py | 30 ++++++++++++++++--------------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index 964dc542c..42759eae8 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -1,13 +1,13 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import json
 import uuid
 
 from .common import InfoExtractor
 from .ooyala import OoyalaIE
 from ..compat import (
     compat_str,
-    compat_urllib_parse_urlencode,
     compat_urlparse,
 )
 from ..utils import (
@@ -42,31 +42,33 @@ class MiTeleBaseIE(InfoExtractor):
                 duration = int_or_none(mmc.get('duration'))
             for location in mmc['locations']:
                 gat = self._proto_relative_url(location.get('gat'), 'http:')
-                bas = location.get('bas')
-                loc = location.get('loc')
+                gcp = location.get('gcp')
                 ogn = location.get('ogn')
-                if None in (gat, bas, loc, ogn):
+                if None in (gat, gcp, ogn):
                     continue
                 token_data = {
-                    'bas': bas,
-                    'icd': loc,
+                    'gcp': gcp,
                     'ogn': ogn,
-                    'sta': '0',
+                    'sta': 0,
                 }
                 media = self._download_json(
-                    '%s/?%s' % (gat, compat_urllib_parse_urlencode(token_data)),
-                    video_id, 'Downloading %s JSON' % location['loc'])
-                file_ = media.get('file')
-                if not file_:
+                    gat, video_id, data=json.dumps(token_data).encode('utf-8'),
+                    headers={
+                        'Content-Type': 'application/json;charset=utf-8',
+                        'Referer': url,
+                    })
+                stream = media.get('stream') or media.get('file')
+                if not stream:
                     continue
-                ext = determine_ext(file_)
+                ext = determine_ext(stream)
                 if ext == 'f4m':
                     formats.extend(self._extract_f4m_formats(
-                        file_ + '&hdcore=3.2.0&plugin=aasp-3.2.0.77.18',
+                        stream + '&hdcore=3.2.0&plugin=aasp-3.2.0.77.18',
                         video_id, f4m_id='hds', fatal=False))
                 elif ext == 'm3u8':
                     formats.extend(self._extract_m3u8_formats(
-                        file_, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+                        stream, video_id, 'mp4', 'm3u8_native',
+                        m3u8_id='hls', fatal=False))
         self._sort_formats(formats)
 
         return {

From a133eb7764594b830cb975e3925972214e932704 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Jan 2018 00:02:41 +0700
Subject: [PATCH 1695/1696] [motherless:group] Capture leading slash of video
 path

---
 youtube_dl/extractor/motherless.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/motherless.py b/youtube_dl/extractor/motherless.py
index 90ed91ba6..4adac691c 100644
--- a/youtube_dl/extractor/motherless.py
+++ b/youtube_dl/extractor/motherless.py
@@ -153,7 +153,7 @@ class MotherlessGroupIE(InfoExtractor):
                 compat_urlparse.urljoin(base, video_path),
                 MotherlessIE.ie_key(), video_title=title)
             for video_path, title in orderedSet(re.findall(
-                r'href="/([^"]+)"[^>]+>\s+<img[^>]+alt="[^-]+-\s([^"]+)"',
+                r'href="(/[^"]+)"[^>]+>\s+<img[^>]+alt="[^-]+-\s([^"]+)"',
                 webpage))
         ]
 

From 0a5b1295b7c1aa6395b65ee137087c540b37b32b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Jan 2018 00:31:53 +0700
Subject: [PATCH 1696/1696] [motherless:group] Relax entry extraction and add a
 fallback scenario

---
 youtube_dl/extractor/motherless.py | 29 +++++++++++++++++++++--------
 1 file changed, 21 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/motherless.py b/youtube_dl/extractor/motherless.py
index 4adac691c..e24396e79 100644
--- a/youtube_dl/extractor/motherless.py
+++ b/youtube_dl/extractor/motherless.py
@@ -148,14 +148,27 @@ class MotherlessGroupIE(InfoExtractor):
                 else super(MotherlessGroupIE, cls).suitable(url))
 
     def _extract_entries(self, webpage, base):
-        return [
-            self.url_result(
-                compat_urlparse.urljoin(base, video_path),
-                MotherlessIE.ie_key(), video_title=title)
-            for video_path, title in orderedSet(re.findall(
-                r'href="(/[^"]+)"[^>]+>\s+<img[^>]+alt="[^-]+-\s([^"]+)"',
-                webpage))
-        ]
+        entries = []
+        for mobj in re.finditer(
+                r'href="(?P<href>/[^"]+)"[^>]*>(?:\s*<img[^>]+alt="[^-]+-\s(?P<title>[^"]+)")?',
+                webpage):
+            video_url = compat_urlparse.urljoin(base, mobj.group('href'))
+            if not MotherlessIE.suitable(video_url):
+                continue
+            video_id = MotherlessIE._match_id(video_url)
+            title = mobj.group('title')
+            entries.append(self.url_result(
+                video_url, ie=MotherlessIE.ie_key(), video_id=video_id,
+                video_title=title))
+        # Alternative fallback
+        if not entries:
+            entries = [
+                self.url_result(
+                    compat_urlparse.urljoin(base, '/' + video_id),
+                    ie=MotherlessIE.ie_key(), video_id=video_id)
+                for video_id in orderedSet(re.findall(
+                    r'data-codename=["\']([A-Z0-9]+)', webpage))]
+        return entries
 
     def _real_extract(self, url):
         group_id = self._match_id(url)