release 2014.08.28.2

This should detect when we're downloading a small text file by accident.

test/test_download.py

@@ -28,6 +28,7 @@ from youtube_dl.utils import (
     compat_HTTPError,
     DownloadError,
     ExtractorError,
+    format_bytes,
     UnavailableVideoError,
 )
 from youtube_dl.extractor import get_info_extractor
@@ -171,6 +172,16 @@ def generator(test_case):
                 if 'md5' in tc:
                     md5_for_file = _file_md5(tc_filename)
                     self.assertEqual(md5_for_file, tc['md5'])
+                expected_minsize = tc.get('file_minsize', 10000)
+                if expected_minsize is not None:
+                    if params.get('test'):
+                        expected_minsize = max(expected_minsize, 10000)
+                    got_fsize = os.path.getsize(tc_filename)
+                    assertGreaterEqual(
+                        self, got_fsize, expected_minsize,
+                        'Expected %s to be at least %s, but it\'s only %s ' %
+                        (tc_filename, format_bytes(expected_minsize),
+                            format_bytes(got_fsize)))
                 with io.open(info_json_fn, encoding='utf-8') as infof:
                     info_dict = json.load(infof)
 

youtube_dl/extractor/common.py

@@ -664,6 +664,9 @@ class InfoExtractor(object):
             elif line.startswith('#') or not line.strip():
                 continue
             else:
+                if last_info is None:
+                    formats.append({'url': line})
+                    continue
                 tbr = int_or_none(last_info.get('BANDWIDTH'), scale=1000)
 
                 f = {

youtube_dl/extractor/dropbox.py

@@ -11,8 +11,7 @@ from ..utils import compat_urllib_parse_unquote
 class DropboxIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?dropbox[.]com/s/(?P<id>[a-zA-Z0-9]{15})/(?P<title>[^?#]*)'
     _TEST = {
-        'url': 'https://www.dropbox.com/s/nelirfsxnmcfbfh/youtube-dl%20test%20video%20%27%C3%A4%22BaW_jenozKc.mp4',
-        'md5': '8a3d905427a6951ccb9eb292f154530b',
+        'url': 'https://www.dropbox.com/s/nelirfsxnmcfbfh/youtube-dl%20test%20video%20%27%C3%A4%22BaW_jenozKc.mp4?dl=0',
         'info_dict': {
             'id': 'nelirfsxnmcfbfh',
             'ext': 'mp4',
@@ -25,7 +24,9 @@ class DropboxIE(InfoExtractor):
         video_id = mobj.group('id')
         fn = compat_urllib_parse_unquote(mobj.group('title'))
         title = os.path.splitext(fn)[0]
-        video_url = url + '?dl=1'
+        video_url = (
+            re.sub(r'[?&]dl=0', '', url) +
+            ('?' if '?' in url else '&') + 'dl=1')
 
         return {
             'id': video_id,

youtube_dl/extractor/rtlnl.py

@@ -12,22 +12,16 @@ class RtlXlIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://www.rtlxl.nl/#!/rtl-nieuws-132237/6e4203a6-0a5e-3596-8424-c599a59e0677',
+        'md5': 'cc16baa36a6c169391f0764fa6b16654',
         'info_dict': {
             'id': '6e4203a6-0a5e-3596-8424-c599a59e0677',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'RTL Nieuws - Laat',
-            'description': 'Dagelijks het laatste nieuws uit binnen- en '
-                'buitenland. Voor nog meer nieuws kunt u ook gebruikmaken van '
-                'onze mobiele apps.',
+            'description': 'md5:6b61f66510c8889923b11f2778c72dc5',
             'timestamp': 1408051800,
             'upload_date': '20140814',
             'duration': 576.880,
         },
-        'params': {
-            # We download the first bytes of the first fragment, it can't be
-            # processed by the f4m downloader beacuse it isn't complete
-            'skip_download': True,
-        },
     }
 
     def _real_extract(self, url):
@@ -41,14 +35,32 @@ class RtlXlIE(InfoExtractor):
         material = info['material'][0]
         episode_info = info['episodes'][0]
 
-        f4m_url = 'http://manifest.us.rtl.nl' + material['videopath']
         progname = info['abstracts'][0]['name']
         subtitle = material['title'] or info['episodes'][0]['name']
 
+        videopath = material['videopath']
+        f4m_url = 'http://manifest.us.rtl.nl' + videopath
+
+        formats = self._extract_f4m_formats(f4m_url, uuid)
+
+        video_urlpart = videopath.split('/flash/')[1][:-4]
+        PG_URL_TEMPLATE = 'http://pg.us.rtl.nl/rtlxl/network/%s/progressive/%s.mp4'
+
+        formats.extend([
+            {
+                'url': PG_URL_TEMPLATE % ('a2m', video_urlpart),
+                'format_id': 'pg-sd',
+            },
+            {
+                'url': PG_URL_TEMPLATE % ('a3m', video_urlpart),
+                'format_id': 'pg-hd',
+            }
+        ])
+
         return {
             'id': uuid,
             'title': '%s - %s' % (progname, subtitle),
-            'formats': self._extract_f4m_formats(f4m_url, uuid),
+            'formats': formats,
             'timestamp': material['original_date'],
             'description': episode_info['synopsis'],
             'duration': parse_duration(material.get('duration')),

youtube_dl/extractor/sportdeutschland.py

@@ -12,7 +12,7 @@ from ..utils import (
 
 class SportDeutschlandIE(InfoExtractor):
     _VALID_URL = r'https?://sportdeutschland\.tv/(?P<sport>[^/?#]+)/(?P<id>[^?#/]+)(?:$|[?#])'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://sportdeutschland.tv/badminton/live-li-ning-badminton-weltmeisterschaft-2014-kopenhagen',
         'info_dict': {
             'id': 'live-li-ning-badminton-weltmeisterschaft-2014-kopenhagen',
@@ -20,15 +20,28 @@ class SportDeutschlandIE(InfoExtractor):
             'title': 'LIVE: Li-Ning Badminton Weltmeisterschaft 2014 Kopenhagen',
             'categories': ['Badminton'],
             'view_count': int,
-            'thumbnail': 're:^https?://.*\.jpg',
+            'thumbnail': 're:^https?://.*\.jpg$',
             'description': 're:^Die Badminton-WM 2014 aus Kopenhagen LIVE',
-            'timestamp': 1409043600,
-            'upload_date': '20140826',
+            'timestamp': int,
+            'upload_date': 're:^201408[23][0-9]$',
         },
         'params': {
             'skip_download': 'Live stream',
         },
-    }
+    }, {
+        'url': 'http://sportdeutschland.tv/li-ning-badminton-wm-2014/lee-li-ning-badminton-weltmeisterschaft-2014-kopenhagen-herren-einzel-wei-vs',
+        'info_dict': {
+            'id': 'lee-li-ning-badminton-weltmeisterschaft-2014-kopenhagen-herren-einzel-wei-vs',
+            'ext': 'mp4',
+            'upload_date': '20140825',
+            'description': 'md5:60a20536b57cee7d9a4ec005e8687504',
+            'timestamp': 1408976060,
+            'title': 'Li-Ning Badminton Weltmeisterschaft 2014 Kopenhagen: Herren Einzel, Wei Lee vs. Keun Lee',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'view_count': int,
+            'categories': ['Li-Ning Badminton WM 2014'],
+        }
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -46,21 +59,27 @@ class SportDeutschlandIE(InfoExtractor):
         categories = list(data.get('section', {}).get('tags', {}).values())
         asset = data['asset']
 
+        formats = []
         smil_url = asset['video']
-        m3u8_url = smil_url.replace('.smil', '.m3u8')
-        formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
+        if '.smil' in smil_url:
+            m3u8_url = smil_url.replace('.smil', '.m3u8')
+            formats.extend(
+                self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4'))
+
+            smil_doc = self._download_xml(
+                smil_url, video_id, note='Downloading SMIL metadata')
+            base_url = smil_doc.find('./head/meta').attrib['base']
+            formats.extend([{
+                'format_id': 'rmtp',
+                'url': base_url,
+                'play_path': n.attrib['src'],
+                'ext': 'flv',
+                'preference': -100,
+                'format_note': 'Seems to fail at example stream',
+            } for n in smil_doc.findall('./body/video')])
+        else:
+            formats.append({'url': smil_url})
 
-        smil_doc = self._download_xml(
-            smil_url, video_id, note='Downloading SMIL metadata')
-        base_url = smil_doc.find('./head/meta').attrib['base']
-        formats.extend([{
-            'format_id': 'rmtp',
-            'url': base_url,
-            'play_path': n.attrib['src'],
-            'ext': 'flv',
-            'preference': -100,
-            'format_note': 'Seems to fail at example stream',
-        } for n in smil_doc.findall('./body/video')])
         self._sort_formats(formats)
 
         return {
@@ -71,7 +90,7 @@ class SportDeutschlandIE(InfoExtractor):
             'description': asset.get('teaser'),
             'categories': categories,
             'view_count': asset.get('views'),
-            'rtmp_live': asset['live'],
+            'rtmp_live': asset.get('live'),
             'timestamp': parse_iso8601(asset.get('date')),
         }
 

youtube_dl/version.py

@@ -1,2 +1,2 @@
 
-__version__ = '2014.08.28'
+__version__ = '2014.08.28.2'