From 8af4ed7b4fb20190ec72e9ab6347e99deabe9bce Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Thu, 29 Nov 2012 16:35:57 +0100 Subject: [PATCH 01/18] Fix 2.6 nosetests --- test/gentests.py | 5 +++-- test/test_download.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/test/gentests.py b/test/gentests.py index da4e3c2f6..6f82ae8fa 100755 --- a/test/gentests.py +++ b/test/gentests.py @@ -52,13 +52,14 @@ try: except AttributeError: # Python 2.6 def _skip_unless(cond, reason='No reason given'): def resfunc(f): - def wfunc(*args, **kwargs): + # Start the function name with test to appease nosetests-2.6 + def test_wfunc(*args, **kwargs): if cond: return f(*args, **kwargs) else: print('Skipped test') return - return wfunc + return test_wfunc return resfunc _skip = lambda *args, **kwargs: _skip_unless(False, *args, **kwargs) diff --git a/test/test_download.py b/test/test_download.py index 0072d4aab..00b9a487b 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -38,13 +38,14 @@ try: except AttributeError: # Python 2.6 def _skip_unless(cond, reason='No reason given'): def resfunc(f): - def wfunc(*args, **kwargs): + # Start the function name with test to appease nosetests-2.6 + def test_wfunc(*args, **kwargs): if cond: return f(*args, **kwargs) else: print('Skipped test') return - return wfunc + return test_wfunc return resfunc _skip = lambda *args, **kwargs: _skip_unless(False, *args, **kwargs) From ad348291bb1e23ded487ba0bbaba3c44c709ef94 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Thu, 29 Nov 2012 19:41:09 +0100 Subject: [PATCH 02/18] Enable travis notifications --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 03947b1eb..696fa9a08 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,8 @@ language: python python: - "2.6" - "2.7" +notifications: + irc: "irc.freenode.org#youtube-dl" #command to install the setup install: # command to run tests From 4b235346d633982afd1f803f671778440cea15c4 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Thu, 29 Nov 2012 19:45:07 +0100 Subject: [PATCH 03/18] Add irc channel notice --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index b218fc3ec..5789ce2e4 100644 --- a/README.md +++ b/README.md @@ -194,3 +194,5 @@ Please include: * The output of `youtube-dl --version` * The output of `python --version` * The name and version of your Operating System ("Ubuntu 11.04 x64" or "Windows 7 x64" is usually enough). + +For discussions, join us in the irc channel #youtube-dl on freenode. From e91d2338d8f5be16d8cd052d38c20bec9d5583ab Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Thu, 29 Nov 2012 20:38:16 +0100 Subject: [PATCH 04/18] Fix MD5 calculation --- test/gentests.py | 13 ------------- test/test_download.py | 23 +++++------------------ test/tests.json | 10 +++++----- 3 files changed, 10 insertions(+), 36 deletions(-) diff --git a/test/gentests.py b/test/gentests.py index 6f82ae8fa..9352d6d94 100755 --- a/test/gentests.py +++ b/test/gentests.py @@ -34,19 +34,6 @@ import youtube_dl.InfoExtractors def _file_md5(fn): with open(fn, 'rb') as f: return hashlib.md5(f.read()).hexdigest() - -def md5_for_file(filename, block_size=2**20): - with open(filename) as f: - md5 = hashlib.md5() - while True: - data = f.read(block_size) - if not data: - break - md5.update(data) - return md5.hexdigest() -_file_md5 = md5_for_file - - try: _skip_unless = unittest.skipUnless except AttributeError: # Python 2.6 diff --git a/test/test_download.py b/test/test_download.py index 00b9a487b..571fbe9ad 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -20,19 +20,6 @@ import youtube_dl.InfoExtractors def _file_md5(fn): with open(fn, 'rb') as f: return hashlib.md5(f.read()).hexdigest() - -def md5_for_file(filename, block_size=2**20): - with open(filename) as f: - md5 = hashlib.md5() - while True: - data = f.read(block_size) - if not data: - break - md5.update(data) - return md5.hexdigest() -_file_md5 = md5_for_file - - try: _skip_unless = unittest.skipUnless except AttributeError: # Python 2.6 @@ -96,7 +83,7 @@ class DownloadTest(unittest.TestCase): fd.download(['http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352']) self.assertTrue(os.path.exists(filename)) md5_for_file = _file_md5(filename) - self.assertEqual(md5_for_file, '93c24d2f4e0782af13b8a7606ea97ba7') + self.assertEqual(md5_for_file, '4962f94441605832eb1008eb820ef47a') @_skip_unless(youtube_dl.InfoExtractors.XVideosIE._WORKING, "IE marked as not _WORKING") def test_XVideos(self): @@ -106,7 +93,7 @@ class DownloadTest(unittest.TestCase): fd.download(['http://www.xvideos.com/video939581/funny_porns_by_s_-1']) self.assertTrue(os.path.exists(filename)) md5_for_file = _file_md5(filename) - self.assertEqual(md5_for_file, '1ab4dedc01f771cb2a65e91caa801aaf') + self.assertEqual(md5_for_file, 'aecab2ea59b7996110a7e409f0c55da3') @_skip_unless(youtube_dl.InfoExtractors.VimeoIE._WORKING, "IE marked as not _WORKING") @_skip("No output file specified") @@ -127,7 +114,7 @@ class DownloadTest(unittest.TestCase): fd.download(['http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy']) self.assertTrue(os.path.exists(filename)) md5_for_file = _file_md5(filename) - self.assertEqual(md5_for_file, 'ce3775768ebb6432fa8495d446a078ed') + self.assertEqual(md5_for_file, 'c1b9b9ea8bfd620b96b2628664576e1c') @_skip_unless(youtube_dl.InfoExtractors.StanfordOpenClassroomIE._WORKING, "IE marked as not _WORKING") def test_StanfordOpenClassroom(self): @@ -137,7 +124,7 @@ class DownloadTest(unittest.TestCase): fd.download(['http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100']) self.assertTrue(os.path.exists(filename)) md5_for_file = _file_md5(filename) - self.assertEqual(md5_for_file, '22c8206291368c4e2c9c1a307f0ea0f4') + self.assertEqual(md5_for_file, '8aac7873a07dcfaed66b1559ab128514') @_skip_unless(youtube_dl.InfoExtractors.CollegeHumorIE._WORKING, "IE marked as not _WORKING") @_skip("No output file specified") @@ -158,7 +145,7 @@ class DownloadTest(unittest.TestCase): fd.download(['http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_']) self.assertTrue(os.path.exists(filename)) md5_for_file = _file_md5(filename) - self.assertEqual(md5_for_file, '5f0469c8d1dfd1bc38c8e6deb5e0a21d') + self.assertEqual(md5_for_file, 'c5c67df477eb0d9b058200351448ba4c') def tearDown(self): diff --git a/test/tests.json b/test/tests.json index 611780cc3..731c91718 100644 --- a/test/tests.json +++ b/test/tests.json @@ -20,13 +20,13 @@ }, { "name": "BlipTV", - "md5": "93c24d2f4e0782af13b8a7606ea97ba7", + "md5": "4962f94441605832eb1008eb820ef47a", "url": "http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352", "file": "5779306.m4v" }, { "name": "XVideos", - "md5": "1ab4dedc01f771cb2a65e91caa801aaf", + "md5": "aecab2ea59b7996110a7e409f0c55da3", "url": "http://www.xvideos.com/video939581/funny_porns_by_s_-1", "file": "939581.flv" }, @@ -38,13 +38,13 @@ }, { "name": "Soundcloud", - "md5": "ce3775768ebb6432fa8495d446a078ed", + "md5": "c1b9b9ea8bfd620b96b2628664576e1c", "url": "http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy", "file": "n6FLbx6ZzMiu.mp3" }, { "name": "StanfordOpenClassroom", - "md5": "22c8206291368c4e2c9c1a307f0ea0f4", + "md5": "8aac7873a07dcfaed66b1559ab128514", "url": "http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100", "file": "PracticalUnix_intro-environment.mp4" }, @@ -56,7 +56,7 @@ }, { "name": "XNXX", - "md5": "5f0469c8d1dfd1bc38c8e6deb5e0a21d", + "md5": "c5c67df477eb0d9b058200351448ba4c", "url": "http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_", "file": "1135332.flv" } From 15c8d83358a964e8e5d12f1d0c55fe96fc111a01 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Thu, 29 Nov 2012 20:40:12 +0100 Subject: [PATCH 05/18] Fix Soundcloud IE (+ Python3 support) --- youtube_dl/InfoExtractors.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index 925dabc96..e5e8f41f0 100644 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -2818,16 +2818,17 @@ class SoundcloudIE(InfoExtractor): return # extract uploader (which is in the url) - uploader = mobj.group(1).decode('utf-8') + uploader = mobj.group(1) # extract simple title (uploader + slug of song title) - slug_title = mobj.group(2).decode('utf-8') + slug_title = mobj.group(2) simple_title = uploader + u'-' + slug_title self.report_webpage('%s/%s' % (uploader, slug_title)) request = compat_urllib_request.Request('http://soundcloud.com/%s/%s' % (uploader, slug_title)) try: - webpage = compat_urllib_request.urlopen(request).read() + webpage_bytes = compat_urllib_request.urlopen(request).read() + webpage = webpage_bytes.decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) return @@ -2843,7 +2844,7 @@ class SoundcloudIE(InfoExtractor): # extract unsimplified title mobj = re.search('"title":"(.*?)",', webpage) if mobj: - title = mobj.group(1).decode('utf-8') + title = mobj.group(1) else: title = simple_title @@ -2870,13 +2871,13 @@ class SoundcloudIE(InfoExtractor): request = compat_urllib_request.Request('http://media.soundcloud.com/crossdomain.xml', std_headers) return [{ - 'id': video_id.decode('utf-8'), + 'id': video_id, 'url': mediaURL, - 'uploader': uploader.decode('utf-8'), + 'uploader': uploader, 'upload_date': upload_date, 'title': title, 'ext': u'mp3', - 'description': description.decode('utf-8') + 'description': description }] From d78be7e331f3f859f770a37a33286d42680114e7 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Fri, 30 Nov 2012 08:42:11 +0100 Subject: [PATCH 06/18] Add test for Youku (Mentioned in #314) --- test/test_download.py | 12 ++++++++++++ test/tests.json | 8 +++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/test/test_download.py b/test/test_download.py index 571fbe9ad..c1f6ea556 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -147,6 +147,16 @@ class DownloadTest(unittest.TestCase): md5_for_file = _file_md5(filename) self.assertEqual(md5_for_file, 'c5c67df477eb0d9b058200351448ba4c') + @_skip_unless(youtube_dl.InfoExtractors.YoukuIE._WORKING, "IE marked as not _WORKING") + def test_Youku(self): + filename = 'XNDgyMDQ2NTQw_part00.flv' + fd = FileDownloader(self.parameters) + fd.add_info_extractor(youtube_dl.InfoExtractors.YoukuIE()) + fd.download(['http://v.youku.com/v_show/id_XNDgyMDQ2NTQw.html']) + self.assertTrue(os.path.exists(filename)) + md5_for_file = _file_md5(filename) + self.assertEqual(md5_for_file, 'ffe3f2e435663dc2d1eea34faeff5b5b') + def tearDown(self): if os.path.exists('BaW_jenozKc.mp4'): @@ -167,6 +177,8 @@ class DownloadTest(unittest.TestCase): # No file specified for CollegeHumor if os.path.exists('1135332.flv'): os.remove('1135332.flv') + if os.path.exists('XNDgyMDQ2NTQw_part00.flv'): + os.remove('XNDgyMDQ2NTQw_part00.flv') diff --git a/test/tests.json b/test/tests.json index 731c91718..b830cf36f 100644 --- a/test/tests.json +++ b/test/tests.json @@ -54,10 +54,16 @@ "url": "http://www.collegehumor.com/video/6830834/mitt-romney-style-gangnam-style-parody", "file": "" }, - { + { "name": "XNXX", "md5": "c5c67df477eb0d9b058200351448ba4c", "url": "http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_", "file": "1135332.flv" + }, + { + "name": "Youku", + "url": "http://v.youku.com/v_show/id_XNDgyMDQ2NTQw.html", + "file": "XNDgyMDQ2NTQw_part00.flv", + "md5": "ffe3f2e435663dc2d1eea34faeff5b5b" } ] \ No newline at end of file From e575b6821ec185ef4f14c433ccc018d748c0fb60 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sat, 1 Dec 2012 15:52:34 +0100 Subject: [PATCH 07/18] Improve execution tests --- test/test_execution.py | 26 ++++++++++++++++++++++++++ test/test_import.py | 13 ------------- 2 files changed, 26 insertions(+), 13 deletions(-) create mode 100644 test/test_execution.py delete mode 100644 test/test_import.py diff --git a/test/test_execution.py b/test/test_execution.py new file mode 100644 index 000000000..2b115fb31 --- /dev/null +++ b/test/test_execution.py @@ -0,0 +1,26 @@ +import unittest + +import sys +import os +import subprocess + +rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +try: + _DEV_NULL = subprocess.DEVNULL +except AttributeError: + _DEV_NULL = open(os.devnull, 'wb') + +class TestExecution(unittest.TestCase): + def test_import(self): + subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir) + + def test_module_exec(self): + if sys.version_info >= (2,7): # Python 2.6 doesn't support package execution + subprocess.check_call([sys.executable, '-m', 'youtube_dl', '--version'], cwd=rootDir, stdout=_DEV_NULL) + + def test_main_exec(self): + subprocess.check_call([sys.executable, 'youtube_dl/__main__.py', '--version'], cwd=rootDir, stdout=_DEV_NULL) + +if __name__ == '__main__': + unittest.main() diff --git a/test/test_import.py b/test/test_import.py deleted file mode 100644 index acf95444d..000000000 --- a/test/test_import.py +++ /dev/null @@ -1,13 +0,0 @@ -import unittest - -import sys -import os.path -import subprocess - -class TestImport(unittest.TestCase): - def test_import(self): - rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir) - -if __name__ == '__main__': - unittest.main() From 796173d08b514182eedc704541eb55d5c9e1dc0d Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 3 Dec 2012 15:36:24 +0100 Subject: [PATCH 08/18] Keep video IDs verbatim if possible (Closes #571) --- test/test_utils.py | 5 +++++ youtube_dl/FileDownloader.py | 7 +++++-- youtube_dl/utils.py | 20 +++++++++++--------- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/test/test_utils.py b/test/test_utils.py index fd8190f51..8afc30370 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -79,6 +79,11 @@ class TestUtil(unittest.TestCase): self.assertTrue(sanitize_filename('-', restricted=True) != '') self.assertTrue(sanitize_filename(':', restricted=True) != '') + def test_sanitize_ids(self): + self.assertEquals(sanitize_filename('_n_cd26wFpw', is_id=True), '_n_cd26wFpw') + self.assertEquals(sanitize_filename('_BD_eEpuzXw', is_id=True), '_BD_eEpuzXw') + self.assertEquals(sanitize_filename('N0Y__7-UOdI', is_id=True), 'N0Y__7-UOdI') + def test_ordered_set(self): self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7]) self.assertEqual(orderedSet([]), []) diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index 48c8eb126..c471cc160 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -334,8 +334,11 @@ class FileDownloader(object): template_dict['epoch'] = int(time.time()) template_dict['autonumber'] = u'%05d' % self._num_downloads - template_dict = dict((key, u'NA' if val is None else val) for key, val in template_dict.items()) - template_dict = dict((k, sanitize_filename(compat_str(v), self.params.get('restrictfilenames'))) for k,v in template_dict.items()) + sanitize = lambda k,v: sanitize_filename( + u'NA' if v is None else compat_str(v), + restricted=self.params.get('restrictfilenames'), + is_id=(k==u'id')) + template_dict = dict((k, sanitize(k, v)) for k,v in template_dict.items()) filename = self.params['outtmpl'] % template_dict return filename diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 7f73b8476..4dcf18991 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -317,9 +317,10 @@ def timeconvert(timestr): timestamp = email.utils.mktime_tz(timetuple) return timestamp -def sanitize_filename(s, restricted=False): +def sanitize_filename(s, restricted=False, is_id=False): """Sanitizes a string so it could be used as part of a filename. If restricted is set, use a stricter subset of allowed characters. + Set is_id if this is not an arbitrary string, but an ID that should be kept if possible """ def replace_insane(char): if char == '?' or ord(char) < 32 or ord(char) == 127: @@ -337,14 +338,15 @@ def sanitize_filename(s, restricted=False): return char result = u''.join(map(replace_insane, s)) - while '__' in result: - result = result.replace('__', '_') - result = result.strip('_') - # Common case of "Foreign band name - English song title" - if restricted and result.startswith('-_'): - result = result[2:] - if not result: - result = '_' + if not is_id: + while '__' in result: + result = result.replace('__', '_') + result = result.strip('_') + # Common case of "Foreign band name - English song title" + if restricted and result.startswith('-_'): + result = result[2:] + if not result: + result = '_' return result def orderedSet(iterable): From 4083bf81a01f2e580f84f5db0274dd5585705ad6 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 3 Dec 2012 20:17:47 +0100 Subject: [PATCH 09/18] Correct metacafe test filename (happens to start with an underscore) --- test/test_download.py | 6 +++--- test/tests.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test_download.py b/test/test_download.py index c1f6ea556..fd7c1860c 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -67,7 +67,7 @@ class DownloadTest(unittest.TestCase): @_skip_unless(youtube_dl.InfoExtractors.MetacafeIE._WORKING, "IE marked as not _WORKING") def test_Metacafe(self): - filename = 'aUehQsCQtM.flv' + filename = '_aUehQsCQtM.flv' fd = FileDownloader(self.parameters) fd.add_info_extractor(youtube_dl.InfoExtractors.MetacafeIE()) fd.add_info_extractor(youtube_dl.InfoExtractors.YoutubeIE()) @@ -163,8 +163,8 @@ class DownloadTest(unittest.TestCase): os.remove('BaW_jenozKc.mp4') if os.path.exists('x33vw9.mp4'): os.remove('x33vw9.mp4') - if os.path.exists('aUehQsCQtM.flv'): - os.remove('aUehQsCQtM.flv') + if os.path.exists('_aUehQsCQtM.flv'): + os.remove('_aUehQsCQtM.flv') if os.path.exists('5779306.m4v'): os.remove('5779306.m4v') if os.path.exists('939581.flv'): diff --git a/test/tests.json b/test/tests.json index b830cf36f..2c29a07b4 100644 --- a/test/tests.json +++ b/test/tests.json @@ -16,7 +16,7 @@ "size": 5754305, "addIEs": ["Youtube"], "url": "http://www.metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/", - "file": "aUehQsCQtM.flv" + "file": "_aUehQsCQtM.flv" }, { "name": "BlipTV", From e387eb5aba241e55e38e8bd5b57cc76b082ac6b9 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Tue, 4 Dec 2012 10:59:38 +0100 Subject: [PATCH 10/18] Let youtube IE handle IDs starting with PL (Closes #572) --- test/test_utils.py | 2 ++ test/test_youtube_playlist_ids.py | 22 ++++++++++++++++++++++ youtube_dl/InfoExtractors.py | 2 +- 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 test/test_youtube_playlist_ids.py diff --git a/test/test_utils.py b/test/test_utils.py index 8afc30370..796ed4ab2 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # Various small unit tests import sys diff --git a/test/test_youtube_playlist_ids.py b/test/test_youtube_playlist_ids.py new file mode 100644 index 000000000..b4dcedb45 --- /dev/null +++ b/test/test_youtube_playlist_ids.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python + +import sys +import unittest + +# Allow direct execution +import os +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from youtube_dl.InfoExtractors import YoutubeIE, YoutubePlaylistIE + +class TestYoutubePlaylistMatching(unittest.TestCase): + def test_playlist_matching(self): + assert YoutubePlaylistIE().suitable(u'ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8') + assert YoutubePlaylistIE().suitable(u'PL63F0C78739B09958') + assert not YoutubePlaylistIE().suitable(u'PLtS2H6bU1M') + + def test_youtube_matching(self): + assert YoutubeIE().suitable(u'PLtS2H6bU1M') + +if __name__ == '__main__': + unittest.main() diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index e5e8f41f0..1857360d4 100644 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -1671,7 +1671,7 @@ class YahooSearchIE(InfoExtractor): class YoutubePlaylistIE(InfoExtractor): """Information Extractor for YouTube playlists.""" - _VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]+)(?:/.*?/([0-9A-Za-z_-]+))?.*' + _VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]{10,})(?:/.*?/([0-9A-Za-z_-]+))?.*' _TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en' _VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&([^&"]+&)*list=.*?%s' _MORE_PAGES_INDICATOR = r'yt-uix-pager-next' From 077174f4ed2578706cf11a2b2ea74ddc309b11db Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Tue, 4 Dec 2012 11:05:25 +0100 Subject: [PATCH 11/18] Add an example to the -o documentation (#573) --- youtube_dl/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 37d6f8cfd..71c7d29e0 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -279,7 +279,7 @@ def parseOpts(): action='store_true', dest='autonumber', help='number downloaded files starting from 00000', default=False) filesystem.add_option('-o', '--output', - dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout.') + dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout. Can also be used to download to a different directory, for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .') filesystem.add_option('--restrict-filenames', action='store_true', dest='restrictfilenames', help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False) From 0f8d03f81cb20ba0f2a4358b8111146b589d1c5d Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Fri, 7 Dec 2012 00:39:44 +0100 Subject: [PATCH 12/18] Let YoutubeDLHandler (transparent gzip) handle HTTPS URLs as well (Needed for #579) --- youtube_dl/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 4dcf18991..44f939053 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -506,3 +506,6 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler): resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code) resp.msg = old_resp.msg return resp + + https_request = http_request + https_response = http_response From b4cd069d5e3b8dcacd201efd7d31e9fdd99452ea Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Fri, 7 Dec 2012 00:40:13 +0100 Subject: [PATCH 13/18] Better error reporting for SoundCloud IE --- youtube_dl/InfoExtractors.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index 1857360d4..334eb59d8 100644 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -2825,9 +2825,10 @@ class SoundcloudIE(InfoExtractor): self.report_webpage('%s/%s' % (uploader, slug_title)) - request = compat_urllib_request.Request('http://soundcloud.com/%s/%s' % (uploader, slug_title)) + url = 'https://soundcloud.com/%s/%s' % (uploader, slug_title) + request = compat_urllib_request.Request(url) try: - webpage_bytes = compat_urllib_request.urlopen(request).read() + urlo = compat_urllib_request.urlopen(request).read() webpage = webpage_bytes.decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) @@ -2840,6 +2841,9 @@ class SoundcloudIE(InfoExtractor): if mobj: video_id = mobj.group(1) stream_token = mobj.group(2) + else: + self._downloader.trouble(u'ERROR: unable to find video ID in Soundcloud file') + return # extract unsimplified title mobj = re.search('"title":"(.*?)",', webpage) From 633b4a5ff6bf100659695f9f932dd000919a2630 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Fri, 7 Dec 2012 00:50:56 +0100 Subject: [PATCH 14/18] Mark SoundCloud IE as nonfunctional for now (#579) --- youtube_dl/InfoExtractors.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index 334eb59d8..71b6a136e 100644 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -2799,6 +2799,7 @@ class SoundcloudIE(InfoExtractor): _VALID_URL = r'^(?:https?://)?(?:www\.)?soundcloud\.com/([\w\d-]+)/([\w\d-]+)' IE_NAME = u'soundcloud' + _WORKING = False def __init__(self, downloader=None): InfoExtractor.__init__(self, downloader) From f9b2f2b955cb94053e60bdd13b5d813ff76ae59f Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Fri, 7 Dec 2012 00:57:06 +0100 Subject: [PATCH 15/18] Correct accidental rename --- youtube_dl/InfoExtractors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index 71b6a136e..14c32fe67 100644 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -2829,7 +2829,7 @@ class SoundcloudIE(InfoExtractor): url = 'https://soundcloud.com/%s/%s' % (uploader, slug_title) request = compat_urllib_request.Request(url) try: - urlo = compat_urllib_request.urlopen(request).read() + webpage_bytes = compat_urllib_request.urlopen(request).read() webpage = webpage_bytes.decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) From 8fd3afd56cde0919608f3994a04aca8a1aedcf60 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Fri, 7 Dec 2012 01:24:51 +0100 Subject: [PATCH 16/18] More work on soundcloud IE --- youtube_dl/InfoExtractors.py | 39 +++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index 14c32fe67..e7b00ae21 100644 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -2799,18 +2799,17 @@ class SoundcloudIE(InfoExtractor): _VALID_URL = r'^(?:https?://)?(?:www\.)?soundcloud\.com/([\w\d-]+)/([\w\d-]+)' IE_NAME = u'soundcloud' - _WORKING = False def __init__(self, downloader=None): InfoExtractor.__init__(self, downloader) - def report_webpage(self, video_id): + def report_resolve(self, video_id): """Report information extraction.""" - self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id)) + self._downloader.to_screen(u'[%s] %s: Resolving id' % (self.IE_NAME, video_id)) def report_extraction(self, video_id): """Report information extraction.""" - self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id)) + self._downloader.to_screen(u'[%s] %s: Retrieving stream' % (self.IE_NAME, video_id)) def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) @@ -2824,28 +2823,36 @@ class SoundcloudIE(InfoExtractor): slug_title = mobj.group(2) simple_title = uploader + u'-' + slug_title - self.report_webpage('%s/%s' % (uploader, slug_title)) + self.report_resolve('%s/%s' % (uploader, slug_title)) - url = 'https://soundcloud.com/%s/%s' % (uploader, slug_title) - request = compat_urllib_request.Request(url) + url = 'http://soundcloud.com/%s/%s' % (uploader, slug_title) + resolv_url = 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=b45b1aa10f1ac2941910a7f0d10f8e28' + request = compat_urllib_request.Request(resolv_url) try: - webpage_bytes = compat_urllib_request.urlopen(request).read() - webpage = webpage_bytes.decode('utf-8') + info_json_bytes = compat_urllib_request.urlopen(request).read() + info_json = info_json_bytes.decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) return + info = json.loads(info_json) + video_id = info['id'] self.report_extraction('%s/%s' % (uploader, slug_title)) - # extract uid and stream token that soundcloud hands out for access - mobj = re.search('"uid":"([\w\d]+?)".*?stream_token=([\w\d]+)', webpage) - if mobj: - video_id = mobj.group(1) - stream_token = mobj.group(2) - else: - self._downloader.trouble(u'ERROR: unable to find video ID in Soundcloud file') + streams_url = 'https://api.sndcdn.com/i1/tracks/' + str(video_id) + '/streams?client_id=b45b1aa10f1ac2941910a7f0d10f8e28' + request = compat_urllib_request.Request(resolv_url) + try: + stream_json_bytes = compat_urllib_request.urlopen(request).read() + stream_json = stream_json_bytes.decode('utf-8') + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) return + streams = json.loads(stream_json) + print('\n\n\n' + repr(streams)) + assert "http_mp3_128_url" in streams + + # TODO get title etc. from info # extract unsimplified title mobj = re.search('"title":"(.*?)",', webpage) if mobj: From c7214f9a6fd5fb86668b71cbd44187e1babd26a9 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Fri, 7 Dec 2012 01:30:03 +0100 Subject: [PATCH 17/18] Use Soundcloud API (Closes #579) --- youtube_dl/InfoExtractors.py | 45 ++++++------------------------------ 1 file changed, 7 insertions(+), 38 deletions(-) diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index e7b00ae21..49976c25b 100644 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -2840,7 +2840,7 @@ class SoundcloudIE(InfoExtractor): self.report_extraction('%s/%s' % (uploader, slug_title)) streams_url = 'https://api.sndcdn.com/i1/tracks/' + str(video_id) + '/streams?client_id=b45b1aa10f1ac2941910a7f0d10f8e28' - request = compat_urllib_request.Request(resolv_url) + request = compat_urllib_request.Request(streams_url) try: stream_json_bytes = compat_urllib_request.urlopen(request).read() stream_json = stream_json_bytes.decode('utf-8') @@ -2849,47 +2849,16 @@ class SoundcloudIE(InfoExtractor): return streams = json.loads(stream_json) - print('\n\n\n' + repr(streams)) - assert "http_mp3_128_url" in streams - - # TODO get title etc. from info - # extract unsimplified title - mobj = re.search('"title":"(.*?)",', webpage) - if mobj: - title = mobj.group(1) - else: - title = simple_title - - # construct media url (with uid/token) - mediaURL = "http://media.soundcloud.com/stream/%s?stream_token=%s" - mediaURL = mediaURL % (video_id, stream_token) - - # description - description = u'No description available' - mobj = re.search('track-description-value">

(.*?)

', webpage) - if mobj: - description = mobj.group(1) - - # upload date - upload_date = None - mobj = re.search("pretty-date'>on ([\w]+ [\d]+, [\d]+ \d+:\d+)", webpage) - if mobj: - try: - upload_date = datetime.datetime.strptime(mobj.group(1), '%B %d, %Y %H:%M').strftime('%Y%m%d') - except Exception as err: - self._downloader.to_stderr(compat_str(err)) - - # for soundcloud, a request to a cross domain is required for cookies - request = compat_urllib_request.Request('http://media.soundcloud.com/crossdomain.xml', std_headers) + mediaURL = streams['http_mp3_128_url'] return [{ - 'id': video_id, + 'id': info['id'], 'url': mediaURL, - 'uploader': uploader, - 'upload_date': upload_date, - 'title': title, + 'uploader': info['user']['username'], + 'upload_date': info['created_at'], + 'title': info['title'], 'ext': u'mp3', - 'description': description + 'description': info['description'], }] From 863baa16eca7a98c38fb29c5e54f76df224f2351 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Fri, 7 Dec 2012 01:34:40 +0100 Subject: [PATCH 18/18] SoundCloud IDs have changed, fix tests --- test/test_download.py | 6 +++--- test/tests.json | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/test_download.py b/test/test_download.py index fd7c1860c..6926dd04e 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -108,7 +108,7 @@ class DownloadTest(unittest.TestCase): @_skip_unless(youtube_dl.InfoExtractors.SoundcloudIE._WORKING, "IE marked as not _WORKING") def test_Soundcloud(self): - filename = 'n6FLbx6ZzMiu.mp3' + filename = '62986583.mp3' fd = FileDownloader(self.parameters) fd.add_info_extractor(youtube_dl.InfoExtractors.SoundcloudIE()) fd.download(['http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy']) @@ -170,8 +170,8 @@ class DownloadTest(unittest.TestCase): if os.path.exists('939581.flv'): os.remove('939581.flv') # No file specified for Vimeo - if os.path.exists('n6FLbx6ZzMiu.mp3'): - os.remove('n6FLbx6ZzMiu.mp3') + if os.path.exists('62986583.mp3'): + os.remove('62986583.mp3') if os.path.exists('PracticalUnix_intro-environment.mp4'): os.remove('PracticalUnix_intro-environment.mp4') # No file specified for CollegeHumor diff --git a/test/tests.json b/test/tests.json index 2c29a07b4..cb29f2053 100644 --- a/test/tests.json +++ b/test/tests.json @@ -40,7 +40,7 @@ "name": "Soundcloud", "md5": "c1b9b9ea8bfd620b96b2628664576e1c", "url": "http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy", - "file": "n6FLbx6ZzMiu.mp3" + "file": "62986583.mp3" }, { "name": "StanfordOpenClassroom", @@ -66,4 +66,4 @@ "file": "XNDgyMDQ2NTQw_part00.flv", "md5": "ffe3f2e435663dc2d1eea34faeff5b5b" } -] \ No newline at end of file +]