2011-11-22 03:59:59 +09:00
#!/usr/bin/env python
# -*- coding: utf-8 -*-
2012-01-05 08:36:47 +09:00
__authors__ = (
2012-11-28 10:04:46 +09:00
' Ricardo Garcia Gonzalez ' ,
' Danny Colligan ' ,
' Benjamin Johnson ' ,
' Vasyl \' Vavrychuk ' ,
' Witold Baryluk ' ,
' Paweł Paprota ' ,
' Gergely Imreh ' ,
' Rogério Brito ' ,
' Philipp Hagemeister ' ,
' Sören Schulze ' ,
' Kevin Ngo ' ,
' Ori Avtalion ' ,
' shizeeg ' ,
' Filippo Valsorda ' ,
' Christian Albrecht ' ,
2012-12-16 19:16:57 +09:00
' Dave Vasilevsky ' ,
2013-01-02 02:29:43 +09:00
' Jaime Marquínez Ferrándiz ' ,
2013-01-13 00:51:20 +09:00
' Jeff Crouse ' ,
2013-02-08 19:01:09 +09:00
' Osama Khalid ' ,
2013-04-11 17:48:37 +09:00
' Michael Walter ' ,
2013-05-04 03:08:16 +09:00
' M. Yasoob Ullah Khalid ' ,
2013-05-06 04:35:39 +09:00
' Julien Fraichard ' ,
2013-05-06 04:56:38 +09:00
' Johny Mo Swag ' ,
2013-06-26 00:57:47 +09:00
' Axel Noack ' ,
2013-06-28 03:50:34 +09:00
' Albert Kim ' ,
2013-08-22 20:21:32 +09:00
' Pierre Rudloff ' ,
2013-09-05 05:09:22 +09:00
' Huarong Huo ' ,
2013-09-12 00:58:51 +09:00
' Ismael Mejía ' ,
2013-09-19 05:30:22 +09:00
' Steffan \' Ruirize \' James ' ,
2013-06-28 03:50:34 +09:00
)
2011-11-22 03:59:59 +09:00
__license__ = ' Public Domain '
2013-05-04 14:09:50 +09:00
import codecs
2012-02-27 07:53:56 +09:00
import getpass
import optparse
2011-11-22 03:59:59 +09:00
import os
2013-07-02 01:52:19 +09:00
import random
2011-11-22 03:59:59 +09:00
import re
2012-02-27 07:53:56 +09:00
import shlex
2011-11-22 03:59:59 +09:00
import socket
import subprocess
import sys
import warnings
2012-12-27 09:45:40 +09:00
import platform
2011-11-22 03:59:59 +09:00
2013-08-28 19:57:10 +09:00
2012-11-28 11:34:40 +09:00
from . utils import *
2012-12-31 03:49:14 +09:00
from . update import update_self
2012-12-31 02:22:36 +09:00
from . version import __version__
2012-11-28 11:34:40 +09:00
from . FileDownloader import *
2013-06-24 05:42:59 +09:00
from . extractor import gen_extractors
2013-06-19 05:14:21 +09:00
from . YoutubeDL import YoutubeDL
2012-11-28 11:34:40 +09:00
from . PostProcessor import *
2011-11-22 03:59:59 +09:00
2013-04-23 06:05:05 +09:00
def parseOpts ( overrideArguments = None ) :
2012-11-28 10:04:46 +09:00
def _readOptions ( filename_bytes ) :
try :
optionf = open ( filename_bytes )
except IOError :
return [ ] # silently skip if file is not present
try :
res = [ ]
for l in optionf :
res + = shlex . split ( l , comments = True )
finally :
optionf . close ( )
return res
def _format_option_string ( option ) :
''' ( ' -o ' , ' --option ' ) -> -o, --format METAVAR '''
opts = [ ]
if option . _short_opts :
opts . append ( option . _short_opts [ 0 ] )
if option . _long_opts :
opts . append ( option . _long_opts [ 0 ] )
if len ( opts ) > 1 :
opts . insert ( 1 , ' , ' )
if option . takes_value ( ) : opts . append ( ' %s ' % option . metavar )
return " " . join ( opts )
2013-09-06 23:26:22 +09:00
def _comma_separated_values_options_callback ( option , opt_str , value , parser ) :
setattr ( parser . values , option . dest , value . split ( ' , ' ) )
2012-11-28 10:04:46 +09:00
def _find_term_columns ( ) :
columns = os . environ . get ( ' COLUMNS ' , None )
if columns :
return int ( columns )
try :
sp = subprocess . Popen ( [ ' stty ' , ' size ' ] , stdout = subprocess . PIPE , stderr = subprocess . PIPE )
out , err = sp . communicate ( )
return int ( out . split ( ) [ 1 ] )
except :
pass
return None
2013-07-20 19:49:24 +09:00
def _hide_login_info ( opts ) :
opts = list ( opts )
for private_opt in [ ' -p ' , ' --password ' , ' -u ' , ' --username ' ] :
try :
i = opts . index ( private_opt )
opts [ i + 1 ] = ' <PRIVATE> '
except ValueError :
pass
return opts
2012-11-28 10:04:46 +09:00
max_width = 80
max_help_position = 80
# No need to wrap help messages if we're on a wide console
columns = _find_term_columns ( )
if columns : max_width = columns
fmt = optparse . IndentedHelpFormatter ( width = max_width , max_help_position = max_help_position )
fmt . format_option_strings = _format_option_string
kw = {
' version ' : __version__ ,
' formatter ' : fmt ,
' usage ' : ' % prog [options] url [url...] ' ,
' conflict_handler ' : ' resolve ' ,
}
parser = optparse . OptionParser ( * * kw )
# option groups
general = optparse . OptionGroup ( parser , ' General Options ' )
selection = optparse . OptionGroup ( parser , ' Video Selection ' )
authentication = optparse . OptionGroup ( parser , ' Authentication Options ' )
video_format = optparse . OptionGroup ( parser , ' Video Format Options ' )
2013-08-08 16:53:25 +09:00
subtitles = optparse . OptionGroup ( parser , ' Subtitle Options ' )
2013-07-01 21:53:25 +09:00
downloader = optparse . OptionGroup ( parser , ' Download Options ' )
2012-11-28 10:04:46 +09:00
postproc = optparse . OptionGroup ( parser , ' Post-processing Options ' )
filesystem = optparse . OptionGroup ( parser , ' Filesystem Options ' )
verbosity = optparse . OptionGroup ( parser , ' Verbosity / Simulation Options ' )
general . add_option ( ' -h ' , ' --help ' ,
action = ' help ' , help = ' print this help text and exit ' )
general . add_option ( ' -v ' , ' --version ' ,
action = ' version ' , help = ' print program version and exit ' )
general . add_option ( ' -U ' , ' --update ' ,
2013-07-18 19:53:24 +09:00
action = ' store_true ' , dest = ' update_self ' , help = ' update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed) ' )
2012-11-28 10:04:46 +09:00
general . add_option ( ' -i ' , ' --ignore-errors ' ,
2013-09-18 20:23:04 +09:00
action = ' store_true ' , dest = ' ignoreerrors ' , help = ' continue on download errors, for example to to skip unavailable videos in a playlist ' , default = False )
2012-11-28 10:04:46 +09:00
general . add_option ( ' --dump-user-agent ' ,
action = ' store_true ' , dest = ' dump_user_agent ' ,
help = ' display the current browser identification ' , default = False )
general . add_option ( ' --user-agent ' ,
dest = ' user_agent ' , help = ' specify a custom user agent ' , metavar = ' UA ' )
2013-04-24 20:56:04 +09:00
general . add_option ( ' --referer ' ,
2013-04-27 02:25:17 +09:00
dest = ' referer ' , help = ' specify a custom referer, use if the video access is restricted to one domain ' ,
metavar = ' REF ' , default = None )
2012-11-28 10:04:46 +09:00
general . add_option ( ' --list-extractors ' ,
action = ' store_true ' , dest = ' list_extractors ' ,
help = ' List all supported extractors and the URLs they would handle ' , default = False )
2013-07-02 01:59:29 +09:00
general . add_option ( ' --extractor-descriptions ' ,
2013-07-02 01:52:19 +09:00
action = ' store_true ' , dest = ' list_extractor_descriptions ' ,
help = ' Output descriptions of all supported extractors ' , default = False )
2013-05-01 01:27:54 +09:00
general . add_option ( ' --proxy ' , dest = ' proxy ' , default = None , help = ' Use the specified HTTP/HTTPS proxy ' , metavar = ' URL ' )
2013-05-04 19:19:02 +09:00
general . add_option ( ' --no-check-certificate ' , action = ' store_true ' , dest = ' no_check_certificate ' , default = False , help = ' Suppress HTTPS certificate validation. ' )
2013-09-25 04:26:10 +09:00
general . add_option (
2013-10-02 15:41:03 +09:00
' --cache-dir ' , dest = ' cachedir ' , default = get_cachedir ( ) ,
2013-10-02 21:37:19 +09:00
help = ' Location in the filesystem where youtube-dl can store downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . ' )
2013-09-25 04:26:10 +09:00
general . add_option (
' --no-cache-dir ' , action = ' store_const ' , const = None , dest = ' cachedir ' ,
help = ' Disable filesystem caching ' )
2013-07-01 21:53:25 +09:00
2012-11-28 10:04:46 +09:00
selection . add_option ( ' --playlist-start ' ,
dest = ' playliststart ' , metavar = ' NUMBER ' , help = ' playlist video to start at (default is %d efault) ' , default = 1 )
selection . add_option ( ' --playlist-end ' ,
dest = ' playlistend ' , metavar = ' NUMBER ' , help = ' playlist video to end at (default is last) ' , default = - 1 )
selection . add_option ( ' --match-title ' , dest = ' matchtitle ' , metavar = ' REGEX ' , help = ' download only matching titles (regex or caseless sub-string) ' )
selection . add_option ( ' --reject-title ' , dest = ' rejecttitle ' , metavar = ' REGEX ' , help = ' skip download for matching titles (regex or caseless sub-string) ' )
selection . add_option ( ' --max-downloads ' , metavar = ' NUMBER ' , dest = ' max_downloads ' , help = ' Abort after downloading NUMBER files ' , default = None )
2013-02-02 22:44:22 +09:00
selection . add_option ( ' --min-filesize ' , metavar = ' SIZE ' , dest = ' min_filesize ' , help = " Do not download any videos smaller than SIZE (e.g. 50k or 44.6m) " , default = None )
selection . add_option ( ' --max-filesize ' , metavar = ' SIZE ' , dest = ' max_filesize ' , help = " Do not download any videos larger than SIZE (e.g. 50k or 44.6m) " , default = None )
2013-04-27 21:01:55 +09:00
selection . add_option ( ' --date ' , metavar = ' DATE ' , dest = ' date ' , help = ' download only videos uploaded in this date ' , default = None )
selection . add_option ( ' --datebefore ' , metavar = ' DATE ' , dest = ' datebefore ' , help = ' download only videos uploaded before this date ' , default = None )
selection . add_option ( ' --dateafter ' , metavar = ' DATE ' , dest = ' dateafter ' , help = ' download only videos uploaded after this date ' , default = None )
2013-10-01 05:26:25 +09:00
selection . add_option ( ' --no-playlist ' , action = ' store_true ' , dest = ' noplaylist ' , help = ' download only the currently playing video ' , default = False )
2013-01-22 14:50:42 +09:00
2012-11-28 10:04:46 +09:00
authentication . add_option ( ' -u ' , ' --username ' ,
dest = ' username ' , metavar = ' USERNAME ' , help = ' account username ' )
authentication . add_option ( ' -p ' , ' --password ' ,
dest = ' password ' , metavar = ' PASSWORD ' , help = ' account password ' )
authentication . add_option ( ' -n ' , ' --netrc ' ,
action = ' store_true ' , dest = ' usenetrc ' , help = ' use .netrc authentication data ' , default = False )
2013-06-26 05:22:32 +09:00
authentication . add_option ( ' --video-password ' ,
dest = ' videopassword ' , metavar = ' PASSWORD ' , help = ' video password (vimeo only) ' )
2012-11-28 10:04:46 +09:00
video_format . add_option ( ' -f ' , ' --format ' ,
2013-04-23 17:33:54 +09:00
action = ' store ' , dest = ' format ' , metavar = ' FORMAT ' ,
2013-08-31 09:51:50 +09:00
help = ' video format code, specifiy the order of preference using slashes: " -f 22/17/18 " . " -f mp4 " and " -f flv " are also supported ' )
2012-11-28 10:04:46 +09:00
video_format . add_option ( ' --all-formats ' ,
action = ' store_const ' , dest = ' format ' , help = ' download all available video formats ' , const = ' all ' )
video_format . add_option ( ' --prefer-free-formats ' ,
action = ' store_true ' , dest = ' prefer_free_formats ' , default = False , help = ' prefer free video formats unless a specific one is requested ' )
video_format . add_option ( ' --max-quality ' ,
action = ' store ' , dest = ' format_limit ' , metavar = ' FORMAT ' , help = ' highest quality format to download ' )
video_format . add_option ( ' -F ' , ' --list-formats ' ,
action = ' store_true ' , dest = ' listformats ' , help = ' list all available formats (currently youtube only) ' )
2013-08-08 16:53:25 +09:00
subtitles . add_option ( ' --write-sub ' , ' --write-srt ' ,
2012-11-28 10:04:46 +09:00
action = ' store_true ' , dest = ' writesubtitles ' ,
2013-08-08 01:59:11 +09:00
help = ' write subtitle file ' , default = False )
2013-08-08 16:53:25 +09:00
subtitles . add_option ( ' --write-auto-sub ' , ' --write-automatic-sub ' ,
2013-06-26 06:45:16 +09:00
action = ' store_true ' , dest = ' writeautomaticsub ' ,
2013-08-08 01:59:11 +09:00
help = ' write automatic subtitle file (youtube only) ' , default = False )
2013-08-08 16:53:25 +09:00
subtitles . add_option ( ' --all-subs ' ,
2013-02-22 10:52:55 +09:00
action = ' store_true ' , dest = ' allsubtitles ' ,
2013-08-08 01:59:11 +09:00
help = ' downloads all the available subtitles of the video ' , default = False )
2013-08-08 16:53:25 +09:00
subtitles . add_option ( ' --list-subs ' ,
2013-02-22 12:50:05 +09:00
action = ' store_true ' , dest = ' listsubtitles ' ,
2013-08-08 01:59:11 +09:00
help = ' lists all available subtitles for the video ' , default = False )
2013-08-08 16:53:25 +09:00
subtitles . add_option ( ' --sub-format ' ,
2013-06-23 19:59:20 +09:00
action = ' store ' , dest = ' subtitlesformat ' , metavar = ' FORMAT ' ,
2013-08-08 01:59:11 +09:00
help = ' subtitle format (default=srt) ([sbv/vtt] youtube only) ' , default = ' srt ' )
2013-09-06 23:26:22 +09:00
subtitles . add_option ( ' --sub-lang ' , ' --sub-langs ' , ' --srt-lang ' ,
action = ' callback ' , dest = ' subtitleslangs ' , metavar = ' LANGS ' , type = ' str ' ,
default = [ ] , callback = _comma_separated_values_options_callback ,
help = ' languages of the subtitles to download (optional) separated by commas, use IETF language tags like \' en,pt \' ' )
2012-11-28 10:04:46 +09:00
2013-07-01 21:53:25 +09:00
downloader . add_option ( ' -r ' , ' --rate-limit ' ,
dest = ' ratelimit ' , metavar = ' LIMIT ' , help = ' maximum download rate (e.g. 50k or 44.6m) ' )
downloader . add_option ( ' -R ' , ' --retries ' ,
dest = ' retries ' , metavar = ' RETRIES ' , help = ' number of retries (default is %d efault) ' , default = 10 )
downloader . add_option ( ' --buffer-size ' ,
dest = ' buffersize ' , metavar = ' SIZE ' , help = ' size of download buffer (e.g. 1024 or 16k) (default is %d efault) ' , default = " 1024 " )
downloader . add_option ( ' --no-resize-buffer ' ,
action = ' store_true ' , dest = ' noresizebuffer ' ,
help = ' do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE. ' , default = False )
downloader . add_option ( ' --test ' , action = ' store_true ' , dest = ' test ' , default = False , help = optparse . SUPPRESS_HELP )
2012-11-28 10:04:46 +09:00
verbosity . add_option ( ' -q ' , ' --quiet ' ,
action = ' store_true ' , dest = ' quiet ' , help = ' activates quiet mode ' , default = False )
verbosity . add_option ( ' -s ' , ' --simulate ' ,
action = ' store_true ' , dest = ' simulate ' , help = ' do not download the video and do not write anything to disk ' , default = False )
verbosity . add_option ( ' --skip-download ' ,
action = ' store_true ' , dest = ' skip_download ' , help = ' do not download the video ' , default = False )
verbosity . add_option ( ' -g ' , ' --get-url ' ,
action = ' store_true ' , dest = ' geturl ' , help = ' simulate, quiet but print URL ' , default = False )
verbosity . add_option ( ' -e ' , ' --get-title ' ,
action = ' store_true ' , dest = ' gettitle ' , help = ' simulate, quiet but print title ' , default = False )
2013-05-06 14:30:07 +09:00
verbosity . add_option ( ' --get-id ' ,
action = ' store_true ' , dest = ' getid ' , help = ' simulate, quiet but print id ' , default = False )
2012-11-28 10:04:46 +09:00
verbosity . add_option ( ' --get-thumbnail ' ,
action = ' store_true ' , dest = ' getthumbnail ' ,
help = ' simulate, quiet but print thumbnail URL ' , default = False )
verbosity . add_option ( ' --get-description ' ,
action = ' store_true ' , dest = ' getdescription ' ,
help = ' simulate, quiet but print video description ' , default = False )
verbosity . add_option ( ' --get-filename ' ,
action = ' store_true ' , dest = ' getfilename ' ,
help = ' simulate, quiet but print output filename ' , default = False )
verbosity . add_option ( ' --get-format ' ,
action = ' store_true ' , dest = ' getformat ' ,
help = ' simulate, quiet but print output format ' , default = False )
2013-02-13 10:02:31 +09:00
verbosity . add_option ( ' --newline ' ,
2013-02-19 02:52:06 +09:00
action = ' store_true ' , dest = ' progress_with_newline ' , help = ' output progress bar as new lines ' , default = False )
2012-11-28 10:04:46 +09:00
verbosity . add_option ( ' --no-progress ' ,
action = ' store_true ' , dest = ' noprogress ' , help = ' do not print progress bar ' , default = False )
verbosity . add_option ( ' --console-title ' ,
action = ' store_true ' , dest = ' consoletitle ' ,
help = ' display progress in console titlebar ' , default = False )
verbosity . add_option ( ' -v ' , ' --verbose ' ,
action = ' store_true ' , dest = ' verbose ' , help = ' print various debugging information ' , default = False )
2013-04-12 01:31:35 +09:00
verbosity . add_option ( ' --dump-intermediate-pages ' ,
action = ' store_true ' , dest = ' dump_intermediate_pages ' , default = False ,
help = ' print downloaded pages to debug problems(very verbose) ' )
2013-09-22 17:30:02 +09:00
verbosity . add_option ( ' --youtube-print-sig-code ' ,
action = ' store_true ' , dest = ' youtube_print_sig_code ' , default = False ,
help = optparse . SUPPRESS_HELP )
2012-11-28 10:04:46 +09:00
filesystem . add_option ( ' -t ' , ' --title ' ,
2013-04-28 23:26:11 +09:00
action = ' store_true ' , dest = ' usetitle ' , help = ' use title in file name (default) ' , default = False )
2012-11-28 10:04:46 +09:00
filesystem . add_option ( ' --id ' ,
2013-04-28 23:26:11 +09:00
action = ' store_true ' , dest = ' useid ' , help = ' use only video ID in file name ' , default = False )
2012-11-28 10:04:46 +09:00
filesystem . add_option ( ' -l ' , ' --literal ' ,
action = ' store_true ' , dest = ' usetitle ' , help = ' [deprecated] alias of --title ' , default = False )
filesystem . add_option ( ' -A ' , ' --auto-number ' ,
action = ' store_true ' , dest = ' autonumber ' ,
help = ' number downloaded files starting from 00000 ' , default = False )
filesystem . add_option ( ' -o ' , ' --output ' ,
2013-04-22 17:06:07 +09:00
dest = ' outtmpl ' , metavar = ' TEMPLATE ' ,
help = ( ' output filename template. Use %(title)s to get the title, '
' %(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
' %(autonumber)s to get an automatically incremented number, '
' %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), '
' %(extractor)s for the provider (youtube, metacafe, etc), '
' %(id)s for the video id , %(playlist)s for the playlist the video is in, '
' %(playlist_index)s for the position in the playlist and %% for a literal percent. '
' Use - to output to stdout. Can also be used to download to a different directory, '
' for example with -o \' /my/downloads/ %(uploader)s / %(title)s - %(id)s . %(ext)s \' . ' ) )
2013-04-02 13:40:07 +09:00
filesystem . add_option ( ' --autonumber-size ' ,
dest = ' autonumber_size ' , metavar = ' NUMBER ' ,
help = ' Specifies the number of digits in %(autonumber)s when it is present in output filename template or --autonumber option is given ' )
2012-11-28 10:04:46 +09:00
filesystem . add_option ( ' --restrict-filenames ' ,
action = ' store_true ' , dest = ' restrictfilenames ' ,
help = ' Restrict filenames to only ASCII characters, and avoid " & " and spaces in filenames ' , default = False )
filesystem . add_option ( ' -a ' , ' --batch-file ' ,
dest = ' batchfile ' , metavar = ' FILE ' , help = ' file containing URLs to download ( \' - \' for stdin) ' )
filesystem . add_option ( ' -w ' , ' --no-overwrites ' ,
action = ' store_true ' , dest = ' nooverwrites ' , help = ' do not overwrite files ' , default = False )
filesystem . add_option ( ' -c ' , ' --continue ' ,
action = ' store_true ' , dest = ' continue_dl ' , help = ' resume partially downloaded files ' , default = True )
filesystem . add_option ( ' --no-continue ' ,
action = ' store_false ' , dest = ' continue_dl ' ,
help = ' do not resume partially downloaded files (restart from beginning) ' )
filesystem . add_option ( ' --cookies ' ,
dest = ' cookiefile ' , metavar = ' FILE ' , help = ' file to read cookies from and dump cookie jar in ' )
filesystem . add_option ( ' --no-part ' ,
action = ' store_true ' , dest = ' nopart ' , help = ' do not use .part files ' , default = False )
filesystem . add_option ( ' --no-mtime ' ,
action = ' store_false ' , dest = ' updatetime ' ,
help = ' do not use the Last-modified header to set the file modification time ' , default = True )
filesystem . add_option ( ' --write-description ' ,
action = ' store_true ' , dest = ' writedescription ' ,
help = ' write video description to a .description file ' , default = False )
filesystem . add_option ( ' --write-info-json ' ,
action = ' store_true ' , dest = ' writeinfojson ' ,
help = ' write video metadata to a .info.json file ' , default = False )
2013-05-01 21:04:33 +09:00
filesystem . add_option ( ' --write-thumbnail ' ,
action = ' store_true ' , dest = ' writethumbnail ' ,
help = ' write thumbnail image to disk ' , default = False )
2012-11-28 10:04:46 +09:00
postproc . add_option ( ' -x ' , ' --extract-audio ' , action = ' store_true ' , dest = ' extractaudio ' , default = False ,
help = ' convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe) ' )
postproc . add_option ( ' --audio-format ' , metavar = ' FORMAT ' , dest = ' audioformat ' , default = ' best ' ,
2013-01-11 03:15:04 +09:00
help = ' " best " , " aac " , " vorbis " , " mp3 " , " m4a " , " opus " , or " wav " ; best by default ' )
2012-11-28 10:04:46 +09:00
postproc . add_option ( ' --audio-quality ' , metavar = ' QUALITY ' , dest = ' audioquality ' , default = ' 5 ' ,
help = ' ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5) ' )
2013-01-12 23:07:59 +09:00
postproc . add_option ( ' --recode-video ' , metavar = ' FORMAT ' , dest = ' recodevideo ' , default = None ,
help = ' Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm) ' )
2012-11-28 10:04:46 +09:00
postproc . add_option ( ' -k ' , ' --keep-video ' , action = ' store_true ' , dest = ' keepvideo ' , default = False ,
help = ' keeps the video file on disk after the post-processing; the video is erased by default ' )
2012-12-24 02:36:48 +09:00
postproc . add_option ( ' --no-post-overwrites ' , action = ' store_true ' , dest = ' nopostoverwrites ' , default = False ,
help = ' do not overwrite post-processed files; the post-processed files are overwritten by default ' )
2013-07-20 19:48:57 +09:00
postproc . add_option ( ' --embed-subs ' , action = ' store_true ' , dest = ' embedsubtitles ' , default = False ,
help = ' embed subtitles in the video (only for mp4 videos) ' )
2012-11-28 10:04:46 +09:00
parser . add_option_group ( general )
parser . add_option_group ( selection )
2013-07-01 21:53:25 +09:00
parser . add_option_group ( downloader )
2012-11-28 10:04:46 +09:00
parser . add_option_group ( filesystem )
parser . add_option_group ( verbosity )
parser . add_option_group ( video_format )
2013-08-08 16:53:25 +09:00
parser . add_option_group ( subtitles )
2012-11-28 10:04:46 +09:00
parser . add_option_group ( authentication )
parser . add_option_group ( postproc )
2013-04-23 06:05:05 +09:00
if overrideArguments is not None :
opts , args = parser . parse_args ( overrideArguments )
if opts . verbose :
2013-09-16 13:55:33 +09:00
write_string ( u ' [debug] Override config: ' + repr ( overrideArguments ) + ' \n ' )
2012-11-28 10:04:46 +09:00
else :
2013-04-23 06:05:05 +09:00
xdg_config_home = os . environ . get ( ' XDG_CONFIG_HOME ' )
if xdg_config_home :
2013-09-30 11:50:46 +09:00
userConfFile = os . path . join ( xdg_config_home , ' youtube-dl ' , ' config ' )
if not os . path . isfile ( userConfFile ) :
userConfFile = os . path . join ( xdg_config_home , ' youtube-dl.conf ' )
2013-04-23 06:05:05 +09:00
else :
2013-09-30 11:50:46 +09:00
userConfFile = os . path . join ( os . path . expanduser ( ' ~ ' ) , ' .config ' , ' youtube-dl ' , ' config ' )
if not os . path . isfile ( userConfFile ) :
userConfFile = os . path . join ( os . path . expanduser ( ' ~ ' ) , ' .config ' , ' youtube-dl.conf ' )
2013-04-23 06:05:05 +09:00
systemConf = _readOptions ( ' /etc/youtube-dl.conf ' )
userConf = _readOptions ( userConfFile )
2013-08-08 16:53:25 +09:00
commandLineConf = sys . argv [ 1 : ]
2013-04-23 06:05:05 +09:00
argv = systemConf + userConf + commandLineConf
opts , args = parser . parse_args ( argv )
2013-04-23 06:15:05 +09:00
if opts . verbose :
2013-09-16 13:55:33 +09:00
write_string ( u ' [debug] System config: ' + repr ( _hide_login_info ( systemConf ) ) + ' \n ' )
write_string ( u ' [debug] User config: ' + repr ( _hide_login_info ( userConf ) ) + ' \n ' )
write_string ( u ' [debug] Command-line args: ' + repr ( _hide_login_info ( commandLineConf ) ) + ' \n ' )
2013-03-12 08:10:05 +09:00
2012-11-28 10:04:46 +09:00
return parser , opts , args
2011-11-22 03:59:59 +09:00
2013-04-17 02:26:48 +09:00
def _real_main ( argv = None ) :
2013-05-04 14:09:50 +09:00
# Compatibility fixes for Windows
if sys . platform == ' win32 ' :
# https://github.com/rg3/youtube-dl/issues/820
codecs . register ( lambda name : codecs . lookup ( ' utf-8 ' ) if name == ' cp65001 ' else None )
2013-04-17 02:26:48 +09:00
parser , opts , args = parseOpts ( argv )
2012-11-28 10:04:46 +09:00
# Open appropriate CookieJar
if opts . cookiefile is None :
jar = compat_cookiejar . CookieJar ( )
else :
try :
jar = compat_cookiejar . MozillaCookieJar ( opts . cookiefile )
2013-02-02 21:51:54 +09:00
if os . access ( opts . cookiefile , os . R_OK ) :
2012-11-28 10:04:46 +09:00
jar . load ( )
except ( IOError , OSError ) as err :
2013-02-02 21:51:54 +09:00
if opts . verbose :
traceback . print_exc ( )
2013-09-16 13:55:33 +09:00
write_string ( u ' ERROR: unable to open cookie file \n ' )
2013-02-02 21:51:54 +09:00
sys . exit ( 101 )
2012-11-28 10:04:46 +09:00
# Set user agent
if opts . user_agent is not None :
std_headers [ ' User-Agent ' ] = opts . user_agent
2013-08-08 16:53:25 +09:00
2013-04-24 20:56:04 +09:00
# Set referer
if opts . referer is not None :
std_headers [ ' Referer ' ] = opts . referer
2012-11-28 10:04:46 +09:00
# Dump user agent
if opts . dump_user_agent :
2013-06-24 22:57:53 +09:00
compat_print ( std_headers [ ' User-Agent ' ] )
2012-11-28 10:04:46 +09:00
sys . exit ( 0 )
# Batch file verification
batchurls = [ ]
if opts . batchfile is not None :
try :
if opts . batchfile == ' - ' :
batchfd = sys . stdin
else :
batchfd = open ( opts . batchfile , ' r ' )
batchurls = batchfd . readlines ( )
batchurls = [ x . strip ( ) for x in batchurls ]
batchurls = [ x for x in batchurls if len ( x ) > 0 and not re . search ( r ' ^[#/;] ' , x ) ]
2013-07-31 06:11:44 +09:00
if opts . verbose :
2013-09-16 13:55:33 +09:00
write_string ( u ' [debug] Batch file urls: ' + repr ( batchurls ) + u ' \n ' )
2012-11-28 10:04:46 +09:00
except IOError :
sys . exit ( u ' ERROR: batch file could not be read ' )
all_urls = batchurls + args
all_urls = [ url . strip ( ) for url in all_urls ]
# General configuration
cookie_processor = compat_urllib_request . HTTPCookieProcessor ( jar )
2013-06-10 06:43:18 +09:00
if opts . proxy is not None :
if opts . proxy == ' ' :
proxies = { }
else :
proxies = { ' http ' : opts . proxy , ' https ' : opts . proxy }
2013-05-01 00:57:13 +09:00
else :
proxies = compat_urllib_request . getproxies ( )
# Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
if ' http ' in proxies and ' https ' not in proxies :
proxies [ ' https ' ] = proxies [ ' http ' ]
2013-04-28 22:41:05 +09:00
proxy_handler = compat_urllib_request . ProxyHandler ( proxies )
2013-05-04 19:19:02 +09:00
https_handler = make_HTTPS_handler ( opts )
2013-05-04 03:07:35 +09:00
opener = compat_urllib_request . build_opener ( https_handler , proxy_handler , cookie_processor , YoutubeDLHandler ( ) )
2013-08-28 06:31:24 +09:00
# Delete the default user-agent header, which would otherwise apply in
# cases where our custom HTTP handler doesn't come into play
# (See https://github.com/rg3/youtube-dl/issues/1309 for details)
opener . addheaders = [ ]
2012-11-28 10:04:46 +09:00
compat_urllib_request . install_opener ( opener )
socket . setdefaulttimeout ( 300 ) # 5 minutes should be enough (famous last words)
extractors = gen_extractors ( )
if opts . list_extractors :
2013-07-02 01:21:29 +09:00
for ie in sorted ( extractors , key = lambda ie : ie . IE_NAME . lower ( ) ) :
2013-06-24 22:57:53 +09:00
compat_print ( ie . IE_NAME + ( ' (CURRENTLY BROKEN) ' if not ie . _WORKING else ' ' ) )
2012-12-20 22:18:23 +09:00
matchedUrls = [ url for url in all_urls if ie . suitable ( url ) ]
all_urls = [ url for url in all_urls if url not in matchedUrls ]
2012-11-28 10:04:46 +09:00
for mu in matchedUrls :
2013-06-24 22:57:53 +09:00
compat_print ( u ' ' + mu )
2012-11-28 10:04:46 +09:00
sys . exit ( 0 )
2013-07-02 01:52:19 +09:00
if opts . list_extractor_descriptions :
for ie in sorted ( extractors , key = lambda ie : ie . IE_NAME . lower ( ) ) :
if not ie . _WORKING :
continue
desc = getattr ( ie , ' IE_DESC ' , ie . IE_NAME )
if hasattr ( ie , ' SEARCH_KEY ' ) :
_SEARCHES = ( u ' cute kittens ' , u ' slithering pythons ' , u ' falling cat ' , u ' angry poodle ' , u ' purple fish ' , u ' running tortoise ' )
_COUNTS = ( u ' ' , u ' 5 ' , u ' 10 ' , u ' all ' )
desc + = u ' (Example: " %s %s : %s " ) ' % ( ie . SEARCH_KEY , random . choice ( _COUNTS ) , random . choice ( _SEARCHES ) )
compat_print ( desc )
sys . exit ( 0 )
2012-11-28 10:04:46 +09:00
# Conflicting, missing and erroneous options
if opts . usenetrc and ( opts . username is not None or opts . password is not None ) :
parser . error ( u ' using .netrc conflicts with giving username/password ' )
if opts . password is not None and opts . username is None :
2013-06-26 05:22:32 +09:00
parser . error ( u ' account username missing \n ' )
2012-11-28 10:04:46 +09:00
if opts . outtmpl is not None and ( opts . usetitle or opts . autonumber or opts . useid ) :
parser . error ( u ' using output template conflicts with using title, video ID or auto number ' )
if opts . usetitle and opts . useid :
parser . error ( u ' using title conflicts with using video ID ' )
if opts . username is not None and opts . password is None :
opts . password = getpass . getpass ( u ' Type account password and press return: ' )
if opts . ratelimit is not None :
numeric_limit = FileDownloader . parse_bytes ( opts . ratelimit )
if numeric_limit is None :
parser . error ( u ' invalid rate limit specified ' )
opts . ratelimit = numeric_limit
2013-01-22 14:50:42 +09:00
if opts . min_filesize is not None :
numeric_limit = FileDownloader . parse_bytes ( opts . min_filesize )
if numeric_limit is None :
parser . error ( u ' invalid min_filesize specified ' )
opts . min_filesize = numeric_limit
if opts . max_filesize is not None :
numeric_limit = FileDownloader . parse_bytes ( opts . max_filesize )
if numeric_limit is None :
parser . error ( u ' invalid max_filesize specified ' )
opts . max_filesize = numeric_limit
2012-11-28 10:04:46 +09:00
if opts . retries is not None :
try :
opts . retries = int ( opts . retries )
except ( TypeError , ValueError ) as err :
parser . error ( u ' invalid retry count specified ' )
if opts . buffersize is not None :
numeric_buffersize = FileDownloader . parse_bytes ( opts . buffersize )
if numeric_buffersize is None :
parser . error ( u ' invalid buffer size specified ' )
opts . buffersize = numeric_buffersize
try :
opts . playliststart = int ( opts . playliststart )
if opts . playliststart < = 0 :
raise ValueError ( u ' Playlist start must be positive ' )
except ( TypeError , ValueError ) as err :
parser . error ( u ' invalid playlist start number specified ' )
try :
opts . playlistend = int ( opts . playlistend )
if opts . playlistend != - 1 and ( opts . playlistend < = 0 or opts . playlistend < opts . playliststart ) :
raise ValueError ( u ' Playlist end must be greater than playlist start ' )
except ( TypeError , ValueError ) as err :
parser . error ( u ' invalid playlist end number specified ' )
if opts . extractaudio :
2013-01-11 03:15:04 +09:00
if opts . audioformat not in [ ' best ' , ' aac ' , ' mp3 ' , ' m4a ' , ' opus ' , ' vorbis ' , ' wav ' ] :
2012-11-28 10:04:46 +09:00
parser . error ( u ' invalid audio format specified ' )
if opts . audioquality :
opts . audioquality = opts . audioquality . strip ( ' k ' ) . strip ( ' K ' )
if not opts . audioquality . isdigit ( ) :
parser . error ( u ' invalid audio quality specified ' )
2013-01-12 23:07:59 +09:00
if opts . recodevideo is not None :
if opts . recodevideo not in [ ' mp4 ' , ' flv ' , ' webm ' , ' ogg ' ] :
parser . error ( u ' invalid video recode format specified ' )
2013-04-27 21:01:55 +09:00
if opts . date is not None :
date = DateRange . day ( opts . date )
else :
date = DateRange ( opts . dateafter , opts . datebefore )
2012-11-28 10:04:46 +09:00
2013-09-14 18:14:40 +09:00
# --all-sub automatically sets --write-sub if --write-auto-sub is not given
# this was the old behaviour if only --all-sub was given.
if opts . allsubtitles and ( opts . writeautomaticsub == False ) :
opts . writesubtitles = True
2012-12-16 01:44:48 +09:00
if sys . version_info < ( 3 , ) :
# In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
2012-12-16 07:54:44 +09:00
if opts . outtmpl is not None :
opts . outtmpl = opts . outtmpl . decode ( preferredencoding ( ) )
2012-12-16 01:44:48 +09:00
outtmpl = ( ( opts . outtmpl is not None and opts . outtmpl )
or ( opts . format == ' -1 ' and opts . usetitle and u ' %(title)s - %(id)s - %(format)s . %(ext)s ' )
or ( opts . format == ' -1 ' and u ' %(id)s - %(format)s . %(ext)s ' )
or ( opts . usetitle and opts . autonumber and u ' %(autonumber)s - %(title)s - %(id)s . %(ext)s ' )
or ( opts . usetitle and u ' %(title)s - %(id)s . %(ext)s ' )
or ( opts . useid and u ' %(id)s . %(ext)s ' )
or ( opts . autonumber and u ' %(autonumber)s - %(id)s . %(ext)s ' )
2013-04-28 23:26:11 +09:00
or u ' %(title)s - %(id)s . %(ext)s ' )
2013-09-21 18:10:22 +09:00
if ' %(ext)s ' not in outtmpl and opts . extractaudio :
parser . error ( u ' Cannot download a video and extract audio into the same '
u ' file! Use " %% (ext)s " instead of %r ' %
determine_ext ( outtmpl , u ' ' ) )
2013-09-25 04:55:25 +09:00
2013-06-19 05:14:21 +09:00
# YoutubeDL
ydl = YoutubeDL ( {
2012-11-28 10:04:46 +09:00
' usenetrc ' : opts . usenetrc ,
' username ' : opts . username ,
' password ' : opts . password ,
2013-06-26 05:22:32 +09:00
' videopassword ' : opts . videopassword ,
2013-05-06 14:30:07 +09:00
' quiet ' : ( opts . quiet or opts . geturl or opts . gettitle or opts . getid or opts . getthumbnail or opts . getdescription or opts . getfilename or opts . getformat ) ,
2012-11-28 10:04:46 +09:00
' forceurl ' : opts . geturl ,
' forcetitle ' : opts . gettitle ,
2013-05-06 14:30:07 +09:00
' forceid ' : opts . getid ,
2012-11-28 10:04:46 +09:00
' forcethumbnail ' : opts . getthumbnail ,
' forcedescription ' : opts . getdescription ,
' forcefilename ' : opts . getfilename ,
' forceformat ' : opts . getformat ,
' simulate ' : opts . simulate ,
2013-05-06 14:30:07 +09:00
' skip_download ' : ( opts . skip_download or opts . simulate or opts . geturl or opts . gettitle or opts . getid or opts . getthumbnail or opts . getdescription or opts . getfilename or opts . getformat ) ,
2012-11-28 10:04:46 +09:00
' format ' : opts . format ,
' format_limit ' : opts . format_limit ,
' listformats ' : opts . listformats ,
2012-12-16 01:44:48 +09:00
' outtmpl ' : outtmpl ,
2013-04-02 13:40:07 +09:00
' autonumber_size ' : opts . autonumber_size ,
2012-11-28 10:04:46 +09:00
' restrictfilenames ' : opts . restrictfilenames ,
' ignoreerrors ' : opts . ignoreerrors ,
' ratelimit ' : opts . ratelimit ,
' nooverwrites ' : opts . nooverwrites ,
' retries ' : opts . retries ,
' buffersize ' : opts . buffersize ,
' noresizebuffer ' : opts . noresizebuffer ,
' continuedl ' : opts . continue_dl ,
' noprogress ' : opts . noprogress ,
2013-02-19 02:52:06 +09:00
' progress_with_newline ' : opts . progress_with_newline ,
2012-11-28 10:04:46 +09:00
' playliststart ' : opts . playliststart ,
' playlistend ' : opts . playlistend ,
2013-10-01 05:26:25 +09:00
' noplaylist ' : opts . noplaylist ,
2012-11-28 10:04:46 +09:00
' logtostderr ' : opts . outtmpl == ' - ' ,
' consoletitle ' : opts . consoletitle ,
' nopart ' : opts . nopart ,
' updatetime ' : opts . updatetime ,
' writedescription ' : opts . writedescription ,
' writeinfojson ' : opts . writeinfojson ,
2013-05-01 21:04:33 +09:00
' writethumbnail ' : opts . writethumbnail ,
2012-11-28 10:04:46 +09:00
' writesubtitles ' : opts . writesubtitles ,
2013-06-26 06:45:16 +09:00
' writeautomaticsub ' : opts . writeautomaticsub ,
2013-02-22 10:52:55 +09:00
' allsubtitles ' : opts . allsubtitles ,
2013-02-22 12:50:05 +09:00
' listsubtitles ' : opts . listsubtitles ,
2013-02-22 11:53:54 +09:00
' subtitlesformat ' : opts . subtitlesformat ,
2013-09-06 23:26:22 +09:00
' subtitleslangs ' : opts . subtitleslangs ,
2013-02-22 01:09:39 +09:00
' matchtitle ' : decodeOption ( opts . matchtitle ) ,
' rejecttitle ' : decodeOption ( opts . rejecttitle ) ,
2012-11-28 10:04:46 +09:00
' max_downloads ' : opts . max_downloads ,
' prefer_free_formats ' : opts . prefer_free_formats ,
' verbose ' : opts . verbose ,
2013-04-12 01:31:35 +09:00
' dump_intermediate_pages ' : opts . dump_intermediate_pages ,
2012-12-11 17:57:40 +09:00
' test ' : opts . test ,
2013-01-12 23:07:59 +09:00
' keepvideo ' : opts . keepvideo ,
2013-01-22 14:50:42 +09:00
' min_filesize ' : opts . min_filesize ,
2013-04-27 21:01:55 +09:00
' max_filesize ' : opts . max_filesize ,
2013-05-01 21:04:33 +09:00
' daterange ' : date ,
2013-09-25 04:26:10 +09:00
' cachedir ' : opts . cachedir ,
2013-09-22 17:50:12 +09:00
' youtube_print_sig_code ' : opts . youtube_print_sig_code ,
2012-11-28 10:04:46 +09:00
} )
if opts . verbose :
2013-09-16 13:55:33 +09:00
write_string ( u ' [debug] youtube-dl version ' + __version__ + u ' \n ' )
2012-12-27 09:45:40 +09:00
try :
2013-07-02 15:35:39 +09:00
sp = subprocess . Popen (
[ ' git ' , ' rev-parse ' , ' --short ' , ' HEAD ' ] ,
stdout = subprocess . PIPE , stderr = subprocess . PIPE ,
cwd = os . path . dirname ( os . path . abspath ( __file__ ) ) )
2012-12-27 09:45:40 +09:00
out , err = sp . communicate ( )
out = out . decode ( ) . strip ( )
if re . match ( ' [0-9a-f]+ ' , out ) :
2013-09-16 13:55:33 +09:00
write_string ( u ' [debug] Git HEAD: ' + out + u ' \n ' )
2012-12-27 09:45:40 +09:00
except :
2013-07-17 08:33:28 +09:00
try :
sys . exc_clear ( )
except :
pass
2013-09-16 13:55:33 +09:00
write_string ( u ' [debug] Python version %s - %s ' % ( platform . python_version ( ) , platform_name ( ) ) + u ' \n ' )
write_string ( u ' [debug] Proxy map: ' + str ( proxy_handler . proxies ) + u ' \n ' )
2012-11-28 10:04:46 +09:00
2013-06-28 06:51:06 +09:00
ydl . add_default_info_extractors ( )
2012-11-28 10:04:46 +09:00
# PostProcessors
if opts . extractaudio :
2013-06-19 05:14:21 +09:00
ydl . add_post_processor ( FFmpegExtractAudioPP ( preferredcodec = opts . audioformat , preferredquality = opts . audioquality , nopostoverwrites = opts . nopostoverwrites ) )
2013-01-12 23:07:59 +09:00
if opts . recodevideo :
2013-06-19 05:14:21 +09:00
ydl . add_post_processor ( FFmpegVideoConvertor ( preferedformat = opts . recodevideo ) )
2013-07-20 19:48:57 +09:00
if opts . embedsubtitles :
ydl . add_post_processor ( FFmpegEmbedSubtitlePP ( subtitlesformat = opts . subtitlesformat ) )
2012-11-28 10:04:46 +09:00
2013-01-13 06:10:36 +09:00
# Update version
if opts . update_self :
2013-09-29 18:17:38 +09:00
update_self ( ydl . to_screen , opts . verbose )
2013-01-13 06:10:36 +09:00
2012-11-28 10:04:46 +09:00
# Maybe do nothing
if len ( all_urls ) < 1 :
if not opts . update_self :
parser . error ( u ' you must provide at least one URL ' )
else :
sys . exit ( )
try :
2013-06-19 05:14:21 +09:00
retcode = ydl . download ( all_urls )
2012-11-28 10:04:46 +09:00
except MaxDownloadsReached :
2013-06-19 05:14:21 +09:00
ydl . to_screen ( u ' --max-download limit reached, aborting. ' )
2012-11-28 10:04:46 +09:00
retcode = 101
# Dump cookie jar if requested
if opts . cookiefile is not None :
try :
jar . save ( )
except ( IOError , OSError ) as err :
sys . exit ( u ' ERROR: unable to save cookie jar ' )
sys . exit ( retcode )
2011-11-22 03:59:59 +09:00
2013-04-17 02:26:48 +09:00
def main ( argv = None ) :
2012-11-28 10:04:46 +09:00
try :
2013-04-17 02:26:48 +09:00
_real_main ( argv )
2012-11-28 10:04:46 +09:00
except DownloadError :
sys . exit ( 1 )
except SameFileError :
sys . exit ( u ' ERROR: fixed output name but more than one file to download ' )
except KeyboardInterrupt :
sys . exit ( u ' \n ERROR: Interrupted by user ' )