From 8230a3575e1aade93403e53eea3077d5383c5515 Mon Sep 17 00:00:00 2001 From: jakeogh Date: Thu, 27 Dec 2018 17:08:20 -0700 Subject: [PATCH] modify sanitize_open() to use locked_file(), preventing silent corruption when a second youtube-dl instance is attempting to write the same file. There is still a corner case, if a .part file is being used (--no-part is not enabled), in that the .part file is closed before it's renamed to remove the .part, in that window, another process could modify the .part file before it's renamed. Using --no-part prevents this corner case. --- youtube_dl/utils.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 8cefafd79..bca590d4c 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -2051,7 +2051,8 @@ def sanitize_open(filename, open_mode): import msvcrt msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename) - stream = open(encodeFilename(filename), open_mode) + stream = locked_file(encodeFilename(filename), open_mode, block=False) + stream = stream.__enter__() return (stream, filename) except (IOError, OSError) as err: if err.errno in (errno.EACCES,): @@ -2063,7 +2064,8 @@ def sanitize_open(filename, open_mode): raise else: # An exception here should be caught in the caller - stream = open(encodeFilename(alt_filename), open_mode) + stream = locked_file(encodeFilename(filename), open_mode, block=False) + stream = stream.__enter__() return (stream, alt_filename) @@ -3258,15 +3260,18 @@ else: try: import fcntl - def _lock_file(f, exclusive): - fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH) + def _lock_file(f, exclusive, block): + if block: + fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH) + else: + fcntl.flock(f, fcntl.LOCK_EX|fcntl.LOCK_NB if exclusive else fcntl.LOCK_SH) def _unlock_file(f): fcntl.flock(f, fcntl.LOCK_UN) except ImportError: UNSUPPORTED_MSG = 'file locking is not supported on this platform' - def _lock_file(f, exclusive): + def _lock_file(f, exclusive, block): raise IOError(UNSUPPORTED_MSG) def _unlock_file(f): @@ -3274,15 +3279,16 @@ else: class locked_file(object): - def __init__(self, filename, mode, encoding=None): - assert mode in ['r', 'a', 'w'] + def __init__(self, filename, mode, block=True, encoding=None): + assert mode in ['r', 'rb', 'a', 'ab', 'w', 'wb'] self.f = io.open(filename, mode, encoding=encoding) self.mode = mode + self.block = block def __enter__(self): - exclusive = self.mode != 'r' + exclusive = self.mode not in ['r', 'rb'] try: - _lock_file(self.f, exclusive) + _lock_file(self.f, exclusive, self.block) except IOError: self.f.close() raise @@ -3303,6 +3309,8 @@ class locked_file(object): def read(self, *args): return self.f.read(*args) + def close(self, *args): + self.__exit__(self, *args, value=False, traceback=False) def get_filesystem_encoding(): encoding = sys.getfilesystemencoding()