[utils, etc] Kill child processes when yt-dl is killed

* derived from PR #26592, closes #26592

Authored by: Unrud

youtube_dl/YoutubeDL.py

@@ -73,6 +73,7 @@ from .utils import (
     PostProcessingError,
     preferredencoding,
     prepend_extension,
+    process_communicate_or_kill,
     register_socks_protocols,
     render_table,
     replace_extension,
@@ -2323,7 +2324,7 @@ class YoutubeDL(object):
                 ['git', 'rev-parse', '--short', 'HEAD'],
                 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                 cwd=os.path.dirname(os.path.abspath(__file__)))
-            out, err = sp.communicate()
+            out, err = process_communicate_or_kill(sp)
             out = out.decode().strip()
             if re.match('[0-9a-f]+', out):
                 self._write_string('[debug] Git HEAD: ' + out + '\n')

youtube_dl/compat.py

@@ -2890,6 +2890,7 @@ else:
     _terminal_size = collections.namedtuple('terminal_size', ['columns', 'lines'])
 
     def compat_get_terminal_size(fallback=(80, 24)):
+        from .utils import process_communicate_or_kill
         columns = compat_getenv('COLUMNS')
         if columns:
             columns = int(columns)
@@ -2906,7 +2907,7 @@ else:
                 sp = subprocess.Popen(
                     ['stty', 'size'],
                     stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-                out, err = sp.communicate()
+                out, err = process_communicate_or_kill(sp)
                 _lines, _columns = map(int, out.split())
             except Exception:
                 _columns, _lines = _terminal_size(*fallback)

youtube_dl/downloader/external.py

@@ -22,6 +22,7 @@ from ..utils import (
     handle_youtubedl_headers,
     check_executable,
     is_outdated_version,
+    process_communicate_or_kill,
 )
 
 
@@ -104,7 +105,7 @@ class ExternalFD(FileDownloader):
 
         p = subprocess.Popen(
             cmd, stderr=subprocess.PIPE)
-        _, stderr = p.communicate()
+        _, stderr = process_communicate_or_kill(p)
         if p.returncode != 0:
             self.to_stderr(stderr.decode('utf-8', 'replace'))
         return p.returncode
@@ -141,7 +142,7 @@ class CurlFD(ExternalFD):
 
         # curl writes the progress to stderr so don't capture it.
         p = subprocess.Popen(cmd)
-        p.communicate()
+        process_communicate_or_kill(p)
         return p.returncode
 
 
@@ -336,14 +337,17 @@ class FFmpegFD(ExternalFD):
         proc = subprocess.Popen(args, stdin=subprocess.PIPE, env=env)
         try:
             retval = proc.wait()
-        except KeyboardInterrupt:
+        except BaseException as e:
-            # subprocces.run would send the SIGKILL signal to ffmpeg and the
+            # subprocess.run would send the SIGKILL signal to ffmpeg and the
             # mp4 file couldn't be played, but if we ask ffmpeg to quit it
             # produces a file that is playable (this is mostly useful for live
             # streams). Note that Windows is not affected and produces playable
             # files (see https://github.com/ytdl-org/youtube-dl/issues/8300).
-            if sys.platform != 'win32':
+            if isinstance(e, KeyboardInterrupt) and sys.platform != 'win32':
-                proc.communicate(b'q')
+                process_communicate_or_kill(proc, b'q')
+            else:
+                proc.kill()
+                proc.wait()
             raise
         return retval
 

youtube_dl/downloader/rtmp.py

@@ -89,11 +89,13 @@ class RtmpFD(FileDownloader):
                                 self.to_screen('')
                             cursor_in_new_line = True
                             self.to_screen('[rtmpdump] ' + line)
-            finally:
-                proc.wait()
                 if not cursor_in_new_line:
                     self.to_screen('')
-            return proc.returncode
+                return proc.wait()
+            except BaseException:  # Including KeyboardInterrupt
+                proc.kill()
+                proc.wait()
+                raise
 
         url = info_dict['url']
         player_url = info_dict.get('player_url')

youtube_dl/extractor/openload.py

@@ -16,6 +16,7 @@ from ..utils import (
     ExtractorError,
     get_exe_version,
     is_outdated_version,
+    process_communicate_or_kill,
     std_headers,
 )
 
@@ -226,7 +227,7 @@ class PhantomJSwrapper(object):
             self.exe, '--ssl-protocol=any',
             self._TMP_FILES['script'].name
         ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        out, err = p.communicate()
+        out, err = process_communicate_or_kill(p)
         if p.returncode != 0:
             raise ExtractorError(
                 'Executing JS failed\n:' + encodeArgument(err))

youtube_dl/postprocessor/embedthumbnail.py

@@ -13,8 +13,9 @@ from ..utils import (
     encodeFilename,
     PostProcessingError,
     prepend_extension,
+    process_communicate_or_kill,
     replace_extension,
-    shell_quote
+    shell_quote,
 )
 
 
@@ -109,7 +110,7 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
                 self._downloader.to_screen('[debug] AtomicParsley command line: %s' % shell_quote(cmd))
 
             p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            stdout, stderr = p.communicate()
+            stdout, stderr = process_communicate_or_kill(p)
 
             if p.returncode != 0:
                 msg = stderr.decode('utf-8', 'replace').strip()

youtube_dl/postprocessor/ffmpeg.py

@@ -16,6 +16,7 @@ from ..utils import (
     is_outdated_version,
     PostProcessingError,
     prepend_extension,
+    process_communicate_or_kill,
     shell_quote,
     subtitles_filename,
     dfxp2srt,
@@ -180,7 +181,7 @@ class FFmpegPostProcessor(PostProcessor):
             handle = subprocess.Popen(
                 cmd, stderr=subprocess.PIPE,
                 stdout=subprocess.PIPE, stdin=subprocess.PIPE)
-            stdout_data, stderr_data = handle.communicate()
+            stdout_data, stderr_data = process_communicate_or_kill(handle)
             expected_ret = 0 if self.probe_available else 1
             if handle.wait() != expected_ret:
                 return None
@@ -228,7 +229,7 @@ class FFmpegPostProcessor(PostProcessor):
         if self._downloader.params.get('verbose', False):
             self._downloader.to_screen('[debug] ffmpeg command line: %s' % shell_quote(cmd))
         p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
-        stdout, stderr = p.communicate()
+        stdout, stderr = process_communicate_or_kill(p)
         if p.returncode != 0:
             stderr = stderr.decode('utf-8', 'replace')
             msgs = stderr.strip().split('\n')

youtube_dl/utils.py

@@ -2212,6 +2212,15 @@ def unescapeHTML(s):
         r'&([^&;]+;)', lambda m: _htmlentity_transform(m.group(1)), s)
 
 
+def process_communicate_or_kill(p, *args, **kwargs):
+    try:
+        return p.communicate(*args, **kwargs)
+    except BaseException:  # Including KeyboardInterrupt
+        p.kill()
+        p.wait()
+        raise
+
+
 def get_subprocess_encoding():
     if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
         # For subprocess calls, encode with locale encoding
@@ -3788,7 +3797,8 @@ def check_executable(exe, args=[]):
     """ Checks if the given binary is installed somewhere in PATH, and returns its name.
     args can be a list of arguments for a short output (like -version) """
     try:
-        subprocess.Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
+        process_communicate_or_kill(subprocess.Popen(
+            [exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE))
     except OSError:
         return False
     return exe
@@ -3802,10 +3812,10 @@ def get_exe_version(exe, args=['--version'],
         # STDIN should be redirected too. On UNIX-like systems, ffmpeg triggers
         # SIGTTOU if youtube-dl is run in the background.
         # See https://github.com/ytdl-org/youtube-dl/issues/955#issuecomment-209789656
-        out, _ = subprocess.Popen(
+        out, _ = process_communicate_or_kill(subprocess.Popen(
             [encodeArgument(exe)] + args,
             stdin=subprocess.PIPE,
-            stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
+            stdout=subprocess.PIPE, stderr=subprocess.STDOUT))
     except OSError:
         return False
     if isinstance(out, bytes):  # Python 2.x
@@ -5744,7 +5754,7 @@ def write_xattr(path, key, value):
                         cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
                 except EnvironmentError as e:
                     raise XAttrMetadataError(e.errno, e.strerror)
-                stdout, stderr = p.communicate()
+                stdout, stderr = process_communicate_or_kill(p)
                 stderr = stderr.decode('utf-8', 'replace')
                 if p.returncode != 0:
                     raise XAttrMetadataError(p.returncode, stderr)