diff options
Diffstat (limited to 'Lib/subprocess.py')
| -rw-r--r-- | Lib/subprocess.py | 481 |
1 files changed, 324 insertions, 157 deletions
diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 299f73e711..28dd691090 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -27,9 +27,10 @@ This module defines one class called Popen: class Popen(args, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, - preexec_fn=None, close_fds=False, shell=False, + preexec_fn=None, close_fds=True, shell=False, cwd=None, env=None, universal_newlines=False, - startupinfo=None, creationflags=0): + startupinfo=None, creationflags=0, + restore_signals=True, start_new_session=False, pass_fds=()): Arguments are: @@ -38,12 +39,12 @@ args should be a string, or a sequence of program arguments. The program to execute is normally the first item in the args sequence or string, but can be explicitly set by using the executable argument. -On UNIX, with shell=False (default): In this case, the Popen class +On POSIX, with shell=False (default): In this case, the Popen class uses os.execvp() to execute the child program. args should normally be a sequence. A string will be treated as a sequence with the string as the only item (the program to execute). -On UNIX, with shell=True: If args is a string, it specifies the +On POSIX, with shell=True: If args is a string, it specifies the command string to execute through the shell. If args is a sequence, the first item specifies the command string, and any additional items will be treated as additional shell arguments. @@ -72,11 +73,19 @@ parent. Additionally, stderr can be STDOUT, which indicates that the stderr data from the applications should be captured into the same file handle as for stdout. -If preexec_fn is set to a callable object, this object will be called -in the child process just before the child is executed. +On POSIX, if preexec_fn is set to a callable object, this object will be +called in the child process just before the child is executed. The use +of preexec_fn is not thread safe, using it in the presence of threads +could lead to a deadlock in the child process before the new executable +is executed. If close_fds is true, all file descriptors except 0, 1 and 2 will be -closed before the child process is executed. +closed before the child process is executed. The default for close_fds +varies by platform: Always true on POSIX. True when stdin/stdout/stderr +are None on Windows, false otherwise. + +pass_fds is an optional sequence of file descriptors to keep open between the +parent and child. Providing any pass_fds implicitly sets close_fds to true. if shell is true, the specified command will be executed through the shell. @@ -84,12 +93,20 @@ shell. If cwd is not None, the current directory will be changed to cwd before the child is executed. +On POSIX, if restore_signals is True all signals that Python sets to +SIG_IGN are restored to SIG_DFL in the child process before the exec. +Currently this includes the SIGPIPE, SIGXFZ and SIGXFSZ signals. This +parameter does nothing on Windows. + +On POSIX, if start_new_session is True, the setsid() system call will be made +in the child process prior to executing the command. + If env is not None, it defines the environment variables for the new process. If universal_newlines is true, the file objects stdout and stderr are opened as a text files, but lines may be terminated by any of '\n', -the Unix end-of-line convention, '\r', the Macintosh convention or +the Unix end-of-line convention, '\r', the old Macintosh convention or '\r\n', the Windows convention. All of these external representations are seen as '\n' by the Python program. Note: This feature is only available if Python is built with universal newline support (the @@ -110,7 +127,7 @@ call(*popenargs, **kwargs): The arguments are the same as for the Popen constructor. Example: - >>> retcode = call(["ls", "-l"]) + >>> retcode = subprocess.call(["ls", "-l"]) check_call(*popenargs, **kwargs): Run command with arguments. Wait for command to complete. If the @@ -120,7 +137,7 @@ check_call(*popenargs, **kwargs): The arguments are the same as for the Popen constructor. Example: - >>> check_call(["ls", "-l"]) + >>> subprocess.check_call(["ls", "-l"]) 0 getstatusoutput(cmd): @@ -230,7 +247,7 @@ pid returncode The child return code. A None value indicates that the process hasn't terminated yet. A negative value -N indicates that the - child was terminated by signal N (UNIX only). + child was terminated by signal N (POSIX only). Replacing older functions with the subprocess module @@ -326,6 +343,8 @@ import os import traceback import gc import signal +import builtins +import warnings import errno # Exception classes used by this module. @@ -361,22 +380,47 @@ else: import fcntl import pickle + try: + import _posixsubprocess + except ImportError: + _posixsubprocess = None + warnings.warn("The _posixsubprocess module is not being used. " + "Child process reliability may suffer if your " + "program uses threads.", RuntimeWarning) + # When select or poll has indicated that the file is writable, # we can write up to _PIPE_BUF bytes without risk of blocking. # POSIX defines PIPE_BUF as >= 512. _PIPE_BUF = getattr(select, 'PIPE_BUF', 512) + _FD_CLOEXEC = getattr(fcntl, 'FD_CLOEXEC', 1) + + def _set_cloexec(fd, cloexec): + old = fcntl.fcntl(fd, fcntl.F_GETFD) + if cloexec: + fcntl.fcntl(fd, fcntl.F_SETFD, old | _FD_CLOEXEC) + else: + fcntl.fcntl(fd, fcntl.F_SETFD, old & ~_FD_CLOEXEC) + + if _posixsubprocess: + _create_pipe = _posixsubprocess.cloexec_pipe + else: + def _create_pipe(): + fds = os.pipe() + _set_cloexec(fds[0], True) + _set_cloexec(fds[1], True) + return fds __all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput", "getoutput", "check_output", "CalledProcessError"] if mswindows: - from _subprocess import (CREATE_NEW_CONSOLE, + from _subprocess import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, STD_ERROR_HANDLE, SW_HIDE, STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW) - - __all__.extend(["CREATE_NEW_CONSOLE", + + __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP", "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE", "STD_ERROR_HANDLE", "SW_HIDE", "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW"]) @@ -547,7 +591,7 @@ def list2cmdline(seq): # Various tools for executing commands and looking at their output and status. # -# NB This only works (and is only relevant) for UNIX. +# NB This only works (and is only relevant) for POSIX. def getstatusoutput(cmd): """Return (status, output) of executing cmd in a shell. @@ -587,12 +631,17 @@ def getoutput(cmd): return getstatusoutput(cmd)[1] +_PLATFORM_DEFAULT_CLOSE_FDS = object() + + class Popen(object): def __init__(self, args, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, - preexec_fn=None, close_fds=False, shell=False, - cwd=None, env=None, universal_newlines=False, - startupinfo=None, creationflags=0): + preexec_fn=None, close_fds=_PLATFORM_DEFAULT_CLOSE_FDS, + shell=False, cwd=None, env=None, universal_newlines=False, + startupinfo=None, creationflags=0, + restore_signals=True, start_new_session=False, + pass_fds=()): """Create new Popen instance.""" _cleanup() @@ -606,12 +655,24 @@ class Popen(object): if preexec_fn is not None: raise ValueError("preexec_fn is not supported on Windows " "platforms") - if close_fds and (stdin is not None or stdout is not None or - stderr is not None): - raise ValueError("close_fds is not supported on Windows " - "platforms if you redirect stdin/stdout/stderr") + any_stdio_set = (stdin is not None or stdout is not None or + stderr is not None) + if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS: + if any_stdio_set: + close_fds = False + else: + close_fds = True + elif close_fds and any_stdio_set: + raise ValueError( + "close_fds is not supported on Windows platforms" + " if you redirect stdin/stdout/stderr") else: # POSIX + if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS: + close_fds = True + if pass_fds and not close_fds: + warnings.warn("pass_fds overriding close_fds.", RuntimeWarning) + close_fds = True if startupinfo is not None: raise ValueError("startupinfo is only supported on Windows " "platforms") @@ -638,46 +699,73 @@ class Popen(object): # On POSIX, the child objects are file descriptors. On # Windows, these are Windows file handles. The parent objects # are file descriptors on both platforms. The parent objects - # are None when not using PIPEs. The child objects are None + # are -1 when not using PIPEs. The child objects are -1 # when not redirecting. (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) = self._get_handles(stdin, stdout, stderr) - self._execute_child(args, executable, preexec_fn, close_fds, - cwd, env, universal_newlines, - startupinfo, creationflags, shell, - p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite) + # We wrap OS handles *before* launching the child, otherwise a + # quickly terminating child could make our fds unwrappable + # (see #8458). if mswindows: - if p2cwrite is not None: + if p2cwrite != -1: p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0) - if c2pread is not None: + if c2pread != -1: c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0) - if errread is not None: + if errread != -1: errread = msvcrt.open_osfhandle(errread.Detach(), 0) - if p2cwrite is not None: + if p2cwrite != -1: self.stdin = io.open(p2cwrite, 'wb', bufsize) if self.universal_newlines: self.stdin = io.TextIOWrapper(self.stdin) - if c2pread is not None: + if c2pread != -1: self.stdout = io.open(c2pread, 'rb', bufsize) if universal_newlines: self.stdout = io.TextIOWrapper(self.stdout) - if errread is not None: + if errread != -1: self.stderr = io.open(errread, 'rb', bufsize) if universal_newlines: self.stderr = io.TextIOWrapper(self.stderr) + try: + self._execute_child(args, executable, preexec_fn, close_fds, + pass_fds, cwd, env, universal_newlines, + startupinfo, creationflags, shell, + p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite, + restore_signals, start_new_session) + except: + # Cleanup if the child failed starting + for f in filter(None, [self.stdin, self.stdout, self.stderr]): + try: + f.close() + except EnvironmentError: + # Ignore EBADF or other errors + pass + raise + def _translate_newlines(self, data, encoding): data = data.replace(b"\r\n", b"\n").replace(b"\r", b"\n") return data.decode(encoding) + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + if self.stdout: + self.stdout.close() + if self.stderr: + self.stderr.close() + if self.stdin: + self.stdin.close() + # Wait for the process to terminate, to avoid zombies. + self.wait() def __del__(self, _maxsize=sys.maxsize, _active=_active): if not self._child_created: @@ -737,11 +825,11 @@ class Popen(object): p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ if stdin is None and stdout is None and stderr is None: - return (None, None, None, None, None, None) + return (-1, -1, -1, -1, -1, -1) - p2cread, p2cwrite = None, None - c2pread, c2pwrite = None, None - errread, errwrite = None, None + p2cread, p2cwrite = -1, -1 + c2pread, c2pwrite = -1, -1 + errread, errwrite = -1, -1 if stdin is None: p2cread = _subprocess.GetStdHandle(_subprocess.STD_INPUT_HANDLE) @@ -814,20 +902,23 @@ class Popen(object): def _execute_child(self, args, executable, preexec_fn, close_fds, - cwd, env, universal_newlines, + pass_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, - errread, errwrite): + errread, errwrite, + unused_restore_signals, unused_start_new_session): """Execute program (MS Windows version)""" + assert not pass_fds, "pass_fds not supported on Windows." + if not isinstance(args, str): args = list2cmdline(args) # Process startup details if startupinfo is None: startupinfo = STARTUPINFO() - if None not in (p2cread, c2pwrite, errwrite): + if -1 not in (p2cread, c2pwrite, errwrite): startupinfo.dwFlags |= _subprocess.STARTF_USESTDHANDLES startupinfo.hStdInput = p2cread startupinfo.hStdOutput = c2pwrite @@ -877,11 +968,11 @@ class Popen(object): # output pipe are maintained in this process or else the # pipe will not close when the child process exits and the # ReadFile will hang. - if p2cread is not None: + if p2cread != -1: p2cread.Close() - if c2pwrite is not None: + if c2pwrite != -1: c2pwrite.Close() - if errwrite is not None: + if errwrite != -1: errwrite.Close() # Retain the process handle, but close the thread handle @@ -919,6 +1010,7 @@ class Popen(object): def _readerthread(self, fh, buffer): buffer.append(fh.read()) + fh.close() def _communicate(self, input): @@ -966,8 +1058,12 @@ class Popen(object): """ if sig == signal.SIGTERM: self.terminate() + elif sig == signal.CTRL_C_EVENT: + os.kill(self.pid, signal.CTRL_C_EVENT) + elif sig == signal.CTRL_BREAK_EVENT: + os.kill(self.pid, signal.CTRL_BREAK_EVENT) else: - raise ValueError("Only SIGTERM is supported on Windows") + raise ValueError("Unsupported signal: {}".format(sig)) def terminate(self): """Terminates the process @@ -984,14 +1080,14 @@ class Popen(object): """Construct and return tuple with IO objects: p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ - p2cread, p2cwrite = None, None - c2pread, c2pwrite = None, None - errread, errwrite = None, None + p2cread, p2cwrite = -1, -1 + c2pread, c2pwrite = -1, -1 + errread, errwrite = -1, -1 if stdin is None: pass elif stdin == PIPE: - p2cread, p2cwrite = os.pipe() + p2cread, p2cwrite = _create_pipe() elif isinstance(stdin, int): p2cread = stdin else: @@ -1001,7 +1097,7 @@ class Popen(object): if stdout is None: pass elif stdout == PIPE: - c2pread, c2pwrite = os.pipe() + c2pread, c2pwrite = _create_pipe() elif isinstance(stdout, int): c2pwrite = stdout else: @@ -1011,7 +1107,7 @@ class Popen(object): if stderr is None: pass elif stderr == PIPE: - errread, errwrite = os.pipe() + errread, errwrite = _create_pipe() elif stderr == STDOUT: errwrite = c2pwrite elif isinstance(stderr, int): @@ -1025,30 +1121,23 @@ class Popen(object): errread, errwrite) - def _set_cloexec_flag(self, fd, cloexec=True): - try: - cloexec_flag = fcntl.FD_CLOEXEC - except AttributeError: - cloexec_flag = 1 - - old = fcntl.fcntl(fd, fcntl.F_GETFD) - if cloexec: - fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag) - else: - fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag) - - - def _close_fds(self, but): - os.closerange(3, but) - os.closerange(but + 1, MAXFD) + def _close_fds(self, fds_to_keep): + start_fd = 3 + for fd in sorted(fds_to_keep): + if fd >= start_fd: + os.closerange(start_fd, fd) + start_fd = fd + 1 + if start_fd <= MAXFD: + os.closerange(start_fd, MAXFD) def _execute_child(self, args, executable, preexec_fn, close_fds, - cwd, env, universal_newlines, + pass_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, - errread, errwrite): + errread, errwrite, + restore_signals, start_new_session): """Execute program (POSIX version)""" if isinstance(args, str): @@ -1064,104 +1153,165 @@ class Popen(object): if executable is None: executable = args[0] - # For transferring possible exec failure from child to parent - # The first char specifies the exception type: 0 means - # OSError, 1 means some other error. - errpipe_read, errpipe_write = os.pipe() + # For transferring possible exec failure from child to parent. + # Data format: "exception name:hex errno:description" + # Pickle is not used; it is complex and involves memory allocation. + errpipe_read, errpipe_write = _create_pipe() try: try: - self._set_cloexec_flag(errpipe_write) - gc_was_enabled = gc.isenabled() - # Disable gc to avoid bug where gc -> file_dealloc -> - # write to stderr -> hang. http://bugs.python.org/issue1336 - gc.disable() - try: - self.pid = os.fork() - except: - if gc_was_enabled: - gc.enable() - raise - self._child_created = True - if self.pid == 0: - # Child - try: - # Close parent's pipe ends - if p2cwrite is not None: - os.close(p2cwrite) - if c2pread is not None: - os.close(c2pread) - if errread is not None: - os.close(errread) - os.close(errpipe_read) - - # Dup fds for child - def _dup2(a, b): - # dup2() removes the CLOEXEC flag but - # we must do it ourselves if dup2() - # would be a no-op (issue #10806). - if a == b: - self._set_cloexec_flag(a, False) - elif a is not None: - os.dup2(a, b) - _dup2(p2cread, 0) - _dup2(c2pwrite, 1) - _dup2(errwrite, 2) - - # Close pipe fds. Make sure we don't close the - # same fd more than once, or standard fds. - closed = { None } - for fd in [p2cread, c2pwrite, errwrite]: - if fd not in closed and fd > 2: - os.close(fd) - closed.add(fd) - - # Close all other fds, if asked for - if close_fds: - self._close_fds(but=errpipe_write) - - if cwd is not None: - os.chdir(cwd) - - if preexec_fn: - preexec_fn() - - if env is None: - os.execvp(executable, args) - else: - os.execvpe(executable, args, env) + if _posixsubprocess: + # We must avoid complex work that could involve + # malloc or free in the child process to avoid + # potential deadlocks, thus we do all this here. + # and pass it to fork_exec() + if env: + env_list = [os.fsencode(k) + b'=' + os.fsencode(v) + for k, v in env.items()] + else: + env_list = None # Use execv instead of execve. + executable = os.fsencode(executable) + if os.path.dirname(executable): + executable_list = (executable,) + else: + # This matches the behavior of os._execvpe(). + executable_list = tuple( + os.path.join(os.fsencode(dir), executable) + for dir in os.get_exec_path(env)) + fds_to_keep = set(pass_fds) + fds_to_keep.add(errpipe_write) + self.pid = _posixsubprocess.fork_exec( + args, executable_list, + close_fds, sorted(fds_to_keep), cwd, env_list, + p2cread, p2cwrite, c2pread, c2pwrite, + errread, errwrite, + errpipe_read, errpipe_write, + restore_signals, start_new_session, preexec_fn) + else: + # Pure Python implementation: It is not thread safe. + # This implementation may deadlock in the child if your + # parent process has any other threads running. + + gc_was_enabled = gc.isenabled() + # Disable gc to avoid bug where gc -> file_dealloc -> + # write to stderr -> hang. See issue1336 + gc.disable() + try: + self.pid = os.fork() except: - exc_type, exc_value, tb = sys.exc_info() - # Save the traceback and attach it to the exception - # object - exc_lines = traceback.format_exception(exc_type, - exc_value, - tb) - exc_value.child_traceback = ''.join(exc_lines) - os.write(errpipe_write, pickle.dumps(exc_value)) - - # This exitcode won't be reported to applications, so - # it really doesn't matter what we return. - os._exit(255) - - # Parent - if gc_was_enabled: - gc.enable() + if gc_was_enabled: + gc.enable() + raise + self._child_created = True + if self.pid == 0: + # Child + try: + # Close parent's pipe ends + if p2cwrite != -1: + os.close(p2cwrite) + if c2pread != -1: + os.close(c2pread) + if errread != -1: + os.close(errread) + os.close(errpipe_read) + + # Dup fds for child + def _dup2(a, b): + # dup2() removes the CLOEXEC flag but + # we must do it ourselves if dup2() + # would be a no-op (issue #10806). + if a == b: + _set_cloexec(a, False) + elif a != -1: + os.dup2(a, b) + _dup2(p2cread, 0) + _dup2(c2pwrite, 1) + _dup2(errwrite, 2) + + # Close pipe fds. Make sure we don't close the + # same fd more than once, or standard fds. + closed = set() + for fd in [p2cread, c2pwrite, errwrite]: + if fd > 2 and fd not in closed: + os.close(fd) + closed.add(fd) + + # Close all other fds, if asked for + if close_fds: + fds_to_keep = set(pass_fds) + fds_to_keep.add(errpipe_write) + self._close_fds(fds_to_keep) + + + if cwd is not None: + os.chdir(cwd) + + # This is a copy of Python/pythonrun.c + # _Py_RestoreSignals(). If that were exposed + # as a sys._py_restoresignals func it would be + # better.. but this pure python implementation + # isn't likely to be used much anymore. + if restore_signals: + signals = ('SIGPIPE', 'SIGXFZ', 'SIGXFSZ') + for sig in signals: + if hasattr(signal, sig): + signal.signal(getattr(signal, sig), + signal.SIG_DFL) + + if start_new_session and hasattr(os, 'setsid'): + os.setsid() + + if preexec_fn: + preexec_fn() + + if env is None: + os.execvp(executable, args) + else: + os.execvpe(executable, args, env) + + except: + try: + exc_type, exc_value = sys.exc_info()[:2] + if isinstance(exc_value, OSError): + errno_num = exc_value.errno + else: + errno_num = 0 + message = '%s:%x:%s' % (exc_type.__name__, + errno_num, exc_value) + message = message.encode(errors="surrogatepass") + os.write(errpipe_write, message) + except Exception: + # We MUST not allow anything odd happening + # above to prevent us from exiting below. + pass + + # This exitcode won't be reported to applications + # so it really doesn't matter what we return. + os._exit(255) + + # Parent + if gc_was_enabled: + gc.enable() finally: # be sure the FD is closed no matter what os.close(errpipe_write) - if p2cread is not None and p2cwrite is not None: + if p2cread != -1 and p2cwrite != -1: os.close(p2cread) - if c2pwrite is not None and c2pread is not None: + if c2pwrite != -1 and c2pread != -1: os.close(c2pwrite) - if errwrite is not None and errread is not None: + if errwrite != -1 and errread != -1: os.close(errwrite) # Wait for exec to fail or succeed; possibly raising an - # exception (limited to 1 MB) - data = _eintr_retry_call(os.read, errpipe_read, 1048576) + # exception (limited in size) + data = bytearray() + while True: + part = _eintr_retry_call(os.read, errpipe_read, 50000) + data += part + if not part or len(data) > 50000: + break finally: # be sure the FD is closed no matter what os.close(errpipe_read) @@ -1172,11 +1322,28 @@ class Popen(object): except OSError as e: if e.errno != errno.ECHILD: raise - child_exception = pickle.loads(data) + try: + exception_name, hex_errno, err_msg = data.split(b':', 2) + except ValueError: + print('Bad exception data:', repr(data)) + exception_name = b'RuntimeError' + hex_errno = b'0' + err_msg = b'Unknown' + child_exception_type = getattr( + builtins, exception_name.decode('ascii'), + RuntimeError) for fd in (p2cwrite, c2pread, errread): - if fd is not None: + if fd != -1: os.close(fd) - raise child_exception + err_msg = err_msg.decode(errors="surrogatepass") + if issubclass(child_exception_type, OSError) and hex_errno: + errno_num = int(hex_errno, 16) + if errno_num != 0: + err_msg = os.strerror(errno_num) + if errno_num == errno.ENOENT: + err_msg += ': ' + repr(args[0]) + raise child_exception_type(errno_num, err_msg) + raise child_exception_type(err_msg) def _handle_exitstatus(self, sts, _WIFSIGNALED=os.WIFSIGNALED, |