diff options
Diffstat (limited to 'Lib/subprocess.py')
| -rw-r--r-- | Lib/subprocess.py | 605 |
1 files changed, 418 insertions, 187 deletions
diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 299f73e711..725564567c 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -25,11 +25,12 @@ Using the subprocess module =========================== This module defines one class called Popen: -class Popen(args, bufsize=0, executable=None, +class Popen(args, bufsize=-1, executable=None, stdin=None, stdout=None, stderr=None, - preexec_fn=None, close_fds=False, shell=False, + preexec_fn=None, close_fds=True, shell=False, cwd=None, env=None, universal_newlines=False, - startupinfo=None, creationflags=0): + startupinfo=None, creationflags=0, + restore_signals=True, start_new_session=False, pass_fds=()): Arguments are: @@ -38,12 +39,12 @@ args should be a string, or a sequence of program arguments. The program to execute is normally the first item in the args sequence or string, but can be explicitly set by using the executable argument. -On UNIX, with shell=False (default): In this case, the Popen class +On POSIX, with shell=False (default): In this case, the Popen class uses os.execvp() to execute the child program. args should normally be a sequence. A string will be treated as a sequence with the string as the only item (the program to execute). -On UNIX, with shell=True: If args is a string, it specifies the +On POSIX, with shell=True: If args is a string, it specifies the command string to execute through the shell. If args is a sequence, the first item specifies the command string, and any additional items will be treated as additional shell arguments. @@ -55,12 +56,12 @@ not all MS Windows applications interpret the command line the same way: The list2cmdline is designed for applications using the same rules as the MS C runtime. -bufsize, if given, has the same meaning as the corresponding argument -to the built-in open() function: 0 means unbuffered, 1 means line -buffered, any other positive value means use a buffer of -(approximately) that size. A negative bufsize means to use the system -default, which usually means fully buffered. The default value for -bufsize is 0 (unbuffered). +bufsize will be supplied as the corresponding argument to the io.open() +function when creating the stdin/stdout/stderr pipe file objects: +0 means unbuffered (read & write are one system call and can return short), +1 means line buffered, any other positive value means use a buffer of +approximately that size. A negative bufsize, the default, means the system +default of io.DEFAULT_BUFFER_SIZE will be used. stdin, stdout and stderr specify the executed programs' standard input, standard output and standard error file handles, respectively. @@ -72,11 +73,19 @@ parent. Additionally, stderr can be STDOUT, which indicates that the stderr data from the applications should be captured into the same file handle as for stdout. -If preexec_fn is set to a callable object, this object will be called -in the child process just before the child is executed. +On POSIX, if preexec_fn is set to a callable object, this object will be +called in the child process just before the child is executed. The use +of preexec_fn is not thread safe, using it in the presence of threads +could lead to a deadlock in the child process before the new executable +is executed. If close_fds is true, all file descriptors except 0, 1 and 2 will be -closed before the child process is executed. +closed before the child process is executed. The default for close_fds +varies by platform: Always true on POSIX. True when stdin/stdout/stderr +are None on Windows, false otherwise. + +pass_fds is an optional sequence of file descriptors to keep open between the +parent and child. Providing any pass_fds implicitly sets close_fds to true. if shell is true, the specified command will be executed through the shell. @@ -84,17 +93,24 @@ shell. If cwd is not None, the current directory will be changed to cwd before the child is executed. +On POSIX, if restore_signals is True all signals that Python sets to +SIG_IGN are restored to SIG_DFL in the child process before the exec. +Currently this includes the SIGPIPE, SIGXFZ and SIGXFSZ signals. This +parameter does nothing on Windows. + +On POSIX, if start_new_session is True, the setsid() system call will be made +in the child process prior to executing the command. + If env is not None, it defines the environment variables for the new process. If universal_newlines is true, the file objects stdout and stderr are opened as a text files, but lines may be terminated by any of '\n', -the Unix end-of-line convention, '\r', the Macintosh convention or +the Unix end-of-line convention, '\r', the old Macintosh convention or '\r\n', the Windows convention. All of these external representations -are seen as '\n' by the Python program. Note: This feature is only -available if Python is built with universal newline support (the -default). Also, the newlines attribute of the file objects stdout, -stdin and stderr are not updated by the communicate() method. +are seen as '\n' by the Python program. Also, the newlines attribute +of the file objects stdout, stdin and stderr are not updated by the +communicate() method. The startupinfo and creationflags, if given, will be passed to the underlying CreateProcess() function. They can specify things such as @@ -110,7 +126,7 @@ call(*popenargs, **kwargs): The arguments are the same as for the Popen constructor. Example: - >>> retcode = call(["ls", "-l"]) + >>> retcode = subprocess.call(["ls", "-l"]) check_call(*popenargs, **kwargs): Run command with arguments. Wait for command to complete. If the @@ -120,7 +136,7 @@ check_call(*popenargs, **kwargs): The arguments are the same as for the Popen constructor. Example: - >>> check_call(["ls", "-l"]) + >>> subprocess.check_call(["ls", "-l"]) 0 getstatusoutput(cmd): @@ -230,7 +246,7 @@ pid returncode The child return code. A None value indicates that the process hasn't terminated yet. A negative value -N indicates that the - child was terminated by signal N (UNIX only). + child was terminated by signal N (POSIX only). Replacing older functions with the subprocess module @@ -326,6 +342,8 @@ import os import traceback import gc import signal +import builtins +import warnings import errno # Exception classes used by this module. @@ -361,22 +379,47 @@ else: import fcntl import pickle + try: + import _posixsubprocess + except ImportError: + _posixsubprocess = None + warnings.warn("The _posixsubprocess module is not being used. " + "Child process reliability may suffer if your " + "program uses threads.", RuntimeWarning) + # When select or poll has indicated that the file is writable, # we can write up to _PIPE_BUF bytes without risk of blocking. # POSIX defines PIPE_BUF as >= 512. _PIPE_BUF = getattr(select, 'PIPE_BUF', 512) + _FD_CLOEXEC = getattr(fcntl, 'FD_CLOEXEC', 1) + + def _set_cloexec(fd, cloexec): + old = fcntl.fcntl(fd, fcntl.F_GETFD) + if cloexec: + fcntl.fcntl(fd, fcntl.F_SETFD, old | _FD_CLOEXEC) + else: + fcntl.fcntl(fd, fcntl.F_SETFD, old & ~_FD_CLOEXEC) + + if _posixsubprocess: + _create_pipe = _posixsubprocess.cloexec_pipe + else: + def _create_pipe(): + fds = os.pipe() + _set_cloexec(fds[0], True) + _set_cloexec(fds[1], True) + return fds __all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput", "getoutput", "check_output", "CalledProcessError"] if mswindows: - from _subprocess import (CREATE_NEW_CONSOLE, + from _subprocess import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, STD_ERROR_HANDLE, SW_HIDE, STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW) - - __all__.extend(["CREATE_NEW_CONSOLE", + + __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP", "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE", "STD_ERROR_HANDLE", "SW_HIDE", "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW"]) @@ -385,12 +428,16 @@ try: except: MAXFD = 256 +# This lists holds Popen instances for which the underlying process had not +# exited at the time its __del__ method got called: those processes are wait()ed +# for synchronously from _cleanup() when a new Popen object is created, to avoid +# zombie processes. _active = [] def _cleanup(): for inst in _active[:]: res = inst._internal_poll(_deadstate=sys.maxsize) - if res is not None and res >= 0: + if res is not None: try: _active.remove(inst) except ValueError: @@ -406,7 +453,7 @@ def _eintr_retry_call(func, *args): while True: try: return func(*args) - except OSError as e: + except (OSError, IOError) as e: if e.errno == errno.EINTR: continue raise @@ -547,7 +594,7 @@ def list2cmdline(seq): # Various tools for executing commands and looking at their output and status. # -# NB This only works (and is only relevant) for UNIX. +# NB This only works (and is only relevant) for POSIX. def getstatusoutput(cmd): """Return (status, output) of executing cmd in a shell. @@ -587,18 +634,23 @@ def getoutput(cmd): return getstatusoutput(cmd)[1] +_PLATFORM_DEFAULT_CLOSE_FDS = object() + + class Popen(object): - def __init__(self, args, bufsize=0, executable=None, + def __init__(self, args, bufsize=-1, executable=None, stdin=None, stdout=None, stderr=None, - preexec_fn=None, close_fds=False, shell=False, - cwd=None, env=None, universal_newlines=False, - startupinfo=None, creationflags=0): + preexec_fn=None, close_fds=_PLATFORM_DEFAULT_CLOSE_FDS, + shell=False, cwd=None, env=None, universal_newlines=False, + startupinfo=None, creationflags=0, + restore_signals=True, start_new_session=False, + pass_fds=()): """Create new Popen instance.""" _cleanup() self._child_created = False if bufsize is None: - bufsize = 0 # Restore default + bufsize = -1 # Restore default if not isinstance(bufsize, int): raise TypeError("bufsize must be an integer") @@ -606,12 +658,24 @@ class Popen(object): if preexec_fn is not None: raise ValueError("preexec_fn is not supported on Windows " "platforms") - if close_fds and (stdin is not None or stdout is not None or - stderr is not None): - raise ValueError("close_fds is not supported on Windows " - "platforms if you redirect stdin/stdout/stderr") + any_stdio_set = (stdin is not None or stdout is not None or + stderr is not None) + if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS: + if any_stdio_set: + close_fds = False + else: + close_fds = True + elif close_fds and any_stdio_set: + raise ValueError( + "close_fds is not supported on Windows platforms" + " if you redirect stdin/stdout/stderr") else: # POSIX + if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS: + close_fds = True + if pass_fds and not close_fds: + warnings.warn("pass_fds overriding close_fds.", RuntimeWarning) + close_fds = True if startupinfo is not None: raise ValueError("startupinfo is only supported on Windows " "platforms") @@ -638,49 +702,93 @@ class Popen(object): # On POSIX, the child objects are file descriptors. On # Windows, these are Windows file handles. The parent objects # are file descriptors on both platforms. The parent objects - # are None when not using PIPEs. The child objects are None + # are -1 when not using PIPEs. The child objects are -1 # when not redirecting. (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) = self._get_handles(stdin, stdout, stderr) - self._execute_child(args, executable, preexec_fn, close_fds, - cwd, env, universal_newlines, - startupinfo, creationflags, shell, - p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite) + # We wrap OS handles *before* launching the child, otherwise a + # quickly terminating child could make our fds unwrappable + # (see #8458). if mswindows: - if p2cwrite is not None: + if p2cwrite != -1: p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0) - if c2pread is not None: + if c2pread != -1: c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0) - if errread is not None: + if errread != -1: errread = msvcrt.open_osfhandle(errread.Detach(), 0) - if p2cwrite is not None: + if p2cwrite != -1: self.stdin = io.open(p2cwrite, 'wb', bufsize) if self.universal_newlines: - self.stdin = io.TextIOWrapper(self.stdin) - if c2pread is not None: + self.stdin = io.TextIOWrapper(self.stdin, write_through=True) + if c2pread != -1: self.stdout = io.open(c2pread, 'rb', bufsize) if universal_newlines: self.stdout = io.TextIOWrapper(self.stdout) - if errread is not None: + if errread != -1: self.stderr = io.open(errread, 'rb', bufsize) if universal_newlines: self.stderr = io.TextIOWrapper(self.stderr) + try: + self._execute_child(args, executable, preexec_fn, close_fds, + pass_fds, cwd, env, universal_newlines, + startupinfo, creationflags, shell, + p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite, + restore_signals, start_new_session) + except: + # Cleanup if the child failed starting. + for f in filter(None, (self.stdin, self.stdout, self.stderr)): + try: + f.close() + except EnvironmentError: + pass # Ignore EBADF or other errors. + + # Make sure the child pipes are closed as well. + to_close = [] + if stdin == PIPE: + to_close.append(p2cread) + if stdout == PIPE: + to_close.append(c2pwrite) + if stderr == PIPE: + to_close.append(errwrite) + for fd in to_close: + try: + os.close(fd) + except EnvironmentError: + pass - def _translate_newlines(self, data, encoding): - data = data.replace(b"\r\n", b"\n").replace(b"\r", b"\n") - return data.decode(encoding) + raise + def _translate_newlines(self, data, encoding): + data = data.decode(encoding) + return data.replace("\r\n", "\n").replace("\r", "\n") + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + if self.stdout: + self.stdout.close() + if self.stderr: + self.stderr.close() + if self.stdin: + self.stdin.close() + # Wait for the process to terminate, to avoid zombies. + self.wait() + def __del__(self, _maxsize=sys.maxsize, _active=_active): - if not self._child_created: + # If __init__ hasn't had a chance to execute (e.g. if it + # was passed an undeclared keyword argument), we don't + # have a _child_created attribute at all. + if not getattr(self, '_child_created', False): # We didn't get to successfully create a child process. return # In case the child hasn't been waited on, check if it's done. @@ -713,10 +821,10 @@ class Popen(object): raise self.stdin.close() elif self.stdout: - stdout = self.stdout.read() + stdout = _eintr_retry_call(self.stdout.read) self.stdout.close() elif self.stderr: - stderr = self.stderr.read() + stderr = _eintr_retry_call(self.stderr.read) self.stderr.close() self.wait() return (stdout, stderr) @@ -737,11 +845,11 @@ class Popen(object): p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ if stdin is None and stdout is None and stderr is None: - return (None, None, None, None, None, None) + return (-1, -1, -1, -1, -1, -1) - p2cread, p2cwrite = None, None - c2pread, c2pwrite = None, None - errread, errwrite = None, None + p2cread, p2cwrite = -1, -1 + c2pread, c2pwrite = -1, -1 + errread, errwrite = -1, -1 if stdin is None: p2cread = _subprocess.GetStdHandle(_subprocess.STD_INPUT_HANDLE) @@ -814,20 +922,23 @@ class Popen(object): def _execute_child(self, args, executable, preexec_fn, close_fds, - cwd, env, universal_newlines, + pass_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, - errread, errwrite): + errread, errwrite, + unused_restore_signals, unused_start_new_session): """Execute program (MS Windows version)""" + assert not pass_fds, "pass_fds not supported on Windows." + if not isinstance(args, str): args = list2cmdline(args) # Process startup details if startupinfo is None: startupinfo = STARTUPINFO() - if None not in (p2cread, c2pwrite, errwrite): + if -1 not in (p2cread, c2pwrite, errwrite): startupinfo.dwFlags |= _subprocess.STARTF_USESTDHANDLES startupinfo.hStdInput = p2cread startupinfo.hStdOutput = c2pwrite @@ -877,11 +988,11 @@ class Popen(object): # output pipe are maintained in this process or else the # pipe will not close when the child process exits and the # ReadFile will hang. - if p2cread is not None: + if p2cread != -1: p2cread.Close() - if c2pwrite is not None: + if c2pwrite != -1: c2pwrite.Close() - if errwrite is not None: + if errwrite != -1: errwrite.Close() # Retain the process handle, but close the thread handle @@ -919,6 +1030,7 @@ class Popen(object): def _readerthread(self, fh, buffer): buffer.append(fh.read()) + fh.close() def _communicate(self, input): @@ -966,13 +1078,27 @@ class Popen(object): """ if sig == signal.SIGTERM: self.terminate() + elif sig == signal.CTRL_C_EVENT: + os.kill(self.pid, signal.CTRL_C_EVENT) + elif sig == signal.CTRL_BREAK_EVENT: + os.kill(self.pid, signal.CTRL_BREAK_EVENT) else: - raise ValueError("Only SIGTERM is supported on Windows") + raise ValueError("Unsupported signal: {}".format(sig)) def terminate(self): """Terminates the process """ - _subprocess.TerminateProcess(self._handle, 1) + try: + _subprocess.TerminateProcess(self._handle, 1) + except OSError as e: + # ERROR_ACCESS_DENIED (winerror 5) is received when the + # process already died. + if e.winerror != 5: + raise + rc = _subprocess.GetExitCodeProcess(self._handle) + if rc == _subprocess.STILL_ACTIVE: + raise + self.returncode = rc kill = terminate @@ -984,14 +1110,14 @@ class Popen(object): """Construct and return tuple with IO objects: p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ - p2cread, p2cwrite = None, None - c2pread, c2pwrite = None, None - errread, errwrite = None, None + p2cread, p2cwrite = -1, -1 + c2pread, c2pwrite = -1, -1 + errread, errwrite = -1, -1 if stdin is None: pass elif stdin == PIPE: - p2cread, p2cwrite = os.pipe() + p2cread, p2cwrite = _create_pipe() elif isinstance(stdin, int): p2cread = stdin else: @@ -1001,7 +1127,7 @@ class Popen(object): if stdout is None: pass elif stdout == PIPE: - c2pread, c2pwrite = os.pipe() + c2pread, c2pwrite = _create_pipe() elif isinstance(stdout, int): c2pwrite = stdout else: @@ -1011,7 +1137,7 @@ class Popen(object): if stderr is None: pass elif stderr == PIPE: - errread, errwrite = os.pipe() + errread, errwrite = _create_pipe() elif stderr == STDOUT: errwrite = c2pwrite elif isinstance(stderr, int): @@ -1025,30 +1151,23 @@ class Popen(object): errread, errwrite) - def _set_cloexec_flag(self, fd, cloexec=True): - try: - cloexec_flag = fcntl.FD_CLOEXEC - except AttributeError: - cloexec_flag = 1 - - old = fcntl.fcntl(fd, fcntl.F_GETFD) - if cloexec: - fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag) - else: - fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag) - - - def _close_fds(self, but): - os.closerange(3, but) - os.closerange(but + 1, MAXFD) + def _close_fds(self, fds_to_keep): + start_fd = 3 + for fd in sorted(fds_to_keep): + if fd >= start_fd: + os.closerange(start_fd, fd) + start_fd = fd + 1 + if start_fd <= MAXFD: + os.closerange(start_fd, MAXFD) def _execute_child(self, args, executable, preexec_fn, close_fds, - cwd, env, universal_newlines, + pass_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, - errread, errwrite): + errread, errwrite, + restore_signals, start_new_session): """Execute program (POSIX version)""" if isinstance(args, str): @@ -1063,120 +1182,217 @@ class Popen(object): if executable is None: executable = args[0] + orig_executable = executable - # For transferring possible exec failure from child to parent - # The first char specifies the exception type: 0 means - # OSError, 1 means some other error. - errpipe_read, errpipe_write = os.pipe() + # For transferring possible exec failure from child to parent. + # Data format: "exception name:hex errno:description" + # Pickle is not used; it is complex and involves memory allocation. + errpipe_read, errpipe_write = _create_pipe() try: try: - self._set_cloexec_flag(errpipe_write) - gc_was_enabled = gc.isenabled() - # Disable gc to avoid bug where gc -> file_dealloc -> - # write to stderr -> hang. http://bugs.python.org/issue1336 - gc.disable() - try: - self.pid = os.fork() - except: - if gc_was_enabled: - gc.enable() - raise - self._child_created = True - if self.pid == 0: - # Child - try: - # Close parent's pipe ends - if p2cwrite is not None: - os.close(p2cwrite) - if c2pread is not None: - os.close(c2pread) - if errread is not None: - os.close(errread) - os.close(errpipe_read) - - # Dup fds for child - def _dup2(a, b): - # dup2() removes the CLOEXEC flag but - # we must do it ourselves if dup2() - # would be a no-op (issue #10806). - if a == b: - self._set_cloexec_flag(a, False) - elif a is not None: - os.dup2(a, b) - _dup2(p2cread, 0) - _dup2(c2pwrite, 1) - _dup2(errwrite, 2) - - # Close pipe fds. Make sure we don't close the - # same fd more than once, or standard fds. - closed = { None } - for fd in [p2cread, c2pwrite, errwrite]: - if fd not in closed and fd > 2: - os.close(fd) - closed.add(fd) - - # Close all other fds, if asked for - if close_fds: - self._close_fds(but=errpipe_write) - - if cwd is not None: - os.chdir(cwd) - - if preexec_fn: - preexec_fn() - - if env is None: - os.execvp(executable, args) - else: - os.execvpe(executable, args, env) + if _posixsubprocess: + # We must avoid complex work that could involve + # malloc or free in the child process to avoid + # potential deadlocks, thus we do all this here. + # and pass it to fork_exec() + if env is not None: + env_list = [os.fsencode(k) + b'=' + os.fsencode(v) + for k, v in env.items()] + else: + env_list = None # Use execv instead of execve. + executable = os.fsencode(executable) + if os.path.dirname(executable): + executable_list = (executable,) + else: + # This matches the behavior of os._execvpe(). + executable_list = tuple( + os.path.join(os.fsencode(dir), executable) + for dir in os.get_exec_path(env)) + fds_to_keep = set(pass_fds) + fds_to_keep.add(errpipe_write) + self.pid = _posixsubprocess.fork_exec( + args, executable_list, + close_fds, sorted(fds_to_keep), cwd, env_list, + p2cread, p2cwrite, c2pread, c2pwrite, + errread, errwrite, + errpipe_read, errpipe_write, + restore_signals, start_new_session, preexec_fn) + self._child_created = True + else: + # Pure Python implementation: It is not thread safe. + # This implementation may deadlock in the child if your + # parent process has any other threads running. + + gc_was_enabled = gc.isenabled() + # Disable gc to avoid bug where gc -> file_dealloc -> + # write to stderr -> hang. See issue1336 + gc.disable() + try: + self.pid = os.fork() except: - exc_type, exc_value, tb = sys.exc_info() - # Save the traceback and attach it to the exception - # object - exc_lines = traceback.format_exception(exc_type, - exc_value, - tb) - exc_value.child_traceback = ''.join(exc_lines) - os.write(errpipe_write, pickle.dumps(exc_value)) - - # This exitcode won't be reported to applications, so - # it really doesn't matter what we return. - os._exit(255) - - # Parent - if gc_was_enabled: - gc.enable() + if gc_was_enabled: + gc.enable() + raise + self._child_created = True + if self.pid == 0: + # Child + reached_preexec = False + try: + # Close parent's pipe ends + if p2cwrite != -1: + os.close(p2cwrite) + if c2pread != -1: + os.close(c2pread) + if errread != -1: + os.close(errread) + os.close(errpipe_read) + + # When duping fds, if there arises a situation + # where one of the fds is either 0, 1 or 2, it + # is possible that it is overwritten (#12607). + if c2pwrite == 0: + c2pwrite = os.dup(c2pwrite) + if errwrite == 0 or errwrite == 1: + errwrite = os.dup(errwrite) + + # Dup fds for child + def _dup2(a, b): + # dup2() removes the CLOEXEC flag but + # we must do it ourselves if dup2() + # would be a no-op (issue #10806). + if a == b: + _set_cloexec(a, False) + elif a != -1: + os.dup2(a, b) + _dup2(p2cread, 0) + _dup2(c2pwrite, 1) + _dup2(errwrite, 2) + + # Close pipe fds. Make sure we don't close the + # same fd more than once, or standard fds. + closed = set() + for fd in [p2cread, c2pwrite, errwrite]: + if fd > 2 and fd not in closed: + os.close(fd) + closed.add(fd) + + # Close all other fds, if asked for + if close_fds: + fds_to_keep = set(pass_fds) + fds_to_keep.add(errpipe_write) + self._close_fds(fds_to_keep) + + + if cwd is not None: + os.chdir(cwd) + + # This is a copy of Python/pythonrun.c + # _Py_RestoreSignals(). If that were exposed + # as a sys._py_restoresignals func it would be + # better.. but this pure python implementation + # isn't likely to be used much anymore. + if restore_signals: + signals = ('SIGPIPE', 'SIGXFZ', 'SIGXFSZ') + for sig in signals: + if hasattr(signal, sig): + signal.signal(getattr(signal, sig), + signal.SIG_DFL) + + if start_new_session and hasattr(os, 'setsid'): + os.setsid() + + reached_preexec = True + if preexec_fn: + preexec_fn() + + if env is None: + os.execvp(executable, args) + else: + os.execvpe(executable, args, env) + + except: + try: + exc_type, exc_value = sys.exc_info()[:2] + if isinstance(exc_value, OSError): + errno_num = exc_value.errno + else: + errno_num = 0 + if not reached_preexec: + exc_value = "noexec" + message = '%s:%x:%s' % (exc_type.__name__, + errno_num, exc_value) + message = message.encode(errors="surrogatepass") + os.write(errpipe_write, message) + except Exception: + # We MUST not allow anything odd happening + # above to prevent us from exiting below. + pass + + # This exitcode won't be reported to applications + # so it really doesn't matter what we return. + os._exit(255) + + # Parent + if gc_was_enabled: + gc.enable() finally: # be sure the FD is closed no matter what os.close(errpipe_write) - if p2cread is not None and p2cwrite is not None: + if p2cread != -1 and p2cwrite != -1: os.close(p2cread) - if c2pwrite is not None and c2pread is not None: + if c2pwrite != -1 and c2pread != -1: os.close(c2pwrite) - if errwrite is not None and errread is not None: + if errwrite != -1 and errread != -1: os.close(errwrite) # Wait for exec to fail or succeed; possibly raising an - # exception (limited to 1 MB) - data = _eintr_retry_call(os.read, errpipe_read, 1048576) + # exception (limited in size) + errpipe_data = bytearray() + while True: + part = _eintr_retry_call(os.read, errpipe_read, 50000) + errpipe_data += part + if not part or len(errpipe_data) > 50000: + break finally: # be sure the FD is closed no matter what os.close(errpipe_read) - if data: + if errpipe_data: try: _eintr_retry_call(os.waitpid, self.pid, 0) except OSError as e: if e.errno != errno.ECHILD: raise - child_exception = pickle.loads(data) - for fd in (p2cwrite, c2pread, errread): - if fd is not None: - os.close(fd) - raise child_exception + try: + exception_name, hex_errno, err_msg = ( + errpipe_data.split(b':', 2)) + except ValueError: + exception_name = b'RuntimeError' + hex_errno = b'0' + err_msg = (b'Bad exception data from child: ' + + repr(errpipe_data)) + child_exception_type = getattr( + builtins, exception_name.decode('ascii'), + RuntimeError) + err_msg = err_msg.decode(errors="surrogatepass") + if issubclass(child_exception_type, OSError) and hex_errno: + errno_num = int(hex_errno, 16) + child_exec_never_called = (err_msg == "noexec") + if child_exec_never_called: + err_msg = "" + if errno_num != 0: + err_msg = os.strerror(errno_num) + if errno_num == errno.ENOENT: + if child_exec_never_called: + # The error must be from chdir(cwd). + err_msg += ': ' + repr(cwd) + else: + err_msg += ': ' + repr(orig_executable) + raise child_exception_type(errno_num, err_msg) + raise child_exception_type(err_msg) def _handle_exitstatus(self, sts, _WIFSIGNALED=os.WIFSIGNALED, @@ -1194,7 +1410,7 @@ class Popen(object): def _internal_poll(self, _deadstate=None, _waitpid=os.waitpid, - _WNOHANG=os.WNOHANG, _os_error=os.error): + _WNOHANG=os.WNOHANG, _os_error=os.error, _ECHILD=errno.ECHILD): """Check if child process has terminated. Returns returncode attribute. @@ -1207,16 +1423,23 @@ class Popen(object): pid, sts = _waitpid(self.pid, _WNOHANG) if pid == self.pid: self._handle_exitstatus(sts) - except _os_error: + except _os_error as e: if _deadstate is not None: self.returncode = _deadstate + elif e.errno == _ECHILD: + # This happens if SIGCLD is set to be ignored or + # waiting for child processes has otherwise been + # disabled for our process. This child is dead, we + # can't get the status. + # http://bugs.python.org/issue15756 + self.returncode = 0 return self.returncode def wait(self): """Wait for child process to terminate. Returns returncode attribute.""" - if self.returncode is None: + while self.returncode is None: try: pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0) except OSError as e: @@ -1225,8 +1448,12 @@ class Popen(object): # This happens if SIGCLD is set to be ignored or waiting # for child processes has otherwise been disabled for our # process. This child is dead, we can't get the status. + pid = self.pid sts = 0 - self._handle_exitstatus(sts) + # Check the pid and loop as waitpid has been known to return + # 0 even without WNOHANG in odd situations. issue14396. + if pid == self.pid: + self._handle_exitstatus(sts) return self.returncode @@ -1291,6 +1518,8 @@ class Popen(object): fd2output[self.stderr.fileno()] = stderr = [] input_offset = 0 + if self.universal_newlines and isinstance(input, str): + input = input.encode(self.stdin.encoding) while fd2file: try: ready = poller.poll() @@ -1343,6 +1572,8 @@ class Popen(object): stderr = [] input_offset = 0 + if self.universal_newlines and isinstance(input, str): + input = input.encode(self.stdin.encoding) while read_set or write_set: try: rlist, wlist, xlist = select.select(read_set, write_set, []) |