diff options
Diffstat (limited to 'Lib/subprocess.py')
| -rw-r--r-- | Lib/subprocess.py | 272 |
1 files changed, 174 insertions, 98 deletions
diff --git a/Lib/subprocess.py b/Lib/subprocess.py index f11e538925..b6c437438e 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -104,17 +104,21 @@ in the child process prior to executing the command. If env is not None, it defines the environment variables for the new process. -If universal_newlines is false, the file objects stdin, stdout and stderr +If universal_newlines is False, the file objects stdin, stdout and stderr are opened as binary files, and no line ending conversion is done. -If universal_newlines is true, the file objects stdout and stderr are -opened as a text files, but lines may be terminated by any of '\n', +If universal_newlines is True, the file objects stdout and stderr are +opened as a text file, but lines may be terminated by any of '\n', the Unix end-of-line convention, '\r', the old Macintosh convention or '\r\n', the Windows convention. All of these external representations are seen as '\n' by the Python program. Also, the newlines attribute of the file objects stdout, stdin and stderr are not updated by the communicate() method. +In either case, the process being communicated with should start up +expecting to receive bytes on its standard input and decode them with +the same encoding they are sent in. + The startupinfo and creationflags, if given, will be passed to the underlying CreateProcess() function. They can specify things such as appearance of the main window and priority for the new process. @@ -184,6 +188,9 @@ check_output(*popenargs, **kwargs): pass a string to the subprocess's stdin. If you use this argument you may not also use the Popen constructor's "stdin" argument. + If universal_newlines is set to True, the "input" argument must + be a string rather than bytes, and the return value will be a string. + Exceptions ---------- Exceptions raised in the child process, before the new program has @@ -225,9 +232,13 @@ wait() communicate(input=None) Interact with process: Send data to stdin. Read data from stdout and stderr, until end-of-file is reached. Wait for process to - terminate. The optional input argument should be a string to be + terminate. The optional input argument should be data to be sent to the child process, or None, if no data should be sent to - the child. + the child. If the Popen instance was constructed with universal_newlines + set to True, the input argument should be a string and will be encoded + using the preferred system encoding (see locale.getpreferredencoding); + if universal_newlines is False, the input argument should be a + byte string. communicate() returns a tuple (stdout, stderr). @@ -345,7 +356,7 @@ Popen(["/bin/mycmd", "myarg"], env={"PATH": "/usr/bin"}) """ import sys -mswindows = (sys.platform == "win32") +_mswindows = (sys.platform == "win32") import io import os @@ -354,10 +365,7 @@ import signal import builtins import warnings import errno -try: - from time import monotonic as _time -except ImportError: - from time import time as _time +from time import monotonic as _time # Exception classes used by this module. class SubprocessError(Exception): pass @@ -369,29 +377,53 @@ class CalledProcessError(SubprocessError): The exit status will be stored in the returncode attribute; check_output() will also store the output in the output attribute. """ - def __init__(self, returncode, cmd, output=None): + def __init__(self, returncode, cmd, output=None, stderr=None): self.returncode = returncode self.cmd = cmd self.output = output + self.stderr = stderr + def __str__(self): return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode) + @property + def stdout(self): + """Alias for output attribute, to match stderr""" + return self.output + + @stdout.setter + def stdout(self, value): + # There's no obvious reason to set this, but allow it anyway so + # .stdout is a transparent alias for .output + self.output = value + class TimeoutExpired(SubprocessError): """This exception is raised when the timeout expires while waiting for a child process. """ - def __init__(self, cmd, timeout, output=None): + def __init__(self, cmd, timeout, output=None, stderr=None): self.cmd = cmd self.timeout = timeout self.output = output + self.stderr = stderr def __str__(self): return ("Command '%s' timed out after %s seconds" % (self.cmd, self.timeout)) + @property + def stdout(self): + return self.output + + @stdout.setter + def stdout(self, value): + # There's no obvious reason to set this, but allow it anyway so + # .stdout is a transparent alias for .output + self.output = value -if mswindows: + +if _mswindows: import threading import msvcrt import _winapi @@ -425,9 +457,12 @@ else: __all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput", - "getoutput", "check_output", "CalledProcessError", "DEVNULL"] + "getoutput", "check_output", "run", "CalledProcessError", "DEVNULL", + "SubprocessError", "TimeoutExpired", "CompletedProcess"] + # NOTE: We intentionally exclude list2cmdline as it is + # considered an internal implementation detail. issue10838. -if mswindows: +if _mswindows: from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, STD_ERROR_HANDLE, SW_HIDE, @@ -453,15 +488,11 @@ if mswindows: raise ValueError("already closed") def __repr__(self): - return "Handle(%d)" % int(self) + return "%s(%d)" % (self.__class__.__name__, int(self)) __del__ = Close __str__ = __repr__ -try: - MAXFD = os.sysconf("SC_OPEN_MAX") -except: - MAXFD = 256 # This lists holds Popen instances for which the underlying process had not # exited at the time its __del__ method got called: those processes are wait()ed @@ -485,14 +516,6 @@ STDOUT = -2 DEVNULL = -3 -def _eintr_retry_call(func, *args): - while True: - try: - return func(*args) - except InterruptedError: - continue - - # XXX This function is only used by multiprocessing and the test suite, # but it's here so that it can be imported when Python is compiled without # threads. @@ -591,34 +614,102 @@ def check_output(*popenargs, timeout=None, **kwargs): ... input=b"when in the course of fooman events\n") b'when in the course of barman events\n' - If universal_newlines=True is passed, the return value will be a - string rather than bytes. + If universal_newlines=True is passed, the "input" argument must be a + string and the return value will be a string rather than bytes. """ if 'stdout' in kwargs: raise ValueError('stdout argument not allowed, it will be overridden.') - if 'input' in kwargs: + + if 'input' in kwargs and kwargs['input'] is None: + # Explicitly passing input=None was previously equivalent to passing an + # empty string. That is maintained here for backwards compatibility. + kwargs['input'] = '' if kwargs.get('universal_newlines', False) else b'' + + return run(*popenargs, stdout=PIPE, timeout=timeout, check=True, + **kwargs).stdout + + +class CompletedProcess(object): + """A process that has finished running. + + This is returned by run(). + + Attributes: + args: The list or str args passed to run(). + returncode: The exit code of the process, negative for signals. + stdout: The standard output (None if not captured). + stderr: The standard error (None if not captured). + """ + def __init__(self, args, returncode, stdout=None, stderr=None): + self.args = args + self.returncode = returncode + self.stdout = stdout + self.stderr = stderr + + def __repr__(self): + args = ['args={!r}'.format(self.args), + 'returncode={!r}'.format(self.returncode)] + if self.stdout is not None: + args.append('stdout={!r}'.format(self.stdout)) + if self.stderr is not None: + args.append('stderr={!r}'.format(self.stderr)) + return "{}({})".format(type(self).__name__, ', '.join(args)) + + def check_returncode(self): + """Raise CalledProcessError if the exit code is non-zero.""" + if self.returncode: + raise CalledProcessError(self.returncode, self.args, self.stdout, + self.stderr) + + +def run(*popenargs, input=None, timeout=None, check=False, **kwargs): + """Run command with arguments and return a CompletedProcess instance. + + The returned instance will have attributes args, returncode, stdout and + stderr. By default, stdout and stderr are not captured, and those attributes + will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them. + + If check is True and the exit code was non-zero, it raises a + CalledProcessError. The CalledProcessError object will have the return code + in the returncode attribute, and output & stderr attributes if those streams + were captured. + + If timeout is given, and the process takes too long, a TimeoutExpired + exception will be raised. + + There is an optional argument "input", allowing you to + pass a string to the subprocess's stdin. If you use this argument + you may not also use the Popen constructor's "stdin" argument, as + it will be used internally. + + The other arguments are the same as for the Popen constructor. + + If universal_newlines=True is passed, the "input" argument must be a + string and stdout/stderr in the returned object will be strings rather than + bytes. + """ + if input is not None: if 'stdin' in kwargs: raise ValueError('stdin and input arguments may not both be used.') - inputdata = kwargs['input'] - del kwargs['input'] kwargs['stdin'] = PIPE - else: - inputdata = None - with Popen(*popenargs, stdout=PIPE, **kwargs) as process: + + with Popen(*popenargs, **kwargs) as process: try: - output, unused_err = process.communicate(inputdata, timeout=timeout) + stdout, stderr = process.communicate(input, timeout=timeout) except TimeoutExpired: process.kill() - output, unused_err = process.communicate() - raise TimeoutExpired(process.args, timeout, output=output) + stdout, stderr = process.communicate() + raise TimeoutExpired(process.args, timeout, output=stdout, + stderr=stderr) except: process.kill() process.wait() raise retcode = process.poll() - if retcode: - raise CalledProcessError(retcode, process.args, output=output) - return output + if check and retcode: + raise CalledProcessError(retcode, process.args, + output=stdout, stderr=stderr) + return CompletedProcess(process.args, retcode, stdout, stderr) def list2cmdline(seq): @@ -766,7 +857,7 @@ class Popen(object): if not isinstance(bufsize, int): raise TypeError("bufsize must be an integer") - if mswindows: + if _mswindows: if preexec_fn is not None: raise ValueError("preexec_fn is not supported on Windows " "platforms") @@ -826,7 +917,7 @@ class Popen(object): # quickly terminating child could make our fds unwrappable # (see #8458). - if mswindows: + if _mswindows: if p2cwrite != -1: p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0) if c2pread != -1: @@ -918,14 +1009,35 @@ class Popen(object): self._devnull = os.open(os.devnull, os.O_RDWR) return self._devnull + def _stdin_write(self, input): + if input: + try: + self.stdin.write(input) + except BrokenPipeError: + # communicate() must ignore broken pipe error + pass + except OSError as e: + if e.errno == errno.EINVAL and self.poll() is not None: + # Issue #19612: On Windows, stdin.write() fails with EINVAL + # if the process already exited before the write + pass + else: + raise + self.stdin.close() + def communicate(self, input=None, timeout=None): """Interact with process: Send data to stdin. Read data from stdout and stderr, until end-of-file is reached. Wait for - process to terminate. The optional input argument should be - bytes to be sent to the child process, or None, if no data - should be sent to the child. + process to terminate. - communicate() returns a tuple (stdout, stderr).""" + The optional "input" argument should be data to be sent to the + child process (if self.universal_newlines is True, this should + be a string; if it is False, "input" should be bytes), or + None, if no data should be sent to the child. + + communicate() returns a tuple (stdout, stderr). These will be + bytes or, if self.universal_newlines was True, a string. + """ if self._communication_started and input: raise ValueError("Cannot send input after starting communication") @@ -938,18 +1050,12 @@ class Popen(object): stdout = None stderr = None if self.stdin: - if input: - try: - self.stdin.write(input) - except OSError as e: - if e.errno != errno.EPIPE and e.errno != errno.EINVAL: - raise - self.stdin.close() + self._stdin_write(input) elif self.stdout: - stdout = _eintr_retry_call(self.stdout.read) + stdout = self.stdout.read() self.stdout.close() elif self.stderr: - stderr = _eintr_retry_call(self.stderr.read) + stderr = self.stderr.read() self.stderr.close() self.wait() else: @@ -988,7 +1094,7 @@ class Popen(object): raise TimeoutExpired(self.args, orig_timeout) - if mswindows: + if _mswindows: # # Windows methods # @@ -1193,21 +1299,7 @@ class Popen(object): self.stderr_thread.start() if self.stdin: - if input is not None: - try: - self.stdin.write(input) - except OSError as e: - if e.errno == errno.EPIPE: - # communicate() should ignore pipe full error - pass - elif (e.errno == errno.EINVAL - and self.poll() is not None): - # Issue #19612: stdin.write() fails with EINVAL - # if the process already exited before the write - pass - else: - raise - self.stdin.close() + self._stdin_write(input) # Wait for the reader threads, or time out. If we time out, the # threads remain reading and the fds left open in case the user @@ -1322,16 +1414,6 @@ class Popen(object): errread, errwrite) - def _close_fds(self, fds_to_keep): - start_fd = 3 - for fd in sorted(fds_to_keep): - if fd >= start_fd: - os.closerange(start_fd, fd) - start_fd = fd + 1 - if start_fd <= MAXFD: - os.closerange(start_fd, MAXFD) - - def _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, @@ -1417,7 +1499,7 @@ class Popen(object): # exception (limited in size) errpipe_data = bytearray() while True: - part = _eintr_retry_call(os.read, errpipe_read, 50000) + part = os.read(errpipe_read, 50000) errpipe_data += part if not part or len(errpipe_data) > 50000: break @@ -1427,10 +1509,9 @@ class Popen(object): if errpipe_data: try: - _eintr_retry_call(os.waitpid, self.pid, 0) - except OSError as e: - if e.errno != errno.ECHILD: - raise + os.waitpid(self.pid, 0) + except ChildProcessError: + pass try: exception_name, hex_errno, err_msg = ( errpipe_data.split(b':', 2)) @@ -1513,10 +1594,8 @@ class Popen(object): def _try_wait(self, wait_flags): """All callers to this function MUST hold self._waitpid_lock.""" try: - (pid, sts) = _eintr_retry_call(os.waitpid, self.pid, wait_flags) - except OSError as e: - if e.errno != errno.ECHILD: - raise + (pid, sts) = os.waitpid(self.pid, wait_flags) + except ChildProcessError: # This happens if SIGCLD is set to be ignored or waiting # for child processes has otherwise been disabled for our # process. This child is dead, we can't get the status. @@ -1628,12 +1707,9 @@ class Popen(object): self._input_offset + _PIPE_BUF] try: self._input_offset += os.write(key.fd, chunk) - except OSError as e: - if e.errno == errno.EPIPE: - selector.unregister(key.fileobj) - key.fileobj.close() - else: - raise + except BrokenPipeError: + selector.unregister(key.fileobj) + key.fileobj.close() else: if self._input_offset >= len(self._input): selector.unregister(key.fileobj) |