diff options
| -rw-r--r-- | numpy/distutils/exec_command.py | 308 | ||||
| -rwxr-xr-x | tools/travis-test.sh | 3 |
2 files changed, 56 insertions, 255 deletions
diff --git a/numpy/distutils/exec_command.py b/numpy/distutils/exec_command.py index 4a4bc67f2..9df48cc27 100644 --- a/numpy/distutils/exec_command.py +++ b/numpy/distutils/exec_command.py @@ -57,12 +57,12 @@ __all__ = ['exec_command', 'find_executable'] import os import sys import shlex +import subprocess from numpy.distutils.misc_util import is_sequence, make_temp_file from numpy.distutils import log from numpy.distutils.compat import get_exception -from numpy.compat import open_latin1 def temp_file_name(): fo, name = make_temp_file() @@ -211,28 +211,10 @@ def exec_command(command, execute_in='', use_shell=None, use_tee=None, _update_environment( **env ) try: - # _exec_command is robust but slow, it relies on - # usable sys.std*.fileno() descriptors. If they - # are bad (like in win32 Idle, PyCrust environments) - # then _exec_command_python (even slower) - # will be used as a last resort. - # - # _exec_command_posix uses os.system and is faster - # but not on all platforms os.system will return - # a correct status. - if (_with_python and _supports_fileno(sys.stdout) and - sys.stdout.fileno() == -1): - st = _exec_command_python(command, - exec_command_dir = exec_dir, - **env) - elif os.name=='posix': - st = _exec_command_posix(command, - use_shell=use_shell, - use_tee=use_tee, - **env) - else: - st = _exec_command(command, use_shell=use_shell, - use_tee=use_tee,**env) + st = _exec_command(command, + use_shell=use_shell, + use_tee=use_tee, + **env) finally: if oldcwd!=execute_in: os.chdir(oldcwd) @@ -241,250 +223,66 @@ def exec_command(command, execute_in='', use_shell=None, use_tee=None, return st -def _exec_command_posix( command, - use_shell = None, - use_tee = None, - **env ): - log.debug('_exec_command_posix(...)') - - if is_sequence(command): - command_str = ' '.join(list(command)) - else: - command_str = command - - tmpfile = temp_file_name() - stsfile = None - if use_tee: - stsfile = temp_file_name() - filter = '' - if use_tee == 2: - filter = r'| tr -cd "\n" | tr "\n" "."; echo' - command_posix = '( %s ; echo $? > %s ) 2>&1 | tee %s %s'\ - % (command_str, stsfile, tmpfile, filter) - else: - stsfile = temp_file_name() - command_posix = '( %s ; echo $? > %s ) > %s 2>&1'\ - % (command_str, stsfile, tmpfile) - #command_posix = '( %s ) > %s 2>&1' % (command_str,tmpfile) - - log.debug('Running os.system(%r)' % (command_posix)) - status = os.system(command_posix) - - if use_tee: - if status: - # if command_tee fails then fall back to robust exec_command - log.warn('_exec_command_posix failed (status=%s)' % status) - return _exec_command(command, use_shell=use_shell, **env) - - if stsfile is not None: - f = open_latin1(stsfile, 'r') - status_text = f.read() - status = int(status_text) - f.close() - os.remove(stsfile) - - f = open_latin1(tmpfile, 'r') - text = f.read() - f.close() - os.remove(tmpfile) - - if text[-1:]=='\n': - text = text[:-1] - - return status, text - - -def _exec_command_python(command, - exec_command_dir='', **env): - log.debug('_exec_command_python(...)') - - python_exe = get_pythonexe() - cmdfile = temp_file_name() - stsfile = temp_file_name() - outfile = temp_file_name() - - f = open(cmdfile, 'w') - f.write('import os\n') - f.write('import sys\n') - f.write('sys.path.insert(0,%r)\n' % (exec_command_dir)) - f.write('from exec_command import exec_command\n') - f.write('del sys.path[0]\n') - f.write('cmd = %r\n' % command) - f.write('os.environ = %r\n' % (os.environ)) - f.write('s,o = exec_command(cmd, _with_python=0, **%r)\n' % (env)) - f.write('f=open(%r,"w")\nf.write(str(s))\nf.close()\n' % (stsfile)) - f.write('f=open(%r,"w")\nf.write(o)\nf.close()\n' % (outfile)) - f.close() - - cmd = '%s %s' % (python_exe, cmdfile) - status = os.system(cmd) - if status: - raise RuntimeError("%r failed" % (cmd,)) - os.remove(cmdfile) - - f = open_latin1(stsfile, 'r') - status = int(f.read()) - f.close() - os.remove(stsfile) - - f = open_latin1(outfile, 'r') - text = f.read() - f.close() - os.remove(outfile) - - return status, text - -def quote_arg(arg): - if arg[0]!='"' and ' ' in arg: - return '"%s"' % arg - return arg - -def _exec_command( command, use_shell=None, use_tee = None, **env ): - log.debug('_exec_command(...)') +def _exec_command(command, use_shell=None, use_tee = None, **env): + """ + Internal workhorse for exec_command(). + """ if use_shell is None: use_shell = os.name=='posix' if use_tee is None: use_tee = os.name=='posix' - using_command = 0 - if use_shell: - # We use shell (unless use_shell==0) so that wildcards can be - # used. + + executable = None + + if os.name == 'posix' and use_shell: + # On POSIX, subprocess always uses /bin/sh, override sh = os.environ.get('SHELL', '/bin/sh') if is_sequence(command): - argv = [sh, '-c', ' '.join(list(command))] - else: - argv = [sh, '-c', command] - else: - # On NT, DOS we avoid using command.com as it's exit status is - # not related to the exit status of a command. - if is_sequence(command): - argv = command[:] - else: - argv = shlex.split(command) - - # `spawn*p` family with path (vp, vpe, ...) are not available on windows. - # Also prefer spawn{v,vp} in favor of spawn{ve,vpe} if no env - # modification is actually requested as the *e* functions are not thread - # safe on windows (https://bugs.python.org/issue6476) - if hasattr(os, 'spawnvpe'): - spawn_command = os.spawnvpe if env else os.spawnvp - else: - spawn_command = os.spawnve if env else os.spawnv - argv[0] = find_executable(argv[0]) or argv[0] - if not os.path.isfile(argv[0]): - log.warn('Executable %s does not exist' % (argv[0])) - if os.name in ['nt', 'dos']: - # argv[0] might be internal command - argv = [os.environ['COMSPEC'], '/C'] + argv - using_command = 1 - - _so_has_fileno = _supports_fileno(sys.stdout) - _se_has_fileno = _supports_fileno(sys.stderr) - so_flush = sys.stdout.flush - se_flush = sys.stderr.flush - if _so_has_fileno: - so_fileno = sys.stdout.fileno() - so_dup = os.dup(so_fileno) - if _se_has_fileno: - se_fileno = sys.stderr.fileno() - se_dup = os.dup(se_fileno) - - outfile = temp_file_name() - fout = open(outfile, 'w') - if using_command: - errfile = temp_file_name() - ferr = open(errfile, 'w') - - log.debug('Running %s(%s,%r,%r,os.environ)' \ - % (spawn_command.__name__, os.P_WAIT, argv[0], argv)) - - if env and sys.version_info[0] >= 3 and os.name == 'nt': - # Pre-encode os.environ, discarding un-encodable entries, - # to avoid it failing during encoding as part of spawn. Failure - # is possible if the environment contains entries that are not - # encoded using the system codepage as windows expects. - # - # This is not necessary on unix, where os.environ is encoded - # using the surrogateescape error handler and decoded using - # it as part of spawn. - encoded_environ = {} - for k, v in os.environ.items(): - try: - encoded_environ[k.encode(sys.getfilesystemencoding())] = v.encode( - sys.getfilesystemencoding()) - except UnicodeEncodeError: - log.debug("ignoring un-encodable env entry %s", k) - else: - encoded_environ = os.environ - - argv0 = argv[0] - if not using_command: - argv[0] = quote_arg(argv0) - - so_flush() - se_flush() - if _so_has_fileno: - os.dup2(fout.fileno(), so_fileno) - - if _se_has_fileno: - if using_command: - #XXX: disabled for now as it does not work from cmd under win32. - # Tests fail on msys - os.dup2(ferr.fileno(), se_fileno) + command = [sh, '-c', ' '.join(command)] else: - os.dup2(fout.fileno(), se_fileno) + command = [sh, '-c', command] + use_shell = False + + elif os.name == 'nt' and is_sequence(command): + # On Windows, join the string for CreateProcess() ourselves as + # subprocess does it a bit differently + command = ' '.join(_quote_arg(arg) for arg in command) + + # Inherit environment by default + env = env or None try: - # Use spawnv in favor of spawnve, unless necessary - if env: - status = spawn_command(os.P_WAIT, argv0, argv, encoded_environ) - else: - status = spawn_command(os.P_WAIT, argv0, argv) - except Exception: - errmess = str(get_exception()) - status = 999 - sys.stderr.write('%s: %s'%(errmess, argv[0])) - - so_flush() - se_flush() - if _so_has_fileno: - os.dup2(so_dup, so_fileno) - os.close(so_dup) - if _se_has_fileno: - os.dup2(se_dup, se_fileno) - os.close(se_dup) - - fout.close() - fout = open_latin1(outfile, 'r') - text = fout.read() - fout.close() - os.remove(outfile) - - if using_command: - ferr.close() - ferr = open_latin1(errfile, 'r') - errmess = ferr.read() - ferr.close() - os.remove(errfile) - if errmess and not status: - # Not sure how to handle the case where errmess - # contains only warning messages and that should - # not be treated as errors. - #status = 998 - if text: - text = text + '\n' - #text = '%sCOMMAND %r FAILED: %s' %(text,command,errmess) - text = text + errmess - print (errmess) - if text[-1:]=='\n': + proc = subprocess.Popen(command, shell=use_shell, env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True) + except EnvironmentError: + # Return 127, as os.spawn*() and /bin/sh do + return '', 127 + text, err = proc.communicate() + # Only append stderr if the command failed, as otherwise + # the output may become garbled for parsing + if proc.returncode: + if text: + text += "\n" + text += err + # Another historical oddity + if text[-1:] == '\n': text = text[:-1] - if status is None: - status = 0 - if use_tee: - print (text) + print(text) + return proc.returncode, text - return status, text + +def _quote_arg(arg): + """ + Quote the argument for safe use in a shell command line. + """ + # If there is a quote in the string, assume relevants parts of the + # string are already quoted (e.g. '-I"C:\\Program Files\\..."') + if '"' not in arg and ' ' in arg: + return '"%s"' % arg + return arg def test_nt(**kws): diff --git a/tools/travis-test.sh b/tools/travis-test.sh index e050b4ccb..2a57c9873 100755 --- a/tools/travis-test.sh +++ b/tools/travis-test.sh @@ -175,6 +175,9 @@ elif [ -n "$USE_SDIST" ] && [ $# -eq 0 ]; then elif [ -n "$USE_CHROOT" ] && [ $# -eq 0 ]; then DIR=/chroot setup_chroot $DIR + # the chroot'ed environment will not have the current locale, + # avoid any warnings which may disturb testing + export LANG=C LC_ALL=C # run again in chroot with this time testing sudo linux32 chroot $DIR bash -c \ "cd numpy && PYTHON=python PIP=pip IN_CHROOT=1 $0 test" |