Merge pull request #13023 from eric-wieser/distutils-windows-split

BUG: Changes to string-to-shell parsing behavior broke paths with quotes

2 files changed, 170 insertions, 0 deletions

diff --git a/numpy/distutils/_shell_utils.py b/numpy/distutils/_shell_utils.py
new file mode 100644
index 000000000..5ef874900
--- /dev/null
+++ b/numpy/distutils/_shell_utils.py
@@ -0,0 +1,91 @@
+"""
+Helper functions for interacting with the shell, and consuming shell-style
+parameters provided in config files.
+"""
+import os
+import shlex
+import subprocess
+try:
+    from shlex import quote
+except ImportError:
+    from pipes import quote
+
+__all__ = ['WindowsParser', 'PosixParser', 'NativeParser']
+
+
+class CommandLineParser:
+    """
+    An object that knows how to split and join command-line arguments.
+
+    It must be true that ``argv == split(join(argv))`` for all ``argv``.
+    The reverse neednt be true - `join(split(cmd))` may result in the addition
+    or removal of unnecessary escaping.
+    """
+    @staticmethod
+    def join(argv):
+        """ Join a list of arguments into a command line string """
+        raise NotImplemented
+
+    @staticmethod
+    def split(cmd):
+        """ Split a command line string into a list of arguments """
+        raise NotImplemented
+
+
+class WindowsParser:
+    """
+    The parsing behavior used by `subprocess.call("string")` on Windows, which
+    matches the Microsoft C/C++ runtime.
+
+    Note that this is _not_ the behavior of cmd.
+    """
+    @staticmethod
+    def join(argv):
+        # note that list2cmdline is specific to the windows syntax
+        return subprocess.list2cmdline(argv)
+
+    @staticmethod
+    def split(cmd):
+        import ctypes  # guarded import for systems without ctypes
+        try:
+            ctypes.windll
+        except AttributeError:
+            raise NotImplementedError
+
+        # Windows has special parsing rules for the executable (no quotes),
+        # that we do not care about - insert a dummy element
+        if not cmd:
+            return []
+        cmd = 'dummy ' + cmd
+
+        CommandLineToArgvW = ctypes.windll.shell32.CommandLineToArgvW
+        CommandLineToArgvW.restype = ctypes.POINTER(ctypes.c_wchar_p)
+        CommandLineToArgvW.argtypes = (ctypes.c_wchar_p, ctypes.POINTER(ctypes.c_int))
+
+        nargs = ctypes.c_int()
+        lpargs = CommandLineToArgvW(cmd, ctypes.byref(nargs))
+        args = [lpargs[i] for i in range(nargs.value)]
+        assert not ctypes.windll.kernel32.LocalFree(lpargs)
+
+        # strip the element we inserted
+        assert args[0] == "dummy"
+        return args[1:]
+
+
+class PosixParser:
+    """
+    The parsing behavior used by `subprocess.call("string", shell=True)` on Posix.
+    """
+    @staticmethod
+    def join(argv):
+        return ' '.join(quote(arg) for arg in argv)
+
+    @staticmethod
+    def split(cmd):
+        return shlex.split(cmd, posix=True)
+
+
+if os.name == 'nt':
+    NativeParser = WindowsParser
+elif os.name == 'posix':
+    NativeParser = PosixParser
diff --git a/numpy/distutils/tests/test_shell_utils.py b/numpy/distutils/tests/test_shell_utils.py
new file mode 100644
index 000000000..a0344244f
--- /dev/null
+++ b/numpy/distutils/tests/test_shell_utils.py
@@ -0,0 +1,79 @@
+from __future__ import division, absolute_import, print_function
+
+import pytest
+import subprocess
+import os
+import json
+import sys
+
+from numpy.distutils import _shell_utils
+
+argv_cases = [
+    [r'exe'],
+    [r'path/exe'],
+    [r'path\exe'],
+    [r'\\server\path\exe'],
+    [r'path to/exe'],
+    [r'path to\exe'],
+
+    [r'exe', '--flag'],
+    [r'path/exe', '--flag'],
+    [r'path\exe', '--flag'],
+    [r'path to/exe', '--flag'],
+    [r'path to\exe', '--flag'],
+
+    # flags containing literal quotes in their name
+    [r'path to/exe', '--flag-"quoted"'],
+    [r'path to\exe', '--flag-"quoted"'],
+    [r'path to/exe', '"--flag-quoted"'],
+    [r'path to\exe', '"--flag-quoted"'],
+]
+
+
+@pytest.fixture(params=[
+    _shell_utils.WindowsParser,
+    _shell_utils.PosixParser
+])
+def Parser(request):
+    return request.param
+
+
+@pytest.fixture
+def runner(Parser):
+    if Parser != _shell_utils.NativeParser:
+        pytest.skip('Unable to run with non-native parser')
+
+    if Parser == _shell_utils.WindowsParser:
+        return lambda cmd: subprocess.check_output(cmd)
+    elif Parser == _shell_utils.PosixParser:
+        # posix has no non-shell string parsing
+        return lambda cmd: subprocess.check_output(cmd, shell=True)
+    else:
+        raise NotImplementedError
+
+
+@pytest.mark.parametrize('argv', argv_cases)
+def test_join_matches_subprocess(Parser, runner, argv):
+    """
+    Test that join produces strings understood by subprocess
+    """
+    # invoke python to return its arguments as json
+    cmd = [
+        sys.executable, '-c',
+        'import json, sys; print(json.dumps(sys.argv[1:]))'
+    ]
+    joined = Parser.join(cmd + argv)
+    json_out = runner(joined).decode()
+    assert json.loads(json_out) == argv
+
+
+@pytest.mark.parametrize('argv', argv_cases)
+def test_roundtrip(Parser, argv):
+    """
+    Test that split is the inverse operation of join
+    """
+    try:
+        joined = Parser.join(argv)
+        assert argv == Parser.split(joined)
+    except NotImplementedError:
+        pytest.skip("Not implemented")