macOs Python 2 Framework support (#1641)

5 files changed, 325 insertions, 41 deletions

diff --git a/src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py b/src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py
index a9bd52c..887263e 100644
--- a/src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py
+++ b/src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py
@@ -50,8 +50,17 @@ class CPython2(CPython, Python2):
         return dirs
 
 
+def is_mac_os_framework(interpreter):
+    framework = bool(interpreter.sysconfig_vars["PYTHONFRAMEWORK"])
+    return framework and interpreter.platform == "darwin"
+
+
 class CPython2Posix(CPython2, CPythonPosix):
-    """CPython 2 on POSIX"""
+    """CPython 2 on POSIX (excluding macOs framework builds)"""
+
+    @classmethod
+    def can_describe(cls, interpreter):
+        return is_mac_os_framework(interpreter) is False and super(CPython2Posix, cls).can_describe(interpreter)
 
     @classmethod
     def sources(cls, interpreter):
diff --git a/src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py b/src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py
new file mode 100644
index 0000000..ff28958
--- /dev/null
+++ b/src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py
@@ -0,0 +1,216 @@
+# -*- coding: utf-8 -*-
+"""The Apple Framework builds require their own customization"""
+import logging
+import os
+import struct
+import subprocess
+
+from virtualenv.create.via_global_ref.builtin.cpython.common import CPythonPosix
+from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest
+from virtualenv.util.path import Path
+from virtualenv.util.six import ensure_text
+
+from .cpython2 import CPython2, is_mac_os_framework
+
+
+class CPython2macOsFramework(CPython2, CPythonPosix):
+    @classmethod
+    def can_describe(cls, interpreter):
+        return is_mac_os_framework(interpreter) and super(CPython2macOsFramework, cls).can_describe(interpreter)
+
+    def create(self):
+        super(CPython2macOsFramework, self).create()
+
+        # change the install_name of the copied python executable
+        current = os.path.join(self.interpreter.prefix, "Python")
+        fix_mach_o(str(self.exe), current, "@executable_path/../.Python", self.interpreter.max_size)
+
+    @classmethod
+    def sources(cls, interpreter):
+        for src in super(CPython2macOsFramework, cls).sources(interpreter):
+            yield src
+
+        # landmark for exec_prefix
+        name = "lib-dynload"
+        yield PathRefToDest(Path(interpreter.system_stdlib) / name, dest=cls.to_stdlib)
+
+        # this must symlink to the host prefix Python
+        marker = Path(interpreter.prefix) / "Python"
+        ref = PathRefToDest(marker, dest=lambda self, _: self.dest / ".Python", must_symlink=True)
+        yield ref
+
+    @classmethod
+    def _executables(cls, interpreter):
+        for _, targets in super(CPython2macOsFramework, cls)._executables(interpreter):
+            # Make sure we use the embedded interpreter inside the framework, even if sys.executable points to the
+            # stub executable in ${sys.prefix}/bin.
+            # See http://groups.google.com/group/python-virtualenv/browse_thread/thread/17cab2f85da75951
+            fixed_host_exe = Path(interpreter.prefix) / "Resources" / "Python.app" / "Contents" / "MacOS" / "Python"
+            yield fixed_host_exe, targets
+
+
+def fix_mach_o(exe, current, new, max_size):
+    """
+    https://en.wikipedia.org/wiki/Mach-O
+
+    Mach-O, short for Mach object file format, is a file format for executables, object code, shared libraries,
+    dynamically-loaded code, and core dumps. A replacement for the a.out format, Mach-O offers more extensibility and
+    faster access to information in the symbol table.
+
+    Each Mach-O file is made up of one Mach-O header, followed by a series of load commands, followed by one or more
+    segments, each of which contains between 0 and 255 sections. Mach-O uses the REL relocation format to handle
+    references to symbols. When looking up symbols Mach-O uses a two-level namespace that encodes each symbol into an
+    'object/symbol name' pair that is then linearly searched for by first the object and then the symbol name.
+
+    The basic structure—a list of variable-length "load commands" that reference pages of data elsewhere in the file—was
+    also used in the executable file format for Accent. The Accent file format was in turn, based on an idea from Spice
+    Lisp.
+
+    With the introduction of Mac OS X 10.6 platform the Mach-O file underwent a significant modification that causes
+    binaries compiled on a computer running 10.6 or later to be (by default) executable only on computers running Mac
+    OS X 10.6 or later. The difference stems from load commands that the dynamic linker, in previous Mac OS X versions,
+    does not understand. Another significant change to the Mach-O format is the change in how the Link Edit tables
+    (found in the __LINKEDIT section) function. In 10.6 these new Link Edit tables are compressed by removing unused and
+    unneeded bits of information, however Mac OS X 10.5 and earlier cannot read this new Link Edit table format.
+    """
+    try:
+        logging.debug(u"change Mach-O for %s from %s to %s", ensure_text(exe), current, ensure_text(new))
+        _builtin_change_mach_o(max_size)(exe, current, new)
+    except Exception as e:
+        logging.warning("Could not call _builtin_change_mac_o: %s. " "Trying to call install_name_tool instead.", e)
+        try:
+            cmd = ["install_name_tool", "-change", current, new, exe]
+            subprocess.check_call(cmd)
+        except Exception:
+            logging.fatal("Could not call install_name_tool -- you must " "have Apple's development tools installed")
+            raise
+
+
+def _builtin_change_mach_o(maxint):
+    MH_MAGIC = 0xFEEDFACE
+    MH_CIGAM = 0xCEFAEDFE
+    MH_MAGIC_64 = 0xFEEDFACF
+    MH_CIGAM_64 = 0xCFFAEDFE
+    FAT_MAGIC = 0xCAFEBABE
+    BIG_ENDIAN = ">"
+    LITTLE_ENDIAN = "<"
+    LC_LOAD_DYLIB = 0xC
+
+    class FileView(object):
+        """A proxy for file-like objects that exposes a given view of a file. Modified from macholib."""
+
+        def __init__(self, file_obj, start=0, size=maxint):
+            if isinstance(file_obj, FileView):
+                self._file_obj = file_obj._file_obj
+            else:
+                self._file_obj = file_obj
+            self._start = start
+            self._end = start + size
+            self._pos = 0
+
+        def __repr__(self):
+            return "<fileview [{:d}, {:d}] {!r}>".format(self._start, self._end, self._file_obj)
+
+        def tell(self):
+            return self._pos
+
+        def _checkwindow(self, seek_to, op):
+            if not (self._start <= seek_to <= self._end):
+                msg = "{} to offset {:d} is outside window [{:d}, {:d}]".format(op, seek_to, self._start, self._end)
+                raise IOError(msg)
+
+        def seek(self, offset, whence=0):
+            seek_to = offset
+            if whence == os.SEEK_SET:
+                seek_to += self._start
+            elif whence == os.SEEK_CUR:
+                seek_to += self._start + self._pos
+            elif whence == os.SEEK_END:
+                seek_to += self._end
+            else:
+                raise IOError("Invalid whence argument to seek: {!r}".format(whence))
+            self._checkwindow(seek_to, "seek")
+            self._file_obj.seek(seek_to)
+            self._pos = seek_to - self._start
+
+        def write(self, content):
+            here = self._start + self._pos
+            self._checkwindow(here, "write")
+            self._checkwindow(here + len(content), "write")
+            self._file_obj.seek(here, os.SEEK_SET)
+            self._file_obj.write(content)
+            self._pos += len(content)
+
+        def read(self, size=maxint):
+            assert size >= 0
+            here = self._start + self._pos
+            self._checkwindow(here, "read")
+            size = min(size, self._end - here)
+            self._file_obj.seek(here, os.SEEK_SET)
+            read_bytes = self._file_obj.read(size)
+            self._pos += len(read_bytes)
+            return read_bytes
+
+    def read_data(file, endian, num=1):
+        """Read a given number of 32-bits unsigned integers from the given file with the given endianness."""
+        res = struct.unpack(endian + "L" * num, file.read(num * 4))
+        if len(res) == 1:
+            return res[0]
+        return res
+
+    def mach_o_change(at_path, what, value):
+        """Replace a given name (what) in any LC_LOAD_DYLIB command found in the given binary with a new name (value),
+         provided it's shorter."""
+
+        def do_macho(file, bits, endian):
+            # Read Mach-O header (the magic number is assumed read by the caller)
+            cpu_type, cpu_sub_type, file_type, n_commands, size_of_commands, flags = read_data(file, endian, 6)
+            # 64-bits header has one more field.
+            if bits == 64:
+                read_data(file, endian)
+            # The header is followed by n commands
+            for _ in range(n_commands):
+                where = file.tell()
+                # Read command header
+                cmd, cmd_size = read_data(file, endian, 2)
+                if cmd == LC_LOAD_DYLIB:
+                    # The first data field in LC_LOAD_DYLIB commands is the offset of the name, starting from the
+                    # beginning of the  command.
+                    name_offset = read_data(file, endian)
+                    file.seek(where + name_offset, os.SEEK_SET)
+                    # Read the NUL terminated string
+                    load = file.read(cmd_size - name_offset).decode()
+                    load = load[: load.index("\0")]
+                    # If the string is what is being replaced, overwrite it.
+                    if load == what:
+                        file.seek(where + name_offset, os.SEEK_SET)
+                        file.write(value.encode() + b"\0")
+                # Seek to the next command
+                file.seek(where + cmd_size, os.SEEK_SET)
+
+        def do_file(file, offset=0, size=maxint):
+            file = FileView(file, offset, size)
+            # Read magic number
+            magic = read_data(file, BIG_ENDIAN)
+            if magic == FAT_MAGIC:
+                # Fat binaries contain nfat_arch Mach-O binaries
+                n_fat_arch = read_data(file, BIG_ENDIAN)
+                for _ in range(n_fat_arch):
+                    # Read arch header
+                    cpu_type, cpu_sub_type, offset, size, align = read_data(file, BIG_ENDIAN, 5)
+                    do_file(file, offset, size)
+            elif magic == MH_MAGIC:
+                do_macho(file, 32, BIG_ENDIAN)
+            elif magic == MH_CIGAM:
+                do_macho(file, 32, LITTLE_ENDIAN)
+            elif magic == MH_MAGIC_64:
+                do_macho(file, 64, BIG_ENDIAN)
+            elif magic == MH_CIGAM_64:
+                do_macho(file, 64, LITTLE_ENDIAN)
+
+        assert len(what) >= len(value)
+
+        with open(at_path, "r+b") as f:
+            do_file(f)
+
+    return mach_o_change
diff --git a/src/virtualenv/create/via_global_ref/builtin/python2/python2.py b/src/virtualenv/create/via_global_ref/builtin/python2/python2.py
index e917ca5..1f57aef 100644
--- a/src/virtualenv/create/via_global_ref/builtin/python2/python2.py
+++ b/src/virtualenv/create/via_global_ref/builtin/python2/python2.py
@@ -32,7 +32,8 @@ class Python2(ViaGlobalRefVirtualenvBuiltin, Python2Supports):
         else:
             custom_site_text = custom_site.read_text()
         expected = json.dumps([os.path.relpath(ensure_text(str(i)), ensure_text(str(site_py))) for i in self.libs])
-        site_py.write_text(custom_site_text.replace("___EXPECTED_SITE_PACKAGES___", expected))
+        custom_site_text = custom_site_text.replace("___EXPECTED_SITE_PACKAGES___", expected)
+        site_py.write_text(custom_site_text)
 
     @classmethod
     def sources(cls, interpreter):
diff --git a/src/virtualenv/create/via_global_ref/builtin/python2/site.py b/src/virtualenv/create/via_global_ref/builtin/python2/site.py
index 6ec0152..44654d2 100644
--- a/src/virtualenv/create/via_global_ref/builtin/python2/site.py
+++ b/src/virtualenv/create/via_global_ref/builtin/python2/site.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 """
 A simple shim module to fix up things on Python 2 only.
 
@@ -18,6 +19,7 @@ def main():
     load_host_site()
     if global_site_package_enabled:
         add_global_site_package()
+    fix_install()
 
 
 def load_host_site():
@@ -37,8 +39,7 @@ def load_host_site():
 
     here = __file__  # the distutils.install patterns will be injected relative to this site.py, save it here
 
-    with PatchForAppleFrameworkBuilds():
-        reload(sys.modules["site"])  # noqa
+    reload(sys.modules["site"])  # noqa # call system site.py to setup import libraries
 
     # and then if the distutils site packages are not on the sys.path we add them via add_site_dir; note we must add
     # them by invoking add_site_dir to trigger the processing of pth files
@@ -56,28 +57,12 @@ def load_host_site():
             add_site_dir(full_path)
 
 
-class PatchForAppleFrameworkBuilds(object):
-    """Apple Framework builds unconditionally add the global site-package, escape this behaviour"""
-
-    framework = None
-
-    def __enter__(self):
-        if sys.platform == "darwin":
-            from sysconfig import get_config_var
-
-            self.framework = get_config_var("PYTHONFRAMEWORK")
-            if self.framework:
-                sys.modules["sysconfig"]._CONFIG_VARS["PYTHONFRAMEWORK"] = None
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        if self.framework:
-            sys.modules["sysconfig"]._CONFIG_VARS["PYTHONFRAMEWORK"] = self.framework
+sep = "\\" if sys.platform == "win32" else "/"  # no os module here yet - poor mans version
 
 
 def read_pyvenv():
     """read pyvenv.cfg"""
-    os_sep = "\\" if sys.platform == "win32" else "/"  # no os module here yet - poor mans version
-    config_file = "{}{}pyvenv.cfg".format(sys.prefix, os_sep)
+    config_file = "{}{}pyvenv.cfg".format(sys.prefix, sep)
     with open(config_file) as file_handler:
         lines = file_handler.readlines()
     config = {}
@@ -93,23 +78,41 @@ def read_pyvenv():
 
 def rewrite_standard_library_sys_path():
     """Once this site file is loaded the standard library paths have already been set, fix them up"""
-    sep = "\\" if sys.platform == "win32" else "/"
-    exe_dir = sys.executable[: sys.executable.rfind(sep)]
+    exe = abs_path(sys.executable)
+    exe_dir = exe[: exe.rfind(sep)]
+    prefix, exec_prefix = abs_path(sys.prefix), abs_path(sys.exec_prefix)
+    base_prefix, base_exec_prefix = abs_path(sys.base_prefix), abs_path(sys.base_exec_prefix)
+    base_executable = abs_path(sys.base_executable)
     for at, value in enumerate(sys.path):
+        value = abs_path(value)
         # replace old sys prefix path starts with new
         if value == exe_dir:
             pass  # don't fix the current executable location, notably on Windows this gets added
         elif value.startswith(exe_dir):
             # content inside the exe folder needs to remap to original executables folder
-            orig_exe_folder = sys.base_executable[: sys.base_executable.rfind(sep)]
+            orig_exe_folder = base_executable[: base_executable.rfind(sep)]
             value = "{}{}".format(orig_exe_folder, value[len(exe_dir) :])
-        elif value.startswith(sys.prefix):
-            value = "{}{}".format(sys.base_prefix, value[len(sys.prefix) :])
-        elif value.startswith(sys.exec_prefix):
-            value = "{}{}".format(sys.base_exec_prefix, value[len(sys.exec_prefix) :])
+        elif value.startswith(prefix):
+            value = "{}{}".format(base_prefix, value[len(prefix) :])
+        elif value.startswith(exec_prefix):
+            value = "{}{}".format(base_exec_prefix, value[len(exec_prefix) :])
         sys.path[at] = value
 
 
+def abs_path(value):
+    keep = []
+    values = value.split(sep)
+    i = len(values) - 1
+    while i >= 0:
+        if values[i] == "..":
+            i -= 1
+        else:
+            keep.append(values[i])
+        i -= 1
+    value = sep.join(keep[::-1])
+    return value
+
+
 def disable_user_site_package():
     """Flip the switch on enable user site package"""
     # sys.flags is a c-extension type, so we cannot monkeypatch it, replace it with a python class to flip it
@@ -140,4 +143,29 @@ def add_global_site_package():
         site.PREFIXES = orig_prefixes
 
 
+def fix_install():
+    def patch(dist_of):
+        # we cannot allow the prefix override as that would get packages installed outside of the virtual environment
+        old_parse_config_files = dist_of.Distribution.parse_config_files
+
+        def parse_config_files(self, *args, **kwargs):
+            result = old_parse_config_files(self, *args, **kwargs)
+            install_dict = self.get_option_dict("install")
+            if "prefix" in install_dict:
+                install_dict["prefix"] = "virtualenv.patch", abs_path(sys.prefix)
+            return result
+
+        dist_of.Distribution.parse_config_files = parse_config_files
+
+    from distutils import dist
+
+    patch(dist)
+    try:
+        from setuptools import dist
+
+        patch(dist)
+    except ImportError:
+        pass  # if setuptools is not around that's alright, just don't patch
+
+
 main()
diff --git a/src/virtualenv/create/via_global_ref/builtin/ref.py b/src/virtualenv/create/via_global_ref/builtin/ref.py
index a579c4b..15a644a 100644
--- a/src/virtualenv/create/via_global_ref/builtin/ref.py
+++ b/src/virtualenv/create/via_global_ref/builtin/ref.py
@@ -1,3 +1,8 @@
+"""
+Virtual environments in the traditional sense are built as reference to the host python. This file allows declarative
+references to elements on the file system, allowing our system to automatically detect what modes it can support given
+the constraints: e.g. can the file system symlink, can the files be read, executed, etc.
+"""
 from __future__ import absolute_import, unicode_literals
 
 import os
@@ -14,15 +19,21 @@ from virtualenv.util.six import ensure_text
 
 @add_metaclass(ABCMeta)
 class PathRef(object):
+    """Base class that checks if a file reference can be symlink/copied"""
+
     FS_SUPPORTS_SYMLINK = fs_supports_symlink()
     FS_CASE_SENSITIVE = fs_is_case_sensitive()
 
-    def __init__(self, src):
+    def __init__(self, src, must_symlink, must_copy):
+        self.must_symlink = must_symlink
+        self.must_copy = must_copy
         self.src = src
         self.exists = src.exists()
         self._can_read = None if self.exists else False
         self._can_copy = None if self.exists else False
         self._can_symlink = None if self.exists else False
+        if self.must_copy is True and self.must_symlink is True:
+            raise ValueError("can copy and symlink at the same time")
 
     def __repr__(self):
         return "{}(src={})".format(self.__class__.__name__, self.src)
@@ -43,24 +54,39 @@ class PathRef(object):
     @property
     def can_copy(self):
         if self._can_copy is None:
-            self._can_copy = self.can_read
+            if self.must_symlink:
+                self._can_copy = self.can_symlink
+            else:
+                self._can_copy = self.can_read
         return self._can_copy
 
     @property
     def can_symlink(self):
         if self._can_symlink is None:
-            self._can_symlink = self.FS_SUPPORTS_SYMLINK and self.can_read
+            if self.must_copy:
+                self._can_symlink = self.can_copy
+            else:
+                self._can_symlink = self.FS_SUPPORTS_SYMLINK and self.can_read
         return self._can_symlink
 
     @abstractmethod
     def run(self, creator, symlinks):
         raise NotImplementedError
 
+    def method(self, symlinks):
+        if self.must_symlink:
+            return symlink
+        if self.must_copy:
+            return copy
+        return symlink if symlinks else copy
+
 
 @add_metaclass(ABCMeta)
 class ExePathRef(PathRef):
-    def __init__(self, src):
-        super(ExePathRef, self).__init__(src)
+    """Base class that checks if a executable can be references via symlink/copy"""
+
+    def __init__(self, src, must_symlink, must_copy):
+        super(ExePathRef, self).__init__(src, must_symlink, must_copy)
         self._can_run = None
 
     @property
@@ -83,22 +109,26 @@ class ExePathRef(PathRef):
 
 
 class PathRefToDest(PathRef):
-    def __init__(self, src, dest):
-        super(PathRefToDest, self).__init__(src)
+    """Link a path on the file system"""
+
+    def __init__(self, src, dest, must_symlink=False, must_copy=False):
+        super(PathRefToDest, self).__init__(src, must_symlink, must_copy)
         self.dest = dest
 
     def run(self, creator, symlinks):
         dest = self.dest(creator, self.src)
-        method = symlink if symlinks else copy
+        method = self.method(symlinks)
         dest_iterable = dest if isinstance(dest, list) else (dest,)
         for dst in dest_iterable:
             method(self.src, dst)
 
 
 class ExePathRefToDest(PathRefToDest, ExePathRef):
-    def __init__(self, src, targets, dest, must_copy=False):
-        ExePathRef.__init__(self, src)
-        PathRefToDest.__init__(self, src, dest)
+    """Link a exe path on the file system"""
+
+    def __init__(self, src, targets, dest, must_symlink=False, must_copy=False):
+        ExePathRef.__init__(self, src, must_symlink, must_copy)
+        PathRefToDest.__init__(self, src, dest, must_symlink, must_copy)
         if not self.FS_CASE_SENSITIVE:
             targets = list(OrderedDict((i.lower(), None) for i in targets).keys())
         self.base = targets[0]
@@ -108,8 +138,8 @@ class ExePathRefToDest(PathRefToDest, ExePathRef):
 
     def run(self, creator, symlinks):
         bin_dir = self.dest(creator, self.src).parent
-        method = symlink if self.must_copy is False and symlinks else copy
         dest = bin_dir / self.base
+        method = self.method(symlinks)
         method(self.src, dest)
         make_exe(dest)
         for extra in self.aliases: