support for c-extension builds within virtualenv (#1503)

* test include folders

- add test to check if it works

Signed-off-by: Bernat Gabor <bgabor8@bloomberg.net>

* pypy add lib on Linux

Signed-off-by: Bernat Gabor <bgabor8@bloomberg.net>

* fix Windows

* fix

Signed-off-by: Bernat Gabor <bgabor8@bloomberg.net>

* debug macos

Signed-off-by: Bernat Gabor <bgabor8@bloomberg.net>

* try fix pypy windows

Signed-off-by: Bernat Gabor <bgabor8@bloomberg.net>

* fix Windows

* fix

* fix

Signed-off-by: Bernat Gabor <bgabor8@bloomberg.net>

* Windows PyPy just does not understand non-ascii PATHS :-(

* allow pypy3 to fail

Signed-off-by: Bernat Gabor <bgabor8@bloomberg.net>

37 files changed, 552 insertions, 303 deletions

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 2de3f7b..4280986 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -37,13 +37,13 @@ jobs:
   parameters:
     jobs:
       py38:
-        image: [linux, windows, macOs]
+        image: [linux, vs2017-win2016, macOs]
       py37:
-        image: [linux, windows, macOs]
+        image: [linux, vs2017-win2016, macOs]
       py36:
-        image: [linux, windows, macOs]
+        image: [linux, vs2017-win2016, macOs]
       py35:
-        image: [linux, windows, macOs]
+        image: [linux, vs2017-win2016, macOs]
       py27:
         image: [linux, windows, macOs]
       pypy:
@@ -88,6 +88,9 @@ jobs:
         displayName: provision pypy 2
         inputs:
           versionSpec: 'pypy2'
+      - script: choco install vcpython27 --yes -f
+        condition: and(succeeded(), eq(variables['image_name'], 'windows'), in(variables['TOXENV'], 'py27', 'pypy'))
+        displayName: Install Visual C++ for Python 2.7
     coverage:
       with_toxenv: 'coverage' # generate .tox/.coverage, .tox/coverage.xml after test run
       for_envs: [py38, py37, py36, py35, py27, pypy, pypy3]
diff --git a/src/virtualenv/activation/batch/__init__.py b/src/virtualenv/activation/batch/__init__.py
index 977dd38..4149712 100644
--- a/src/virtualenv/activation/batch/__init__.py
+++ b/src/virtualenv/activation/batch/__init__.py
@@ -17,7 +17,7 @@ class BatchActivator(ViaTemplateActivator):
         yield Path("deactivate.bat")
         yield Path("pydoc.bat")
 
-    def instantiate_template(self, replacements, template):
+    def instantiate_template(self, replacements, template, creator):
         # ensure the text has all newlines as \r\n - required by batch
-        base = super(BatchActivator, self).instantiate_template(replacements, template)
+        base = super(BatchActivator, self).instantiate_template(replacements, template, creator)
         return base.replace(os.linesep, "\n").replace("\n", os.linesep)
diff --git a/src/virtualenv/activation/python/__init__.py b/src/virtualenv/activation/python/__init__.py
index 8d7e80e..1376243 100644
--- a/src/virtualenv/activation/python/__init__.py
+++ b/src/virtualenv/activation/python/__init__.py
@@ -1,8 +1,11 @@
 from __future__ import absolute_import, unicode_literals
 
-import json
 import os
+from collections import OrderedDict
 
+import six
+
+from virtualenv.info import WIN_CPYTHON_2
 from virtualenv.util.path import Path
 
 from ..via_template import ViaTemplateActivator
@@ -14,6 +17,18 @@ class PythonActivator(ViaTemplateActivator):
 
     def replacements(self, creator, dest_folder):
         replacements = super(PythonActivator, self).replacements(creator, dest_folder)
-        site_dump = json.dumps(list({os.path.relpath(str(i), str(dest_folder)) for i in creator.libs}), indent=2)
-        replacements.update({"__SITE_PACKAGES__": site_dump})
+        lib_folders = OrderedDict((os.path.relpath(str(i), str(dest_folder)), None) for i in creator.libs)
+        replacements.update(
+            {
+                "__LIB_FOLDERS__": six.ensure_text(os.pathsep.join(lib_folders.keys())),
+                "__DECODE_PATH__": ("yes" if WIN_CPYTHON_2 else ""),
+            }
+        )
         return replacements
+
+    @staticmethod
+    def _repr_unicode(creator, value):
+        py2 = creator.interpreter.version_info.major == 2
+        if py2:  # on Python 2 we need to encode this into explicit utf-8, py3 supports unicode literals
+            value = six.ensure_text(repr(value.encode("utf-8"))[1:-1])
+        return value
diff --git a/src/virtualenv/activation/python/activate_this.py b/src/virtualenv/activation/python/activate_this.py
index 19e373b..29debe3 100644
--- a/src/virtualenv/activation/python/activate_this.py
+++ b/src/virtualenv/activation/python/activate_this.py
@@ -5,51 +5,28 @@ Use exec(open(this_file).read(), {'__file__': this_file}).
 
 This can be used when you must use an existing Python interpreter, not the virtualenv bin/python.
 """
-import json
 import os
 import site
 import sys
 
 try:
-    __file__
+    abs_file = os.path.abspath(__file__)
 except NameError:
     raise AssertionError("You must use exec(open(this_file).read(), {'__file__': this_file}))")
 
-
-def set_env(key, value, encoding):
-    if sys.version_info[0] == 2:
-        value = value.encode(encoding)
-    os.environ[key] = value
-
+bin_dir = os.path.dirname(abs_file)
+base = bin_dir[: -len("__BIN_NAME__") - 1]  # strip away the bin part from the __file__, plus the path separator
 
 # prepend bin to PATH (this file is inside the bin directory)
-bin_dir = os.path.dirname(os.path.abspath(__file__))
-set_env("PATH", os.pathsep.join([bin_dir] + os.environ.get("PATH", "").split(os.pathsep)), sys.getfilesystemencoding())
-
-base = os.path.dirname(bin_dir)
-
-# virtual env is right above bin directory
-set_env("VIRTUAL_ENV", base, sys.getfilesystemencoding())
-
-# add the virtual environments site-packages to the host python import mechanism
-prev = set(sys.path)
-
-site_packages = r"""
-__SITE_PACKAGES__
-"""
-
-for site_package in json.loads(site_packages):
-    if sys.version_info[0] == 2:
-        site_package = site_package.encode("utf-8").decode(sys.getfilesystemencoding())
-    path = os.path.realpath(os.path.join(os.path.dirname(__file__), site_package))
-    if sys.version_info[0] == 2:
-        path = path.encode(sys.getfilesystemencoding())
-    site.addsitedir(path)
+os.environ["PATH"] = os.pathsep.join([bin_dir] + os.environ.get("PATH", "").split(os.pathsep))
+os.environ["VIRTUAL_ENV"] = base  # virtual env is right above bin directory
 
+# add the virtual environments libraries to the host python import mechanism
+prev_length = len(sys.path)
+for lib in "__LIB_FOLDERS__".split(os.pathsep):
+    path = os.path.realpath(os.path.join(bin_dir, lib))
+    site.addsitedir(path.decode("utf-8") if "__DECODE_PATH__" else path)
+sys.path[:] = sys.path[prev_length:] + sys.path[0:prev_length]
 
 sys.real_prefix = sys.prefix
 sys.prefix = base
-
-# Move the added items to the front of the path, in place
-new = list(sys.path)
-sys.path[:] = [i for i in new if i not in prev] + [i for i in new if i in prev]
diff --git a/src/virtualenv/activation/via_template.py b/src/virtualenv/activation/via_template.py
index 710126c..3f6b46b 100644
--- a/src/virtualenv/activation/via_template.py
+++ b/src/virtualenv/activation/via_template.py
@@ -22,7 +22,8 @@ class ViaTemplateActivator(Activator):
 
     def generate(self, creator):
         dest_folder = creator.bin_dir
-        self._generate(self.replacements(creator, dest_folder), self.templates(), dest_folder)
+        replacements = self.replacements(creator, dest_folder)
+        self._generate(replacements, self.templates(), dest_folder, creator)
         if self.flag_prompt is not None:
             creator.pyenv_cfg["prompt"] = self.flag_prompt
 
@@ -32,17 +33,23 @@ class ViaTemplateActivator(Activator):
             "__VIRTUAL_ENV__": six.ensure_text(str(creator.dest)),
             "__VIRTUAL_NAME__": creator.env_name,
             "__BIN_NAME__": six.ensure_text(str(creator.bin_dir.relative_to(creator.dest))),
-            "__PATH_SEP__": os.pathsep,
+            "__PATH_SEP__": six.ensure_text(os.pathsep),
         }
 
-    def _generate(self, replacements, templates, to_folder):
+    def _generate(self, replacements, templates, to_folder, creator):
         for template in templates:
-            text = self.instantiate_template(replacements, template)
+            text = self.instantiate_template(replacements, template, creator)
             (to_folder / template).write_text(text, encoding="utf-8")
 
-    def instantiate_template(self, replacements, template):
+    def instantiate_template(self, replacements, template, creator):
         # read text and do replacements
         text = read_text(self.__module__, str(template), encoding="utf-8", errors="strict")
-        for start, end in replacements.items():
-            text = text.replace(start, end)
+        for key, value in replacements.items():
+            value = self._repr_unicode(creator, value)
+            text = text.replace(key, value)
         return text
+
+    @staticmethod
+    def _repr_unicode(creator, value):
+        # by default we just let it be unicode
+        return value
diff --git a/src/virtualenv/create/creator.py b/src/virtualenv/create/creator.py
index 2ec19ae..8548271 100644
--- a/src/virtualenv/create/creator.py
+++ b/src/virtualenv/create/creator.py
@@ -15,7 +15,7 @@ import six
 from six import add_metaclass
 
 from virtualenv.discovery.py_info import Cmd
-from virtualenv.info import IS_ZIPAPP
+from virtualenv.info import IS_PYPY, IS_ZIPAPP
 from virtualenv.pyenv_cfg import PyEnvCfg
 from virtualenv.util.path import Path
 from virtualenv.util.subprocess import run_cmd
@@ -173,8 +173,10 @@ class Creator(object):
 def get_env_debug_info(env_exe, debug_script):
     if IS_ZIPAPP:
         debug_script = extract_to_app_data(debug_script)
-    cmd = [six.ensure_text(str(env_exe)), six.ensure_text(str(debug_script))]
-    logging.debug("debug via %r", Cmd(cmd))
+    cmd = [str(env_exe), str(debug_script)]
+    if not IS_PYPY and six.PY2:
+        cmd = [six.ensure_text(i) for i in cmd]
+    logging.debug(str("debug via %r"), Cmd(cmd))
     env = os.environ.copy()
     env.pop(str("PYTHONPATH"), None)
     code, out, err = run_cmd(cmd)
diff --git a/src/virtualenv/create/debug.py b/src/virtualenv/create/debug.py
index 674d1bb..d015be6 100644
--- a/src/virtualenv/create/debug.py
+++ b/src/virtualenv/create/debug.py
@@ -1,14 +1,19 @@
 """Inspect a target Python interpreter virtual environment wise"""
 import sys  # built-in
 
+PYPY2_WIN = hasattr(sys, "pypy_version_info") and sys.platform != "win32" and sys.version_info[0] == 2
+
 
 def encode_path(value):
     if value is None:
         return None
-    if isinstance(value, bytes):
-        return value.decode(sys.getfilesystemencoding())
-    elif not isinstance(value, str):
-        return repr(value if isinstance(value, type) else type(value))
+    if not isinstance(value, (str, bytes)):
+        if isinstance(value, type):
+            value = repr(value)
+        else:
+            value = repr(type(value))
+    if isinstance(value, bytes) and not PYPY2_WIN:
+        value = value.decode(sys.getfilesystemencoding())
     return value
 
 
diff --git a/src/virtualenv/create/describe.py b/src/virtualenv/create/describe.py
index 0c3a985..b2686a6 100644
--- a/src/virtualenv/create/describe.py
+++ b/src/virtualenv/create/describe.py
@@ -77,6 +77,9 @@ class Describe(object):
         """executable name without suffix - there seems to be no standard way to get this without creating it"""
         raise NotImplementedError
 
+    def script(self, name):
+        return self.script_dir / "{}{}".format(name, self.suffix)
+
 
 @add_metaclass(ABCMeta)
 class Python2Supports(Describe):
diff --git a/src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py b/src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py
index a235164..cd791f4 100644
--- a/src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py
+++ b/src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py
@@ -1,10 +1,11 @@
 from __future__ import absolute_import, unicode_literals
 
 import abc
+import logging
 
 import six
 
-from virtualenv.create.via_global_ref.builtin.ref import RefToDest
+from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest
 from virtualenv.util.path import Path
 
 from ..python2.python2 import Python2
@@ -16,22 +17,49 @@ class CPython2(CPython, Python2):
     """Create a CPython version 2  virtual environment"""
 
     @classmethod
+    def sources(cls, interpreter):
+        for src in super(CPython2, cls).sources(interpreter):
+            yield src
+        # include folder needed on Python 2 as we don't have pyenv.cfg
+        host_include_marker = cls.host_include_marker(interpreter)
+        if host_include_marker.exists():
+            yield PathRefToDest(host_include_marker.parent, dest=lambda self, _: self.include)
+
+    @classmethod
+    def host_include_marker(cls, interpreter):
+        return Path(interpreter.system_include) / "Python.h"
+
+    @property
+    def include(self):
+        # the pattern include the distribution name too at the end, remove that via the parent call
+        return (self.dest / self.interpreter.distutils_install["headers"]).parent
+
+    @classmethod
     def modules(cls):
         return [
             "os",  # landmark to set sys.prefix
         ]
 
+    def ensure_directories(self):
+        dirs = super(CPython2, self).ensure_directories()
+        host_include_marker = self.host_include_marker(self.interpreter)
+        if host_include_marker.exists():
+            dirs.add(self.include.parent)
+        else:
+            logging.debug("no include folders as can't find include marker %s", host_include_marker)
+        return dirs
+
 
 class CPython2Posix(CPython2, CPythonPosix):
     """CPython 2 on POSIX"""
 
     @classmethod
     def sources(cls, interpreter):
-        for src in super(CPythonPosix, cls).sources(interpreter):
+        for src in super(CPython2Posix, cls).sources(interpreter):
             yield src
         # landmark for exec_prefix
         name = "lib-dynload"
-        yield RefToDest(Path(interpreter.system_stdlib) / name, dest=cls.to_stdlib)
+        yield PathRefToDest(Path(interpreter.system_stdlib) / name, dest=cls.to_stdlib)
 
 
 class CPython2Windows(CPython2, CPythonWindows):
@@ -43,4 +71,8 @@ class CPython2Windows(CPython2, CPythonWindows):
             yield src
         py27_dll = Path(interpreter.system_executable).parent / "python27.dll"
         if py27_dll.exists():  # this might be global in the Windows folder in which case it's alright to be missing
-            yield RefToDest(py27_dll, dest=cls.to_bin)
+            yield PathRefToDest(py27_dll, dest=cls.to_bin)
+
+        libs = Path(interpreter.system_prefix) / "libs"
+        if libs.exists():
+            yield PathRefToDest(libs, dest=lambda self, s: self.dest / s.name)
diff --git a/src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py b/src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py
index 07888b7..5d89f1f 100644
--- a/src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py
+++ b/src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py
@@ -5,7 +5,7 @@ import abc
 import six
 
 from virtualenv.create.describe import Python3Supports
-from virtualenv.create.via_global_ref.builtin.ref import RefToDest
+from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest
 from virtualenv.util.path import Path
 
 from .common import CPython, CPythonPosix, CPythonWindows
@@ -37,7 +37,7 @@ class CPython3Windows(CPythonWindows, CPython3):
         for folder in [host_exe_folder, dll_folder]:
             for file in folder.iterdir():
                 if file.suffix in (".pyd", ".dll"):
-                    yield RefToDest(file, dest=cls.to_dll_and_pyd)
+                    yield PathRefToDest(file, dest=cls.to_dll_and_pyd)
 
     def to_dll_and_pyd(self, src):
         return self.bin_dir / src.name
diff --git a/src/virtualenv/create/via_global_ref/builtin/pypy/common.py b/src/virtualenv/create/via_global_ref/builtin/pypy/common.py
index a5383b1..c5bc849 100644
--- a/src/virtualenv/create/via_global_ref/builtin/pypy/common.py
+++ b/src/virtualenv/create/via_global_ref/builtin/pypy/common.py
@@ -4,7 +4,7 @@ import abc
 
 import six
 
-from virtualenv.create.via_global_ref.builtin.ref import RefToDest
+from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest
 from virtualenv.util.path import Path
 
 from ..via_global_self_do import ViaGlobalRefVirtualenvBuiltin
@@ -35,7 +35,7 @@ class PyPy(ViaGlobalRefVirtualenvBuiltin):
         for src in super(PyPy, cls).sources(interpreter):
             yield src
         for host in cls._add_shared_libs(interpreter):
-            yield RefToDest(host, dest=cls.to_shared_lib)
+            yield PathRefToDest(host, dest=lambda self, s: self.bin_dir / s.name)
 
     @classmethod
     def _add_shared_libs(cls, interpreter):
@@ -46,9 +46,6 @@ class PyPy(ViaGlobalRefVirtualenvBuiltin):
             if src.exists():
                 yield src
 
-    def to_shared_lib(self, src):
-        return [self.bin_dir]
-
     @classmethod
     def _shared_libs(cls):
         raise NotImplementedError
diff --git a/src/virtualenv/create/via_global_ref/builtin/pypy/pypy2.py b/src/virtualenv/create/via_global_ref/builtin/pypy/pypy2.py
index 9903c95..c00ff4f 100644
--- a/src/virtualenv/create/via_global_ref/builtin/pypy/pypy2.py
+++ b/src/virtualenv/create/via_global_ref/builtin/pypy/pypy2.py
@@ -1,10 +1,13 @@
 from __future__ import absolute_import, unicode_literals
 
 import abc
+import logging
 
 import six
 
 from virtualenv.create.describe import PosixSupports, WindowsSupports
+from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest
+from virtualenv.util.path import Path
 
 from ..python2.python2 import Python2
 from .common import PyPy
@@ -18,26 +21,50 @@ class PyPy2(PyPy, Python2):
     def exe_stem(cls):
         return "pypy"
 
-    @property
-    def lib_pypy(self):
-        return self.dest / "lib_pypy"
+    @classmethod
+    def sources(cls, interpreter):
+        for src in super(PyPy2, cls).sources(interpreter):
+            yield src
+        # include folder needed on Python 2 as we don't have pyenv.cfg
+        host_include_marker = cls.host_include_marker(interpreter)
+        if host_include_marker.exists():
+            yield PathRefToDest(host_include_marker.parent, dest=lambda self, _: self.include)
 
-    def ensure_directories(self):
-        return super(PyPy, self).ensure_directories() | {self.lib_pypy}
+    @classmethod
+    def host_include_marker(cls, interpreter):
+        return Path(interpreter.system_include) / "PyPy.h"
+
+    @property
+    def include(self):
+        return self.dest / self.interpreter.distutils_install["headers"]
 
     @classmethod
     def modules(cls):
         # pypy2 uses some modules before the site.py loads, so we need to include these too
         return super(PyPy2, cls).modules() + [
+            "os",
             "copy_reg",
             "genericpath",
             "linecache",
-            "os",
             "stat",
             "UserDict",
             "warnings",
         ]
 
+    @property
+    def lib_pypy(self):
+        return self.dest / "lib_pypy"
+
+    def ensure_directories(self):
+        dirs = super(PyPy2, self).ensure_directories()
+        dirs.add(self.lib_pypy)
+        host_include_marker = self.host_include_marker(self.interpreter)
+        if host_include_marker.exists():
+            dirs.add(self.include.parent)
+        else:
+            logging.debug("no include folders as can't find include marker %s", host_include_marker)
+        return dirs
+
 
 class PyPy2Posix(PyPy2, PosixSupports):
     """PyPy 2 on POSIX"""
@@ -50,6 +77,18 @@ class PyPy2Posix(PyPy2, PosixSupports):
     def _shared_libs(cls):
         return ["libpypy-c.so", "libpypy-c.dylib"]
 
+    @property
+    def lib(self):
+        return self.dest / "lib"
+
+    @classmethod
+    def sources(cls, interpreter):
+        for src in super(PyPy2Posix, cls).sources(interpreter):
+            yield src
+        host_lib = Path(interpreter.system_prefix) / "lib"
+        if host_lib.exists():
+            yield PathRefToDest(host_lib, dest=lambda self, _: self.lib)
+
 
 class Pypy2Windows(PyPy2, WindowsSupports):
     """PyPy 2 on Windows"""
@@ -61,3 +100,9 @@ class Pypy2Windows(PyPy2, WindowsSupports):
     @classmethod
     def _shared_libs(cls):
         return ["libpypy-c.dll"]
+
+    @classmethod
+    def sources(cls, interpreter):
+        for src in super(Pypy2Windows, cls).sources(interpreter):
+            yield src
+        yield PathRefToDest(Path(interpreter.system_prefix) / "libs", dest=lambda self, s: self.dest / s.name)
diff --git a/src/virtualenv/create/via_global_ref/builtin/pypy/pypy3.py b/src/virtualenv/create/via_global_ref/builtin/pypy/pypy3.py
index b2cddb1..a6d64f8 100644
--- a/src/virtualenv/create/via_global_ref/builtin/pypy/pypy3.py
+++ b/src/virtualenv/create/via_global_ref/builtin/pypy/pypy3.py
@@ -5,6 +5,8 @@ import abc
 import six
 
 from virtualenv.create.describe import PosixSupports, Python3Supports, WindowsSupports
+from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest
+from virtualenv.util.path import Path
 
 from .common import PyPy
 
@@ -35,8 +37,17 @@ class PyPy3Posix(PyPy3, PosixSupports):
     def _shared_libs(cls):
         return ["libpypy3-c.so", "libpypy3-c.dylib"]
 
-    def to_shared_lib(self, src):
-        return super(PyPy3, self).to_shared_lib(src) + [self.stdlib.parent.parent]
+    def to_lib(self, src):
+        return self.dest / "lib" / src.name
+
+    @classmethod
+    def sources(cls, interpreter):
+        for src in super(PyPy3Posix, cls).sources(interpreter):
+            yield src
+        host_lib = Path(interpreter.system_prefix) / "lib"
+        if host_lib.exists() and host_lib.is_dir():
+            for path in host_lib.iterdir():
+                yield PathRefToDest(path, dest=cls.to_lib)
 
 
 class Pypy3Windows(PyPy3, WindowsSupports):
diff --git a/src/virtualenv/create/via_global_ref/builtin/python2/python2.py b/src/virtualenv/create/via_global_ref/builtin/python2/python2.py
index f9cc006..9b14c2e 100644
--- a/src/virtualenv/create/via_global_ref/builtin/python2/python2.py
+++ b/src/virtualenv/create/via_global_ref/builtin/python2/python2.py
@@ -7,11 +7,11 @@ import os
 import six
 
 from virtualenv.create.describe import Python2Supports
+from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest
 from virtualenv.info import IS_ZIPAPP
 from virtualenv.util.path import Path
 from virtualenv.util.zipapp import read as read_from_zipapp
 
-from ..ref import RefToDest
 from ..via_global_self_do import ViaGlobalRefVirtualenvBuiltin
 
 HERE = Path(__file__).absolute().parent
@@ -41,9 +41,10 @@ class Python2(ViaGlobalRefVirtualenvBuiltin, Python2Supports):
             yield src
         # install files needed to run site.py
         for req in cls.modules():
-            for ext in ["py", "pyc"]:
-                file_path = "{}.{}".format(req, ext)
-                yield RefToDest(Path(interpreter.system_stdlib) / file_path, dest=cls.to_stdlib)
+            yield PathRefToDest(Path(interpreter.system_stdlib) / "{}.py".format(req), dest=cls.to_stdlib)
+            comp = Path(interpreter.system_stdlib) / "{}.pyc".format(req)
+            if comp.exists():
+                yield PathRefToDest(comp, dest=cls.to_stdlib)
 
     def to_stdlib(self, src):
         return self.stdlib / src.name
diff --git a/src/virtualenv/create/via_global_ref/builtin/ref.py b/src/virtualenv/create/via_global_ref/builtin/ref.py
index 24ef206..0c9c361 100644
--- a/src/virtualenv/create/via_global_ref/builtin/ref.py
+++ b/src/virtualenv/create/via_global_ref/builtin/ref.py
@@ -1,6 +1,5 @@
 from __future__ import absolute_import, unicode_literals
 
-import logging
 import os
 from abc import ABCMeta, abstractmethod
 from collections import OrderedDict
@@ -9,14 +8,11 @@ from stat import S_IXGRP, S_IXOTH, S_IXUSR
 from six import add_metaclass, ensure_text
 
 from virtualenv.info import PY3, fs_is_case_sensitive, fs_supports_symlink
-from virtualenv.util.path import copy, make_exe, symlink
-
-if PY3:
-    from os import link
+from virtualenv.util.path import copy, link, make_exe, symlink
 
 
 @add_metaclass(ABCMeta)
-class Ref(object):
+class PathRef(object):
     FS_SUPPORTS_SYMLINK = fs_supports_symlink()
     FS_CASE_SENSITIVE = fs_is_case_sensitive()
 
@@ -59,14 +55,11 @@ class Ref(object):
     def run(self, creator, symlinks):
         raise NotImplementedError
 
-    def method(self, via_symlink):
-        pass
-
 
 @add_metaclass(ABCMeta)
-class ExeRef(Ref):
+class ExePathRef(PathRef):
     def __init__(self, src):
-        super(ExeRef, self).__init__(src)
+        super(ExePathRef, self).__init__(src)
         self._can_run = None
 
     @property
@@ -88,22 +81,26 @@ class ExeRef(Ref):
         return self._can_run
 
 
-class RefToDest(Ref):
+class PathRefToDest(PathRef):
     def __init__(self, src, dest):
-        super(RefToDest, self).__init__(src)
+        super(PathRefToDest, self).__init__(src)
         self.dest = dest
 
-    def run(self, creator, method):
+    def run(self, creator, symlinks):
         dest = self.dest(creator, self.src)
-        if not isinstance(dest, list):
-            dest = [dest]
-        for dst in dest:
+        method = symlink if symlinks else copy
+        dest_iterable = dest if isinstance(dest, list) else (dest,)
+        for dst in dest_iterable:
             method(self.src, dst)
 
 
-class ExeRefToDest(ExeRef):
+alias_via = link if PY3 else (symlink if PathRef.FS_SUPPORTS_SYMLINK else copy)
+
+
+class ExePathRefToDest(PathRefToDest, ExePathRef):
     def __init__(self, src, targets, dest, must_copy=False):
-        super(ExeRefToDest, self).__init__(src)
+        ExePathRef.__init__(self, src)
+        PathRefToDest.__init__(self, src, dest)
         if not self.FS_CASE_SENSITIVE:
             targets = list(OrderedDict((i.lower(), None) for i in targets).keys())
         self.base = targets[0]
@@ -111,24 +108,15 @@ class ExeRefToDest(ExeRef):
         self.dest = dest
         self.must_copy = must_copy
 
-    def run(self, creator, method):
-        to = self.dest(creator, self.src).parent
-        dest = to / self.base
-        if self.must_copy:
-            method = copy
+    def run(self, creator, symlinks):
+        bin_dir = self.dest(creator, self.src).parent
+        method = symlink if self.must_copy is False and symlinks else copy
+        dest = bin_dir / self.base
         method(self.src, dest)
         make_exe(dest)
-
         for extra in self.aliases:
-            link_file = to / extra
+            link_file = bin_dir / extra
             if link_file.exists():
                 link_file.unlink()
-            self.alias_via(dest, link_file)
+            alias_via(dest, link_file)
             make_exe(link_file)
-
-    @staticmethod
-    def do_link(src, dst):
-        logging.debug("hard link %s as %s", dst.name, src)
-        link(ensure_text(str(src)), ensure_text(str(dst)))
-
-    alias_via = do_link if PY3 else (symlink if Ref.FS_SUPPORTS_SYMLINK else copy)
diff --git a/src/virtualenv/create/via_global_ref/builtin/via_global_self_do.py b/src/virtualenv/create/via_global_ref/builtin/via_global_self_do.py
index 9c19252..bc1cc44 100644
--- a/src/virtualenv/create/via_global_ref/builtin/via_global_self_do.py
+++ b/src/virtualenv/create/via_global_ref/builtin/via_global_self_do.py
@@ -4,14 +4,14 @@ import logging
 from abc import ABCMeta
 from collections import namedtuple
 
-from six import PY2, add_metaclass
+from six import add_metaclass
 
+from virtualenv.create.via_global_ref.builtin.ref import ExePathRefToDest
 from virtualenv.info import fs_supports_symlink
-from virtualenv.util.path import copy, ensure_dir, symlink
+from virtualenv.util.path import ensure_dir
 
 from ..api import ViaGlobalRefApi
 from .builtin_way import VirtualenvBuiltin
-from .ref import ExeRefToDest
 
 Meta = namedtuple("Meta", ["sources", "can_copy", "can_symlink"])
 
@@ -50,8 +50,9 @@ class ViaGlobalRefVirtualenvBuiltin(ViaGlobalRefApi, VirtualenvBuiltin):
 
     @classmethod
     def sources(cls, interpreter):
+        is_py2 = interpreter.version_info.major == 2
         for host_exe, targets in cls._executables(interpreter):
-            yield ExeRefToDest(host_exe, dest=cls.to_bin, targets=targets, must_copy=PY2)
+            yield ExePathRefToDest(host_exe, dest=cls.to_bin, targets=targets, must_copy=is_py2)
 
     def to_bin(self, src):
         return self.bin_dir / src.name
@@ -73,9 +74,8 @@ class ViaGlobalRefVirtualenvBuiltin(ViaGlobalRefApi, VirtualenvBuiltin):
         true_system_site = self.enable_system_site_package
         try:
             self.enable_system_site_package = False
-            method = symlink if self.symlinks else copy
             for src in self._sources:
-                src.run(self, method)
+                src.run(self, self.symlinks)
         finally:
             if true_system_site != self.enable_system_site_package:
                 self.enable_system_site_package = true_system_site
diff --git a/src/virtualenv/create/via_global_ref/venv.py b/src/virtualenv/create/via_global_ref/venv.py
index ef2e439..52a3729 100644
--- a/src/virtualenv/create/via_global_ref/venv.py
+++ b/src/virtualenv/create/via_global_ref/venv.py
@@ -73,6 +73,6 @@ class Venv(ViaGlobalRefApi):
         describe = object.__getattribute__(self, "describe")
         if describe is not None and hasattr(describe, item):
             element = getattr(describe, item)
-            if not callable(element):
+            if not callable(element) or item in ("script",):
                 return element
         return object.__getattribute__(self, item)
diff --git a/src/virtualenv/discovery/py_info.py b/src/virtualenv/discovery/py_info.py
index 57da58d..3b3f987 100644
--- a/src/virtualenv/discovery/py_info.py
+++ b/src/virtualenv/discovery/py_info.py
@@ -16,6 +16,7 @@ import sysconfig
 from collections import OrderedDict, namedtuple
 from distutils.command.install import SCHEME_KEYS
 from distutils.dist import Distribution
+from string import digits
 
 VersionInfo = namedtuple("VersionInfo", ["major", "minor", "micro", "releaselevel", "serial"])
 
@@ -127,7 +128,9 @@ class PythonInfo(object):
         return content
 
     def __repr__(self):
-        return "{}({!r})".format(self.__class__.__name__, self.__dict__)
+        return "{}({!r})".format(
+            self.__class__.__name__, {k: v for k, v in self.__dict__.items() if not k.startswith("_")}
+        )
 
     def __str__(self):
         content = "{}({})".format(
@@ -214,7 +217,9 @@ class PythonInfo(object):
     def _find_possible_folders(self, inside_folder):
         candidate_folder = OrderedDict()
         executables = OrderedDict()
+        executables[os.path.realpath(self.executable)] = None
         executables[self.executable] = None
+        executables[os.path.realpath(self.original_executable)] = None
         executables[self.original_executable] = None
         for exe in executables.keys():
             base = os.path.dirname(exe)
@@ -241,7 +246,8 @@ class PythonInfo(object):
 
     def _possible_base(self):
         possible_base = OrderedDict()
-        possible_base[os.path.splitext(os.path.basename(self.executable))[0]] = None
+        basename = os.path.splitext(os.path.basename(self.executable))[0].rstrip(digits)
+        possible_base[basename] = None
         possible_base[self.implementation] = None
         # python is always the final option as in practice is used by multiple implementation as exe name
         if "python" in possible_base:
@@ -365,6 +371,13 @@ class PythonInfo(object):
             self._creators = CreatorSelector.for_interpreter(self)
         return self._creators
 
+    @property
+    def system_include(self):
+        return self.sysconfig_path(
+            "include",
+            {k: (self.system_prefix if v.startswith(self.prefix) else v) for k, v in self.sysconfig_vars.items()},
+        )
+
 
 class Cmd(object):
     def __init__(self, cmd, env=None):
@@ -378,8 +391,16 @@ class Cmd(object):
         cmd_repr = e(" ").join(pipes.quote(e(c)) for c in self.cmd)
         if self.env is not None:
             cmd_repr += e(" env of {!r}").format(self.env)
+        if sys.version_info[0] == 2:
+            return cmd_repr.encode("utf-8")
         return cmd_repr
 
+    def __unicode__(self):
+        raw = repr(self)
+        if sys.version_info[0] == 2:
+            return raw.decode("utf-8")
+        return raw
+
 
 CURRENT = PythonInfo()
 
diff --git a/src/virtualenv/info.py b/src/virtualenv/info.py
index b878728..ba00218 100644
--- a/src/virtualenv/info.py
+++ b/src/virtualenv/info.py
@@ -12,9 +12,11 @@ IMPLEMENTATION = platform.python_implementation()
 IS_PYPY = IMPLEMENTATION == "PyPy"
 IS_CPYTHON = IMPLEMENTATION == "CPython"
 PY3 = sys.version_info[0] == 3
+PY2 = sys.version_info[0] == 2
 IS_WIN = sys.platform == "win32"
 ROOT = os.path.realpath(os.path.join(os.path.abspath(__file__), os.path.pardir, os.path.pardir))
 IS_ZIPAPP = os.path.isfile(ROOT)
+WIN_CPYTHON_2 = IS_CPYTHON and IS_WIN and PY2
 
 _CAN_SYMLINK = _FS_CASE_SENSITIVE = _CFG_DIR = _DATA_DIR = None
 
@@ -82,4 +84,5 @@ __all__ = (
     "fs_supports_symlink",
     "ROOT",
     "IS_ZIPAPP",
+    "WIN_CPYTHON_2",
 )
diff --git a/src/virtualenv/pyenv_cfg.py b/src/virtualenv/pyenv_cfg.py
index 218c759..923316b 100644
--- a/src/virtualenv/pyenv_cfg.py
+++ b/src/virtualenv/pyenv_cfg.py
@@ -30,13 +30,14 @@ class PyEnvCfg(object):
         return content
 
     def write(self):
-        with open(six.ensure_text(str(self.path)), "wb") as file_handler:
-            logging.debug("write %s", six.ensure_text(str(self.path)))
-            for key, value in self.content.items():
-                line = "{} = {}".format(key, value)
-                logging.debug("\t%s", line)
-                file_handler.write(line.encode("utf-8"))
-                file_handler.write(b"\n")
+        logging.debug("write %s", six.ensure_text(str(self.path)))
+        text = ""
+        for key, value in self.content.items():
+            line = "{} = {}".format(key, value)
+            logging.debug("\t%s", line)
+            text += line
+            text += "\n"
+        self.path.write_text(text, encoding="utf-8")
 
     def refresh(self):
         self.content = self._read_values(self.path)
diff --git a/src/virtualenv/report.py b/src/virtualenv/report.py
index be7506e..fc190db 100644
--- a/src/virtualenv/report.py
+++ b/src/virtualenv/report.py
@@ -26,7 +26,7 @@ def setup_report(verbose, quiet):
     level = LEVELS[verbosity]
     msg_format = "%(message)s"
     if level <= logging.DEBUG:
-        locate = "pathname" if level < logging.DEBUG else "module"
+        locate = "module"
         msg_format = "%(relativeCreated)d {} [%(levelname)s %({})s:%(lineno)d]".format(msg_format, locate)
 
     formatter = logging.Formatter(six.ensure_str(msg_format))
diff --git a/src/virtualenv/util/path/__init__.py b/src/virtualenv/util/path/__init__.py
index e00acd5..6cae265 100644
--- a/src/virtualenv/util/path/__init__.py
+++ b/src/virtualenv/util/path/__init__.py
@@ -2,6 +2,6 @@ from __future__ import absolute_import, unicode_literals
 
 from ._pathlib import Path
 from ._permission import make_exe
-from ._sync import copy, ensure_dir, symlink, symlink_or_copy
+from ._sync import copy, ensure_dir, link, symlink
 
-__all__ = ("ensure_dir", "symlink_or_copy", "symlink", "copy", "Path", "make_exe")
+__all__ = ("ensure_dir", "link", "symlink", "copy", "Path", "make_exe")
diff --git a/src/virtualenv/util/path/_pathlib/via_os_path.py b/src/virtualenv/util/path/_pathlib/via_os_path.py
index 3eece17..44bd2bf 100644
--- a/src/virtualenv/util/path/_pathlib/via_os_path.py
+++ b/src/virtualenv/util/path/_pathlib/via_os_path.py
@@ -1,26 +1,41 @@
 from __future__ import absolute_import, unicode_literals
 
 import os
+import platform
 from contextlib import contextmanager
 
 import six
 
+IS_PYPY = platform.python_implementation() == "PyPy"
+
 
 class Path(object):
     def __init__(self, path):
-        self._path = path._path if isinstance(path, Path) else six.ensure_text(path)
+        if isinstance(path, Path):
+            _path = path._path
+        else:
+            _path = six.ensure_text(path)
+            if IS_PYPY:
+                _path = _path.encode("utf-8")
+        self._path = _path
 
     def __repr__(self):
-        return six.ensure_str("Path({})".format(self._path))
+        return six.ensure_str("Path({})".format(six.ensure_text(self._path)))
 
     def __unicode__(self):
-        return self._path
+        return six.ensure_text(self._path)
 
     def __str__(self):
         return six.ensure_str(self._path)
 
     def __div__(self, other):
-        return Path(os.path.join(self._path, other._path if isinstance(other, Path) else six.ensure_text(other)))
+        if isinstance(other, Path):
+            right = other._path
+        else:
+            right = six.ensure_text(other)
+            if IS_PYPY:
+                right = right.encode("utf-8")
+        return Path(os.path.join(self._path, right))
 
     def __truediv__(self, other):
         return self.__div__(other)
diff --git a/src/virtualenv/util/path/_sync.py b/src/virtualenv/util/path/_sync.py
index 2759b32..9bb2067 100644
--- a/src/virtualenv/util/path/_sync.py
+++ b/src/virtualenv/util/path/_sync.py
@@ -3,61 +3,68 @@ from __future__ import absolute_import, unicode_literals
 import logging
 import os
 import shutil
-import sys
-from functools import partial
 
-import six
+from six import PY2, PY3, ensure_text
 
-from virtualenv.info import IS_PYPY, fs_supports_symlink
+from virtualenv.info import IS_CPYTHON, IS_WIN
 
-HAS_SYMLINK = fs_supports_symlink()
+if PY3:
+    from os import link as os_link
+
+if PY2 and IS_CPYTHON and IS_WIN:  # CPython2 on Windows supports unicode paths if passed as unicode
+    norm = lambda src: ensure_text(str(src))  # noqa
+else:
+    norm = str
 
 
 def ensure_dir(path):
     if not path.exists():
-        logging.debug("create folder %s", six.ensure_text(str(path)))
-        os.makedirs(six.ensure_text(str(path)))
-
-
-def symlink_or_copy(do_copy, src, dst, relative_symlinks_ok=False):
-    """
-    Try symlinking a target, and if that fails, fall back to copying.
-    """
-    if not src.exists():
-        raise RuntimeError("source {} does not exists".format(src))
-    if src == dst:
-        raise RuntimeError("source {} is same as destination ".format(src))
-
-    def norm(val):
-        if IS_PYPY and six.PY3:
-            return str(val).encode(sys.getfilesystemencoding())
-        return six.ensure_text(str(val))
-
-    if do_copy is False and HAS_SYMLINK is False:  # if no symlink, always use copy
-        do_copy = True
-    if not do_copy:
-        try:
-            if not dst.is_symlink():  # can't link to itself!
-                if relative_symlinks_ok:
-                    assert src.parent == dst.parent
-                    os.symlink(norm(src.name), norm(dst))
-                else:
-                    os.symlink(norm(str(src)), norm(dst))
-        except OSError as exception:
-            logging.warning(
-                "symlink failed %r, for %s to %s, will try copy",
-                exception,
-                six.ensure_text(str(src)),
-                six.ensure_text(str(dst)),
-            )
-            do_copy = True
-    if do_copy:
-        copier = shutil.copy2 if src.is_file() else shutil.copytree
-        copier(norm(src), norm(dst))
-    logging.debug("%s %s to %s", "copy" if do_copy else "symlink", six.ensure_text(str(src)), six.ensure_text(str(dst)))
-
-
-symlink = partial(symlink_or_copy, False)
-copy = partial(symlink_or_copy, True)
-
-__all__ = ("ensure_dir", "symlink", "copy", "symlink_or_copy")
+        logging.debug("create folder %s", ensure_text(str(path)))
+        os.makedirs(norm(path))
+
+
+def ensure_safe_to_do(src, dest):
+    if src == dest:
+        raise ValueError("source and destination is the same {}".format(src))
+    if not dest.exists():
+        return
+    if dest.is_dir() and not dest.is_symlink():
+        shutil.rmtree(norm(dest))
+        logging.debug("remove directory %s", dest)
+    else:
+        logging.debug("remove file %s", dest)
+        dest.unlink()
+
+
+def symlink(src, dest):
+    ensure_safe_to_do(src, dest)
+    logging.debug("symlink %s", _Debug(src, dest))
+    dest.symlink_to(src, target_is_directory=src.is_dir())
+
+
+def copy(src, dest):
+    ensure_safe_to_do(src, dest)
+    is_dir = src.is_dir()
+    method = shutil.copytree if is_dir else shutil.copy2
+    logging.debug("copy %s", _Debug(src, dest))
+    method(norm(src), norm(dest))
+
+
+def link(src, dest):
+    ensure_safe_to_do(src, dest)
+    logging.debug("hard link %s", _Debug(src, dest.name))
+    os_link(norm(src), norm(dest))
+
+
+class _Debug(object):
+    def __init__(self, src, dest):
+        self.src = src
+        self.dest = dest
+
+    def __str__(self):
+        return "{}{} to {}".format(
+            "directory " if self.src.is_dir() else "", ensure_text(str(self.src)), ensure_text(str(self.dest))
+        )
+
+
+__all__ = ("ensure_dir", "symlink", "copy", "link", "symlink", "link")
diff --git a/tests/conftest.py b/tests/conftest.py
index 6984a0c..a8b2a31 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -11,7 +11,7 @@ import pytest
 import six
 
 from virtualenv.discovery.py_info import CURRENT, PythonInfo
-from virtualenv.info import IS_PYPY, fs_supports_symlink
+from virtualenv.info import IS_PYPY, IS_WIN, fs_supports_symlink
 from virtualenv.util.path import Path
 
 
@@ -237,7 +237,7 @@ def is_inside_ci():
 def special_char_name():
     base = "e-$ èрт🚒♞中片-j"
     # workaround for pypy3 https://bitbucket.org/pypy/pypy/issues/3147/venv-non-ascii-support-windows
-    encoding = "ascii" if IS_PYPY and six.PY3 else sys.getfilesystemencoding()
+    encoding = "ascii" if IS_PYPY and IS_WIN else sys.getfilesystemencoding()
     # let's not include characters that the file system cannot encode)
     result = ""
     for char in base:
@@ -255,7 +255,7 @@ def special_char_name():
 def special_name_dir(tmp_path, special_char_name):
     dest = Path(str(tmp_path)) / special_char_name
     yield dest
-    if six.PY2 and sys.platform == "win32":  # pytest python2 windows does not support unicode delete
+    if six.PY2 and sys.platform == "win32" and not IS_PYPY:  # pytest python2 windows does not support unicode delete
         shutil.rmtree(six.ensure_text(str(dest)))
 
 
diff --git a/tests/unit/activation/conftest.py b/tests/unit/activation/conftest.py
index 8aa437e..0ea6b30 100644
--- a/tests/unit/activation/conftest.py
+++ b/tests/unit/activation/conftest.py
@@ -6,12 +6,12 @@ import re
 import shutil
 import subprocess
 import sys
-from os.path import dirname, normcase, realpath
+from os.path import dirname, normcase
 
 import pytest
 import six
 
-from virtualenv.info import IS_PYPY
+from virtualenv.info import IS_PYPY, WIN_CPYTHON_2
 from virtualenv.run import run_via_cli
 from virtualenv.util.path import Path
 from virtualenv.util.subprocess import Popen
@@ -71,13 +71,12 @@ class ActivationTester(object):
         try:
             process = Popen(invoke, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env)
             _raw, _ = process.communicate()
-            encoding = sys.getfilesystemencoding() if IS_PYPY else "utf-8"
-            raw = "\n{}".format(_raw.decode(encoding)).replace("\r\n", "\n")
+            raw = _raw.decode("utf-8")
         except subprocess.CalledProcessError as exception:
             assert not exception.returncode, six.ensure_text(exception.output)
             return
 
-        out = re.sub(r"pydev debugger: process \d+ is connecting\n\n", "", raw, re.M).strip().split("\n")
+        out = re.sub(r"pydev debugger: process \d+ is connecting\n\n", "", raw, re.M).strip().splitlines()
         self.assert_output(out, raw, tmp_path)
         return env, activate_script
 
@@ -164,7 +163,9 @@ class ActivationTester(object):
     @staticmethod
     def norm_path(path):
         # python may return Windows short paths, normalize
-        path = realpath(six.ensure_text(str(path)) if isinstance(path, Path) else path)
+        if not isinstance(path, Path):
+            path = Path(path)
+        path = six.ensure_text(str(path.resolve()))
         if sys.platform != "win32":
             result = path
         else:
@@ -187,9 +188,10 @@ class RaiseOnNonSourceCall(ActivationTester):
         process = Popen(
             self.non_source_activate(activate_script), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env,
         )
-        out, err = process.communicate()
+        out, _err = process.communicate()
+        err = _err.decode("utf-8")
         assert process.returncode
-        assert self.non_source_fail_message in err.decode("utf-8")
+        assert self.non_source_fail_message in err
 
 
 @pytest.fixture(scope="session")
@@ -205,12 +207,11 @@ def raise_on_non_source_class():
 @pytest.fixture(scope="session")
 def activation_python(tmp_path_factory, special_char_name, current_fastest):
     dest = os.path.join(six.ensure_text(str(tmp_path_factory.mktemp("activation-tester-env"))), special_char_name)
-    session = run_via_cli(["--seed", "none", dest, "--prompt", special_char_name, "--creator", current_fastest])
+    session = run_via_cli(["--seed", "none", dest, "--prompt", special_char_name, "--creator", current_fastest, "-vv"])
     pydoc_test = session.creator.purelib / "pydoc_test.py"
-    with open(six.ensure_text(str(pydoc_test)), "wb") as file_handler:
-        file_handler.write(b'"""This is pydoc_test.py"""')
+    pydoc_test.write_text('"""This is pydoc_test.py"""')
     yield session
-    if six.PY2 and sys.platform == "win32":  # PY2 windows does not support unicode delete
+    if WIN_CPYTHON_2:  # PY2 windows does not support unicode delete
         shutil.rmtree(dest)
 
 
diff --git a/tests/unit/activation/test_powershell.py b/tests/unit/activation/test_powershell.py
index 820d340..b2d4cda 100644
--- a/tests/unit/activation/test_powershell.py
+++ b/tests/unit/activation/test_powershell.py
@@ -1,22 +1,14 @@
 from __future__ import absolute_import, unicode_literals
 
-import os
 import pipes
 import sys
 
 import pytest
-from six import PY2
 
 from virtualenv.activation import PowerShellActivator
-from virtualenv.info import IS_PYPY, IS_WIN
 
 
 @pytest.mark.slow
-@pytest.mark.xfail(
-    condition=IS_PYPY and PY2 and IS_WIN and bool(os.environ.get(str("CI_RUN"))),
-    strict=False,
-    reason="this fails in the CI only, nor sure how, if anyone can reproduce help",
-)
 def test_powershell(activation_tester_class, activation_tester):
     class PowerShell(activation_tester_class):
         def __init__(self, session):
diff --git a/tests/unit/activation/test_python_activator.py b/tests/unit/activation/test_python_activator.py
index 344fe0b..2edc70b 100644
--- a/tests/unit/activation/test_python_activator.py
+++ b/tests/unit/activation/test_python_activator.py
@@ -2,20 +2,15 @@ from __future__ import absolute_import, unicode_literals
 
 import os
 import sys
+from ast import literal_eval
 from textwrap import dedent
 
-import pytest
 import six
 
 from virtualenv.activation import PythonActivator
-from virtualenv.info import IS_PYPY, IS_WIN
+from virtualenv.info import WIN_CPYTHON_2
 
 
-@pytest.mark.xfail(
-    condition=IS_PYPY and six.PY2 and IS_WIN and bool(os.environ.get(str("CI_RUN"))),
-    strict=False,
-    reason="this fails in the CI only, nor sure how, if anyone can reproduce help",
-)
 def test_python(raise_on_non_source_class, activation_tester):
     class Python(raise_on_non_source_class):
         def __init__(self, session):
@@ -41,61 +36,64 @@ def test_python(raise_on_non_source_class, activation_tester):
             raw = """
             import os
             import sys
+            import platform
 
-            def print_path(value):
-                if value is not None and (
-                    sys.version_info[0] == 2 and isinstance(value, str) and not hasattr(sys, "pypy_version_info")
-                ):
-                    value = value.decode(sys.getfilesystemencoding())
-                print(value)
-
-            print_path(os.environ.get("VIRTUAL_ENV"))
-            print_path(os.environ.get("PATH"))
-            print_path(os.pathsep.join(sys.path))
-            file_at = {}
-            with open(file_at, "rb") as file_handler:
+            def print_r(value):
+                print(repr(value))
+
+            print_r(os.environ.get("VIRTUAL_ENV"))
+            print_r(os.environ.get("PATH").split(os.pathsep))
+            print_r(sys.path)
+
+            file_at = {!r}
+            # CPython 2 requires non-ascii path open to be unicode
+            with open(file_at{}, "r") as file_handler:
                 content = file_handler.read()
             exec(content, {{"__file__": file_at}})
-            print_path(os.environ.get("VIRTUAL_ENV"))
-            print_path(os.environ.get("PATH"))
-            print_path(os.pathsep.join(sys.path))
-            import inspect
-            import pydoc_test
 
-            print_path(inspect.getsourcefile(pydoc_test))
+            print_r(os.environ.get("VIRTUAL_ENV"))
+            print_r(os.environ.get("PATH").split(os.pathsep))
+            print_r(sys.path)
+
+            import pydoc_test
+            print_r(pydoc_test.__file__)
             """.format(
-                repr(six.ensure_text(str(activate_script)))
+                str(activate_script), ".decode('utf-8')" if WIN_CPYTHON_2 else ""
             )
             result = dedent(raw).splitlines()
             return result
 
         def assert_output(self, out, raw, tmp_path):
-            assert out[0] == "None"  # start with VIRTUAL_ENV None
-
-            prev_path = out[1].split(os.path.pathsep)
-            prev_sys_path = out[2].split(os.path.pathsep)
+            out = [literal_eval(i) for i in out]
+            assert out[0] is None  # start with VIRTUAL_ENV None
 
-            assert out[3] == six.ensure_text(
-                str(self._creator.dest)
-            )  # VIRTUAL_ENV now points to the virtual env folder
+            prev_path = out[1]
+            prev_sys_path = out[2]
+            assert out[3] == str(self._creator.dest)  # VIRTUAL_ENV now points to the virtual env folder
 
-            new_path = out[4].split(os.pathsep)  # PATH now starts with bin path of current
-            assert ([six.ensure_text(str(self._creator.bin_dir))] + prev_path) == new_path
+            new_path = out[4]  # PATH now starts with bin path of current
+            assert ([str(self._creator.bin_dir)] + prev_path) == new_path
 
             # sys path contains the site package at its start
-            new_sys_path = out[5].split(os.path.pathsep)
+            new_sys_path = out[5]
 
-            new_lib_paths = {six.ensure_text(str(i)) for i in self._creator.libs}
+            new_lib_paths = {six.ensure_text(j) if WIN_CPYTHON_2 else j for j in {str(i) for i in self._creator.libs}}
             assert prev_sys_path == new_sys_path[len(new_lib_paths) :]
             assert new_lib_paths == set(new_sys_path[: len(new_lib_paths)])
 
             # manage to import from activate site package
-            assert self.norm_path(out[6]) == self.norm_path(self._creator.purelib / "pydoc_test.py")
+            dest = self.norm_path(self._creator.purelib / "pydoc_test.py")
+            found = self.norm_path(out[6].decode(sys.getfilesystemencoding()) if WIN_CPYTHON_2 else out[6])
+            assert found.startswith(dest)
 
         def non_source_activate(self, activate_script):
-            return self._invoke_script + [
+            act = str(activate_script)
+            if WIN_CPYTHON_2:
+                act = six.ensure_text(act)
+            cmd = self._invoke_script + [
                 "-c",
-                'exec(open(r"{}").read())'.format(six.ensure_text(str(activate_script))),
+                "exec(open({}).read())".format(repr(act)),
             ]
+            return cmd
 
     activation_tester(Python)
diff --git a/tests/unit/create/conftest.py b/tests/unit/create/conftest.py
index 872c436..82d1ea4 100644
--- a/tests/unit/create/conftest.py
+++ b/tests/unit/create/conftest.py
@@ -47,7 +47,7 @@ def old_virtualenv(tmp_path_factory):
         try:
             process = Popen(
                 [
-                    str(result.creator.exe.parent / "pip"),
+                    str(result.creator.script("pip")),
                     "install",
                     "--no-index",
                     "--disable-pip-version-check",
@@ -65,7 +65,7 @@ def old_virtualenv(tmp_path_factory):
         try:
             old_virtualenv_at = tmp_path_factory.mktemp("old-virtualenv")
             cmd = [
-                str(result.creator.exe.parent / "virtualenv"),
+                str(result.creator.script("virtualenv")),
                 str(old_virtualenv_at),
                 "--no-pip",
                 "--no-setuptools",
diff --git a/tests/unit/create/test_creator.py b/tests/unit/create/test_creator.py
index ff61d67..476b7a3 100644
--- a/tests/unit/create/test_creator.py
+++ b/tests/unit/create/test_creator.py
@@ -2,9 +2,11 @@ from __future__ import absolute_import, unicode_literals
 
 import difflib
 import gc
+import logging
 import os
 import stat
 import sys
+from itertools import product
 
 import pytest
 import six
@@ -78,11 +80,31 @@ def system():
 
 
 CURRENT_CREATORS = list(i for i in CURRENT.creators().key_to_class.keys() if i != "builtin")
-
-
-@pytest.mark.parametrize("isolated", [True, False], ids=["isolated", "with_global_site"])
-@pytest.mark.parametrize("method", (["copies"] + (["symlinks"] if fs_supports_symlink() else [])))
-@pytest.mark.parametrize("creator", CURRENT_CREATORS)
+_VENV_BUG_ON = (
+    IS_PYPY
+    and CURRENT.version_info[0:3] == (3, 6, 9)
+    and CURRENT.pypy_version_info[0:2] == (7, 3)
+    and CURRENT.platform == "linux"
+)
+
+
+@pytest.mark.parametrize(
+    "creator, method, isolated",
+    [
+        pytest.param(
+            *i,
+            marks=pytest.mark.xfail(
+                reason="https://bitbucket.org/pypy/pypy/issues/3159/pypy36-730-venv-fails-with-copies-on-linux",
+                strict=True,
+            )
+        )
+        if _VENV_BUG_ON and i[0] == "venv" and i[1] == "copies"
+        else i
+        for i in product(
+            CURRENT_CREATORS, (["copies"] + (["symlinks"] if fs_supports_symlink() else [])), ["isolated", "global"]
+        )
+    ],
+)
 def test_create_no_seed(python, creator, isolated, system, coverage_env, special_name_dir, method):
     dest = special_name_dir
     cmd = [
@@ -98,7 +120,7 @@ def test_create_no_seed(python, creator, isolated, system, coverage_env, special
         creator,
         "--{}".format(method),
     ]
-    if not isolated:
+    if isolated == "global":
         cmd.append("--system-site-packages")
     result = run_via_cli(cmd)
     coverage_env()
@@ -130,7 +152,7 @@ def test_create_no_seed(python, creator, isolated, system, coverage_env, special
 
     # ensure the global site package is added or not, depending on flag
     last_from_system_path = next(i for i in reversed(system_sys_path) if str(i).startswith(system["sys"]["prefix"]))
-    if isolated:
+    if isolated == "isolated":
         assert last_from_system_path not in sys_path
     else:
         common = []
@@ -183,17 +205,18 @@ def test_debug_bad_virtualenv(tmp_path):
     debug_info = result.creator.debug
     assert debug_info["returncode"]
     assert debug_info["err"].startswith("std-err")
-    assert debug_info["out"] == "'std-out'"
+    assert "std-out" in debug_info["out"]
     assert debug_info["exception"]
 
 
 @pytest.mark.parametrize("creator", CURRENT_CREATORS)
 @pytest.mark.parametrize("clear", [True, False], ids=["clear", "no_clear"])
-def test_create_clear_resets(tmp_path, creator, clear):
+def test_create_clear_resets(tmp_path, creator, clear, caplog):
+    caplog.set_level(logging.DEBUG)
     if creator == "venv" and clear is False:
         pytest.skip("venv without clear might fail")
     marker = tmp_path / "magic"
-    cmd = [str(tmp_path), "--seeder", "none", "--creator", creator]
+    cmd = [str(tmp_path), "--seeder", "none", "--creator", creator, "-vvv"]
     run_via_cli(cmd)
 
     marker.write_text("")  # if we a marker file this should be gone on a clear run, remain otherwise
diff --git a/tests/unit/seed/greet/greet2.c b/tests/unit/seed/greet/greet2.c
new file mode 100644
index 0000000..7dc421c
--- /dev/null
+++ b/tests/unit/seed/greet/greet2.c
@@ -0,0 +1,30 @@
+#include <stdio.h>
+#include <Python.h>
+
+static PyObject * greet(PyObject * self, PyObject * args) {
+  const char * name;
+  if (!PyArg_ParseTuple(args, "s", & name)) {
+    return NULL;
+  }
+  printf("Hello %s!\n", name);
+  Py_RETURN_NONE;
+}
+
+static PyMethodDef GreetMethods[] = {
+  {
+    "greet",
+    greet,
+    METH_VARARGS,
+    "Greet an entity."
+  },
+  {
+    NULL,
+    NULL,
+    0,
+    NULL
+  }
+};
+
+PyMODINIT_FUNC initgreet(void) {
+  (void) Py_InitModule("greet", GreetMethods);
+}
diff --git a/tests/unit/seed/greet/greet3.c b/tests/unit/seed/greet/greet3.c
new file mode 100644
index 0000000..3ec017d
--- /dev/null
+++ b/tests/unit/seed/greet/greet3.c
@@ -0,0 +1,38 @@
+#include <stdio.h>
+#include <Python.h>
+
+static PyObject * greet(PyObject * self, PyObject * args) {
+  const char * name;
+  if (!PyArg_ParseTuple(args, "s", & name)) {
+    return NULL;
+  }
+  printf("Hello %s!\n", name);
+  Py_RETURN_NONE;
+}
+
+static PyMethodDef GreetMethods[] = {
+  {
+    "greet",
+    greet,
+    METH_VARARGS,
+    "Greet an entity."
+  },
+  {
+    NULL,
+    NULL,
+    0,
+    NULL
+  }
+};
+
+static struct PyModuleDef greet_definition = {
+  PyModuleDef_HEAD_INIT,
+  "greet",
+  "A Python module that prints 'greet world' from C code.",
+  -1,
+  GreetMethods
+};
+
+PyMODINIT_FUNC PyInit_greet(void) {
+  return PyModule_Create( & greet_definition);
+}
diff --git a/tests/unit/seed/greet/setup.py b/tests/unit/seed/greet/setup.py
new file mode 100644
index 0000000..856cf90
--- /dev/null
+++ b/tests/unit/seed/greet/setup.py
@@ -0,0 +1,13 @@
+import sys
+
+from setuptools import Extension, setup
+
+setup(
+    name="greet",  # package name
+    version="1.0",  # package version
+    ext_modules=[
+        Extension(
+            "greet", ["greet{}.c".format(sys.version_info[0])]  # extension to package
+        )  # C code to compile to run as extension
+    ],
+)
diff --git a/tests/unit/seed/test_boostrap_link_via_app_data.py b/tests/unit/seed/test_boostrap_link_via_app_data.py
index 7e5ff8d..75c17ad 100644
--- a/tests/unit/seed/test_boostrap_link_via_app_data.py
+++ b/tests/unit/seed/test_boostrap_link_via_app_data.py
@@ -37,7 +37,7 @@ def test_base_bootstrap_link_via_app_data(tmp_path, coverage_env, mocker, curren
     coverage_env()
     assert result
 
-    # uninstalling pip/setuptools now should leave us with a clean env
+    # uninstalling pip/setuptools now should leave us with a ensure_safe_to_do env
     site_package = result.creator.purelib
     pip = site_package / "pip"
     setuptools = site_package / "setuptools"
diff --git a/tests/unit/seed/test_extra_install.py b/tests/unit/seed/test_extra_install.py
new file mode 100644
index 0000000..7c97979
--- /dev/null
+++ b/tests/unit/seed/test_extra_install.py
@@ -0,0 +1,60 @@
+from __future__ import absolute_import, unicode_literals
+
+import os
+import subprocess
+
+import pytest
+
+from virtualenv.discovery.py_info import CURRENT
+from virtualenv.run import run_via_cli
+from virtualenv.util.path import Path
+from virtualenv.util.subprocess import Popen
+
+CREATOR_CLASSES = CURRENT.creators().key_to_class
+
+
+def builtin_shows_marker_missing():
+    builtin_classs = CREATOR_CLASSES.get("builtin")
+    if builtin_classs is None:
+        return False
+    host_include_marker = getattr(builtin_classs, "host_include_marker", None)
+    if host_include_marker is None:
+        return False
+    marker = host_include_marker(CURRENT)
+    return not marker.exists()
+
+
+@pytest.mark.xfail(
+    condition=bool(os.environ.get(str("CI_RUN"))),
+    strict=False,
+    reason="did not manage to setup CI to run with VC 14.1 C++ compiler, but passes locally",
+)
+@pytest.mark.skipif(
+    not Path(CURRENT.system_include).exists() and not builtin_shows_marker_missing(),
+    reason="Building C-Extensions requires header files with host python",
+)
+@pytest.mark.parametrize("creator", list(i for i in CREATOR_CLASSES.keys() if i != "builtin"))
+def test_can_build_c_extensions(creator, tmp_path, coverage_env):
+    session = run_via_cli(["--creator", creator, "--seed", "app-data", str(tmp_path), "-vvv"])
+    coverage_env()
+    cmd = [
+        str(session.creator.script("pip")),
+        "install",
+        "--no-index",
+        "--no-deps",
+        "--disable-pip-version-check",
+        "-vvv",
+        str(Path(__file__).parent.resolve() / "greet"),
+    ]
+    process = Popen(cmd)
+    process.communicate()
+    assert process.returncode == 0
+
+    process = Popen(
+        [str(session.creator.exe), "-c", "import greet; greet.greet('World')"],
+        universal_newlines=True,
+        stdout=subprocess.PIPE,
+    )
+    out, _ = process.communicate()
+    assert process.returncode == 0
+    assert out == "Hello World!\n"
diff --git a/tests/unit/seed/test_pip_invoke.py b/tests/unit/seed/test_pip_invoke.py
index 6c8b6c7..9bbb7e2 100644
--- a/tests/unit/seed/test_pip_invoke.py
+++ b/tests/unit/seed/test_pip_invoke.py
@@ -26,7 +26,6 @@ def test_base_bootstrap_via_pip_invoke(tmp_path, coverage_env, current_fastest):
     coverage_env()
     assert result
 
-    # uninstalling pip/setuptools now should leave us with a clean env
     site_package = result.creator.purelib
     pip = site_package / "pip"
     setuptools = site_package / "setuptools"
diff --git a/tests/unit/test_util.py b/tests/unit/test_util.py
index a035a6b..351d528 100644
--- a/tests/unit/test_util.py
+++ b/tests/unit/test_util.py
@@ -1,46 +1,8 @@
 from __future__ import absolute_import, unicode_literals
 
-import logging
-
-import pytest
-
-from virtualenv.info import fs_supports_symlink
-from virtualenv.util.path import symlink_or_copy
 from virtualenv.util.subprocess import run_cmd
 
 
-@pytest.mark.skipif(not fs_supports_symlink(), reason="symlink is not supported")
-def test_fallback_to_copy_if_symlink_fails(caplog, capsys, tmp_path, mocker):
-    caplog.set_level(logging.DEBUG)
-    mocker.patch("os.symlink", side_effect=OSError())
-    dst, src = _try_symlink(caplog, tmp_path, level=logging.WARNING)
-    msg = "symlink failed {!r}, for {} to {}, will try copy".format(OSError(), src, dst)
-    assert len(caplog.messages) == 1, caplog.text
-    message = caplog.messages[0]
-    assert msg == message
-    out, err = capsys.readouterr()
-    assert not out
-    assert err
-
-
-def _try_symlink(caplog, tmp_path, level):
-    caplog.set_level(level)
-    src = tmp_path / "src"
-    src.write_text("a")
-    dst = tmp_path / "dst"
-    symlink_or_copy(do_copy=False, src=src, dst=dst)
-    assert dst.exists()
-    assert not dst.is_symlink()
-    assert dst.read_text() == "a"
-    return dst, src
-
-
-@pytest.mark.skipif(fs_supports_symlink(), reason="symlink is supported")
-def test_os_no_symlink_use_copy(caplog, tmp_path):
-    dst, src = _try_symlink(caplog, tmp_path, level=logging.DEBUG)
-    assert caplog.messages == ["copy {} to {}".format(src, dst)]
-
-
 def test_run_fail(tmp_path):
     code, out, err = run_cmd([str(tmp_path)])
     assert err