Allow missing .py files if .pyc is present (#1714)

* Allow missing .py files if .pyc is present

* Document packaging types we support

Signed-off-by: Bernat Gabor <bgabor8@bloomberg.net>

* improve test

Signed-off-by: Bernat Gabor <bgabor8@bloomberg.net>

* PyPy requires the standard library source files

Signed-off-by: Bernat Gabor <bgabor8@bloomberg.net>

Co-authored-by: Bernat Gabor <bgabor8@bloomberg.net>

7 files changed, 101 insertions, 10 deletions

diff --git a/docs/changelog/1714.bugfix.rst b/docs/changelog/1714.bugfix.rst
new file mode 100644
index 0000000..8a9bcaa
--- /dev/null
+++ b/docs/changelog/1714.bugfix.rst
@@ -0,0 +1 @@
+Allow missing ``.py`` files if a compiled ``.pyc`` version is available - by :user:`tucked`.
diff --git a/docs/changelog/1714.doc.rst b/docs/changelog/1714.doc.rst
new file mode 100644
index 0000000..e741744
--- /dev/null
+++ b/docs/changelog/1714.doc.rst
@@ -0,0 +1,2 @@
+:ref:`supports <compatibility-requirements>` details now explicitly what Python installations we support
+- by :user:`gaborbernat`.
diff --git a/docs/installation.rst b/docs/installation.rst
index 2611fca..dffd3f5 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -89,8 +89,39 @@ virtualenv works with the following Python interpreter implementations:
 - `PyPy <https://pypy.org/>`_ 2.7 and 3.4+.
 
 This means virtualenv works on the latest patch version of each of these minor versions. Previous patch versions are
-supported on a best effort approach. virtualenv works on the following platforms:
+supported on a best effort approach.
 
-- Unix/Linux,
-- macOS,
-- Windows.
+CPython is shipped in multiple forms, and each OS repackages it, often applying some customization along the way.
+Therefore we cannot say universally that we support all platforms, but rather specify some we test against. In case
+of ones not specified here the support is unknown, though likely will work. If you find some cases please open a feature
+request on our issue tracker.
+
+Linux
+~~~~~
+- installations from `python.org <https://www.python.org/downloads/>`_
+- Ubuntu 16.04+ (both upstream and `deasnakes <https://launchpad.net/~deadsnakes/+archive/ubuntu/ppa>`_ builds)
+- Fedora
+- RHEL and CentOS
+- OpenSuse
+- Arch Linux
+
+macOS
+~~~~~
+In case of macOs we support:
+- installations from `python.org <https://www.python.org/downloads/>`_
+- python versions installed via `brew <https://docs.brew.sh/Homebrew-and-Python>`_ (both older python2.7 and python3)
+- Python 3 part of XCode (Python framework - ``/Library/Frameworks/Python3.framework/``)
+- Python 2 part of the OS (``/System/Library/Frameworks/Python.framework/Versions/``)
+
+Windows
+~~~~~~~
+- Installations from `python.org <https://www.python.org/downloads/>`_
+- Windows Store Python - note only `version 3.8+ <https://www.microsoft.com/en-us/p/python-38/9mssztt1n39l>`_ (``3.7``
+  was marked experimental and contains many bugs that would make it very hard for us to support it)
+
+Packaging variants
+~~~~~~~~~~~~~~~~~~
+- Normal variant (file structure as comes from `python.org <https://www.python.org/downloads/>`_).
+- We support CPython 2 system installations that do not contain the python files for the standard library if the
+  respective compiled files are present (e.g. only ``os.pyc``, not ``os.py``). This can be used by custom systems may
+  want to maximize available storage or obfuscate source code by removing ``.py`` files.
diff --git a/src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py b/src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py
index c46a95e..6a8871b 100644
--- a/src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py
+++ b/src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py
@@ -26,6 +26,10 @@ class CPython2(CPython, Python2):
             yield PathRefToDest(host_include_marker.parent, dest=lambda self, _: self.include)
 
     @classmethod
+    def needs_stdlib_py_module(cls):
+        return False
+
+    @classmethod
     def host_include_marker(cls, interpreter):
         return Path(interpreter.system_include) / "Python.h"
 
diff --git a/src/virtualenv/create/via_global_ref/builtin/pypy/pypy2.py b/src/virtualenv/create/via_global_ref/builtin/pypy/pypy2.py
index c7288da..020000b 100644
--- a/src/virtualenv/create/via_global_ref/builtin/pypy/pypy2.py
+++ b/src/virtualenv/create/via_global_ref/builtin/pypy/pypy2.py
@@ -32,6 +32,10 @@ class PyPy2(PyPy, Python2):
             yield PathRefToDest(host_include_marker.parent, dest=lambda self, _: self.include)
 
     @classmethod
+    def needs_stdlib_py_module(cls):
+        return True
+
+    @classmethod
     def host_include_marker(cls, interpreter):
         return Path(interpreter.system_include) / "PyPy.h"
 
diff --git a/src/virtualenv/create/via_global_ref/builtin/python2/python2.py b/src/virtualenv/create/via_global_ref/builtin/python2/python2.py
index 7c8acf3..22d1da0 100644
--- a/src/virtualenv/create/via_global_ref/builtin/python2/python2.py
+++ b/src/virtualenv/create/via_global_ref/builtin/python2/python2.py
@@ -57,11 +57,27 @@ class Python2(ViaGlobalRefVirtualenvBuiltin, Python2Supports):
             yield src
         # install files needed to run site.py
         for req in cls.modules():
-            stdlib_path = interpreter.stdlib_path("{}.py".format(req))
-            yield PathRefToDest(stdlib_path, dest=cls.to_stdlib)
-            comp = stdlib_path.parent / "{}.pyc".format(req)
-            if comp.exists():
-                yield PathRefToDest(comp, dest=cls.to_stdlib)
+
+            # the compiled path is optional, but refer to it if exists
+            module_compiled_path = interpreter.stdlib_path("{}.pyc".format(req))
+            has_compile = module_compiled_path.exists()
+            if has_compile:
+                yield PathRefToDest(module_compiled_path, dest=cls.to_stdlib)
+
+            # stdlib module src may be missing if the interpreter allows it by falling back to the compiled
+            module_path = interpreter.stdlib_path("{}.py".format(req))
+            add_py_module = cls.needs_stdlib_py_module()
+            if add_py_module is False:
+                if module_path.exists():  # if present add it
+                    add_py_module = True
+                else:
+                    add_py_module = not has_compile  # otherwise only add it if the pyc is not present
+            if add_py_module:
+                yield PathRefToDest(module_path, dest=cls.to_stdlib)
+
+    @classmethod
+    def needs_stdlib_py_module(cls):
+        raise NotImplementedError
 
     def to_stdlib(self, src):
         return self.stdlib / src.name
diff --git a/tests/unit/create/test_creator.py b/tests/unit/create/test_creator.py
index e2d2f1b..cf30f2b 100644
--- a/tests/unit/create/test_creator.py
+++ b/tests/unit/create/test_creator.py
@@ -21,7 +21,7 @@ from virtualenv.__main__ import run, run_with_catch
 from virtualenv.create.creator import DEBUG_SCRIPT, Creator, get_env_debug_info
 from virtualenv.discovery.builtin import get_interpreter
 from virtualenv.discovery.py_info import PythonInfo
-from virtualenv.info import IS_PYPY, IS_WIN, PY3, fs_is_case_sensitive, fs_supports_symlink
+from virtualenv.info import IS_PYPY, IS_WIN, PY2, PY3, fs_is_case_sensitive, fs_supports_symlink
 from virtualenv.pyenv_cfg import PyEnvCfg
 from virtualenv.run import cli_run, session_via_cli
 from virtualenv.util.path import Path
@@ -446,3 +446,36 @@ def test_python_path(monkeypatch, tmp_path, python_path_on):
         assert non_python_path == [i for i in base if i not in extra_as_python_path]
     else:
         assert base == extra_all
+
+
+@pytest.mark.skipif(
+    not (CURRENT.implementation == "CPython" and PY2),
+    reason="stdlib components without py files only possible on CPython2",
+)
+def test_pyc_only(tmp_path, mocker, session_app_data):
+    """Ensure that creation can succeed if os.pyc exists (even if os.py has been deleted)"""
+    interpreter = PythonInfo.from_exe(sys.executable, session_app_data)
+    host_pyc = interpreter.stdlib_path("os.pyc")
+    if not host_pyc.exists():
+        pytest.skip("missing system os.pyc at {}".format(host_pyc))
+    previous = interpreter.stdlib_path
+
+    def stdlib_path(name):
+        path = previous(name)
+        if name.endswith(".py"):
+
+            class _Path(type(path)):
+                @staticmethod
+                def exists():
+                    return False
+
+            return _Path(path)
+        return path
+
+    mocker.patch.object(interpreter, "stdlib_path", side_effect=stdlib_path)
+
+    result = cli_run([ensure_text(str(tmp_path)), "--without-pip", "--activators", ""])
+
+    assert not (result.creator.stdlib / "os.py").exists()
+    assert (result.creator.stdlib / "os.pyc").exists()
+    assert "os.pyc" in result.creator.debug["os"]