diff options
| author | Bernát Gábor <bgabor8@bloomberg.net> | 2020-01-16 11:09:43 +0000 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2020-01-16 11:09:43 +0000 |
| commit | b5f618f352557ddea5ec0e0bfe7188690b51e373 (patch) | |
| tree | feb3fd0e4a36d101aa35aa5973c33f8c43fca939 | |
| parent | dbcc95683d00df3e5d7befff431db4bceb52aebc (diff) | |
| download | virtualenv-b5f618f352557ddea5ec0e0bfe7188690b51e373.tar.gz | |
add zipapp support with bundled dependencies (#1491)
Signed-off-by: Bernat Gabor <bgabor8@bloomberg.net>
33 files changed, 702 insertions, 78 deletions
diff --git a/.coveragerc b/.coveragerc index f7bf90c..000f9a5 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,5 +1,5 @@ [coverage:report] -skip_covered = False +skip_covered = True show_missing = True exclude_lines = \#\s*pragma: no cover @@ -6,7 +6,7 @@ dist .eggs # python -*.py[cod] +*.py[codz] *$py.class # tools @@ -24,3 +24,5 @@ dist /src/virtualenv/out /*env* .python-version + +*wheel-store* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2a63f2d..71f275c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: rev: v1.4.0 hooks: - id: blacken-docs - additional_dependencies: [black==19.3b0] + additional_dependencies: [black==19.10b0] language_version: python3.8 - repo: https://github.com/asottile/seed-isort-config rev: v1.9.4 diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 869a6d9..2de3f7b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -52,6 +52,8 @@ jobs: image: [linux, windows, macOs] fix_lint: image: [linux, windows] + zipapp: + image: [linux, windows] docs: image: [linux, windows] package_readme: @@ -38,7 +38,7 @@ project_urls = [options] packages = find: install_requires = - appdirs>=1.4.3 + appdirs>=1.4.3,<2 six>=1.12.0,<2 distlib>=0.3.0,<1;sys.platform == 'win32' importlib-metadata>=0.12,<2;python_version<"3.8" @@ -83,9 +83,11 @@ docs = towncrier >= 18.5.0 sphinx_rtd_theme >= 0.4.2, < 1 testing = + packaging>=20.0;python_version>"3.4" pytest >= 4.0.0, <6 coverage >= 4.5.1, <6 pytest-mock >= 1.12.1, <2 + pytest-env >= 0.6.2, <1 xonsh >= 0.9.13, <1; python_version > '3.4' [options.package_data] @@ -113,5 +115,9 @@ markers = fish pwsh xonsh + slow junit_family = xunit2 addopts = --tb=auto -ra --showlocals +env = + PYTHONWARNINGS=ignore:DEPRECATION::pip._internal.cli.base_command + PYTHONIOENCODING=utf-8 @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import, unicode_literals + import textwrap from setuptools import setup diff --git a/src/__init__.py b/src/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/src/__init__.py +++ /dev/null diff --git a/src/virtualenv/info.py b/src/virtualenv/info.py index cc8dbd8..573a97f 100644 --- a/src/virtualenv/info.py +++ b/src/virtualenv/info.py @@ -13,7 +13,8 @@ IS_PYPY = IMPLEMENTATION == "PyPy" IS_CPYTHON = IMPLEMENTATION == "CPython" PY3 = sys.version_info[0] == 3 IS_WIN = sys.platform == "win32" - +ROOT = os.path.realpath(os.path.join(os.path.abspath(__file__), os.path.pardir, os.path.pardir)) +IS_ZIPAPP = os.path.isfile(ROOT) _FS_CASE_SENSITIVE = _CFG_DIR = _DATA_DIR = None @@ -22,7 +23,9 @@ def get_default_data_dir(): global _DATA_DIR if _DATA_DIR is None: - _DATA_DIR = Path(user_data_dir(appname="virtualenv", appauthor="pypa")) + key = str("_VIRTUALENV_OVERRIDE_APP_DATA") + folder = os.environ[key] if key in os.environ else user_data_dir(appname="virtualenv", appauthor="pypa") + _DATA_DIR = Path(folder) return _DATA_DIR @@ -47,4 +50,14 @@ def is_fs_case_sensitive(): return _FS_CASE_SENSITIVE -__all__ = ("IS_PYPY", "PY3", "IS_WIN", "get_default_data_dir", "get_default_config_dir", "_FS_CASE_SENSITIVE") +__all__ = ( + "IS_PYPY", + "IS_CPYTHON", + "PY3", + "IS_WIN", + "get_default_data_dir", + "get_default_config_dir", + "is_fs_case_sensitive", + "ROOT", + "IS_ZIPAPP", +) diff --git a/src/virtualenv/interpreters/create/creator.py b/src/virtualenv/interpreters/create/creator.py index b171327..1185b56 100644 --- a/src/virtualenv/interpreters/create/creator.py +++ b/src/virtualenv/interpreters/create/creator.py @@ -14,9 +14,12 @@ from stat import S_IWUSR import six from six import add_metaclass +from virtualenv.info import IS_ZIPAPP +from virtualenv.interpreters.discovery.py_info import Cmd from virtualenv.pyenv_cfg import PyEnvCfg from virtualenv.util.path import Path from virtualenv.util.subprocess import run_cmd +from virtualenv.util.zipapp import extract_to_app_data from virtualenv.version import __version__ HERE = Path(__file__).absolute().parent @@ -174,8 +177,10 @@ class Creator(object): def get_env_debug_info(env_exe, debug_script): + if IS_ZIPAPP: + debug_script = extract_to_app_data(debug_script) cmd = [six.ensure_text(str(env_exe)), six.ensure_text(str(debug_script))] - logging.debug(" ".join(six.ensure_text(i) for i in cmd)) + logging.debug("debug via %r", Cmd(cmd)) env = os.environ.copy() env.pop(str("PYTHONPATH"), None) code, out, err = run_cmd(cmd) diff --git a/src/virtualenv/interpreters/create/venv.py b/src/virtualenv/interpreters/create/venv.py index b0b69b9..afb9471 100644 --- a/src/virtualenv/interpreters/create/venv.py +++ b/src/virtualenv/interpreters/create/venv.py @@ -41,13 +41,9 @@ class Venv(ViaGlobalRefApi): from venv import EnvBuilder builder = EnvBuilder( - system_site_packages=self.enable_system_site_package, - clear=False, - symlinks=self.symlinks, - with_pip=False, - prompt=None, + system_site_packages=self.enable_system_site_package, clear=False, symlinks=self.symlinks, with_pip=False, ) - builder.create(self.dest_dir) + builder.create(str(self.dest_dir)) def create_via_sub_process(self): cmd = self.get_host_create_cmd() diff --git a/src/virtualenv/interpreters/create/via_global_ref/python2.py b/src/virtualenv/interpreters/create/via_global_ref/python2.py index cf195a9..4ff6e84 100644 --- a/src/virtualenv/interpreters/create/via_global_ref/python2.py +++ b/src/virtualenv/interpreters/create/via_global_ref/python2.py @@ -6,9 +6,11 @@ import os import six +from virtualenv.info import IS_ZIPAPP from virtualenv.interpreters.create.support import Python2Supports from virtualenv.interpreters.create.via_global_ref.via_global_self_do import ViaGlobalRefVirtualenvBuiltin from virtualenv.util.path import Path, copy +from virtualenv.util.zipapp import read as read_from_zipapp HERE = Path(__file__).absolute().parent @@ -29,9 +31,12 @@ class Python2(ViaGlobalRefVirtualenvBuiltin, Python2Supports): relative_site_packages = [ os.path.relpath(six.ensure_text(str(s)), six.ensure_text(str(site_py))) for s in self.site_packages ] - site_py.write_text( - get_custom_site().read_text().replace("___EXPECTED_SITE_PACKAGES___", json.dumps(relative_site_packages)) - ) + custom_site = get_custom_site() + if IS_ZIPAPP: + custom_site_text = read_from_zipapp(custom_site) + else: + custom_site_text = custom_site.read_text() + site_py.write_text(custom_site_text.replace("___EXPECTED_SITE_PACKAGES___", json.dumps(relative_site_packages))) @abc.abstractmethod def modules(self): diff --git a/src/virtualenv/interpreters/create/via_global_ref/via_global_self_do.py b/src/virtualenv/interpreters/create/via_global_ref/via_global_self_do.py index cfe6160..7e259f3 100644 --- a/src/virtualenv/interpreters/create/via_global_ref/via_global_self_do.py +++ b/src/virtualenv/interpreters/create/via_global_ref/via_global_self_do.py @@ -3,15 +3,13 @@ from __future__ import absolute_import, unicode_literals import abc from abc import ABCMeta from collections import OrderedDict -from os import chmod, stat -from stat import S_IXGRP, S_IXOTH, S_IXUSR import six from six import add_metaclass from virtualenv.info import is_fs_case_sensitive from virtualenv.interpreters.create.builtin_way import VirtualenvBuiltin -from virtualenv.util.path import Path, copy, ensure_dir, symlink +from virtualenv.util.path import Path, copy, ensure_dir, make_exe, symlink @add_metaclass(ABCMeta) @@ -80,9 +78,7 @@ class ViaGlobalRefVirtualenvBuiltin(VirtualenvBuiltin): @staticmethod def symlink_exe(src, dest): symlink(src, dest) - dest_str = six.ensure_text(str(dest)) - original_mode = stat(dest_str).st_mode - chmod(dest_str, original_mode | S_IXUSR | S_IXGRP | S_IXOTH) + make_exe(dest) @property def lib_base(self): diff --git a/src/virtualenv/interpreters/discovery/py_info.py b/src/virtualenv/interpreters/discovery/py_info.py index fe8d291..9feecb4 100644 --- a/src/virtualenv/interpreters/discovery/py_info.py +++ b/src/virtualenv/interpreters/discovery/py_info.py @@ -8,6 +8,7 @@ from __future__ import absolute_import, print_function import json import logging import os +import pipes import platform import sys from collections import OrderedDict, namedtuple @@ -245,19 +246,10 @@ class PythonInfo(object): def _load_for_exe(cls, exe): from virtualenv.util.subprocess import subprocess, Popen - path = "{}.py".format(os.path.splitext(__file__)[0]) - cmd = [exe, "-s", path] - + cmd = cls._get_exe_cmd(exe) # noinspection DuplicatedCode # this is duplicated here because this file is executed on its own, so cannot be refactored otherwise - - class Cmd(object): - def __repr__(self): - import pipes - - return " ".join(pipes.quote(c) for c in cmd) - - logging.debug("get interpreter info via cmd: %s", Cmd()) + logging.debug("get interpreter info via cmd: %s", Cmd(cmd)) try: process = Popen( cmd, universal_newlines=True, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE @@ -277,6 +269,22 @@ class PythonInfo(object): failure = RuntimeError(msg) return failure, result + @classmethod + def _get_exe_cmd(cls, exe): + cmd = [exe, "-s"] + from virtualenv.info import IS_ZIPAPP + + self_path = os.path.abspath(__file__) + if IS_ZIPAPP: + from virtualenv.util.zipapp import extract_to_app_data + from virtualenv.util.path import Path + + path = str(extract_to_app_data(Path(self_path))) + else: + path = "{}.py".format(os.path.splitext(self_path)[0]) + cmd.append(path) + return cmd + def satisfies(self, spec, impl_must_match): """check if a given specification can be satisfied by the this python interpreter instance""" if self.executable == spec.path: # if the path is a our own executable path we're done @@ -300,6 +308,18 @@ class PythonInfo(object): return True +class Cmd(object): + def __init__(self, cmd, env=None): + self.cmd = cmd + self.env = env + + def __repr__(self): + cmd_repr = " ".join(pipes.quote(c) for c in self.cmd) + if self.env is not None: + cmd_repr += " env of {!r}".format(self.env) + return cmd_repr + + CURRENT = PythonInfo() if __name__ == "__main__": diff --git a/src/virtualenv/seed/embed/pip_invoke.py b/src/virtualenv/seed/embed/pip_invoke.py index 662415f..ee5ac9b 100644 --- a/src/virtualenv/seed/embed/pip_invoke.py +++ b/src/virtualenv/seed/embed/pip_invoke.py @@ -1,7 +1,10 @@ from __future__ import absolute_import, unicode_literals +import logging + +from virtualenv.interpreters.discovery.py_info import Cmd from virtualenv.seed.embed.base_embed import BaseEmbed -from virtualenv.seed.embed.wheels.acquire import get_bundled_wheel, pip_wheel_env_run +from virtualenv.seed.embed.wheels.acquire import get_bundled_wheel_non_zipped, pip_wheel_env_run from virtualenv.util.subprocess import Popen @@ -12,12 +15,15 @@ class PipInvoke(BaseEmbed): def run(self, creator): cmd = self.get_pip_install_cmd(creator.exe, creator.interpreter.version_release_str) env = pip_wheel_env_run(creator.interpreter.version_release_str) + logging.debug("pip seed by running: %s", Cmd(cmd, env)) process = Popen(cmd, env=env) process.communicate() + if process.returncode != 0: + raise RuntimeError("failed seed") def get_pip_install_cmd(self, exe, version): - cmd = [str(exe), "-m", "pip", "install", "--only-binary", ":all:"] - for folder in {get_bundled_wheel(p, version).parent for p in self.packages}: + cmd = [str(exe), "-m", "pip", "-q", "install", "--only-binary", ":all:"] + for folder in {get_bundled_wheel_non_zipped(p, version).parent for p in self.packages}: cmd.extend(["--find-links", str(folder)]) cmd.extend(self.extra_search_dir) if not self.download: diff --git a/src/virtualenv/seed/embed/wheels/acquire.py b/src/virtualenv/seed/embed/wheels/acquire.py index 1d92b07..40d8ad3 100644 --- a/src/virtualenv/seed/embed/wheels/acquire.py +++ b/src/virtualenv/seed/embed/wheels/acquire.py @@ -11,12 +11,14 @@ from zipfile import ZipFile import six +from virtualenv.info import IS_ZIPAPP from virtualenv.util.path import Path from virtualenv.util.subprocess import Popen, subprocess +from virtualenv.util.zipapp import extract_to_app_data from . import BUNDLE_SUPPORT, MAX -BUNDLE_FOLDER = Path(__file__).parent +BUNDLE_FOLDER = Path(os.path.abspath(__file__)).parent def get_wheels(for_py_version, wheel_cache_dir, extra_search_dir, download, packages): @@ -47,13 +49,25 @@ def acquire_from_bundle(packages, for_py_version, to_folder): bundled_wheel_file = to_folder / bundle.name if not bundled_wheel_file.exists(): logging.debug("get bundled wheel %s", bundle) - copy2(str(bundle), str(bundled_wheel_file)) + if IS_ZIPAPP: + from virtualenv.util.zipapp import extract + + extract(bundle, bundled_wheel_file) + else: + copy2(str(bundle), str(bundled_wheel_file)) def get_bundled_wheel(package, version_release): return BUNDLE_FOLDER / (BUNDLE_SUPPORT.get(version_release, {}) or BUNDLE_SUPPORT[MAX]).get(package) +def get_bundled_wheel_non_zipped(package, version_release): + bundle = get_bundled_wheel(package, version_release) + if not IS_ZIPAPP: + return bundle + return extract_to_app_data(bundle) + + def acquire_from_dir(packages, for_py_version, to_folder, extra_search_dir): if not packages: return @@ -139,6 +153,8 @@ def download_wheel(packages, for_py_version, to_folder): # pip has no interface in python - must be a new sub-process process = Popen(cmd, env=pip_wheel_env_run("{}{}".format(*sys.version_info[0:2])), stdout=subprocess.PIPE) process.communicate() + if process.returncode != 0: + raise RuntimeError("failed to download wheels") def pip_wheel_env_run(version): @@ -148,7 +164,7 @@ def pip_wheel_env_run(version): six.ensure_str(k): str(v) # python 2 requires these to be string only (non-unicode) for k, v in { # put the bundled wheel onto the path, and use it to do the bootstrap operation - "PYTHONPATH": get_bundled_wheel("pip", version), + "PYTHONPATH": get_bundled_wheel_non_zipped("pip", version), "PIP_USE_WHEEL": "1", "PIP_USER": "0", "PIP_NO_INPUT": "1", diff --git a/src/virtualenv/seed/via_app_data/pip_install/base.py b/src/virtualenv/seed/via_app_data/pip_install/base.py index b74282c..ba65e9a 100644 --- a/src/virtualenv/seed/via_app_data/pip_install/base.py +++ b/src/virtualenv/seed/via_app_data/pip_install/base.py @@ -6,7 +6,6 @@ import re import shutil import zipfile from abc import ABCMeta, abstractmethod -from stat import S_IXGRP, S_IXOTH, S_IXUSR from tempfile import mkdtemp from textwrap import dedent @@ -15,7 +14,7 @@ from six import PY3 from virtualenv.info import IS_WIN from virtualenv.util import ConfigParser -from virtualenv.util.path import Path +from virtualenv.util.path import Path, make_exe @six.add_metaclass(ABCMeta) @@ -38,7 +37,7 @@ class PipInstall(object): site_package = self._creator.site_packages[0] for filename in self._image_dir.iterdir(): into = site_package / filename.name - logging.debug("link %s of %s", filename, into) + logging.debug("%s %s from %s", self.__class__.__name__, into, filename) if into.exists(): if into.is_dir() and not into.is_symlink(): shutil.rmtree(str(into)) @@ -175,7 +174,7 @@ class PipInstall(object): ): exe = to_folder / new_name exe.write_text(content, encoding="utf-8") - exe.chmod(exe.stat().st_mode | S_IXUSR | S_IXGRP | S_IXOTH) + make_exe(exe) result.append(exe) return result diff --git a/src/virtualenv/seed/via_app_data/via_app_data.py b/src/virtualenv/seed/via_app_data/via_app_data.py index c5695a8..f38e112 100644 --- a/src/virtualenv/seed/via_app_data/via_app_data.py +++ b/src/virtualenv/seed/via_app_data/via_app_data.py @@ -36,7 +36,7 @@ class FromAppData(BaseEmbed): name_to_whl = self._get_seed_wheels(creator, base_cache) installer_class = self.installer_class(name_to_whl["pip"].stem.split("-")[1]) for name, wheel in name_to_whl.items(): - logging.debug("install %s from wheel %s", name, wheel) + logging.debug("install %s from wheel %s via %s", name, wheel, installer_class.__name__) image_folder = base_cache / "image" / installer_class.__name__ / wheel.stem installer = installer_class(wheel, creator, image_folder) if self.clear: @@ -68,3 +68,6 @@ class FromAppData(BaseEmbed): if pip_version_int >= (19, 3): return SymlinkPipInstall return CopyPipInstall + + def __unicode__(self): + return super(FromAppData, self).__unicode__() + " app_data_dir={}".format(self.app_data_dir) diff --git a/src/virtualenv/util/path/__init__.py b/src/virtualenv/util/path/__init__.py index ab5db8e..e00acd5 100644 --- a/src/virtualenv/util/path/__init__.py +++ b/src/virtualenv/util/path/__init__.py @@ -1,12 +1,7 @@ from __future__ import absolute_import, unicode_literals from ._pathlib import Path +from ._permission import make_exe from ._sync import copy, ensure_dir, symlink, symlink_or_copy -__all__ = ( - "ensure_dir", - "symlink_or_copy", - "symlink", - "copy", - "Path", -) +__all__ = ("ensure_dir", "symlink_or_copy", "symlink", "copy", "Path", "make_exe") diff --git a/src/virtualenv/util/path/_permission.py b/src/virtualenv/util/path/_permission.py new file mode 100644 index 0000000..1356b6d --- /dev/null +++ b/src/virtualenv/util/path/_permission.py @@ -0,0 +1,20 @@ +from __future__ import absolute_import, unicode_literals + +from stat import S_IXGRP, S_IXOTH, S_IXUSR + + +def make_exe(filename): + original_mode = filename.stat().st_mode + levels = [S_IXUSR, S_IXGRP, S_IXOTH] + for at in range(len(levels), 0, -1): + try: + mode = original_mode + for level in levels[:at]: + mode |= level + filename.chmod(mode) + break + except PermissionError: + continue + + +__all__ = ("make_exe",) diff --git a/src/virtualenv/util/zipapp.py b/src/virtualenv/util/zipapp.py new file mode 100644 index 0000000..fa3ed4b --- /dev/null +++ b/src/virtualenv/util/zipapp.py @@ -0,0 +1,43 @@ +from __future__ import absolute_import, unicode_literals + +import logging +import os +import zipfile + +import six + +from virtualenv.info import IS_WIN, ROOT, get_default_data_dir +from virtualenv.version import __version__ + + +def read(full_path): + sub_file = _get_path_within_zip(full_path) + with zipfile.ZipFile(ROOT, "r") as zip_file: + with zip_file.open(sub_file) as file_handler: + return file_handler.read().decode("utf-8") + + +def extract(full_path, dest): + logging.debug("extract %s to %s", full_path, dest) + sub_file = _get_path_within_zip(full_path) + with zipfile.ZipFile(ROOT, "r") as zip_file: + info = zip_file.getinfo(sub_file) + info.filename = dest.name + zip_file.extract(info, six.ensure_text(str(dest.parent))) + + +def _get_path_within_zip(full_path): + sub_file = str(full_path)[len(ROOT) + 1 :] + if IS_WIN: + # paths are always UNIX separators, even on Windows, though __file__ still follows platform default + sub_file = sub_file.replace(os.sep, "/") + return sub_file + + +def extract_to_app_data(full_path): + base = get_default_data_dir() / "zipapp" / "extract" / __version__ + base.mkdir(parents=True, exist_ok=True) + dest = base / full_path.name + if not dest.exists(): + extract(full_path, dest) + return dest diff --git a/tasks/__main__zipapp.py b/tasks/__main__zipapp.py new file mode 100644 index 0000000..3cae294 --- /dev/null +++ b/tasks/__main__zipapp.py @@ -0,0 +1,169 @@ +import json +import os +import sys +import zipfile + +ABS_HERE = os.path.abspath(os.path.dirname(__file__)) +NEW_IMPORT_SYSTEM = sys.version_info[0:2] > (3, 4) + + +class VersionPlatformSelect(object): + def __init__(self): + self.archive = ABS_HERE + self._zip_file = zipfile.ZipFile(ABS_HERE, "r") + self.modules = self._load("modules.json") + self.distributions = self._load("distributions.json") + self.__cache = {} + + def _load(self, of_file): + version = ".".join(str(i) for i in sys.version_info[0:2]) + per_version = json.loads(self.get_data(of_file).decode("utf-8")) + all_platforms = per_version[version] if version in per_version else per_version["3.9"] + content = all_platforms.get("==any", {}) # start will all platforms + not_us = "!={}".format(sys.platform) + for key, value in all_platforms.items(): # now override that with not platform + if key.startswith("!=") and key != not_us: + content.update(value) + content.update(all_platforms.get("=={}".format(sys.platform), {})) # and finish it off with our platform + return content + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self._zip_file.close() + + def find_mod(self, fullname): + if fullname in self.modules: + result = self.modules[fullname] + return result + + def get_filename(self, fullname): + zip_path = self.find_mod(fullname) + return None if zip_path is None else os.path.join(ABS_HERE, zip_path) + + def get_data(self, filename): + if filename.startswith(ABS_HERE): + # keep paths relative from the zipfile + filename = filename[len(ABS_HERE) + 1 :] + filename = filename.lstrip(os.sep) + if sys.platform == "win32": + # paths within the zipfile is always /, fixup on Windows to transform \ to / + filename = "/".join(filename.split(os.sep)) + with self._zip_file.open(filename) as file_handler: + return file_handler.read() + + def find_distributions(self, context): + dist_class = versioned_distribution_class() + name = context.name + if name in self.distributions: + result = dist_class(file_loader=self.get_data, dist_path=self.distributions[name]) + yield result + + def __repr__(self): + return "{}(path={})".format(self.__class__.__name__, ABS_HERE) + + def _register_distutils_finder(self): + if "distlib" not in self.modules: + return + + class DistlibFinder(object): + def __init__(self, path, loader): + self.path = path + self.loader = loader + + def find(self, name): + class Resource(object): + def __init__(self, content): + self.bytes = content + + full_path = os.path.join(self.path, name) + return Resource(self.loader.get_data(full_path)) + + # noinspection PyPackageRequirements + from distlib.resources import register_finder + + register_finder(self, lambda module: DistlibFinder(os.path.dirname(module.__file__), self)) + + +_VER_DISTRIBUTION_CLASS = None + + +def versioned_distribution_class(): + global _VER_DISTRIBUTION_CLASS + if _VER_DISTRIBUTION_CLASS is None: + if sys.version_info >= (3, 8): + # noinspection PyCompatibility + from importlib.metadata import Distribution + else: + # noinspection PyUnresolvedReferences + from importlib_metadata import Distribution + + class VersionedDistribution(Distribution): + def __init__(self, file_loader, dist_path): + self.file_loader = file_loader + self.dist_path = dist_path + + def read_text(self, filename): + return self.file_loader(self.locate_file(filename)).decode("utf-8") + + def locate_file(self, path): + return os.path.join(self.dist_path, path) + + _VER_DISTRIBUTION_CLASS = VersionedDistribution + return _VER_DISTRIBUTION_CLASS + + +if NEW_IMPORT_SYSTEM: + # noinspection PyCompatibility + from importlib.util import spec_from_file_location + + # noinspection PyCompatibility + from importlib.abc import SourceLoader + + class VersionedFindLoad(VersionPlatformSelect, SourceLoader): + def find_spec(self, fullname, path, target=None): + zip_path = self.find_mod(fullname) + if zip_path is not None: + spec = spec_from_file_location(name=fullname, loader=self) + return spec + + def module_repr(self, module): + raise NotImplementedError + + +else: + # noinspection PyDeprecation + from imp import new_module + + class VersionedFindLoad(VersionPlatformSelect): + def find_module(self, fullname, path=None): + return self if self.find_mod(fullname) else None + + def load_module(self, fullname): + filename = self.get_filename(fullname) + code = self.get_data(filename) + mod = sys.modules.setdefault(fullname, new_module(fullname)) + mod.__file__ = filename + mod.__loader__ = self + is_package = filename.endswith("__init__.py") + if is_package: + mod.__path__ = [os.path.dirname(filename)] + mod.__package__ = fullname + else: + mod.__package__ = fullname.rpartition(".")[0] + exec(code, mod.__dict__) + return mod + + +def run(): + with VersionedFindLoad() as finder: + sys.meta_path.insert(0, finder) + finder._register_distutils_finder() + from virtualenv.__main__ import run as run_virtualenv + + run_virtualenv() + + +if __name__ == "__main__": + run() diff --git a/tasks/make_zipapp.py b/tasks/make_zipapp.py index 7e5bee3..dae6336 100644 --- a/tasks/make_zipapp.py +++ b/tasks/make_zipapp.py @@ -1,38 +1,249 @@ """https://docs.python.org/3/library/zipapp.html""" import argparse import io -import os.path +import json +import os +import pipes +import shutil +import subprocess +import sys import zipapp import zipfile +from collections import defaultdict, deque +from email import message_from_string +from pathlib import Path, PurePosixPath +from tempfile import TemporaryDirectory + +from packaging.markers import Marker +from packaging.requirements import Requirement + +HERE = Path(__file__).parent + +VERSIONS = ["3.{}".format(i) for i in range(9, 3, -1)] + ["2.7"] def main(): parser = argparse.ArgumentParser() - parser.add_argument("--root", default=".") - parser.add_argument("--dest") + parser.add_argument("--dest", default="virtualenv.pyz") args = parser.parse_args() + with TemporaryDirectory() as folder: + packages = get_wheels_for_support_versions(Path(folder)) + create_zipapp(os.path.abspath(args.dest), packages) - if args.dest is not None: - dest = args.dest - else: - dest = os.path.join(args.root, "virtualenv.pyz") - - filenames = {"LICENSE.txt": "LICENSE.txt", os.path.join("src", "virtualenv.py"): "virtualenv.py"} - for support in os.listdir(os.path.join(args.root, "src", "virtualenv_support")): - support_file = os.path.join("virtualenv_support", support) - filenames[os.path.join("src", support_file)] = support_file +def create_zipapp(dest, packages): bio = io.BytesIO() - with zipfile.ZipFile(bio, "w") as zip_file: - for filename in filenames: - zip_file.write(os.path.join(args.root, filename), filename) - - zip_file.writestr("__main__.py", "import virtualenv; virtualenv.main()") - + base = PurePosixPath("__virtualenv__") + modules = defaultdict(lambda: defaultdict(dict)) + dist = defaultdict(lambda: defaultdict(dict)) + with zipfile.ZipFile(bio, "w") as zip_app: + write_packages_to_zipapp(base, dist, modules, packages, zip_app) + modules_json = json.dumps(modules, indent=2) + zip_app.writestr("modules.json", modules_json) + distributions_json = json.dumps(dist, indent=2) + zip_app.writestr("distributions.json", distributions_json) + zip_app.writestr("__main__.py", (HERE / "__main__zipapp.py").read_bytes()) bio.seek(0) zipapp.create_archive(bio, dest) print("zipapp created at {}".format(dest)) +def write_packages_to_zipapp(base, dist, modules, packages, zip_app): + has = set() + for name, p_w_v in packages.items(): + for platform, w_v in p_w_v.items(): + for wheel_data in w_v.values(): + wheel = wheel_data.wheel + with zipfile.ZipFile(str(wheel)) as wheel_zip: + for filename in wheel_zip.namelist(): + if name in ("virtualenv",): + dest = PurePosixPath(filename) + else: + dest = base / wheel.stem / filename + if dest.suffix in (".so", ".pyi"): + continue + if dest.suffix == ".py": + key = filename[:-3].replace("/", ".").replace("__init__", "").rstrip(".") + for version in wheel_data.versions: + modules[version][platform][key] = str(dest) + if dest.parent.suffix == ".dist-info": + for version in wheel_data.versions: + dist[version][platform][dest.parent.stem.split("-")[0]] = str(dest.parent) + dest_str = str(dest) + if dest_str in has: + continue + has.add(dest_str) + if "/tests/" in dest_str or "/docs/" in dest_str: + continue + print(dest_str) + content = wheel_zip.read(filename) + zip_app.writestr(dest_str, content) + del content + + +class WheelDownloader(object): + def __init__(self, into): + if into.exists(): + shutil.rmtree(into) + into.mkdir(parents=True) + self.into = into + self.collected = defaultdict(lambda: defaultdict(dict)) + self.pip_cmd = [str(Path(sys.executable).parent / "pip")] + self._cmd = self.pip_cmd + ["download", "-q", "--no-deps", "--dest", str(self.into)] + + def run(self, target, versions): + whl = self.build_sdist(target) + todo = deque((version, None, whl) for version in versions) + wheel_store = dict() + while todo: + version, platform, dep = todo.popleft() + dep_str = dep.name.split("-")[0] if isinstance(dep, Path) else dep.name + if dep_str in self.collected[version] and platform in self.collected[version][dep_str]: + continue + whl = self._get_wheel(dep, platform[2:] if platform and platform.startswith("==") else None, version) + if whl is None: + if dep_str not in wheel_store: + raise RuntimeError("failed to get {}, have {}".format(dep_str, wheel_store)) + whl = wheel_store[dep_str] + else: + wheel_store[dep_str] = whl + self.collected[version][dep_str][platform] = whl + todo.extend(self.get_dependencies(whl, version)) + + def _get_wheel(self, dep, platform, version): + if isinstance(dep, Requirement): + before = set(self.into.iterdir()) + if self._download(platform, False, "--python-version", version, "--only-binary", ":all:", str(dep)): + self._download(platform, True, "--python-version", version, str(dep)) + after = set(self.into.iterdir()) + new_files = after - before + # print(dep, new_files) + assert len(new_files) <= 1 + if not len(new_files): + return None + new_file = next(iter(new_files)) + if new_file.suffix == ".whl": + return new_file + dep = new_file + new_file = self.build_sdist(dep) + assert new_file.suffix == ".whl" + return new_file + + def _download(self, platform, stop_print_on_fail, *args): + exe_cmd = self._cmd + list(args) + if platform is not None: + exe_cmd.extend(["--platform", platform]) + return run_suppress_output(exe_cmd, stop_print_on_fail=stop_print_on_fail) + + @staticmethod + def get_dependencies(whl, version): + with zipfile.ZipFile(str(whl), "r") as zip_file: + name = "/".join(["{}.dist-info".format("-".join(whl.name.split("-")[0:2])), "METADATA"]) + with zip_file.open(name) as file_handler: + metadata = message_from_string(file_handler.read().decode("utf-8")) + deps = metadata.get_all("Requires-Dist") + if deps is None: + return + for dep in deps: + req = Requirement(dep) + markers = getattr(req.marker, "_markers", tuple()) or tuple() + if any(m for m in markers if isinstance(m, tuple) and len(m) == 3 and m[0].value == "extra"): + continue + py_versions = WheelDownloader._marker_at(markers, "python_version") + if py_versions: + marker = Marker('python_version < "1"') + marker._markers = [ + markers[ver] + for ver in sorted(list(i for i in set(py_versions) | {i - 1 for i in py_versions} if i >= 0)) + ] + matches_python = marker.evaluate({"python_version": version}) + if not matches_python: + continue + deleted = 0 + for ver in py_versions: + deleted += WheelDownloader._del_marker_at(markers, ver - deleted) + platforms = [] + platform_positions = WheelDownloader._marker_at(markers, "sys_platform") + deleted = 0 + for pos in platform_positions: # can only be ore meaningfully + platform = "{}{}".format(markers[pos][1].value, markers[pos][2].value) + deleted += WheelDownloader._del_marker_at(markers, pos - deleted) + platforms.append(platform) + if not platforms: + platforms.append(None) + for platform in platforms: + yield version, platform, req + + @staticmethod + def _marker_at(markers, key): + positions = [] + for i, m in enumerate(markers): + if isinstance(m, tuple) and len(m) == 3 and m[0].value == key: + positions.append(i) + return positions + + @staticmethod + def _del_marker_at(markers, at): + del markers[at] + deleted = 1 + op = max(at - 1, 0) + if markers and isinstance(markers[op], str): + del markers[op] + deleted += 1 + return deleted + + def build_sdist(self, target): + if target.is_dir(): + folder = self.into + else: + folder = target.parent / target.stem + if not folder.exists() or not list(folder.iterdir()): + cmd = self.pip_cmd + ["wheel", "-w", str(folder), "--no-deps", str(target), "-q"] + run_suppress_output(cmd, stop_print_on_fail=True) + return list(folder.iterdir())[0] + + +def run_suppress_output(cmd, stop_print_on_fail=False): + process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + out, err = process.communicate() + if stop_print_on_fail and process.returncode != 0: + print("exit with {} of {}".format(process.returncode, " ".join(pipes.quote(i) for i in cmd)), file=sys.stdout) + if out: + print(out, file=sys.stdout) + if err: + print(err, file=sys.stderr) + raise SystemExit(process.returncode) + return process.returncode + + +def get_wheels_for_support_versions(folder): + downloader = WheelDownloader(folder / "wheel-store") + downloader.run(HERE.parent, VERSIONS) + packages = defaultdict(lambda: defaultdict(lambda: defaultdict(WheelForVersion))) + for version, collected in downloader.collected.items(): + for pkg, platform_to_wheel in collected.items(): + name = Requirement(pkg).name + for platform, wheel in platform_to_wheel.items(): + platform = platform or "==any" + wheel_versions = packages[name][platform][wheel.name] + wheel_versions.versions.append(version) + wheel_versions.wheel = wheel + for name, p_w_v in packages.items(): + for platform, w_v in p_w_v.items(): + print("{} - {}".format(name, platform)) + for wheel, wheel_versions in w_v.items(): + print("{} of {} (use {})".format(" ".join(wheel_versions.versions), wheel, wheel_versions.wheel)) + return packages + + +class WheelForVersion(object): + def __init__(self, wheel=None, versions=None): + self.wheel = wheel + self.versions = versions if versions else [] + + def __repr__(self): + return "{}({!r}, {!r})".format(self.__class__.__name__, self.wheel, self.versions) + + if __name__ == "__main__": - exit(main()) + main() diff --git a/tests/conftest.py b/tests/conftest.py index 25a456d..606c9c8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,6 +15,23 @@ from virtualenv.interpreters.discovery.py_info import PythonInfo from virtualenv.util.path import Path +def pytest_addoption(parser): + parser.addoption("--int", action="store_true", default=False, help="run integration tests") + + +def pytest_collection_modifyitems(config, items): + int_location = os.path.join("tests", "integration", "").rstrip() + if len(items) == 1: + return + + items.sort(key=lambda i: 2 if i.location[0].startswith(int_location) else (1 if "slow" in i.keywords else 0)) + + if not config.getoption("--int"): + for item in items: + if item.location[0].startswith(int_location): + item.add_marker(pytest.mark.skip(reason="need --int option to run")) + + @pytest.fixture(scope="session") def has_symlink_support(tmp_path_factory): platform_supports = hasattr(os, "symlink") diff --git a/tests/integration/test_zipapp.py b/tests/integration/test_zipapp.py new file mode 100644 index 0000000..3a21327 --- /dev/null +++ b/tests/integration/test_zipapp.py @@ -0,0 +1,78 @@ +from __future__ import absolute_import, unicode_literals + +import shutil +import subprocess +import sys + +import pytest +import six + +from virtualenv.interpreters.discovery.py_info import CURRENT +from virtualenv.run import run_via_cli +from virtualenv.util.path import Path + +HERE = Path(__file__).parent + + +@pytest.fixture(scope="session") +def zipapp_build_env(tmp_path_factory): + create_env_path = None + if sys.version_info[0:2] >= (3, 5): + exe = CURRENT.executable # guaranteed to contain a recent enough pip (tox.ini) + else: + create_env_path = tmp_path_factory.mktemp("zipapp-create-env") + for version in range(8, 4, -1): + try: + # create a virtual environment which is also guaranteed to contain a recent enough pip (bundled) + session = run_via_cli(["-v", "-p", "3.{}".format(version), "--activators", "", str(create_env_path)]) + exe = str(session.creator.exe) + break + except Exception: + pass + else: + raise RuntimeError("could not find a python to build zipapp") + cmd = [str(Path(exe).parent / "pip"), "install", "pip>=19.3", "packaging>=20"] + subprocess.check_call(cmd) + yield exe + if create_env_path is not None: + shutil.rmtree(str(create_env_path)) + + +@pytest.fixture(scope="session") +def zipapp(zipapp_build_env, tmp_path_factory): + into = tmp_path_factory.mktemp("zipapp") + path = Path(HERE).parent.parent / "tasks" / "make_zipapp.py" + filename = into / "virtualenv.pyz" + cmd = [zipapp_build_env, str(path), "--dest", str(filename)] + subprocess.check_call(cmd) + yield filename + shutil.rmtree(str(into)) + + +@pytest.fixture(scope="session") +def zipapp_test_env(tmp_path_factory): + base_path = tmp_path_factory.mktemp("zipapp-test") + session = run_via_cli(["-v", "--activators", "", "--seed", "none", str(base_path / "env")]) + yield session.creator.exe + shutil.rmtree(str(base_path)) + + +@pytest.fixture() +def call_zipapp(zipapp, monkeypatch, tmp_path, zipapp_test_env): + def _run(*args): + monkeypatch.setenv(str("_VIRTUALENV_OVERRIDE_APP_DATA"), str(tmp_path / "app_data")) + cmd = [str(zipapp_test_env), str(zipapp), "-vv", six.ensure_text(str(tmp_path / "env"))] + list(args) + subprocess.check_call(cmd) + + return _run + + +def test_zipapp_help(call_zipapp, capsys): + call_zipapp("-h") + out, err = capsys.readouterr() + assert not err + + +@pytest.mark.parametrize("seeder", ["none", "app-data", "pip"]) +def test_zipapp_create(call_zipapp, seeder): + call_zipapp("--seeder", seeder) diff --git a/tests/unit/activation/test_activation_support.py b/tests/unit/activation/test_activation_support.py index cff940e..3681977 100644 --- a/tests/unit/activation/test_activation_support.py +++ b/tests/unit/activation/test_activation_support.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import, unicode_literals + from argparse import Namespace import pytest diff --git a/tests/unit/activation/test_powershell.py b/tests/unit/activation/test_powershell.py index 78669c3..820d340 100644 --- a/tests/unit/activation/test_powershell.py +++ b/tests/unit/activation/test_powershell.py @@ -7,10 +7,11 @@ import sys import pytest from six import PY2 -from src.virtualenv.info import IS_PYPY, IS_WIN from virtualenv.activation import PowerShellActivator +from virtualenv.info import IS_PYPY, IS_WIN +@pytest.mark.slow @pytest.mark.xfail( condition=IS_PYPY and PY2 and IS_WIN and bool(os.environ.get(str("CI_RUN"))), strict=False, diff --git a/tests/unit/activation/test_python_activator.py b/tests/unit/activation/test_python_activator.py index 798295f..3db0b93 100644 --- a/tests/unit/activation/test_python_activator.py +++ b/tests/unit/activation/test_python_activator.py @@ -7,8 +7,8 @@ import sys import pytest import six -from src.virtualenv.info import IS_PYPY, IS_WIN from virtualenv.activation import PythonActivator +from virtualenv.info import IS_PYPY, IS_WIN @pytest.mark.xfail( diff --git a/tests/unit/activation/test_xonosh.py b/tests/unit/activation/test_xonosh.py index 33596eb..501cb7c 100644 --- a/tests/unit/activation/test_xonosh.py +++ b/tests/unit/activation/test_xonosh.py @@ -8,6 +8,7 @@ from virtualenv.activation import XonshActivator from virtualenv.info import IS_PYPY, PY3 +@pytest.mark.slow @pytest.mark.skipif(sys.platform == "win32" and IS_PYPY and PY3, reason="xonsh on Windows blocks indefinitely") def test_xonsh(activation_tester_class, activation_tester): class Xonsh(activation_tester_class): diff --git a/tests/unit/interpreters/boostrap/test_boostrap_link_via_app_data.py b/tests/unit/interpreters/boostrap/test_boostrap_link_via_app_data.py index f82647e..618d281 100644 --- a/tests/unit/interpreters/boostrap/test_boostrap_link_via_app_data.py +++ b/tests/unit/interpreters/boostrap/test_boostrap_link_via_app_data.py @@ -3,16 +3,20 @@ from __future__ import absolute_import, unicode_literals import os import sys +import pytest import six from virtualenv.interpreters.discovery.py_info import CURRENT from virtualenv.run import run_via_cli from virtualenv.seed.embed.wheels import BUNDLE_SUPPORT from virtualenv.seed.embed.wheels.acquire import BUNDLE_FOLDER +from virtualenv.util.path import Path from virtualenv.util.subprocess import Popen -def test_base_bootstrap_link_via_app_data(tmp_path, coverage_env): +@pytest.mark.slow +def test_base_bootstrap_link_via_app_data(tmp_path, coverage_env, mocker): + mocker.patch("virtualenv.seed.via_app_data.via_app_data.get_default_data_dir", return_value=Path(str(tmp_path))) bundle_ver = BUNDLE_SUPPORT[CURRENT.version_release_str] create_cmd = [ six.ensure_text(str(tmp_path / "env")), diff --git a/tests/unit/interpreters/boostrap/test_pip_invoke.py b/tests/unit/interpreters/boostrap/test_pip_invoke.py index 5b8e683..8904c13 100644 --- a/tests/unit/interpreters/boostrap/test_pip_invoke.py +++ b/tests/unit/interpreters/boostrap/test_pip_invoke.py @@ -1,10 +1,13 @@ from __future__ import absolute_import, unicode_literals +import pytest + from virtualenv.interpreters.discovery.py_info import CURRENT from virtualenv.run import run_via_cli from virtualenv.seed.embed.wheels import BUNDLE_SUPPORT +@pytest.mark.slow def test_base_bootstrap_via_pip_invoke(tmp_path, coverage_env): bundle_ver = BUNDLE_SUPPORT[CURRENT.version_release_str] create_cmd = [ diff --git a/tests/unit/interpreters/create/test_creator.py b/tests/unit/interpreters/create/test_creator.py index 4a58d31..a3d0dec 100644 --- a/tests/unit/interpreters/create/test_creator.py +++ b/tests/unit/interpreters/create/test_creator.py @@ -81,7 +81,7 @@ def system(): @pytest.mark.parametrize( "use_venv", [False, True] if six.PY3 else [False], ids=["no_venv", "venv"] if six.PY3 else ["no_venv"] ) -def test_create(python, use_venv, global_access, system, coverage_env, special_name_dir): +def test_create_no_seed(python, use_venv, global_access, system, coverage_env, special_name_dir): dest = special_name_dir cmd = [ "-v", @@ -190,8 +190,6 @@ def test_debug_bad_virtualenv(tmp_path): ) @pytest.mark.parametrize("clear", [True, False], ids=["clear", "no_clear"]) def test_create_clear_resets(tmp_path, use_venv, clear): - if sys.version_info[0:2] == (3, 4) and use_venv and clear is False and CURRENT.implementation == "CPython": - pytest.skip("python 3.4 does not support overwrite venv without clear") marker = tmp_path / "magic" cmd = [str(tmp_path), "--seeder", "none", "--creator", "venv" if use_venv else "builtin"] run_via_cli(cmd) @@ -235,6 +233,7 @@ def cross_python(is_inside_ci): yield interpreter +@pytest.mark.slow def test_cross_major(cross_python, coverage_env, tmp_path): cmd = [ "-v", diff --git a/tests/unit/interpreters/test_interpreters.py b/tests/unit/interpreters/test_interpreters.py index feb1bc6..a7dacf9 100644 --- a/tests/unit/interpreters/test_interpreters.py +++ b/tests/unit/interpreters/test_interpreters.py @@ -9,6 +9,7 @@ from virtualenv.interpreters.discovery.py_info import CURRENT from virtualenv.run import run_via_cli +@pytest.mark.slow def test_failed_to_find_bad_spec(): of_id = uuid4().hex with pytest.raises(RuntimeError) as context: @@ -33,7 +33,7 @@ commands = coverage run\ -m pytest \ --junitxml {toxworkdir}/junit.{envname}.xml \ - tests {posargs} + tests {posargs:--int} coverage combine coverage report @@ -144,3 +144,12 @@ deps = changedir = {toxinidir}/tasks commands = python release.py --version {posargs} + +[testenv:zipapp] +description = generate a zipapp +skip_install = true +deps = + {[testenv]deps} + packaging >= 20.0.0 +commands = + python tasks/make_zipapp.py |