diff options
| author | Seungmin Ryu <yakkle@gmail.com> | 2020-02-26 17:17:01 +0900 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2020-02-26 08:17:01 +0000 |
| commit | c3453b6c381d28377c8e0115bf1124b2ca7b3c2b (patch) | |
| tree | a0fba9154991f4d4c6d89158a97550e02b8de64f | |
| parent | 45d2802651cda42f3202945fee73835253782b4e (diff) | |
| download | virtualenv-c3453b6c381d28377c8e0115bf1124b2ca7b3c2b.tar.gz | |
handle application data folder is read only (#1661)
* fixed FileNotFoundError when directory isn't writable (#1640)
- when using docker, if `user_data_dir()` isn't writable directory,
`default_data_dir()` use `system temp directory` + `virtualenv`.
for example, tempdir is `/tmp`, it use `/tmp/virtualenv`
* start making the app-data more explicit and robust
Signed-off-by: Bernat Gabor <bgabor8@bloomberg.net>
* fix Windows
* fix docs
Signed-off-by: Bernat Gabor <bgabor8@bloomberg.net>
Co-authored-by: Bernát Gábor <gaborjbernat@gmail.com>
41 files changed, 436 insertions, 300 deletions
diff --git a/docs/_static/custom.css b/docs/_static/custom.css index d941353..cf97bf6 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -14,8 +14,8 @@ } .wy-table-responsive table { - width: calc(100% + 3em); - margin-left: 20px !important; + width: 100%; + margin-left: 0 !important; } .rst-content table.docutils td ol { diff --git a/docs/changelog/1640.bugfix.rst b/docs/changelog/1640.bugfix.rst new file mode 100644 index 0000000..4e710ea --- /dev/null +++ b/docs/changelog/1640.bugfix.rst @@ -0,0 +1,9 @@ +Handle the case when the application data folder is read-only: + +- the application data folder is now controllable via :option:`app-data`, +- :option:`clear-app-data` now cleans the entire application data folder, not just the ``app-data`` seeder path, +- check if the application data path passed in does not exist or is read-only, and fallback to a temporary directory, +- temporary directory application data is automatically cleaned up at the end of execution, +- :option:`symlink-app-data` is always ``False`` when the application data is temporary + +by :user:`gaborbernat`. diff --git a/docs/cli_interface.rst b/docs/cli_interface.rst index b4d190e..32d8be3 100644 --- a/docs/cli_interface.rst +++ b/docs/cli_interface.rst @@ -33,8 +33,8 @@ Configuration file ^^^^^^^^^^^^^^^^^^ virtualenv looks for a standard ini configuration file. The exact location depends on the operating system you're using, -as determined by :pypi:`appdirs` application data definition. The configuration file location is printed as at the end of -the output when ``--help`` is passed. +as determined by :pypi:`appdirs` application configuration definition. The configuration file location is printed as at +the end of the output when ``--help`` is passed. The keys of the settings are derived from the long command line option. For example, :option:`--python <python>` would be specified as: @@ -42,7 +42,7 @@ would be specified as: .. code-block:: ini [virtualenv] - python = /opt/python-3.3/bin/python + python = /opt/python-3.8/bin/python Options that take multiple values, like :option:`extra-search-dir` can be specified as: diff --git a/docs/render_cli.py b/docs/render_cli.py index 4417c63..88549ec 100644 --- a/docs/render_cli.py +++ b/docs/render_cli.py @@ -166,7 +166,7 @@ class CliTable(SphinxDirective): @staticmethod def _get_help_text(row): name = row.names[0] - if name in ("--creator", "--clear-app-data"): + if name in ("--creator",): content = row.help[: row.help.index("(") - 1] else: content = row.help @@ -196,6 +196,8 @@ class CliTable(SphinxDirective): name = row.names[0] if name == "-p": default_body = n.Text("the python executable virtualenv is installed into") + elif name == "--app-data": + default_body = n.Text("platform specific application data folder") elif name == "--activators": default_body = n.Text("comma separated list of activators supported") elif name == "--creator": diff --git a/src/virtualenv/config/cli/parser.py b/src/virtualenv/config/cli/parser.py index aff7f44..9cfa63f 100644 --- a/src/virtualenv/config/cli/parser.py +++ b/src/virtualenv/config/cli/parser.py @@ -24,6 +24,7 @@ class VirtualEnvConfigParser(ArgumentParser): self._verbosity = None self._options = options self._interpreter = None + self._app_data = None def _fix_defaults(self): for action in self._actions: @@ -56,7 +57,7 @@ class VirtualEnvConfigParser(ArgumentParser): class HelpFormatter(ArgumentDefaultsHelpFormatter): def __init__(self, prog): - super(HelpFormatter, self).__init__(prog, max_help_position=35, width=240) + super(HelpFormatter, self).__init__(prog, max_help_position=32, width=240) def _get_help_string(self, action): # noinspection PyProtectedMember diff --git a/src/virtualenv/config/ini.py b/src/virtualenv/config/ini.py index c910375..c878947 100644 --- a/src/virtualenv/config/ini.py +++ b/src/virtualenv/config/ini.py @@ -3,7 +3,8 @@ from __future__ import absolute_import, unicode_literals import logging import os -from virtualenv.dirs import default_config_dir +from appdirs import user_config_dir + from virtualenv.info import PY3 from virtualenv.util import ConfigParser from virtualenv.util.path import Path @@ -21,7 +22,12 @@ class IniConfig(object): def __init__(self): config_file = os.environ.get(self.VIRTUALENV_CONFIG_FILE_ENV_VAR, None) self.is_env_var = config_file is not None - self.config_file = Path(config_file) if config_file is not None else (default_config_dir() / "virtualenv.ini") + config_file = ( + Path(config_file) + if config_file is not None + else Path(user_config_dir(appname="virtualenv", appauthor="pypa")) / "virtualenv.ini" + ) + self.config_file = config_file self._cache = {} exception = None diff --git a/src/virtualenv/create/creator.py b/src/virtualenv/create/creator.py index 79e489f..4c920dc 100644 --- a/src/virtualenv/create/creator.py +++ b/src/virtualenv/create/creator.py @@ -3,20 +3,18 @@ from __future__ import absolute_import, print_function, unicode_literals import json import logging import os -import shutil import sys from abc import ABCMeta, abstractmethod from argparse import ArgumentTypeError from ast import literal_eval from collections import OrderedDict -from stat import S_IWUSR from six import add_metaclass from virtualenv.discovery.cached_py_info import LogCmd from virtualenv.info import WIN_CPYTHON_2 from virtualenv.pyenv_cfg import PyEnvCfg -from virtualenv.util.path import Path +from virtualenv.util.path import Path, safe_delete from virtualenv.util.six import ensure_str, ensure_text from virtualenv.util.subprocess import run_cmd from virtualenv.util.zipapp import ensure_file_on_disk @@ -41,6 +39,7 @@ class Creator(object): self.dest = Path(options.dest) self.clear = options.clear self.pyenv_cfg = PyEnvCfg.from_folder(self.dest) + self.app_data = options.app_data.folder def __repr__(self): return ensure_str(self.__unicode__()) @@ -65,7 +64,7 @@ class Creator(object): return True @classmethod - def add_parser_arguments(cls, parser, interpreter, meta): + def add_parser_arguments(cls, parser, interpreter, meta, app_data): """Add CLI arguments for the creator. :param parser: the CLI parser @@ -147,15 +146,7 @@ class Creator(object): def run(self): if self.dest.exists() and self.clear: logging.debug("delete %s", self.dest) - - def onerror(func, path, exc_info): - if not os.access(path, os.W_OK): - os.chmod(path, S_IWUSR) - func(path) - else: - raise - - shutil.rmtree(str(self.dest), ignore_errors=True, onerror=onerror) + safe_delete(self.dest) self.create() self.set_pyenv_cfg() @@ -172,7 +163,7 @@ class Creator(object): :return: debug information about the virtual environment (only valid after :meth:`create` has run) """ if self._debug is None and self.exe is not None: - self._debug = get_env_debug_info(self.exe, self.debug_script()) + self._debug = get_env_debug_info(self.exe, self.debug_script(), self.app_data) return self._debug # noinspection PyMethodMayBeStatic @@ -180,11 +171,11 @@ class Creator(object): return DEBUG_SCRIPT -def get_env_debug_info(env_exe, debug_script): +def get_env_debug_info(env_exe, debug_script, app_data): env = os.environ.copy() env.pop(str("PYTHONPATH"), None) - with ensure_file_on_disk(debug_script) as debug_script: + with ensure_file_on_disk(debug_script, app_data) as debug_script: cmd = [str(env_exe), str(debug_script)] if WIN_CPYTHON_2: cmd = [ensure_text(i) for i in cmd] diff --git a/src/virtualenv/create/via_global_ref/api.py b/src/virtualenv/create/via_global_ref/api.py index 33c606b..8ee0c5c 100644 --- a/src/virtualenv/create/via_global_ref/api.py +++ b/src/virtualenv/create/via_global_ref/api.py @@ -20,8 +20,8 @@ class ViaGlobalRefApi(Creator): self.enable_system_site_package = options.system_site @classmethod - def add_parser_arguments(cls, parser, interpreter, meta): - super(ViaGlobalRefApi, cls).add_parser_arguments(parser, interpreter, meta) + def add_parser_arguments(cls, parser, interpreter, meta, app_data): + super(ViaGlobalRefApi, cls).add_parser_arguments(parser, interpreter, meta, app_data) parser.add_argument( "--system-site-packages", default=False, @@ -54,7 +54,7 @@ class ViaGlobalRefApi(Creator): def patch_distutils_via_pth(self): """Patch the distutils package to not be derailed by its configuration files""" patch_file = Path(__file__).parent / "_distutils_patch_virtualenv.py" - with ensure_file_on_disk(patch_file) as resolved_path: + with ensure_file_on_disk(patch_file, self.app_data) as resolved_path: text = resolved_path.read_text() text = text.replace('"__SCRIPT_DIR__"', repr(os.path.relpath(str(self.script_dir), str(self.purelib)))) patch_path = self.purelib / "_distutils_patch_virtualenv.py" diff --git a/src/virtualenv/create/via_global_ref/venv.py b/src/virtualenv/create/via_global_ref/venv.py index 0cbf1d1..5b774ef 100644 --- a/src/virtualenv/create/via_global_ref/venv.py +++ b/src/virtualenv/create/via_global_ref/venv.py @@ -38,9 +38,6 @@ class Venv(ViaGlobalRefApi): self.create_inline() else: self.create_via_sub_process() - - # TODO: cleanup activation scripts - for lib in self.libs: ensure_dir(lib) super(Venv, self).create() diff --git a/src/virtualenv/dirs.py b/src/virtualenv/dirs.py deleted file mode 100644 index 50919ea..0000000 --- a/src/virtualenv/dirs.py +++ /dev/null @@ -1,41 +0,0 @@ -import os - -from appdirs import user_config_dir, user_data_dir - -from virtualenv.util.lock import ReentrantFileLock - -_DATA_DIR = None -_CFG_DIR = None - - -def default_data_dir(): - - global _DATA_DIR - if _DATA_DIR is None: - folder = _get_default_data_folder() - _DATA_DIR = ReentrantFileLock(folder) - return _DATA_DIR - - -def _get_default_data_folder(): - key = str("VIRTUALENV_OVERRIDE_APP_DATA") - if key in os.environ: - folder = os.environ[key] - else: - folder = user_data_dir(appname="virtualenv", appauthor="pypa") - return folder - - -def default_config_dir(): - from virtualenv.util.path import Path - - global _CFG_DIR - if _CFG_DIR is None: - _CFG_DIR = Path(user_config_dir(appname="virtualenv", appauthor="pypa")) - return _CFG_DIR - - -__all__ = ( - "default_data_dir", - "default_config_dir", -) diff --git a/src/virtualenv/discovery/builtin.py b/src/virtualenv/discovery/builtin.py index fc8eca3..35734ce 100644 --- a/src/virtualenv/discovery/builtin.py +++ b/src/virtualenv/discovery/builtin.py @@ -16,6 +16,7 @@ class Builtin(Discover): def __init__(self, options): super(Builtin, self).__init__(options) self.python_spec = options.python + self.app_data = options.app_data @classmethod def add_parser_arguments(cls, parser): @@ -29,7 +30,7 @@ class Builtin(Discover): ) def run(self): - return get_interpreter(self.python_spec) + return get_interpreter(self.python_spec, self.app_data.folder) def __repr__(self): return ensure_str(self.__unicode__()) @@ -38,11 +39,11 @@ class Builtin(Discover): return "{} discover of python_spec={!r}".format(self.__class__.__name__, self.python_spec) -def get_interpreter(key): +def get_interpreter(key, app_data=None): spec = PythonSpec.from_string_spec(key) logging.info("find interpreter for spec %r", spec) proposed_paths = set() - for interpreter, impl_must_match in propose_interpreters(spec): + for interpreter, impl_must_match in propose_interpreters(spec, app_data): key = interpreter.system_executable, impl_must_match if key in proposed_paths: continue @@ -53,19 +54,19 @@ def get_interpreter(key): proposed_paths.add(key) -def propose_interpreters(spec): +def propose_interpreters(spec, app_data): # 1. if it's an absolute path and exists, use that if spec.is_abs and os.path.exists(spec.path): - yield PythonInfo.from_exe(spec.path), True + yield PythonInfo.from_exe(spec.path, app_data), True # 2. try with the current - yield PythonInfo.current_system(), True + yield PythonInfo.current_system(app_data), True # 3. otherwise fallback to platform default logic if IS_WIN: from .windows import propose_interpreters - for interpreter in propose_interpreters(spec): + for interpreter in propose_interpreters(spec, app_data): yield interpreter, True paths = get_paths() @@ -80,7 +81,7 @@ def propose_interpreters(spec): exe = os.path.abspath(found) if exe not in tested_exes: tested_exes.add(exe) - interpreter = PathPythonInfo.from_exe(exe, raise_on_error=False) + interpreter = PathPythonInfo.from_exe(exe, app_data, raise_on_error=False) if interpreter is not None: yield interpreter, match diff --git a/src/virtualenv/discovery/cached_py_info.py b/src/virtualenv/discovery/cached_py_info.py index 2306f28..e2f1d54 100644 --- a/src/virtualenv/discovery/cached_py_info.py +++ b/src/virtualenv/discovery/cached_py_info.py @@ -14,7 +14,6 @@ import sys from collections import OrderedDict from hashlib import sha256 -from virtualenv.dirs import default_data_dir from virtualenv.discovery.py_info import PythonInfo from virtualenv.info import PY2, PY3 from virtualenv.util.path import Path @@ -25,12 +24,12 @@ from virtualenv.version import __version__ _CACHE = OrderedDict() _CACHE[Path(sys.executable)] = PythonInfo() -_FS_PATH = None -def from_exe(cls, exe, raise_on_error=True, ignore_cache=False): +def from_exe(cls, app_data, exe, raise_on_error=True, ignore_cache=False): """""" - result = _get_from_cache(cls, exe, ignore_cache=ignore_cache) + py_info_cache = _get_py_info_cache(app_data) + result = _get_from_cache(cls, py_info_cache, app_data, exe, ignore_cache=ignore_cache) if isinstance(result, Exception): if raise_on_error: raise result @@ -40,14 +39,21 @@ def from_exe(cls, exe, raise_on_error=True, ignore_cache=False): return result -def _get_from_cache(cls, exe, ignore_cache=True): +def _get_py_info_cache(app_data): + return None if app_data is None else app_data / "py_info" / __version__ + + +def _get_from_cache(cls, py_info_cache, app_data, exe, ignore_cache=True): # note here we cannot resolve symlinks, as the symlink may trigger different prefix information if there's a # pyenv.cfg somewhere alongside on python3.4+ exe_path = Path(exe) if not ignore_cache and exe_path in _CACHE: # check in the in-memory cache result = _CACHE[exe_path] + elif py_info_cache is None: # cache disabled + failure, py_info = _run_subprocess(cls, exe, app_data) + result = py_info if failure is None else failure else: # then check the persisted cache - py_info = _get_via_file_cache(cls, exe_path, exe) + py_info = _get_via_file_cache(cls, py_info_cache, app_data, exe_path, exe) result = _CACHE[exe_path] = py_info # independent if it was from the file or in-memory cache fix the original executable location if isinstance(result, PythonInfo): @@ -55,15 +61,13 @@ def _get_from_cache(cls, exe, ignore_cache=True): return result -def _get_via_file_cache(cls, resolved_path, exe): +def _get_via_file_cache(cls, py_info_cache, app_data, resolved_path, exe): key = sha256(str(resolved_path).encode("utf-8") if PY3 else str(resolved_path)).hexdigest() py_info = None resolved_path_text = ensure_text(str(resolved_path)) resolved_path_modified_timestamp = resolved_path.stat().st_mtime - fs_path = _get_fs_path() - data_file = fs_path / "{}.json".format(key) - - with fs_path.lock_for_key(key): + data_file = py_info_cache / "{}.json".format(key) + with py_info_cache.lock_for_key(key): data_file_path = data_file.path if data_file_path.exists(): # if exists and matches load try: @@ -72,12 +76,12 @@ def _get_via_file_cache(cls, resolved_path, exe): logging.debug("get PythonInfo from %s for %s", data_file_path, exe) py_info = cls._from_dict({k: v for k, v in data["content"].items()}) else: - raise ValueError("force cleanup as stale") + raise ValueError("force close as stale") except (KeyError, ValueError, OSError): logging.debug("remove PythonInfo %s for %s", data_file_path, exe) - data_file_path.unlink() # cleanup out of date files + data_file_path.unlink() # close out of date files if py_info is None: # if not loaded run and save - failure, py_info = _run_subprocess(cls, exe) + failure, py_info = _run_subprocess(cls, exe, app_data) if failure is None: file_cache_content = { "st_mtime": resolved_path_modified_timestamp, @@ -91,22 +95,13 @@ def _get_via_file_cache(cls, resolved_path, exe): return py_info -def _get_fs_path(): - global _FS_PATH - if _FS_PATH is None: - _FS_PATH = default_data_dir() / "py-info" / __version__ - return _FS_PATH - - -def _run_subprocess(cls, exe): +def _run_subprocess(cls, exe, app_data): resolved_path = Path(os.path.abspath(__file__)).parent / "py_info.py" - with ensure_file_on_disk(resolved_path) as resolved_path: - - cmd = [exe, "-s", str(resolved_path)] + with ensure_file_on_disk(resolved_path, app_data) as resolved_path: + cmd = [exe, str(resolved_path)] # prevent sys.prefix from leaking into the child process - see https://bugs.python.org/issue22490 env = os.environ.copy() env.pop("__PYVENV_LAUNCHER__", None) - logging.debug("get interpreter info via cmd: %s", LogCmd(cmd)) try: process = Popen( @@ -156,14 +151,15 @@ class LogCmd(object): return raw -def clear(): - fs_path = _get_fs_path() - with fs_path: - for filename in fs_path.path.iterdir(): - if filename.suffix == ".json": - with fs_path.lock_for_key(filename.stem): - if filename.exists(): - filename.unlink() +def clear(app_data): + py_info_cache = _get_py_info_cache(app_data) + if py_info_cache is not None: + with py_info_cache: + for filename in py_info_cache.path.iterdir(): + if filename.suffix == ".json": + with py_info_cache.lock_for_key(filename.stem): + if filename.exists(): + filename.unlink() _CACHE.clear() diff --git a/src/virtualenv/discovery/py_info.py b/src/virtualenv/discovery/py_info.py index 64b91dc..352a17c 100644 --- a/src/virtualenv/discovery/py_info.py +++ b/src/virtualenv/discovery/py_info.py @@ -220,11 +220,11 @@ class PythonInfo(object): return "{}{}-{}".format(self.implementation, ".".join(str(i) for i in self.version_info), self.architecture) @classmethod - def clear_cache(cls): + def clear_cache(cls, app_data): # this method is not used by itself, so here and called functions can import stuff locally from virtualenv.discovery.cached_py_info import clear - clear() + clear(app_data) cls._cache_exe_discovery.clear() def satisfies(self, spec, impl_must_match): @@ -253,23 +253,23 @@ class PythonInfo(object): _current = None @classmethod - def current(cls): + def current(cls, app_data=None): """ This locates the current host interpreter information. This might be different than what we run into in case the host python has been upgraded from underneath us. """ if cls._current is None: - cls._current = cls.from_exe(sys.executable, raise_on_error=True, resolve_to_host=False) + cls._current = cls.from_exe(sys.executable, app_data, raise_on_error=True, resolve_to_host=False) return cls._current @classmethod - def current_system(cls): + def current_system(cls, app_data=None): """ This locates the current host interpreter information. This might be different than what we run into in case the host python has been upgraded from underneath us. """ if cls._current_system is None: - cls._current_system = cls.from_exe(sys.executable, raise_on_error=True, resolve_to_host=True) + cls._current_system = cls.from_exe(sys.executable, app_data, raise_on_error=True, resolve_to_host=True) return cls._current_system def _to_json(self): @@ -283,15 +283,15 @@ class PythonInfo(object): return data @classmethod - def from_exe(cls, exe, raise_on_error=True, ignore_cache=False, resolve_to_host=True): + def from_exe(cls, exe, app_data=None, raise_on_error=True, ignore_cache=False, resolve_to_host=True): """Given a path to an executable get the python information""" # this method is not used by itself, so here and called functions can import stuff locally from virtualenv.discovery.cached_py_info import from_exe - proposed = from_exe(cls, exe, raise_on_error=raise_on_error, ignore_cache=ignore_cache) + proposed = from_exe(cls, app_data, exe, raise_on_error=raise_on_error, ignore_cache=ignore_cache) # noinspection PyProtectedMember if isinstance(proposed, PythonInfo) and resolve_to_host: - proposed = proposed._resolve_to_system(proposed) + proposed = proposed._resolve_to_system(app_data, proposed) return proposed @classmethod @@ -308,7 +308,7 @@ class PythonInfo(object): return result @classmethod - def _resolve_to_system(cls, target): + def _resolve_to_system(cls, app_data, target): start_executable = target.executable prefixes = OrderedDict() while target.system_executable is None: @@ -324,63 +324,60 @@ class PythonInfo(object): logging.error("%d: prefix=%s, info=%r", len(prefixes) + 1, prefix, target) raise RuntimeError("prefixes are causing a circle {}".format("|".join(prefixes.keys()))) prefixes[prefix] = target - target = target.discover_exe(prefix=prefix, exact=False) + target = target.discover_exe(app_data, prefix=prefix, exact=False) if target.executable != target.system_executable: - target = cls.from_exe(target.system_executable) + target = cls.from_exe(target.system_executable, app_data) target.executable = start_executable return target _cache_exe_discovery = {} - def discover_exe(self, prefix, exact=True): + def discover_exe(self, app_data, prefix, exact=True): key = prefix, exact if key in self._cache_exe_discovery and prefix: - logging.debug("discover exe cache %r via %r", key, self._cache_exe_discovery[key]) + logging.debug("discover exe from cache %s - exact %s: %r", prefix, exact, self._cache_exe_discovery[key]) return self._cache_exe_discovery[key] - logging.debug("discover system for %s in %s", self, prefix) + logging.debug("discover exe for %s in %s", self, prefix) # we don't know explicitly here, do some guess work - our executable name should tell possible_names = self._find_possible_exe_names() possible_folders = self._find_possible_folders(prefix) discovered = [] for folder in possible_folders: for name in possible_names: - exe_path = os.path.join(folder, name) - if os.path.exists(exe_path): - info = self.from_exe(exe_path, resolve_to_host=False, raise_on_error=False) - if info is None: # ignore if for some reason we can't query - continue - for item in ["implementation", "architecture", "version_info"]: - found = getattr(info, item) - searched = getattr(self, item) - if found != searched: - if item == "version_info": - found, searched = ".".join(str(i) for i in found), ".".join(str(i) for i in searched) - logging.debug( - "refused interpreter %s because %s differs %s != %s", - info.executable, - item, - found, - searched, - ) - if exact is False: - discovered.append(info) - break - else: - self._cache_exe_discovery[key] = info - return info + info = self._check_exe(app_data, folder, name, exact, discovered) + if info is not None: + self._cache_exe_discovery[key] = info + return info if exact is False and discovered: info = self._select_most_likely(discovered, self) - logging.debug( - "no exact match found, chosen most similar of %s within base folders %s", - info, - os.pathsep.join(possible_folders), - ) + folders = os.pathsep.join(possible_folders) self._cache_exe_discovery[key] = info + logging.debug("no exact match found, chosen most similar of %s within base folders %s", info, folders) return info - what = "|".join(possible_names) # pragma: no cover - raise RuntimeError( - "failed to detect {} in {}".format(what, os.pathsep.join(possible_folders)) - ) # pragma: no cover + msg = "failed to detect {} in {}".format("|".join(possible_names), os.pathsep.join(possible_folders)) + raise RuntimeError(msg) + + def _check_exe(self, app_data, folder, name, exact, discovered): + exe_path = os.path.join(folder, name) + if not os.path.exists(exe_path): + return None + info = self.from_exe(exe_path, app_data, resolve_to_host=False, raise_on_error=False) + if info is None: # ignore if for some reason we can't query + return None + for item in ["implementation", "architecture", "version_info"]: + found = getattr(info, item) + searched = getattr(self, item) + if found != searched: + if item == "version_info": + found, searched = ".".join(str(i) for i in found), ".".join(str(i) for i in searched) + executable = info.executable + logging.debug("refused interpreter %s because %s differs %s != %s", executable, item, found, searched) + if exact is False: + discovered.append(info) + break + else: + return info + return None @staticmethod def _select_most_likely(discovered, target): diff --git a/src/virtualenv/discovery/windows/__init__.py b/src/virtualenv/discovery/windows/__init__.py index f321d7f..48983e6 100644 --- a/src/virtualenv/discovery/windows/__init__.py +++ b/src/virtualenv/discovery/windows/__init__.py @@ -9,13 +9,13 @@ class Pep514PythonInfo(PythonInfo): """""" -def propose_interpreters(spec): +def propose_interpreters(spec, cache_dir): # see if PEP-514 entries are good for name, major, minor, arch, exe, _ in discover_pythons(): # pre-filter registry_spec = PythonSpec(None, name, major, minor, None, arch, exe) if registry_spec.satisfies(spec): - interpreter = Pep514PythonInfo.from_exe(exe, raise_on_error=False) + interpreter = Pep514PythonInfo.from_exe(exe, cache_dir, raise_on_error=False) if interpreter is not None: if interpreter.satisfies(spec, impl_must_match=True): yield interpreter diff --git a/src/virtualenv/run/__init__.py b/src/virtualenv/run/__init__.py index 443ba66..6983e33 100644 --- a/src/virtualenv/run/__init__.py +++ b/src/virtualenv/run/__init__.py @@ -1,7 +1,10 @@ from __future__ import absolute_import, unicode_literals +import argparse import logging +from virtualenv.run.app_data import AppDataAction + from ..config.cli.parser import VirtualEnvConfigParser from ..report import LEVELS, setup_report from ..session import Session @@ -19,17 +22,20 @@ def cli_run(args, options=None): :param options: passing in a ``argparse.Namespace`` object allows return of the parsed options :return: the session object of the creation (its structure for now is experimental and might change on short notice) """ + if options is None: + options = argparse.Namespace() session = session_via_cli(args, options) - session.run() + with session: + session.run() return session # noinspection PyProtectedMember -def session_via_cli(args, options=None): +def session_via_cli(args, options): parser = build_parser(args, options) parser.parse_args(args, namespace=parser._options) creator, seeder, activators = tuple(e.create(parser._options) for e in parser._elements) # create types - session = Session(parser._verbosity, parser._interpreter, creator, seeder, activators) + session = Session(parser._verbosity, options.app_data, parser._interpreter, creator, seeder, activators) return session @@ -45,6 +51,22 @@ def build_parser(args=None, options=None): help="on failure also display the stacktrace internals of virtualenv", ) parser._options, parser._verbosity = _do_report_setup(parser, args) + # here we need a write-able application data (e.g. the zipapp might need this for discovery cache) + default_app_data = AppDataAction.default() + parser.add_argument( + "--app-data", + dest="app_data", + action=AppDataAction, + default="<temp folder>" if default_app_data is None else default_app_data, + help="a data folder used as cache by the virtualenv", + ) + parser.add_argument( + "--clear-app-data", + dest="clear_app_data", + action="store_true", + help="start with empty app data folder", + default=False, + ) discover = get_discover(parser, args, parser._options) parser._interpreter = interpreter = discover.interpreter if interpreter is None: diff --git a/src/virtualenv/run/app_data.py b/src/virtualenv/run/app_data.py new file mode 100644 index 0000000..00a3940 --- /dev/null +++ b/src/virtualenv/run/app_data.py @@ -0,0 +1,83 @@ +import logging +import os +from argparse import Action, ArgumentError +from tempfile import mkdtemp + +from appdirs import user_data_dir + +from virtualenv.util.lock import ReentrantFileLock +from virtualenv.util.path import safe_delete + + +class AppData(object): + def __init__(self, folder): + self.folder = ReentrantFileLock(folder) + self.transient = False + + def __repr__(self): + return "{}".format(self.folder.path) + + def clean(self): + logging.debug("clean app data folder %s", self.folder.path) + safe_delete(self.folder.path) + + def close(self): + """""" + + +class TempAppData(AppData): + def __init__(self): + super(TempAppData, self).__init__(folder=mkdtemp()) + self.transient = True + logging.debug("created temporary app data folder %s", self.folder.path) + + def close(self): + logging.debug("remove temporary app data folder %s", self.folder.path) + safe_delete(self.folder.path) + + +class AppDataAction(Action): + def __call__(self, parser, namespace, values, option_string=None): + folder = self._check_folder(values) + if folder is None: + raise ArgumentError("app data path {} is not valid".format(values)) + setattr(namespace, self.dest, AppData(folder)) + + @staticmethod + def _check_folder(folder): + folder = os.path.abspath(folder) + if not os.path.exists(folder): + try: + os.mkdir(folder) + logging.debug("created app data folder %s", folder) + except OSError as exception: + logging.info("could not create app data folder %s due to %r", folder, exception) + return None + write_enabled = os.access(folder, os.W_OK) + if write_enabled: + return folder + logging.debug("app data folder %s has no write access", folder) + return None + + @staticmethod + def default(): + for folder in AppDataAction._app_data_candidates(): + folder = AppDataAction._check_folder(folder) + if folder is not None: + return AppData(folder) + return None + + @staticmethod + def _app_data_candidates(): + key = str("VIRTUALENV_OVERRIDE_APP_DATA") + if key in os.environ: + yield os.environ[key] + else: + yield user_data_dir(appname="virtualenv", appauthor="pypa") + + +__all__ = ( + "AppData", + "TempAppData", + "AppDataAction", +) diff --git a/src/virtualenv/run/plugin/activators.py b/src/virtualenv/run/plugin/activators.py index 69b3050..dea2827 100644 --- a/src/virtualenv/run/plugin/activators.py +++ b/src/virtualenv/run/plugin/activators.py @@ -21,7 +21,7 @@ class ActivationSelector(ComponentBuilder): self.parser.add_argument( "--{}".format(name), default=self.default, - metavar="comma_separated_list", + metavar="comma_sep_list", required=False, help="activators to generate - default is all supported", type=self._extract_activators, diff --git a/src/virtualenv/run/plugin/base.py b/src/virtualenv/run/plugin/base.py index 8aa4206..ed10fe0 100644 --- a/src/virtualenv/run/plugin/base.py +++ b/src/virtualenv/run/plugin/base.py @@ -47,12 +47,12 @@ class ComponentBuilder(PluginLoader): if selected not in self.possible: raise RuntimeError("No implementation for {}".format(self.interpreter)) self._impl_class = self.possible[selected] - self.populate_selected_argparse(selected) + self.populate_selected_argparse(selected, options.app_data) return selected - def populate_selected_argparse(self, selected): + def populate_selected_argparse(self, selected, app_data): self.parser.description = "options for {} {}".format(self.name, selected) - self._impl_class.add_parser_arguments(self.parser, self.interpreter) + self._impl_class.add_parser_arguments(self.parser, self.interpreter, app_data) def create(self, options): return self._impl_class(options, self.interpreter) diff --git a/src/virtualenv/run/plugin/creators.py b/src/virtualenv/run/plugin/creators.py index 7c8132d..cbf0a5d 100644 --- a/src/virtualenv/run/plugin/creators.py +++ b/src/virtualenv/run/plugin/creators.py @@ -55,9 +55,9 @@ class CreatorSelector(ComponentBuilder): def _get_default(choices): return next(iter(choices)) - def populate_selected_argparse(self, selected): + def populate_selected_argparse(self, selected, app_data): self.parser.description = "options for {} {}".format(self.name, selected) - self._impl_class.add_parser_arguments(self.parser, self.interpreter, self.key_to_meta[selected]) + self._impl_class.add_parser_arguments(self.parser, self.interpreter, self.key_to_meta[selected], app_data) def create(self, options): options.meta = self.key_to_meta[getattr(options, self.name)] diff --git a/src/virtualenv/run/plugin/discovery.py b/src/virtualenv/run/plugin/discovery.py index 8890150..43d5eb2 100644 --- a/src/virtualenv/run/plugin/discovery.py +++ b/src/virtualenv/run/plugin/discovery.py @@ -1,5 +1,7 @@ from __future__ import absolute_import, unicode_literals +from virtualenv.run.app_data import TempAppData + from .base import PluginLoader @@ -20,6 +22,10 @@ def get_discover(parser, args, options): help="interpreter discovery method", ) options, _ = parser.parse_known_args(args, namespace=options) + if options.app_data == "<temp folder>": + options.app_data = TempAppData() + if options.clear_app_data: + options.app_data.clean() discover_class = discover_types[options.discovery] discover_class.add_parser_arguments(discovery_parser) options, _ = parser.parse_known_args(args, namespace=options) diff --git a/src/virtualenv/seed/embed/base_embed.py b/src/virtualenv/seed/embed/base_embed.py index 753eeb2..98ba65f 100644 --- a/src/virtualenv/seed/embed/base_embed.py +++ b/src/virtualenv/seed/embed/base_embed.py @@ -31,12 +31,13 @@ class BaseEmbed(Seeder): self.no_pip = options.no_pip self.no_setuptools = options.no_setuptools self.no_wheel = options.no_wheel + self.app_data = options.app_data.folder def package_version(self): return {package: getattr(self, "{}_version".format(package)) for package in self.packages} @classmethod - def add_parser_arguments(cls, parser, interpreter): + def add_parser_arguments(cls, parser, interpreter, app_data): group = parser.add_mutually_exclusive_group() group.add_argument( "--download", diff --git a/src/virtualenv/seed/embed/pip_invoke.py b/src/virtualenv/seed/embed/pip_invoke.py index 4b35ee5..c007af3 100644 --- a/src/virtualenv/seed/embed/pip_invoke.py +++ b/src/virtualenv/seed/embed/pip_invoke.py @@ -24,7 +24,7 @@ class PipInvoke(BaseEmbed): if not self.enabled: return with self.get_pip_install_cmd(creator.exe, creator.interpreter.version_release_str) as cmd: - with pip_wheel_env_run(creator.interpreter.version_release_str) as env: + with pip_wheel_env_run(creator.interpreter.version_release_str, self.app_data) as env: logging.debug("pip seed by running: %s", LogCmd(cmd, env)) process = Popen(cmd, env=env) process.communicate() @@ -40,7 +40,7 @@ class PipInvoke(BaseEmbed): cmd.append("{}{}".format(key, "=={}".format(ver) if ver is not None else "")) with ExitStack() as stack: folders = set() - for context in (ensure_file_on_disk(get_bundled_wheel(p, version)) for p in self.packages): + for context in (ensure_file_on_disk(get_bundled_wheel(p, version), self.app_data) for p in self.packages): folders.add(stack.enter_context(context).parent) for folder in folders: cmd.extend(["--find-links", str(folder)]) diff --git a/src/virtualenv/seed/embed/wheels/acquire.py b/src/virtualenv/seed/embed/wheels/acquire.py index 3be475f..dd25acc 100644 --- a/src/virtualenv/seed/embed/wheels/acquire.py +++ b/src/virtualenv/seed/embed/wheels/acquire.py @@ -21,7 +21,7 @@ from . import BUNDLE_SUPPORT, MAX BUNDLE_FOLDER = Path(os.path.abspath(__file__)).parent -def get_wheels(for_py_version, wheel_cache_dir, extra_search_dir, download, packages): +def get_wheels(for_py_version, wheel_cache_dir, extra_search_dir, download, packages, app_data): # not all wheels are compatible with all python versions, so we need to py version qualify it processed = copy(packages) # 1. acquire from bundle @@ -30,7 +30,7 @@ def get_wheels(for_py_version, wheel_cache_dir, extra_search_dir, download, pack acquire_from_dir(processed, for_py_version, wheel_cache_dir, extra_search_dir) # 3. download from the internet if download and processed: - download_wheel(processed, for_py_version, wheel_cache_dir) + download_wheel(processed, for_py_version, wheel_cache_dir, app_data) # in the end just get the wheels wheels = _get_wheels(wheel_cache_dir, packages) @@ -126,7 +126,7 @@ def _get_wheels(from_folder, packages): return wheels -def download_wheel(packages, for_py_version, to_folder): +def download_wheel(packages, for_py_version, to_folder, app_data): to_download = list(p if v is None else "{}={}".format(p, v) for p, v in packages.items()) logging.debug("download wheels %s", to_download) cmd = [ @@ -145,7 +145,7 @@ def download_wheel(packages, for_py_version, to_folder): cmd.extend(to_download) # pip has no interface in python - must be a new sub-process - with pip_wheel_env_run("{}{}".format(*sys.version_info[0:2])) as env: + with pip_wheel_env_run("{}{}".format(*sys.version_info[0:2]), app_data) as env: process = Popen(cmd, env=env, stdout=subprocess.PIPE) process.communicate() if process.returncode != 0: @@ -153,7 +153,7 @@ def download_wheel(packages, for_py_version, to_folder): @contextmanager -def pip_wheel_env_run(version): +def pip_wheel_env_run(version, app_data): env = os.environ.copy() env.update( { @@ -161,7 +161,7 @@ def pip_wheel_env_run(version): for k, v in {"PIP_USE_WHEEL": "1", "PIP_USER": "0", "PIP_NO_INPUT": "1"}.items() } ) - with ensure_file_on_disk(get_bundled_wheel("pip", version)) as pip_wheel_path: + with ensure_file_on_disk(get_bundled_wheel("pip", version), app_data) as pip_wheel_path: # put the bundled wheel onto the path, and use it to do the bootstrap operation env[str("PYTHONPATH")] = str(pip_wheel_path) yield env diff --git a/src/virtualenv/seed/seeder.py b/src/virtualenv/seed/seeder.py index 5ed5e5a..2bcccfc 100644 --- a/src/virtualenv/seed/seeder.py +++ b/src/virtualenv/seed/seeder.py @@ -19,11 +19,12 @@ class Seeder(object): self.enabled = enabled @classmethod - def add_parser_arguments(cls, parser, interpreter): + def add_parser_arguments(cls, parser, interpreter, app_data): """ Add CLI arguments for this seed mechanisms. :param parser: the CLI parser + :param app_data: the CLI parser :param interpreter: the interpreter this virtual environment is based of """ raise NotImplementedError diff --git a/src/virtualenv/seed/via_app_data/pip_install/base.py b/src/virtualenv/seed/via_app_data/pip_install/base.py index 4285574..8963e62 100644 --- a/src/virtualenv/seed/via_app_data/pip_install/base.py +++ b/src/virtualenv/seed/via_app_data/pip_install/base.py @@ -3,7 +3,6 @@ from __future__ import absolute_import, unicode_literals import logging import os import re -import shutil import zipfile from abc import ABCMeta, abstractmethod from contextlib import contextmanager @@ -13,7 +12,7 @@ from threading import Lock from six import PY3, add_metaclass from virtualenv.util import ConfigParser -from virtualenv.util.path import Path +from virtualenv.util.path import Path, safe_delete from virtualenv.util.six import ensure_text @@ -40,7 +39,7 @@ class PipInstall(object): into = self._creator.purelib / filename.name if into.exists(): if into.is_dir() and not into.is_symlink(): - shutil.rmtree(str(into)) + safe_delete(into) else: into.unlink() self._sync(filename, into) @@ -88,7 +87,7 @@ class PipInstall(object): for i in self._create_console_entry_point(name, module, to_folder, version_info) ) finally: - shutil.rmtree(folder, ignore_errors=True) + safe_delete(folder) return new_files @property @@ -169,7 +168,7 @@ class PipInstall(object): def clear(self): if self._image_dir.exists(): - shutil.rmtree(ensure_text(str(self._image_dir))) + safe_delete(self._image_dir) def has_image(self): return self._image_dir.exists() and next(self._image_dir.iterdir()) is not None diff --git a/src/virtualenv/seed/via_app_data/pip_install/symlink.py b/src/virtualenv/seed/via_app_data/pip_install/symlink.py index 578af0c..f958b65 100644 --- a/src/virtualenv/seed/via_app_data/pip_install/symlink.py +++ b/src/virtualenv/seed/via_app_data/pip_install/symlink.py @@ -1,10 +1,10 @@ from __future__ import absolute_import, unicode_literals import os -import shutil import subprocess -from stat import S_IREAD, S_IRGRP, S_IROTH, S_IWUSR +from stat import S_IREAD, S_IRGRP, S_IROTH +from virtualenv.util.path import safe_delete, set_tree from virtualenv.util.six import ensure_text from virtualenv.util.subprocess import Popen @@ -25,13 +25,13 @@ class SymlinkPipInstall(PipInstall): stderr=subprocess.PIPE, ) process.communicate() - # the root pyc is shared, so we'll not symlink that - but still add the pyc files to the RECORD for cleanup + # the root pyc is shared, so we'll not symlink that - but still add the pyc files to the RECORD for close root_py_cache = self._image_dir / "__pycache__" new_files = set() if root_py_cache.exists(): new_files.update(root_py_cache.iterdir()) new_files.add(root_py_cache) - shutil.rmtree(ensure_text(str(root_py_cache))) + safe_delete(root_py_cache) core_new_files = super(SymlinkPipInstall, self)._generate_new_files() # remove files that are within the image folder deeper than one level (as these will be not linked directly) for file in core_new_files: @@ -53,15 +53,9 @@ class SymlinkPipInstall(PipInstall): def build_image(self): super(SymlinkPipInstall, self).build_image() # protect the image by making it read only - self._set_tree(self._image_dir, S_IREAD | S_IRGRP | S_IROTH) + set_tree(self._image_dir, S_IREAD | S_IRGRP | S_IROTH) def clear(self): if self._image_dir.exists(): - self._set_tree(self._image_dir, S_IWUSR) + safe_delete(self._image_dir) super(SymlinkPipInstall, self).clear() - - @staticmethod - def _set_tree(folder, stat): - for root, _, files in os.walk(ensure_text(str(folder))): - for filename in files: - os.chmod(os.path.join(root, filename), stat) diff --git a/src/virtualenv/seed/via_app_data/via_app_data.py b/src/virtualenv/seed/via_app_data/via_app_data.py index 0025a7b..50d3afb 100644 --- a/src/virtualenv/seed/via_app_data/via_app_data.py +++ b/src/virtualenv/seed/via_app_data/via_app_data.py @@ -2,15 +2,13 @@ from __future__ import absolute_import, unicode_literals import logging -import shutil from contextlib import contextmanager from threading import Lock, Thread -from virtualenv.dirs import default_data_dir from virtualenv.info import fs_supports_symlink from virtualenv.seed.embed.base_embed import BaseEmbed from virtualenv.seed.embed.wheels.acquire import get_wheels -from virtualenv.util.six import ensure_text +from virtualenv.util.path import safe_delete from .pip_install.copy import CopyPipInstall from .pip_install.symlink import SymlinkPipInstall @@ -19,21 +17,13 @@ from .pip_install.symlink import SymlinkPipInstall class FromAppData(BaseEmbed): def __init__(self, options): super(FromAppData, self).__init__(options) - self.clear = options.clear_app_data - self.app_data_dir = default_data_dir() / "seed-v1" self.symlinks = options.symlink_app_data + self.base_cache = self.app_data / "seed-app-data" / "v1" @classmethod - def add_parser_arguments(cls, parser, interpreter): - super(FromAppData, cls).add_parser_arguments(parser, interpreter) - parser.add_argument( - "--clear-app-data", - dest="clear_app_data", - action="store_true", - help="clear the app data folder of seed images ({})".format((default_data_dir() / "seed-v1").path), - default=False, - ) - can_symlink = fs_supports_symlink() + def add_parser_arguments(cls, parser, interpreter, app_data): + super(FromAppData, cls).add_parser_arguments(parser, interpreter, app_data) + can_symlink = app_data.transient is False and fs_supports_symlink() parser.add_argument( "--symlink-app-data", dest="symlink_app_data", @@ -47,7 +37,7 @@ class FromAppData(BaseEmbed): def run(self, creator): if not self.enabled: return - base_cache = self.app_data_dir / creator.interpreter.version_release_str + base_cache = self.base_cache / creator.interpreter.version_release_str with self._get_seed_wheels(creator, base_cache) as name_to_whl: pip_version = name_to_whl["pip"].stem.split("-")[1] installer_class = self.installer_class(pip_version) @@ -56,8 +46,6 @@ class FromAppData(BaseEmbed): logging.debug("install %s from wheel %s via %s", name, wheel, installer_class.__name__) image_folder = base_cache.path / "image" / installer_class.__name__ / wheel.stem installer = installer_class(wheel, creator, image_folder) - if self.clear: - installer.clear() if not installer.has_image(): installer.build_image() installer.install(creator.interpreter.version_info) @@ -72,8 +60,8 @@ class FromAppData(BaseEmbed): def _get_seed_wheels(self, creator, base_cache): with base_cache.lock_for_key("wheels"): wheels_to = base_cache.path / "wheels" - if self.clear and wheels_to.exists(): - shutil.rmtree(ensure_text(str(wheels_to))) + if wheels_to.exists(): + safe_delete(wheels_to) wheels_to.mkdir(parents=True, exist_ok=True) name_to_whl, lock = {}, Lock() @@ -84,6 +72,7 @@ class FromAppData(BaseEmbed): self.extra_search_dir, self.download, {package: version}, + self.app_data, ) with lock: name_to_whl.update(result) @@ -106,8 +95,5 @@ class FromAppData(BaseEmbed): def __unicode__(self): base = super(FromAppData, self).__unicode__() - return ( - base[:-1] - + ", via={}, app_data_dir={}".format("symlink" if self.symlinks else "copy", self.app_data_dir.path) - + base[-1] - ) + msg = ", via={}, app_data_dir={}".format("symlink" if self.symlinks else "copy", self.base_cache.path) + return base[:-1] + msg + base[-1] diff --git a/src/virtualenv/session.py b/src/virtualenv/session.py index ffd0c07..2e6a3d6 100644 --- a/src/virtualenv/session.py +++ b/src/virtualenv/session.py @@ -9,8 +9,9 @@ from virtualenv.util.six import ensure_text class Session(object): """Represents a virtual environment creation session""" - def __init__(self, verbosity, interpreter, creator, seeder, activators): + def __init__(self, verbosity, app_data, interpreter, creator, seeder, activators): self._verbosity = verbosity + self._app_data = app_data self._interpreter = interpreter self._creator = creator self._seeder = seeder @@ -66,6 +67,12 @@ class Session(object): for activator in self.activators: activator.generate(self.creator) + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self._app_data.close() + _DEBUG_MARKER = "=" * 30 + " target debug " + "=" * 30 diff --git a/src/virtualenv/util/path/__init__.py b/src/virtualenv/util/path/__init__.py index 2d9d56a..a7f7163 100644 --- a/src/virtualenv/util/path/__init__.py +++ b/src/virtualenv/util/path/__init__.py @@ -1,8 +1,8 @@ from __future__ import absolute_import, unicode_literals from ._pathlib import Path -from ._permission import make_exe -from ._sync import copy, copytree, ensure_dir, symlink +from ._permission import make_exe, set_tree +from ._sync import copy, copytree, ensure_dir, safe_delete, symlink __all__ = ( "ensure_dir", @@ -11,4 +11,6 @@ __all__ = ( "copytree", "Path", "make_exe", + "set_tree", + "safe_delete", ) diff --git a/src/virtualenv/util/path/_permission.py b/src/virtualenv/util/path/_permission.py index 85f93b8..73bb6e8 100644 --- a/src/virtualenv/util/path/_permission.py +++ b/src/virtualenv/util/path/_permission.py @@ -1,7 +1,10 @@ from __future__ import absolute_import, unicode_literals +import os from stat import S_IXGRP, S_IXOTH, S_IXUSR +from virtualenv.util.six import ensure_text + def make_exe(filename): original_mode = filename.stat().st_mode @@ -17,4 +20,13 @@ def make_exe(filename): continue -__all__ = ("make_exe",) +def set_tree(folder, stat): + for root, _, files in os.walk(ensure_text(str(folder))): + for filename in files: + os.chmod(os.path.join(root, filename), stat) + + +__all__ = ( + "make_exe", + "set_tree", +) diff --git a/src/virtualenv/util/path/_sync.py b/src/virtualenv/util/path/_sync.py index 09fd006..9a6eba6 100644 --- a/src/virtualenv/util/path/_sync.py +++ b/src/virtualenv/util/path/_sync.py @@ -3,6 +3,7 @@ from __future__ import absolute_import, unicode_literals import logging import os import shutil +from stat import S_IWUSR from six import PY2 @@ -27,8 +28,8 @@ def ensure_safe_to_do(src, dest): if not dest.exists(): return if dest.is_dir() and not dest.is_symlink(): - shutil.rmtree(norm(dest)) logging.debug("remove directory %s", dest) + safe_delete(dest) else: logging.debug("remove file %s", dest) dest.unlink() @@ -59,6 +60,17 @@ def copytree(src, dest): shutil.copy(src_f, dest_f) +def safe_delete(dest): + def onerror(func, path, exc_info): + if not os.access(path, os.W_OK): + os.chmod(path, S_IWUSR) + func(path) + else: + raise + + shutil.rmtree(ensure_text(str(dest)), ignore_errors=True, onerror=onerror) + + class _Debug(object): def __init__(self, src, dest): self.src = src @@ -76,4 +88,5 @@ __all__ = ( "copy", "symlink", "copytree", + "safe_delete", ) diff --git a/src/virtualenv/util/zipapp.py b/src/virtualenv/util/zipapp.py index d6fe54c..698cc3e 100644 --- a/src/virtualenv/util/zipapp.py +++ b/src/virtualenv/util/zipapp.py @@ -4,9 +4,10 @@ import logging import os import zipfile from contextlib import contextmanager +from tempfile import TemporaryFile -from virtualenv.dirs import default_data_dir from virtualenv.info import IS_WIN, IS_ZIPAPP, ROOT +from virtualenv.util.path import Path from virtualenv.util.six import ensure_text from virtualenv.version import __version__ @@ -36,13 +37,19 @@ def _get_path_within_zip(full_path): @contextmanager -def ensure_file_on_disk(path): +def ensure_file_on_disk(path, app_data): if IS_ZIPAPP: - base = default_data_dir() / "zipapp" / "extract" / __version__ - with base.lock_for_key(path.name): - dest = base.path / path.name - if not dest.exists(): + if app_data is None: + with TemporaryFile() as temp_file: + dest = Path(temp_file.name) extract(path, dest) - yield dest + yield Path(dest) + else: + base = app_data / "zipapp" / "extract" / __version__ + with base.lock_for_key(path.name): + dest = base.path / path.name + if not dest.exists(): + extract(path, dest) + yield dest else: yield path diff --git a/tests/conftest.py b/tests/conftest.py index d3ed9b2..1fd0545 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,28 +1,23 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, unicode_literals -import atexit import logging import os import shutil import sys -import tempfile +from contextlib import contextmanager from functools import partial import coverage import pytest import six -from virtualenv import dirs from virtualenv.discovery.py_info import PythonInfo from virtualenv.info import IS_PYPY, IS_WIN, fs_supports_symlink from virtualenv.report import LOGGER +from virtualenv.run.app_data import AppData from virtualenv.util.path import Path -from virtualenv.util.six import ensure_text - -_TEST_SETUP_DIR = tempfile.mkdtemp() -dirs._DATA_DIR = dirs.ReentrantFileLock(_TEST_SETUP_DIR) -atexit.register(lambda: shutil.rmtree(_TEST_SETUP_DIR)) +from virtualenv.util.six import ensure_str, ensure_text def pytest_addoption(parser): @@ -115,17 +110,15 @@ def check_cwd_not_changed_by_test(): @pytest.fixture(autouse=True) -def ensure_py_info_cache_empty(): - PythonInfo.clear_cache() +def ensure_py_info_cache_empty(session_app_data): + PythonInfo.clear_cache(session_app_data) yield - PythonInfo.clear_cache() + PythonInfo.clear_cache(session_app_data) @pytest.fixture(autouse=True) -def ignore_global_config(tmp_path, mocker, monkeypatch): - mocker.patch("virtualenv.dirs._CFG_DIR", None) - mocker.patch("virtualenv.dirs.user_config_dir", return_value=Path(str(tmp_path / "this-should-never-exist"))) - yield +def ignore_global_config(tmp_path, monkeypatch): + monkeypatch.setenv(ensure_str("VIRTUALENV_CONFIG_FILE"), str(tmp_path / "this-should-never-exist")) @pytest.fixture(autouse=True) @@ -183,7 +176,7 @@ def coverage_env(monkeypatch, link): # we inject right after creation, we cannot collect coverage on site.py - used for helper scripts, such as debug from virtualenv import run - def via_cli(args, options=None): + def via_cli(args, options): session = prev_run(args, options) old_run = session.creator.run @@ -284,10 +277,42 @@ def special_name_dir(tmp_path, special_char_name): @pytest.fixture(scope="session") -def current_creators(): - return PythonInfo.current_system().creators() +def current_creators(session_app_data): + return PythonInfo.current_system(session_app_data).creators() @pytest.fixture(scope="session") def current_fastest(current_creators): return "builtin" if "builtin" in current_creators.key_to_class else next(iter(current_creators.key_to_class)) + + +@pytest.fixture(scope="session") +def session_app_data(tmp_path_factory): + app_data = AppData(folder=str(tmp_path_factory.mktemp("session-app-data"))) + with change_env_var("VIRTUALENV_OVERRIDE_APP_DATA", str(app_data.folder.path)): + yield app_data.folder + + +@contextmanager +def change_env_var(key, value): + """Temporarily change an environment variable. + :param key: the key of the env var + :param value: the value of the env var + """ + already_set = key in os.environ + prev_value = os.environ.get(key) + os.environ[key] = value + try: + yield + finally: + if already_set: + os.environ[key] = prev_value # type: ignore + else: + del os.environ[key] # pragma: no cover + + +@pytest.fixture() +def temp_app_data(monkeypatch, tmp_path): + app_data = tmp_path / "app-data" + monkeypatch.setenv(str("VIRTUALENV_OVERRIDE_APP_DATA"), str(app_data)) + return app_data diff --git a/tests/integration/test_zipapp.py b/tests/integration/test_zipapp.py index 7a90ffc..191fd5a 100644 --- a/tests/integration/test_zipapp.py +++ b/tests/integration/test_zipapp.py @@ -67,9 +67,8 @@ def zipapp_test_env(tmp_path_factory): @pytest.fixture() -def call_zipapp(zipapp, monkeypatch, tmp_path, zipapp_test_env): +def call_zipapp(zipapp, monkeypatch, tmp_path, zipapp_test_env, temp_app_data): def _run(*args): - monkeypatch.setenv(str("VIRTUALENV_OVERRIDE_APP_DATA"), str(tmp_path / "app_data")) cmd = [str(zipapp_test_env), str(zipapp), "-vv", ensure_text(str(tmp_path / "env"))] + list(args) subprocess.check_call(cmd) diff --git a/tests/unit/config/test_env_var.py b/tests/unit/config/test_env_var.py index e23b9fb..a439b6d 100644 --- a/tests/unit/config/test_env_var.py +++ b/tests/unit/config/test_env_var.py @@ -1,5 +1,7 @@ from __future__ import absolute_import, unicode_literals +from argparse import Namespace + import pytest from virtualenv.config.ini import IniConfig @@ -7,7 +9,8 @@ from virtualenv.run import session_via_cli def parse_cli(args): - return session_via_cli(args) + options = Namespace() + return session_via_cli(args, options) @pytest.fixture() diff --git a/tests/unit/create/test_creator.py b/tests/unit/create/test_creator.py index e50ba7c..f66218e 100644 --- a/tests/unit/create/test_creator.py +++ b/tests/unit/create/test_creator.py @@ -9,6 +9,7 @@ import stat import subprocess import sys from itertools import product +from stat import S_IREAD, S_IRGRP, S_IROTH from textwrap import dedent from threading import Thread @@ -54,11 +55,11 @@ def test_destination_exists_file(tmp_path, capsys): assert msg in err, err -@pytest.mark.skipif(sys.platform == "win32", reason="no chmod on Windows") +@pytest.mark.skipif(sys.platform == "win32", reason="Windows only applies R/O to files") def test_destination_not_write_able(tmp_path, capsys): target = tmp_path prev_mod = target.stat().st_mode - target.chmod(0o444) + target.chmod(S_IREAD | S_IRGRP | S_IROTH) try: err = _non_success_exit_code(capsys, str(target)) msg = "the destination . is not write-able at {}".format(str(target)) @@ -80,8 +81,8 @@ def cleanup_sys_path(paths): @pytest.fixture(scope="session") -def system(): - return get_env_debug_info(Path(CURRENT.system_executable), DEBUG_SCRIPT) +def system(session_app_data): + return get_env_debug_info(Path(CURRENT.system_executable), DEBUG_SCRIPT, session_app_data) CURRENT_CREATORS = list(i for i in CURRENT.creators().key_to_class.keys() if i != "builtin") @@ -131,7 +132,7 @@ def test_create_no_seed(python, creator, isolated, system, coverage_env, special coverage_env() if IS_PYPY: # pypy cleans up file descriptors periodically so our (many) subprocess calls impact file descriptor limits - # force a cleanup of these on system where the limit is low-ish (e.g. MacOS 256) + # force a close of these on system where the limit is low-ish (e.g. MacOS 256) gc.collect() purelib = result.creator.purelib patch_files = {purelib / "{}.{}".format("_distutils_patch_virtualenv", i) for i in ("py", "pyc", "pth")} @@ -276,9 +277,9 @@ def test_prompt_set(tmp_path, creator, prompt): @pytest.fixture(scope="session") -def cross_python(is_inside_ci): +def cross_python(is_inside_ci, session_app_data): spec = "{}{}".format(CURRENT.implementation, 2 if CURRENT.version_info.major == 3 else 3) - interpreter = get_interpreter(spec) + interpreter = get_interpreter(spec, session_app_data) if interpreter is None: msg = "could not find {}".format(spec) if is_inside_ci: @@ -288,7 +289,7 @@ def cross_python(is_inside_ci): @pytest.mark.slow -def test_cross_major(cross_python, coverage_env, tmp_path, current_fastest): +def test_cross_major(cross_python, coverage_env, tmp_path, current_fastest, session_app_data): cmd = [ "-v", "-v", @@ -307,13 +308,11 @@ def test_cross_major(cross_python, coverage_env, tmp_path, current_fastest): major, minor = cross_python.version_info[0:2] assert pip_scripts == {"pip", "pip-{}.{}".format(major, minor), "pip{}".format(major)} coverage_env() - env = PythonInfo.from_exe(str(result.creator.exe)) + env = PythonInfo.from_exe(str(result.creator.exe), session_app_data) assert env.version_info.major != CURRENT.version_info.major -def test_create_parallel(tmp_path, monkeypatch): - monkeypatch.setenv(str("VIRTUALENV_OVERRIDE_APP_DATA"), str(tmp_path)) - +def test_create_parallel(tmp_path, monkeypatch, temp_app_data): def create(count): subprocess.check_call([sys.executable, "-m", "virtualenv", str(tmp_path / "venv{}".format(count))]) diff --git a/tests/unit/discovery/py_info/test_py_info.py b/tests/unit/discovery/py_info/test_py_info.py index 26d2e69..11b52c7 100644 --- a/tests/unit/discovery/py_info/test_py_info.py +++ b/tests/unit/discovery/py_info/test_py_info.py @@ -24,19 +24,19 @@ def test_current_as_json(): assert parsed["version_info"] == {"major": a, "minor": b, "micro": c, "releaselevel": d, "serial": e} -def test_bad_exe_py_info_raise(tmp_path): +def test_bad_exe_py_info_raise(tmp_path, session_app_data): exe = str(tmp_path) with pytest.raises(RuntimeError) as context: - PythonInfo.from_exe(exe) + PythonInfo.from_exe(exe, session_app_data) msg = str(context.value) assert "code" in msg assert exe in msg -def test_bad_exe_py_info_no_raise(tmp_path, caplog, capsys): +def test_bad_exe_py_info_no_raise(tmp_path, caplog, capsys, session_app_data): caplog.set_level(logging.NOTSET) exe = str(tmp_path) - result = PythonInfo.from_exe(exe, raise_on_error=False) + result = PythonInfo.from_exe(exe, session_app_data, raise_on_error=False) assert result is None out, _ = capsys.readouterr() assert not out @@ -107,40 +107,40 @@ def test_satisfy_not_version(spec): assert matches is False -def test_py_info_cached_error(mocker, tmp_path): +def test_py_info_cached_error(mocker, tmp_path, session_app_data): spy = mocker.spy(cached_py_info, "_run_subprocess") with pytest.raises(RuntimeError): - PythonInfo.from_exe(str(tmp_path)) + PythonInfo.from_exe(str(tmp_path), session_app_data) with pytest.raises(RuntimeError): - PythonInfo.from_exe(str(tmp_path)) + PythonInfo.from_exe(str(tmp_path), session_app_data) assert spy.call_count == 1 @pytest.mark.skipif(not fs_supports_symlink(), reason="symlink is not supported") -def test_py_info_cached_symlink_error(mocker, tmp_path): +def test_py_info_cached_symlink_error(mocker, tmp_path, session_app_data): spy = mocker.spy(cached_py_info, "_run_subprocess") with pytest.raises(RuntimeError): - PythonInfo.from_exe(str(tmp_path)) + PythonInfo.from_exe(str(tmp_path), session_app_data) symlinked = tmp_path / "a" symlinked.symlink_to(tmp_path) with pytest.raises(RuntimeError): - PythonInfo.from_exe(str(symlinked)) + PythonInfo.from_exe(str(symlinked), session_app_data) assert spy.call_count == 2 -def test_py_info_cache_clear(mocker, tmp_path): +def test_py_info_cache_clear(mocker, tmp_path, session_app_data): spy = mocker.spy(cached_py_info, "_run_subprocess") - assert PythonInfo.from_exe(sys.executable) is not None + assert PythonInfo.from_exe(sys.executable, session_app_data) is not None assert spy.call_count >= 2 # at least two, one for the venv, one more for the host - PythonInfo.clear_cache() - assert PythonInfo.from_exe(sys.executable) is not None + PythonInfo.clear_cache(session_app_data) + assert PythonInfo.from_exe(sys.executable, session_app_data) is not None assert spy.call_count >= 4 @pytest.mark.skipif(not fs_supports_symlink(), reason="symlink is not supported") -def test_py_info_cached_symlink(mocker, tmp_path): +def test_py_info_cached_symlink(mocker, tmp_path, session_app_data): spy = mocker.spy(cached_py_info, "_run_subprocess") - first_result = PythonInfo.from_exe(sys.executable) + first_result = PythonInfo.from_exe(sys.executable, session_app_data) assert first_result is not None count = spy.call_count assert count >= 2 # at least two, one for the venv, one more for the host @@ -148,7 +148,7 @@ def test_py_info_cached_symlink(mocker, tmp_path): new_exe = tmp_path / "a" new_exe.symlink_to(sys.executable) new_exe_str = str(new_exe) - second_result = PythonInfo.from_exe(new_exe_str) + second_result = PythonInfo.from_exe(new_exe_str, session_app_data) assert second_result.executable == new_exe_str assert spy.call_count == count + 1 # no longer needed the host invocation, but the new symlink is must @@ -185,7 +185,7 @@ PyInfoMock = namedtuple("PyInfoMock", ["implementation", "architecture", "versio ), ], ) -def test_system_executable_no_exact_match(target, discovered, position, tmp_path, mocker, caplog): +def test_system_executable_no_exact_match(target, discovered, position, tmp_path, mocker, caplog, session_app_data): """Here we should fallback to other compatible""" caplog.set_level(logging.DEBUG) @@ -216,7 +216,7 @@ def test_system_executable_no_exact_match(target, discovered, position, tmp_path mocker.patch.object(target_py_info, "_find_possible_folders", return_value=[str(tmp_path)]) # noinspection PyUnusedLocal - def func(k, resolve_to_host, raise_on_error): + def func(k, app_data, resolve_to_host, raise_on_error): return discovered_with_path[k] mocker.patch.object(target_py_info, "from_exe", side_effect=func) @@ -224,12 +224,12 @@ def test_system_executable_no_exact_match(target, discovered, position, tmp_path target_py_info.system_executable = None target_py_info.executable = str(tmp_path) - mapped = target_py_info._resolve_to_system(target_py_info) + mapped = target_py_info._resolve_to_system(session_app_data, target_py_info) assert mapped.system_executable == CURRENT.system_executable found = discovered_with_path[mapped.base_executable] assert found is selected - assert caplog.records[0].msg == "discover system for %s in %s" + assert caplog.records[0].msg == "discover exe for %s in %s" for record in caplog.records[1:-1]: assert record.message.startswith("refused interpreter ") assert record.levelno == logging.DEBUG diff --git a/tests/unit/discovery/py_info/test_py_info_exe_based_of.py b/tests/unit/discovery/py_info/test_py_info_exe_based_of.py index 4c559d4..3f88dd5 100644 --- a/tests/unit/discovery/py_info/test_py_info_exe_based_of.py +++ b/tests/unit/discovery/py_info/test_py_info_exe_based_of.py @@ -12,9 +12,9 @@ from virtualenv.util.path import Path CURRENT = PythonInfo.current() -def test_discover_empty_folder(tmp_path, monkeypatch): +def test_discover_empty_folder(tmp_path, monkeypatch, session_app_data): with pytest.raises(RuntimeError): - CURRENT.discover_exe(prefix=str(tmp_path)) + CURRENT.discover_exe(session_app_data, prefix=str(tmp_path)) BASE = {str(Path(CURRENT.executable).parent.relative_to(Path(CURRENT.prefix))), "."} @@ -26,14 +26,14 @@ BASE = {str(Path(CURRENT.executable).parent.relative_to(Path(CURRENT.prefix))), @pytest.mark.parametrize("arch", [CURRENT.architecture, ""]) @pytest.mark.parametrize("version", [".".join(str(i) for i in CURRENT.version_info[0:i]) for i in range(3, 0, -1)]) @pytest.mark.parametrize("impl", [CURRENT.implementation, "python"]) -def test_discover_ok(tmp_path, monkeypatch, suffix, impl, version, arch, into, caplog): +def test_discover_ok(tmp_path, monkeypatch, suffix, impl, version, arch, into, caplog, session_app_data): caplog.set_level(logging.DEBUG) folder = tmp_path / into folder.mkdir(parents=True, exist_ok=True) dest = folder / "{}{}".format(impl, version, arch, suffix) os.symlink(CURRENT.executable, str(dest)) inside_folder = str(tmp_path) - base = CURRENT.discover_exe(inside_folder) + base = CURRENT.discover_exe(session_app_data, inside_folder) found = base.executable dest_str = str(dest) if not fs_is_case_sensitive(): @@ -46,4 +46,4 @@ def test_discover_ok(tmp_path, monkeypatch, suffix, impl, version, arch, into, c dest.rename(dest.parent / (dest.name + "-1")) CURRENT._cache_exe_discovery.clear() with pytest.raises(RuntimeError): - CURRENT.discover_exe(inside_folder) + CURRENT.discover_exe(session_app_data, inside_folder) diff --git a/tests/unit/discovery/test_discovery.py b/tests/unit/discovery/test_discovery.py index b662222..260052d 100644 --- a/tests/unit/discovery/test_discovery.py +++ b/tests/unit/discovery/test_discovery.py @@ -15,9 +15,9 @@ from virtualenv.util.six import ensure_text @pytest.mark.skipif(not fs_supports_symlink(), reason="symlink not supported") @pytest.mark.parametrize("case", ["mixed", "lower", "upper"]) -def test_discovery_via_path(monkeypatch, case, special_name_dir, caplog): +def test_discovery_via_path(monkeypatch, case, special_name_dir, caplog, session_app_data): caplog.set_level(logging.DEBUG) - current = PythonInfo.current_system() + current = PythonInfo.current_system(session_app_data) core = "somethingVeryCryptic{}".format(".".join(str(i) for i in current.version_info[0:3])) name = "somethingVeryCryptic" if case == "lower": diff --git a/tests/unit/discovery/windows/test_windows_pep514.py b/tests/unit/discovery/windows/test_windows_pep514.py index ee0facd..9fbe04c 100644 --- a/tests/unit/discovery/windows/test_windows_pep514.py +++ b/tests/unit/discovery/windows/test_windows_pep514.py @@ -12,7 +12,7 @@ from virtualenv.util.path import Path @pytest.mark.skipif(sys.platform != "win32", reason="no Windows registry") -def test_pep517(_mock_registry): +def test_pep514(_mock_registry): from virtualenv.discovery.windows.pep514 import discover_pythons interpreters = list(discover_pythons()) @@ -31,7 +31,7 @@ def test_pep517(_mock_registry): @pytest.mark.skipif(sys.platform != "win32", reason="no Windows registry") -def test_pep517_run(_mock_registry, capsys, caplog): +def test_pep514_run(_mock_registry, capsys, caplog): from virtualenv.discovery.windows import pep514 pep514._run() diff --git a/tests/unit/seed/test_boostrap_link_via_app_data.py b/tests/unit/seed/test_boostrap_link_via_app_data.py index 436254b..3d8f7dd 100644 --- a/tests/unit/seed/test_boostrap_link_via_app_data.py +++ b/tests/unit/seed/test_boostrap_link_via_app_data.py @@ -2,6 +2,7 @@ from __future__ import absolute_import, unicode_literals import os import sys +from stat import S_IREAD, S_IRGRP, S_IROTH, S_IWUSR import pytest @@ -101,3 +102,20 @@ def test_base_bootstrap_link_via_app_data(tmp_path, coverage_env, current_fastes if sys.version_info[0:2] == (3, 4) and os.environ.get(str("PIP_REQ_TRACKER")): os.environ.pop(str("PIP_REQ_TRACKER")) + + +@pytest.fixture() +def read_only_folder(temp_app_data): + temp_app_data.mkdir() + try: + os.chmod(str(temp_app_data), S_IREAD | S_IRGRP | S_IROTH) + yield temp_app_data + finally: + os.chmod(str(temp_app_data), S_IWUSR | S_IREAD) + + +@pytest.mark.skipif(sys.platform == "win32", reason="Windows only applies R/O to files") +def test_base_bootstrap_link_via_app_data_not_writable(tmp_path, current_fastest, read_only_folder, monkeypatch): + dest = tmp_path / "venv" + result = cli_run(["--seeder", "app-data", "--creator", current_fastest, "--clear-app-data", "-vv", str(dest)]) + assert result |
