summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorBernát Gábor <bgabor8@bloomberg.net>2020-06-21 08:28:50 +0100
committerGitHub <noreply@github.com>2020-06-21 08:28:50 +0100
commit0cd009b5a1338f66397f71c85a75f576a2f3eabf (patch)
treed1a1e6564776ba6123f9e8b245fb58c14ea71df9
parentf99353ca3d0ca9e23cfe4b66e54ba653bf99ab4a (diff)
downloadvirtualenv-0cd009b5a1338f66397f71c85a75f576a2f3eabf.tar.gz
Implement periodic update feature (#1841)
Co-authored-by: Pradyun Gedam <pradyunsg@gmail.com>
-rw-r--r--.coveragerc11
-rw-r--r--.pre-commit-config.yaml14
-rw-r--r--codecov.yaml5
-rw-r--r--docs/changelog/1821.doc.rst1
-rw-r--r--docs/changelog/1821.feature.rst8
-rw-r--r--docs/changelog/1841.feature.rst7
-rw-r--r--docs/cli_interface.rst2
-rw-r--r--docs/user_guide.rst64
-rw-r--r--setup.cfg6
-rw-r--r--setup.py5
-rw-r--r--src/virtualenv/__main__.py41
-rw-r--r--src/virtualenv/app_data/__init__.py (renamed from src/virtualenv/run/app_data.py)47
-rw-r--r--src/virtualenv/app_data/base.py91
-rw-r--r--src/virtualenv/app_data/na.py67
-rw-r--r--src/virtualenv/app_data/via_disk_folder.py172
-rw-r--r--src/virtualenv/app_data/via_tempdir.py28
-rw-r--r--src/virtualenv/config/cli/parser.py1
-rw-r--r--src/virtualenv/create/creator.py9
-rw-r--r--src/virtualenv/create/pyenv_cfg.py (renamed from src/virtualenv/pyenv_cfg.py)0
-rw-r--r--src/virtualenv/create/via_global_ref/api.py3
-rw-r--r--src/virtualenv/create/via_global_ref/store.py7
-rw-r--r--src/virtualenv/create/via_global_ref/venv.py2
-rw-r--r--src/virtualenv/discovery/builtin.py2
-rw-r--r--src/virtualenv/discovery/cached_py_info.py82
-rw-r--r--src/virtualenv/report.py1
-rw-r--r--src/virtualenv/run/__init__.py70
-rw-r--r--src/virtualenv/run/plugin/discovery.py6
-rw-r--r--src/virtualenv/run/session.py (renamed from src/virtualenv/session.py)0
-rw-r--r--src/virtualenv/seed/embed/__init__.py1
-rw-r--r--src/virtualenv/seed/embed/base_embed.py74
-rw-r--r--src/virtualenv/seed/embed/pip_invoke.py48
-rw-r--r--src/virtualenv/seed/embed/via_app_data/__init__.py (renamed from src/virtualenv/seed/via_app_data/__init__.py)0
-rw-r--r--src/virtualenv/seed/embed/via_app_data/pip_install/__init__.py (renamed from src/virtualenv/seed/via_app_data/pip_install/__init__.py)0
-rw-r--r--src/virtualenv/seed/embed/via_app_data/pip_install/base.py (renamed from src/virtualenv/seed/via_app_data/pip_install/base.py)4
-rw-r--r--src/virtualenv/seed/embed/via_app_data/pip_install/copy.py (renamed from src/virtualenv/seed/via_app_data/pip_install/copy.py)0
-rw-r--r--src/virtualenv/seed/embed/via_app_data/pip_install/symlink.py (renamed from src/virtualenv/seed/via_app_data/pip_install/symlink.py)0
-rw-r--r--src/virtualenv/seed/embed/via_app_data/via_app_data.py127
-rw-r--r--src/virtualenv/seed/embed/wheels/acquire.py178
-rw-r--r--src/virtualenv/seed/via_app_data/via_app_data.py128
-rw-r--r--src/virtualenv/seed/wheels/__init__.py11
-rw-r--r--src/virtualenv/seed/wheels/acquire.py114
-rw-r--r--src/virtualenv/seed/wheels/bundle.py51
-rw-r--r--src/virtualenv/seed/wheels/embed/__init__.py (renamed from src/virtualenv/seed/embed/wheels/__init__.py)24
-rw-r--r--src/virtualenv/seed/wheels/embed/pip-19.1.1-py2.py3-none-any.whl (renamed from src/virtualenv/seed/embed/wheels/pip-19.1.1-py2.py3-none-any.whl)bin1360957 -> 1360957 bytes
-rw-r--r--src/virtualenv/seed/wheels/embed/pip-20.1.1-py2.py3-none-any.whl (renamed from src/virtualenv/seed/embed/wheels/pip-20.1.1-py2.py3-none-any.whl)bin1490666 -> 1490666 bytes
-rw-r--r--src/virtualenv/seed/wheels/embed/setuptools-43.0.0-py2.py3-none-any.whl (renamed from src/virtualenv/seed/embed/wheels/setuptools-43.0.0-py2.py3-none-any.whl)bin583228 -> 583228 bytes
-rw-r--r--src/virtualenv/seed/wheels/embed/setuptools-44.1.1-py2.py3-none-any.whl (renamed from src/virtualenv/seed/embed/wheels/setuptools-44.1.1-py2.py3-none-any.whl)bin583493 -> 583493 bytes
-rw-r--r--src/virtualenv/seed/wheels/embed/setuptools-47.1.1-py3-none-any.whl (renamed from src/virtualenv/seed/embed/wheels/setuptools-47.1.1-py3-none-any.whl)bin583233 -> 583233 bytes
-rw-r--r--src/virtualenv/seed/wheels/embed/wheel-0.33.6-py2.py3-none-any.whl (renamed from src/virtualenv/seed/embed/wheels/wheel-0.33.6-py2.py3-none-any.whl)bin21556 -> 21556 bytes
-rw-r--r--src/virtualenv/seed/wheels/embed/wheel-0.34.2-py2.py3-none-any.whl (renamed from src/virtualenv/seed/embed/wheels/wheel-0.34.2-py2.py3-none-any.whl)bin26502 -> 26502 bytes
-rw-r--r--src/virtualenv/seed/wheels/periodic_update.py311
-rw-r--r--src/virtualenv/seed/wheels/util.py116
-rw-r--r--src/virtualenv/util/error.py (renamed from src/virtualenv/error.py)0
-rw-r--r--src/virtualenv/util/lock.py14
-rw-r--r--src/virtualenv/util/path/_pathlib/via_os_path.py8
-rw-r--r--src/virtualenv/util/subprocess/__init__.py4
-rw-r--r--src/virtualenv/util/zipapp.py25
-rw-r--r--tasks/upgrade_wheels.py54
-rw-r--r--tests/conftest.py23
-rw-r--r--tests/integration/test_zipapp.py3
-rw-r--r--tests/unit/activation/conftest.py5
-rw-r--r--tests/unit/activation/test_activate_this.py2
-rw-r--r--tests/unit/activation/test_xonsh.py2
-rw-r--r--tests/unit/config/test___main__.py70
-rw-r--r--tests/unit/create/conftest.py4
-rw-r--r--tests/unit/create/test_creator.py6
-rw-r--r--tests/unit/create/via_global_ref/greet/greet2.c (renamed from tests/unit/seed/greet/greet2.c)0
-rw-r--r--tests/unit/create/via_global_ref/greet/greet3.c (renamed from tests/unit/seed/greet/greet3.c)0
-rw-r--r--tests/unit/create/via_global_ref/greet/setup.py (renamed from tests/unit/seed/greet/setup.py)0
-rw-r--r--tests/unit/create/via_global_ref/test_build_c_ext.py (renamed from tests/unit/seed/test_extra_install.py)0
-rw-r--r--tests/unit/seed/embed/test_base_embed.py (renamed from tests/unit/seed/test_base_embed.py)0
-rw-r--r--tests/unit/seed/embed/test_boostrap_link_via_app_data.py (renamed from tests/unit/seed/test_boostrap_link_via_app_data.py)7
-rw-r--r--tests/unit/seed/embed/test_pip_invoke.py89
-rw-r--r--tests/unit/seed/embed/wheels/test_acquire.py11
-rw-r--r--tests/unit/seed/test_pip_invoke.py63
-rw-r--r--tests/unit/seed/wheels/test_acquire.py66
-rw-r--r--tests/unit/seed/wheels/test_acquire_find_wheel.py30
-rw-r--r--tests/unit/seed/wheels/test_periodic_update.py351
-rw-r--r--tests/unit/seed/wheels/test_wheels_util.py31
-rw-r--r--tox.ini6
80 files changed, 2161 insertions, 652 deletions
diff --git a/.coveragerc b/.coveragerc
index 000f9a5..2b79b26 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -1,5 +1,4 @@
[coverage:report]
-skip_covered = True
show_missing = True
exclude_lines =
\#\s*pragma: no cover
@@ -9,8 +8,10 @@ exclude_lines =
^if __name__ == ['"]__main__['"]:$
omit =
# site.py is ran before the coverage can be enabled, no way to measure coverage on this
- src/virtualenv/interpreters/create/impl/cpython/site.py
- src/virtualenv/seed/embed/wheels/pip-*.whl/*
+ src/virtualenv/create/via_global_ref/builtin/python2/site.py
+ src/virtualenv/create/via_global_ref/_virtualenv.py
+ src/virtualenv/activation/python/activate_this.py
+ src/virtualenv/seed/wheels/embed/pip-*.whl/*
[coverage:paths]
source =
@@ -24,5 +25,9 @@ source =
[coverage:run]
branch = false
parallel = true
+dynamic_context = test_function
source =
${_COVERAGE_SRC}
+
+[coverage:html]
+show_contexts = true
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 921d5e8..63284f7 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,6 +1,6 @@
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
- rev: v2.5.0
+ rev: v3.1.0
hooks:
- id: check-ast
- id: check-builtin-literals
@@ -15,16 +15,12 @@ repos:
rev: v2.0.1
hooks:
- id: add-trailing-comma
-- repo: https://github.com/asottile/yesqa
- rev: v1.1.0
- hooks:
- - id: yesqa
- repo: https://github.com/asottile/pyupgrade
- rev: v2.4.1
+ rev: v2.6.1
hooks:
- id: pyupgrade
- repo: https://github.com/asottile/seed-isort-config
- rev: v2.1.1
+ rev: v2.2.0
hooks:
- id: seed-isort-config
args: [--application-directories, '.:src']
@@ -54,8 +50,8 @@ repos:
- id: setup-cfg-fmt
args: [--min-py3-version, "3.4"]
- repo: https://gitlab.com/pycqa/flake8
- rev: "3.8.1"
+ rev: "3.8.3"
hooks:
- id: flake8
- additional_dependencies: ["flake8-bugbear == 20.1.2"]
+ additional_dependencies: ["flake8-bugbear == 20.1.4"]
language_version: python3.8
diff --git a/codecov.yaml b/codecov.yaml
index 8df1c8e..217805e 100644
--- a/codecov.yaml
+++ b/codecov.yaml
@@ -1,3 +1,8 @@
+ignore:
+- "src/virtualenv/create/via_global_ref/builtin/python2/site.py"
+- "src/virtualenv/create/via_global_ref/_virtualenv.py"
+- "src/virtualenv/activation/python/activate_this.py"
+
coverage:
range: "80...100"
coverage:
diff --git a/docs/changelog/1821.doc.rst b/docs/changelog/1821.doc.rst
new file mode 100644
index 0000000..43e7e57
--- /dev/null
+++ b/docs/changelog/1821.doc.rst
@@ -0,0 +1 @@
+Document how bundled wheels are handled and (potentially automatically) kept up to date - by :user:`gaborbernat`.
diff --git a/docs/changelog/1821.feature.rst b/docs/changelog/1821.feature.rst
new file mode 100644
index 0000000..08070b8
--- /dev/null
+++ b/docs/changelog/1821.feature.rst
@@ -0,0 +1,8 @@
+Better handling of bundled wheel installation:
+
+- display the installed seed package versions in the final summary output
+- add a manual upgrade of embedded wheels feature via :option:`upgrade-embed-wheels` CLI flag
+- periodically (once every 14 days) try to automatically upgrade the embedded wheels in the background, can be disabled
+ via :option:`no-periodic-update`
+
+by :user:`gaborbernat`.
diff --git a/docs/changelog/1841.feature.rst b/docs/changelog/1841.feature.rst
new file mode 100644
index 0000000..38d8081
--- /dev/null
+++ b/docs/changelog/1841.feature.rst
@@ -0,0 +1,7 @@
+Bump embed wheel content:
+
+- ship wheels for Python ``3.9``
+- upgrade embedded setuptools for Python ``3.5+`` from ``46.4.0`` to ``47.1.1``
+- upgrade embedded setuptools for Python ``2.7`` from ``44.1.0`` to ``44.1.1``
+
+by :user:`gaborbernat`.
diff --git a/docs/cli_interface.rst b/docs/cli_interface.rst
index db5da6c..f1c7b7e 100644
--- a/docs/cli_interface.rst
+++ b/docs/cli_interface.rst
@@ -22,7 +22,7 @@ The options that can be passed to virtualenv, along with their default values an
.. table_cli::
:module: virtualenv.run
- :func: build_parser
+ :func: build_parser_only
Defaults
~~~~~~~~
diff --git a/docs/user_guide.rst b/docs/user_guide.rst
index eefbaf7..3dee806 100644
--- a/docs/user_guide.rst
+++ b/docs/user_guide.rst
@@ -120,7 +120,7 @@ enables you to install additional python packages into the created virtual envir
main seed mechanism available:
- ``pip`` - this method uses the bundled pip with virtualenv to install the seed packages (note, a new child process
- needs to be created to do this).
+ needs to be created to do this, which can be expensive especially on Windows).
- ``app-data`` - this method uses the user application data directory to create install images. These images are needed
to be created only once, and subsequent virtual environments can just link/copy those images into their pure python
library path (the ``site-packages`` folder). This allows all but the first virtual environment creation to be blazing
@@ -131,6 +131,66 @@ main seed mechanism available:
To override the filesystem location of the seed cache, one can use the
``VIRTUALENV_OVERRIDE_APP_DATA`` environment variable.
+Wheels
+~~~~~~
+
+To install a seed package via either ``pip`` or ``app-data`` method virtualenv needs to acquire a wheel of the target
+package. These wheels may be acquired from multiple locations as follows:
+
+- ``virtualenv`` ships out of box with a set of embed ``wheels`` for all three seed packages (:pypi:`pip`,
+ :pypi:`setuptools`, :pypi:`wheel`). These are packaged together with the virtualenv source files, and only change upon
+ upgrading virtualenv. Different Python versions require different versions of these, and because virtualenv supports a
+ wide range of Python versions, the number of embedded wheels out of box is greater than 3. Whenever newer versions of
+ these embedded packages are released upstream ``virtualenv`` project upgrades them, and does a new release. Therefore,
+ upgrading virtualenv periodically will also upgrade the version of the seed packages.
+- However, end users might not be able to upgrade virtualenv at the same speed as we do new releases. Therefore, a user
+ might request to upgrade the list of embedded wheels by invoking virtualenv with the :option:`upgrade-embed-wheels`
+ flag. If the operation is triggered in such manual way subsequent runs of virtualenv will always use the upgraded
+ embed wheels.
+
+ The operation can trigger automatically too, as a background process upon invocation of virtualenv, if no such upgrade
+ has been performed in the last 14 days. It will only start using automatically upgraded wheel if they have been
+ released for more than 28 days, and the automatic upgrade finished at least an hour ago:
+
+ - the 28 days period should guarantee end users are not pulling in automatically releases that have known bugs within,
+ - the one hour period after the automatic upgrade finished is implemented so that continuous integration services do
+ not start using a new embedded versions half way through.
+
+
+ The automatic behaviour might be disabled via the :option:`no-periodic-update` configuration flag/option. To acquire
+ the release date of a package virtualenv will perform the following:
+
+ - lookup ``https://pypi.org/pypi/<distribution>/json`` (primary truth source),
+ - save the date the version was first discovered, and wait until 28 days passed.
+- Users can specify a set of local paths containing additional wheels by using the :option:`extra-search-dir` command
+ line argument flag.
+
+When searching for a wheel to use virtualenv performs lookup in the following order:
+
+- embedded wheels,
+- upgraded embedded wheels,
+- extra search dir.
+
+Bundled wheels are all three above together. If neither of the locations contain the requested wheel version or
+:option:`download` option is set will use ``pip`` download to load the latest version available from the index server.
+
+Embed wheels for distributions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Custom distributions often want to use their own set of wheel versions to distribute instead of the one virtualenv
+releases on PyPi. The reason for this is trying to keep the system versions of those package in sync with what
+virtualenv uses. In such cases they should patch the module `virtualenv.seed.wheels.embed
+<https://github.com/pypa/virtualenv/tree/bundle/src/virtualenv/seed/wheels/embed>`_, making sure to provide the function
+``get_embed_wheel`` (which returns the wheel to use given a distribution/python version). The ``BUNDLE_FOLDER``,
+``BUNDLE_SUPPORT`` and ``MAX`` variables are needed if they want to use virtualenvs test suite to validate.
+
+Furthermore, they might want to disable the periodic update by patching the
+`virtualenv.seed.embed.base_embed.PERIODIC_UPDATE_ON_BY_DEFAULT
+<https://github.com/pypa/virtualenv/tree/bundle/src/virtualenv/seed/embed/base_embed.py>`_
+to ``False``, and letting the system update mechanism to handle this. Note in this case the user might still request an
+upgrade of the embedded wheels by invoking virtualenv via :option:`upgrade-embed-wheels`, but no longer happens
+automatically, and will not alter the OS provided wheels.
+
Activators
----------
These are activation scripts that will mangle with your shells settings to ensure that commands from within the python
@@ -201,7 +261,7 @@ about the created virtual environment.
.. automodule:: virtualenv
:members:
-.. currentmodule:: virtualenv.session
+.. currentmodule:: virtualenv.run.session
.. autoclass:: Session
:members:
diff --git a/setup.cfg b/setup.cfg
index f831606..a64a2c3 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -43,7 +43,6 @@ install_requires =
distlib>=0.3.0,<1
filelock>=3.0.0,<4
six>=1.9.0,<2 # keep it >=1.9.0 as it may cause problems on LTS platforms
- contextlib2>=0.6.0,<1;python_version<"3.3"
importlib-metadata>=0.12,<2;python_version<"3.8"
importlib-resources>=1.0;python_version<"3.7"
pathlib2>=2.3.3,<3;python_version < '3.4' and sys.platform != 'win32'
@@ -79,7 +78,7 @@ virtualenv.discovery =
builtin = virtualenv.discovery.builtin:Builtin
virtualenv.seed =
pip = virtualenv.seed.embed.pip_invoke:PipInvoke
- app-data = virtualenv.seed.via_app_data.via_app_data:FromAppData
+ app-data = virtualenv.seed.embed.via_app_data.via_app_data:FromAppData
[options.extras_require]
docs =
@@ -98,6 +97,7 @@ testing =
pytest-env >= 0.6.2
pytest-randomly >= 1
pytest-timeout >= 1
+ pytest-freezegun >= 0.4.1
flaky >= 3
xonsh >= 0.9.16; python_version > '3.4' and python_version != '3.9'
@@ -108,7 +108,7 @@ virtualenv.activation.cshell = *.csh
virtualenv.activation.fish = *.fish
virtualenv.activation.powershell = *.ps1
virtualenv.activation.xonsh = *.xsh
-virtualenv.seed.embed.wheels = *.whl
+virtualenv.seed.wheels.embed = *.whl
[options.packages.find]
where = src
diff --git a/setup.py b/setup.py
index 0f0ab7a..8ba6e37 100644
--- a/setup.py
+++ b/setup.py
@@ -4,6 +4,9 @@ if int(__version__.split(".")[0]) < 41:
raise RuntimeError("setuptools >= 41 required to build")
setup(
- use_scm_version={"write_to": "src/virtualenv/version.py", "write_to_template": '__version__ = "{version}"'},
+ use_scm_version={
+ "write_to": "src/virtualenv/version.py",
+ "write_to_template": 'from __future__ import unicode_literals;\n\n__version__ = "{version}"',
+ },
setup_requires=["setuptools_scm >= 2"],
)
diff --git a/src/virtualenv/__main__.py b/src/virtualenv/__main__.py
index 5d47db8..c87fc41 100644
--- a/src/virtualenv/__main__.py
+++ b/src/virtualenv/__main__.py
@@ -5,13 +5,10 @@ import os
import sys
from datetime import datetime
-from virtualenv.config.cli.parser import VirtualEnvOptions
-from virtualenv.util.six import ensure_text
-
def run(args=None, options=None):
start = datetime.now()
- from virtualenv.error import ProcessCallFailed
+ from virtualenv.util.error import ProcessCallFailed
from virtualenv.run import cli_run
if args is None:
@@ -32,6 +29,8 @@ class LogSession(object):
self.start = start
def __str__(self):
+ from virtualenv.util.six import ensure_text
+
spec = self.session.creator.interpreter.spec
elapsed = (datetime.now() - self.start).total_seconds() * 1000
lines = [
@@ -39,24 +38,40 @@ class LogSession(object):
" creator {}".format(ensure_text(str(self.session.creator))),
]
if self.session.seeder.enabled:
- lines += (" seeder {}".format(ensure_text(str(self.session.seeder))),)
+ lines += (
+ " seeder {}".format(ensure_text(str(self.session.seeder))),
+ " added seed packages: {}".format(
+ ", ".join(
+ sorted(
+ "==".join(i.stem.split("-"))
+ for i in self.session.creator.purelib.iterdir()
+ if i.suffix == ".dist-info"
+ ),
+ ),
+ ),
+ )
if self.session.activators:
lines.append(" activators {}".format(",".join(i.__class__.__name__ for i in self.session.activators)))
return os.linesep.join(lines)
def run_with_catch(args=None):
+ from virtualenv.config.cli.parser import VirtualEnvOptions
+
options = VirtualEnvOptions()
try:
run(args, options)
- except (KeyboardInterrupt, Exception) as exception:
- if getattr(options, "with_traceback", False):
+ except (KeyboardInterrupt, SystemExit, Exception) as exception:
+ try:
+ if getattr(options, "with_traceback", False):
+ raise
+ else:
+ logging.error("%s: %s", type(exception).__name__, exception)
+ code = exception.code if isinstance(exception, SystemExit) else 1
+ sys.exit(code)
+ finally:
logging.shutdown() # force flush of log messages before the trace is printed
- raise
- else:
- logging.error("%s: %s", type(exception).__name__, exception)
- sys.exit(1)
-if __name__ == "__main__":
- run_with_catch()
+if __name__ == "__main__": # pragma: no cov
+ run_with_catch() # pragma: no cov
diff --git a/src/virtualenv/run/app_data.py b/src/virtualenv/app_data/__init__.py
index 68edb93..1d85745 100644
--- a/src/virtualenv/run/app_data.py
+++ b/src/virtualenv/app_data/__init__.py
@@ -1,39 +1,17 @@
+"""
+Application data stored by virtualenv.
+"""
+from __future__ import absolute_import, unicode_literals
+
import logging
import os
from argparse import Action, ArgumentError
-from tempfile import mkdtemp
from appdirs import user_data_dir
-from virtualenv.util.lock import ReentrantFileLock
-from virtualenv.util.path import safe_delete
-
-
-class AppData(object):
- def __init__(self, folder):
- self.folder = ReentrantFileLock(folder)
- self.transient = False
-
- def __repr__(self):
- return "{}".format(self.folder.path)
-
- def clean(self):
- logging.debug("clean app data folder %s", self.folder.path)
- safe_delete(self.folder.path)
-
- def close(self):
- """"""
-
-
-class TempAppData(AppData):
- def __init__(self):
- super(TempAppData, self).__init__(folder=mkdtemp())
- self.transient = True
- logging.debug("created temporary app data folder %s", self.folder.path)
-
- def close(self):
- logging.debug("remove temporary app data folder %s", self.folder.path)
- safe_delete(self.folder.path)
+from .na import AppDataDisabled
+from .via_disk_folder import AppDataDiskFolder
+from .via_tempdir import TempAppData
class AppDataAction(Action):
@@ -41,7 +19,7 @@ class AppDataAction(Action):
folder = self._check_folder(values)
if folder is None:
raise ArgumentError("app data path {} is not valid".format(values))
- setattr(namespace, self.dest, AppData(folder))
+ setattr(namespace, self.dest, AppDataDiskFolder(folder))
@staticmethod
def _check_folder(folder):
@@ -64,8 +42,8 @@ class AppDataAction(Action):
for folder in AppDataAction._app_data_candidates():
folder = AppDataAction._check_folder(folder)
if folder is not None:
- return AppData(folder)
- return None
+ return AppDataDiskFolder(folder)
+ return AppDataDisabled()
@staticmethod
def _app_data_candidates():
@@ -77,7 +55,8 @@ class AppDataAction(Action):
__all__ = (
- "AppData",
+ "AppDataDiskFolder",
"TempAppData",
"AppDataAction",
+ "AppDataDisabled",
)
diff --git a/src/virtualenv/app_data/base.py b/src/virtualenv/app_data/base.py
new file mode 100644
index 0000000..d0da0fc
--- /dev/null
+++ b/src/virtualenv/app_data/base.py
@@ -0,0 +1,91 @@
+"""
+Application data stored by virtualenv.
+"""
+from __future__ import absolute_import, unicode_literals
+
+from abc import ABCMeta, abstractmethod
+from contextlib import contextmanager
+
+import six
+
+from virtualenv.info import IS_ZIPAPP
+
+
+@six.add_metaclass(ABCMeta)
+class AppData(object):
+ """Abstract storage interface for the virtualenv application"""
+
+ @abstractmethod
+ def close(self):
+ """called before virtualenv exits"""
+
+ @abstractmethod
+ def reset(self):
+ """called when the user passes in the reset app data"""
+
+ @abstractmethod
+ def py_info(self, path):
+ raise NotImplementedError
+
+ @abstractmethod
+ def py_info_clear(self):
+ raise NotImplementedError
+
+ @abstractmethod
+ def embed_update_log(self, distribution, for_py_version):
+ raise NotImplementedError
+
+ @property
+ def house(self):
+ raise NotImplementedError
+
+ @property
+ def transient(self):
+ raise NotImplementedError
+
+ @abstractmethod
+ def wheel_image(self, for_py_version, name):
+ raise NotImplementedError
+
+ @contextmanager
+ def ensure_extracted(self, path, to_folder=None):
+ """Some paths might be within the zipapp, unzip these to a path on the disk"""
+ if IS_ZIPAPP:
+ with self.extract(path, to_folder) as result:
+ yield result
+ else:
+ yield path
+
+ @abstractmethod
+ @contextmanager
+ def extract(self, path, to_folder):
+ raise NotImplementedError
+
+ @abstractmethod
+ @contextmanager
+ def locked(self, path):
+ raise NotImplementedError
+
+
+@six.add_metaclass(ABCMeta)
+class ContentStore(object):
+ @abstractmethod
+ def exists(self):
+ raise NotImplementedError
+
+ @abstractmethod
+ def read(self):
+ raise NotImplementedError
+
+ @abstractmethod
+ def write(self, content):
+ raise NotImplementedError
+
+ @abstractmethod
+ def remove(self):
+ raise NotImplementedError
+
+ @abstractmethod
+ @contextmanager
+ def locked(self):
+ pass
diff --git a/src/virtualenv/app_data/na.py b/src/virtualenv/app_data/na.py
new file mode 100644
index 0000000..937aa9a
--- /dev/null
+++ b/src/virtualenv/app_data/na.py
@@ -0,0 +1,67 @@
+from __future__ import absolute_import, unicode_literals
+
+from contextlib import contextmanager
+
+from .base import AppData, ContentStore
+
+
+class AppDataDisabled(AppData):
+ """No application cache available (most likely as we don't have write permissions)"""
+
+ def __init__(self):
+ pass
+
+ error = RuntimeError("no app data folder available, probably no write access to the folder")
+
+ def close(self):
+ """do nothing"""
+
+ def reset(self):
+ """do nothing"""
+
+ def py_info(self, path):
+ return ContentStoreNA()
+
+ def embed_update_log(self, distribution, for_py_version):
+ return ContentStoreNA()
+
+ def extract(self, path, to_folder):
+ raise self.error
+
+ @contextmanager
+ def locked(self, path):
+ """do nothing"""
+ yield
+
+ @property
+ def house(self):
+ raise self.error
+
+ def wheel_image(self, for_py_version, name):
+ raise self.error
+
+ @property
+ def transient(self):
+ return True
+
+ def py_info_clear(self):
+ """"""
+
+
+class ContentStoreNA(ContentStore):
+ def exists(self):
+ return False
+
+ def read(self):
+ """"""
+ return None
+
+ def write(self, content):
+ """"""
+
+ def remove(self):
+ """"""
+
+ @contextmanager
+ def locked(self):
+ yield
diff --git a/src/virtualenv/app_data/via_disk_folder.py b/src/virtualenv/app_data/via_disk_folder.py
new file mode 100644
index 0000000..6b12ef8
--- /dev/null
+++ b/src/virtualenv/app_data/via_disk_folder.py
@@ -0,0 +1,172 @@
+# -*- coding: utf-8 -*-
+"""
+A rough layout of the current storage goes as:
+
+virtualenv-app-data
+├── py - <version> <cache information about python interpreters>
+│   └── *.json/lock
+├── wheel <cache wheels used for seeding>
+│   ├── house
+│ │ └── *.whl <wheels downloaded go here>
+│ └── <python major.minor> -> 3.9
+│ ├── img-<version>
+│ │ └── image
+│ │ └── <install class> -> CopyPipInstall / SymlinkPipInstall
+│ │ └── <wheel name> -> pip-20.1.1-py2.py3-none-any
+│ └── embed
+│ └── 1
+│ └── *.json -> for every distribution contains data about newer embed versions and releases
+└─── unzip <in zip app we cannot refer to some internal files, so first extract them>
+ └── <virtualenv version>
+ ├── py_info.py
+ ├── debug.py
+ └── _virtualenv.py
+"""
+from __future__ import absolute_import, unicode_literals
+
+import json
+import logging
+from abc import ABCMeta
+from contextlib import contextmanager
+from hashlib import sha256
+
+import six
+
+from virtualenv.util.lock import ReentrantFileLock
+from virtualenv.util.path import safe_delete
+from virtualenv.util.six import ensure_text
+from virtualenv.util.zipapp import extract
+from virtualenv.version import __version__
+
+from .base import AppData, ContentStore
+
+
+class AppDataDiskFolder(AppData):
+ """
+ Store the application data on the disk within a folder layout.
+ """
+
+ def __init__(self, folder):
+ self.lock = ReentrantFileLock(folder)
+
+ def __repr__(self):
+ return "{}".format(self.lock.path)
+
+ @property
+ def transient(self):
+ return False
+
+ def reset(self):
+ logging.debug("reset app data folder %s", self.lock.path)
+ safe_delete(self.lock.path)
+
+ def close(self):
+ """do nothing"""
+
+ @contextmanager
+ def locked(self, path):
+ path_lock = self.lock / path
+ with path_lock:
+ yield path_lock.path
+
+ @contextmanager
+ def extract(self, path, to_folder):
+ if to_folder is not None:
+ root = ReentrantFileLock(to_folder())
+ else:
+ root = self.lock / "unzip" / __version__
+ with root.lock_for_key(path.name):
+ dest = root.path / path.name
+ if not dest.exists():
+ extract(path, dest)
+ yield dest
+
+ @property
+ def py_info_at(self):
+ return self.lock / "py_info" / "1"
+
+ def py_info(self, path):
+ return PyInfoStoreDisk(self.py_info_at, path)
+
+ def py_info_clear(self):
+ """"""
+ py_info_folder = self.py_info_at
+ with py_info_folder:
+ for filename in py_info_folder.path.iterdir():
+ if filename.suffix == ".json":
+ with py_info_folder.lock_for_key(filename.stem):
+ if filename.exists():
+ filename.unlink()
+
+ def embed_update_log(self, distribution, for_py_version):
+ return EmbedDistributionUpdateStoreDisk(self.lock / "wheel" / for_py_version / "embed" / "1", distribution)
+
+ @property
+ def house(self):
+ path = self.lock.path / "wheel" / "house"
+ path.mkdir(parents=True, exist_ok=True)
+ return path
+
+ def wheel_image(self, for_py_version, name):
+ return self.lock.path / "wheel" / for_py_version / "image" / "1" / name
+
+
+@six.add_metaclass(ABCMeta)
+class JSONStoreDisk(ContentStore):
+ def __init__(self, in_folder, key, msg, msg_args):
+ self.in_folder = in_folder
+ self.key = key
+ self.msg = msg
+ self.msg_args = msg_args + (self.file,)
+
+ @property
+ def file(self):
+ return self.in_folder.path / "{}.json".format(self.key)
+
+ def exists(self):
+ return self.file.exists()
+
+ def read(self):
+ data, bad_format = None, False
+ try:
+ data = json.loads(self.file.read_text())
+ logging.debug("got {} from %s".format(self.msg), *self.msg_args)
+ return data
+ except ValueError:
+ bad_format = True
+ except Exception: # noqa
+ pass
+ if bad_format:
+ self.remove()
+ return None
+
+ def remove(self):
+ self.file.unlink()
+ logging.debug("removed {} at %s".format(self.msg), *self.msg_args)
+
+ @contextmanager
+ def locked(self):
+ with self.in_folder.lock_for_key(self.key):
+ yield
+
+ def write(self, content):
+ folder = self.file.parent
+ try:
+ folder.mkdir(parents=True, exist_ok=True)
+ except OSError:
+ pass
+ self.file.write_text(ensure_text(json.dumps(content, sort_keys=True, indent=2)))
+ logging.debug("wrote {} at %s".format(self.msg), *self.msg_args)
+
+
+class PyInfoStoreDisk(JSONStoreDisk):
+ def __init__(self, in_folder, path):
+ key = sha256(str(path).encode("utf-8") if six.PY3 else str(path)).hexdigest()
+ super(PyInfoStoreDisk, self).__init__(in_folder, key, "python info of %s", (path,))
+
+
+class EmbedDistributionUpdateStoreDisk(JSONStoreDisk):
+ def __init__(self, in_folder, distribution):
+ super(EmbedDistributionUpdateStoreDisk, self).__init__(
+ in_folder, distribution, "embed update of distribution %s", (distribution,),
+ )
diff --git a/src/virtualenv/app_data/via_tempdir.py b/src/virtualenv/app_data/via_tempdir.py
new file mode 100644
index 0000000..e8b387c
--- /dev/null
+++ b/src/virtualenv/app_data/via_tempdir.py
@@ -0,0 +1,28 @@
+from __future__ import absolute_import, unicode_literals
+
+import logging
+from tempfile import mkdtemp
+
+from virtualenv.util.path import safe_delete
+
+from .via_disk_folder import AppDataDiskFolder
+
+
+class TempAppData(AppDataDiskFolder):
+ def __init__(self):
+ super(TempAppData, self).__init__(folder=mkdtemp())
+ logging.debug("created temporary app data folder %s", self.lock.path)
+
+ def reset(self):
+ """this is a temporary folder, is already empty to start with"""
+
+ def close(self):
+ logging.debug("remove temporary app data folder %s", self.lock.path)
+ safe_delete(self.lock.path)
+
+ def embed_update_log(self, distribution, for_py_version):
+ return None
+
+ @property
+ def transient(self):
+ return True
diff --git a/src/virtualenv/config/cli/parser.py b/src/virtualenv/config/cli/parser.py
index 1c4d62c..eb4db30 100644
--- a/src/virtualenv/config/cli/parser.py
+++ b/src/virtualenv/config/cli/parser.py
@@ -56,7 +56,6 @@ class VirtualEnvConfigParser(ArgumentParser):
kwargs["prog"] = "virtualenv"
super(VirtualEnvConfigParser, self).__init__(*args, **kwargs)
self._fixed = set()
- self._elements = None
if options is not None and not isinstance(options, VirtualEnvOptions):
raise TypeError("options must be of type VirtualEnvOptions")
self.options = VirtualEnvOptions() if options is None else options
diff --git a/src/virtualenv/create/creator.py b/src/virtualenv/create/creator.py
index 81fcd15..0dcc49c 100644
--- a/src/virtualenv/create/creator.py
+++ b/src/virtualenv/create/creator.py
@@ -14,13 +14,13 @@ from six import add_metaclass
from virtualenv.discovery.cached_py_info import LogCmd
from virtualenv.info import WIN_CPYTHON_2
-from virtualenv.pyenv_cfg import PyEnvCfg
from virtualenv.util.path import Path, safe_delete
from virtualenv.util.six import ensure_str, ensure_text
from virtualenv.util.subprocess import run_cmd
-from virtualenv.util.zipapp import ensure_file_on_disk
from virtualenv.version import __version__
+from .pyenv_cfg import PyEnvCfg
+
HERE = Path(os.path.abspath(__file__)).parent
DEBUG_SCRIPT = HERE / "debug.py"
@@ -45,7 +45,7 @@ class Creator(object):
self.dest = Path(options.dest)
self.clear = options.clear
self.pyenv_cfg = PyEnvCfg.from_folder(self.dest)
- self.app_data = options.app_data.folder
+ self.app_data = options.app_data
def __repr__(self):
return ensure_str(self.__unicode__())
@@ -74,6 +74,7 @@ class Creator(object):
"""Add CLI arguments for the creator.
:param parser: the CLI parser
+ :param app_data: the application data folder
:param interpreter: the interpreter we're asked to create virtual environment for
:param meta: value as returned by :meth:`can_create`
"""
@@ -199,7 +200,7 @@ def get_env_debug_info(env_exe, debug_script, app_data):
env = os.environ.copy()
env.pop(str("PYTHONPATH"), None)
- with ensure_file_on_disk(debug_script, app_data) as debug_script:
+ with app_data.ensure_extracted(debug_script) as debug_script:
cmd = [str(env_exe), str(debug_script)]
if WIN_CPYTHON_2:
cmd = [ensure_text(i) for i in cmd]
diff --git a/src/virtualenv/pyenv_cfg.py b/src/virtualenv/create/pyenv_cfg.py
index 1a8d824..1a8d824 100644
--- a/src/virtualenv/pyenv_cfg.py
+++ b/src/virtualenv/create/pyenv_cfg.py
diff --git a/src/virtualenv/create/via_global_ref/api.py b/src/virtualenv/create/via_global_ref/api.py
index 412e871..c9eab3c 100644
--- a/src/virtualenv/create/via_global_ref/api.py
+++ b/src/virtualenv/create/via_global_ref/api.py
@@ -8,7 +8,6 @@ from six import add_metaclass
from virtualenv.info import fs_supports_symlink
from virtualenv.util.path import Path
-from virtualenv.util.zipapp import ensure_file_on_disk
from ..creator import Creator, CreatorMeta
@@ -100,7 +99,7 @@ class ViaGlobalRefApi(Creator):
def env_patch_text(self):
"""Patch the distutils package to not be derailed by its configuration files"""
- with ensure_file_on_disk(Path(__file__).parent / "_virtualenv.py", self.app_data) as resolved_path:
+ with self.app_data.ensure_extracted(Path(__file__).parent / "_virtualenv.py") as resolved_path:
text = resolved_path.read_text()
return text.replace('"__SCRIPT_DIR__"', repr(os.path.relpath(str(self.script_dir), str(self.purelib))))
diff --git a/src/virtualenv/create/via_global_ref/store.py b/src/virtualenv/create/via_global_ref/store.py
index 55d9413..134a535 100644
--- a/src/virtualenv/create/via_global_ref/store.py
+++ b/src/virtualenv/create/via_global_ref/store.py
@@ -1,3 +1,5 @@
+from __future__ import absolute_import, unicode_literals
+
from virtualenv.util.path import Path
@@ -18,4 +20,7 @@ def is_store_python(interpreter):
)
-__all__ = ("handle_store_python", "is_store_python")
+__all__ = (
+ "handle_store_python",
+ "is_store_python",
+)
diff --git a/src/virtualenv/create/via_global_ref/venv.py b/src/virtualenv/create/via_global_ref/venv.py
index 88e74e3..4a4ed77 100644
--- a/src/virtualenv/create/via_global_ref/venv.py
+++ b/src/virtualenv/create/via_global_ref/venv.py
@@ -5,7 +5,7 @@ from copy import copy
from virtualenv.create.via_global_ref.store import handle_store_python
from virtualenv.discovery.py_info import PythonInfo
-from virtualenv.error import ProcessCallFailed
+from virtualenv.util.error import ProcessCallFailed
from virtualenv.util.path import ensure_dir
from virtualenv.util.subprocess import run_cmd
diff --git a/src/virtualenv/discovery/builtin.py b/src/virtualenv/discovery/builtin.py
index 2498555..4d57fa5 100644
--- a/src/virtualenv/discovery/builtin.py
+++ b/src/virtualenv/discovery/builtin.py
@@ -30,7 +30,7 @@ class Builtin(Discover):
)
def run(self):
- return get_interpreter(self.python_spec, self.app_data.folder)
+ return get_interpreter(self.python_spec, self.app_data)
def __repr__(self):
return ensure_str(self.__unicode__())
diff --git a/src/virtualenv/discovery/cached_py_info.py b/src/virtualenv/discovery/cached_py_info.py
index a1fd3f3..13a213d 100644
--- a/src/virtualenv/discovery/cached_py_info.py
+++ b/src/virtualenv/discovery/cached_py_info.py
@@ -6,21 +6,18 @@ caching.
"""
from __future__ import absolute_import, unicode_literals
-import json
import logging
import os
import pipes
import sys
from collections import OrderedDict
-from hashlib import sha256
+from virtualenv.app_data import AppDataDisabled
from virtualenv.discovery.py_info import PythonInfo
-from virtualenv.info import PY2, PY3
+from virtualenv.info import PY2
from virtualenv.util.path import Path
from virtualenv.util.six import ensure_text
from virtualenv.util.subprocess import Popen, subprocess
-from virtualenv.util.zipapp import ensure_file_on_disk
-from virtualenv.version import __version__
_CACHE = OrderedDict()
_CACHE[Path(sys.executable)] = PythonInfo()
@@ -28,8 +25,7 @@ _CACHE[Path(sys.executable)] = PythonInfo()
def from_exe(cls, app_data, exe, raise_on_error=True, ignore_cache=False):
""""""
- py_info_cache = _get_py_info_cache(app_data)
- result = _get_from_cache(cls, py_info_cache, app_data, exe, ignore_cache=ignore_cache)
+ result = _get_from_cache(cls, app_data, exe, ignore_cache=ignore_cache)
if isinstance(result, Exception):
if raise_on_error:
raise result
@@ -39,21 +35,14 @@ def from_exe(cls, app_data, exe, raise_on_error=True, ignore_cache=False):
return result
-def _get_py_info_cache(app_data):
- return None if app_data is None else app_data / "py_info" / __version__
-
-
-def _get_from_cache(cls, py_info_cache, app_data, exe, ignore_cache=True):
+def _get_from_cache(cls, app_data, exe, ignore_cache=True):
# note here we cannot resolve symlinks, as the symlink may trigger different prefix information if there's a
# pyenv.cfg somewhere alongside on python3.4+
exe_path = Path(exe)
if not ignore_cache and exe_path in _CACHE: # check in the in-memory cache
result = _CACHE[exe_path]
- elif py_info_cache is None: # cache disabled
- failure, py_info = _run_subprocess(cls, exe, app_data)
- result = py_info if failure is None else failure
- else: # then check the persisted cache
- py_info = _get_via_file_cache(cls, py_info_cache, app_data, exe_path, exe)
+ else: # otherwise go through the app data cache
+ py_info = _get_via_file_cache(cls, app_data, exe_path, exe)
result = _CACHE[exe_path] = py_info
# independent if it was from the file or in-memory cache fix the original executable location
if isinstance(result, PythonInfo):
@@ -61,47 +50,37 @@ def _get_from_cache(cls, py_info_cache, app_data, exe, ignore_cache=True):
return result
-def _get_via_file_cache(cls, py_info_cache, app_data, resolved_path, exe):
- key = sha256(str(resolved_path).encode("utf-8") if PY3 else str(resolved_path)).hexdigest()
- py_info = None
- resolved_path_text = ensure_text(str(resolved_path))
+def _get_via_file_cache(cls, app_data, path, exe):
+ path_text = ensure_text(str(path))
try:
- resolved_path_modified_timestamp = resolved_path.stat().st_mtime
+ path_modified = path.stat().st_mtime
except OSError:
- resolved_path_modified_timestamp = -1
- data_file = py_info_cache / "{}.json".format(key)
- with py_info_cache.lock_for_key(key):
- data_file_path = data_file.path
- if data_file_path.exists() and resolved_path_modified_timestamp != 1: # if exists and matches load
- try:
- data = json.loads(data_file_path.read_text())
- if data["path"] == resolved_path_text and data["st_mtime"] == resolved_path_modified_timestamp:
- logging.debug("get PythonInfo from %s for %s", data_file_path, exe)
- py_info = cls._from_dict({k: v for k, v in data["content"].items()})
- else:
- raise ValueError("force close as stale")
- except (KeyError, ValueError, OSError):
- logging.debug("remove PythonInfo %s for %s", data_file_path, exe)
- data_file_path.unlink() # close out of date files
+ path_modified = -1
+ if app_data is None:
+ app_data = AppDataDisabled()
+ py_info, py_info_store = None, app_data.py_info(path)
+ with py_info_store.locked():
+ if py_info_store.exists(): # if exists and matches load
+ data = py_info_store.read()
+ of_path, of_st_mtime, of_content = data["path"], data["st_mtime"], data["content"]
+ if of_path == path_text and of_st_mtime == path_modified:
+ py_info = cls._from_dict({k: v for k, v in of_content.items()})
+ else:
+ py_info_store.remove()
if py_info is None: # if not loaded run and save
failure, py_info = _run_subprocess(cls, exe, app_data)
if failure is None:
- file_cache_content = {
- "st_mtime": resolved_path_modified_timestamp,
- "path": resolved_path_text,
- "content": py_info._to_dict(),
- }
- logging.debug("write PythonInfo to %s for %s", data_file_path, exe)
- data_file_path.write_text(ensure_text(json.dumps(file_cache_content, indent=2)))
+ data = {"st_mtime": path_modified, "path": path_text, "content": py_info._to_dict()}
+ py_info_store.write(data)
else:
py_info = failure
return py_info
def _run_subprocess(cls, exe, app_data):
- resolved_path = Path(os.path.abspath(__file__)).parent / "py_info.py"
- with ensure_file_on_disk(resolved_path, app_data) as resolved_path:
- cmd = [exe, str(resolved_path)]
+ py_info_script = Path(os.path.abspath(__file__)).parent / "py_info.py"
+ with app_data.ensure_extracted(py_info_script) as py_info_script:
+ cmd = [exe, str(py_info_script)]
# prevent sys.prefix from leaking into the child process - see https://bugs.python.org/issue22490
env = os.environ.copy()
env.pop("__PYVENV_LAUNCHER__", None)
@@ -155,14 +134,7 @@ class LogCmd(object):
def clear(app_data):
- py_info_cache = _get_py_info_cache(app_data)
- if py_info_cache is not None:
- with py_info_cache:
- for filename in py_info_cache.path.iterdir():
- if filename.suffix == ".json":
- with py_info_cache.lock_for_key(filename.stem):
- if filename.exists():
- filename.unlink()
+ app_data.py_info_clear()
_CACHE.clear()
diff --git a/src/virtualenv/report.py b/src/virtualenv/report.py
index 7ae2f24..8d5c358 100644
--- a/src/virtualenv/report.py
+++ b/src/virtualenv/report.py
@@ -40,6 +40,7 @@ def setup_report(verbosity):
LOGGER.addHandler(stream_handler)
level_name = logging.getLevelName(level)
logging.debug("setup logging to %s", level_name)
+ logging.getLogger("distlib").setLevel(logging.ERROR)
return verbosity
diff --git a/src/virtualenv/run/__init__.py b/src/virtualenv/run/__init__.py
index ed6d808..8de7962 100644
--- a/src/virtualenv/run/__init__.py
+++ b/src/virtualenv/run/__init__.py
@@ -2,11 +2,11 @@ from __future__ import absolute_import, unicode_literals
import logging
-from virtualenv.run.app_data import AppDataAction
-
+from ..app_data import AppDataAction, AppDataDisabled, TempAppData
from ..config.cli.parser import VirtualEnvConfigParser
from ..report import LEVELS, setup_report
-from ..session import Session
+from ..run.session import Session
+from ..seed.wheels.periodic_update import manual_upgrade
from ..version import __version__
from .plugin.activators import ActivationSelector
from .plugin.creators import CreatorSelector
@@ -29,9 +29,9 @@ def cli_run(args, options=None):
# noinspection PyProtectedMember
def session_via_cli(args, options=None):
- parser = build_parser(args, options)
+ parser, elements = build_parser(args, options)
options = parser.parse_args(args)
- creator, seeder, activators = tuple(e.create(options) for e in parser._elements) # create types
+ creator, seeder, activators = tuple(e.create(options) for e in elements) # create types
session = Session(options.verbosity, options.app_data, parser._interpreter, creator, seeder, activators)
return session
@@ -48,13 +48,44 @@ def build_parser(args=None, options=None):
help="on failure also display the stacktrace internals of virtualenv",
)
_do_report_setup(parser, args)
+ options = load_app_data(args, parser, options)
+ handle_extra_commands(options)
+
+ discover = get_discover(parser, args)
+ parser._interpreter = interpreter = discover.interpreter
+ if interpreter is None:
+ raise RuntimeError("failed to find interpreter for {}".format(discover))
+ elements = [
+ CreatorSelector(interpreter, parser),
+ SeederSelector(interpreter, parser),
+ ActivationSelector(interpreter, parser),
+ ]
+ options, _ = parser.parse_known_args(args)
+ for element in elements:
+ element.handle_selected_arg_parse(options)
+ parser.enable_help()
+ return parser, elements
+
+
+def build_parser_only(args=None):
+ """Used to provide a parser for the doc generation"""
+ return build_parser(args)[0]
+
+
+def handle_extra_commands(options):
+ if options.upgrade_embed_wheels:
+ result = manual_upgrade(options.app_data)
+ raise SystemExit(result)
+
+
+def load_app_data(args, parser, options):
# here we need a write-able application data (e.g. the zipapp might need this for discovery cache)
default_app_data = AppDataAction.default()
parser.add_argument(
"--app-data",
dest="app_data",
action=AppDataAction,
- default="<temp folder>" if default_app_data is None else default_app_data,
+ default="<temp folder>" if isinstance(default_app_data, AppDataDisabled) else default_app_data,
help="a data folder used as cache by the virtualenv",
)
parser.add_argument(
@@ -64,20 +95,19 @@ def build_parser(args=None, options=None):
help="start with empty app data folder",
default=False,
)
- discover = get_discover(parser, args)
- parser._interpreter = interpreter = discover.interpreter
- if interpreter is None:
- raise RuntimeError("failed to find interpreter for {}".format(discover))
- parser._elements = [
- CreatorSelector(interpreter, parser),
- SeederSelector(interpreter, parser),
- ActivationSelector(interpreter, parser),
- ]
- options, _ = parser.parse_known_args(args)
- for element in parser._elements:
- element.handle_selected_arg_parse(options)
- parser.enable_help()
- return parser
+ parser.add_argument(
+ "--upgrade-embed-wheels",
+ dest="upgrade_embed_wheels",
+ action="store_true",
+ help="trigger a manual update of the embedded wheels",
+ default=False,
+ )
+ options, _ = parser.parse_known_args(args, namespace=options)
+ if options.app_data == "<temp folder>":
+ options.app_data = TempAppData()
+ if options.reset_app_data:
+ options.app_data.reset()
+ return options
def add_version_flag(parser):
diff --git a/src/virtualenv/run/plugin/discovery.py b/src/virtualenv/run/plugin/discovery.py
index 9365e5e..e2cfe92 100644
--- a/src/virtualenv/run/plugin/discovery.py
+++ b/src/virtualenv/run/plugin/discovery.py
@@ -1,7 +1,5 @@
from __future__ import absolute_import, unicode_literals
-from virtualenv.run.app_data import TempAppData
-
from .base import PluginLoader
@@ -22,10 +20,6 @@ def get_discover(parser, args):
help="interpreter discovery method",
)
options, _ = parser.parse_known_args(args)
- if options.app_data == "<temp folder>":
- options.app_data = TempAppData()
- if options.reset_app_data:
- options.app_data.clean()
discover_class = discover_types[options.discovery]
discover_class.add_parser_arguments(discovery_parser)
options, _ = parser.parse_known_args(args, namespace=options)
diff --git a/src/virtualenv/session.py b/src/virtualenv/run/session.py
index c936089..c936089 100644
--- a/src/virtualenv/session.py
+++ b/src/virtualenv/run/session.py
diff --git a/src/virtualenv/seed/embed/__init__.py b/src/virtualenv/seed/embed/__init__.py
index 01e6d4f..e69de29 100644
--- a/src/virtualenv/seed/embed/__init__.py
+++ b/src/virtualenv/seed/embed/__init__.py
@@ -1 +0,0 @@
-from __future__ import absolute_import, unicode_literals
diff --git a/src/virtualenv/seed/embed/base_embed.py b/src/virtualenv/seed/embed/base_embed.py
index bffd494..f41b5fc 100644
--- a/src/virtualenv/seed/embed/base_embed.py
+++ b/src/virtualenv/seed/embed/base_embed.py
@@ -8,38 +8,45 @@ from virtualenv.util.path import Path
from virtualenv.util.six import ensure_str, ensure_text
from ..seeder import Seeder
+from ..wheels import Version
+
+PERIODIC_UPDATE_ON_BY_DEFAULT = True
@add_metaclass(ABCMeta)
class BaseEmbed(Seeder):
- packages = ["pip", "setuptools", "wheel"]
-
def __init__(self, options):
super(BaseEmbed, self).__init__(options, enabled=options.no_seed is False)
+
self.download = options.download
self.extra_search_dir = [i.resolve() for i in options.extra_search_dir if i.exists()]
- def latest_is_none(key):
- value = getattr(options, key)
- return None if value == "latest" else value
-
- self.pip_version = latest_is_none("pip")
- self.setuptools_version = latest_is_none("setuptools")
- self.wheel_version = latest_is_none("wheel")
+ self.pip_version = options.pip
+ self.setuptools_version = options.setuptools
+ self.wheel_version = options.wheel
self.no_pip = options.no_pip
self.no_setuptools = options.no_setuptools
self.no_wheel = options.no_wheel
- self.app_data = options.app_data.folder
+ self.app_data = options.app_data
+ self.periodic_update = not options.no_periodic_update
- if not self.package_version():
+ if not self.distribution_to_versions():
self.enabled = False
- def package_version(self):
+ @classmethod
+ def distributions(cls):
return {
- package: getattr(self, "{}_version".format(package))
- for package in self.packages
- if getattr(self, "no_{}".format(package)) is False
+ "pip": Version.bundle,
+ "setuptools": Version.bundle,
+ "wheel": Version.bundle,
+ }
+
+ def distribution_to_versions(self):
+ return {
+ distribution: getattr(self, "{}_version".format(distribution))
+ for distribution in self.distributions()
+ if getattr(self, "no_{}".format(distribution)) is False
}
@classmethod
@@ -50,14 +57,14 @@ class BaseEmbed(Seeder):
"--never-download",
dest="download",
action="store_false",
- help="pass to disable download of the latest {} from PyPI".format("/".join(cls.packages)),
+ help="pass to disable download of the latest {} from PyPI".format("/".join(cls.distributions())),
default=True,
)
group.add_argument(
"--download",
dest="download",
action="store_true",
- help="pass to enable download of the latest {} from PyPI".format("/".join(cls.packages)),
+ help="pass to enable download of the latest {} from PyPI".format("/".join(cls.distributions())),
default=False,
)
parser.add_argument(
@@ -65,25 +72,32 @@ class BaseEmbed(Seeder):
metavar="d",
type=Path,
nargs="+",
- help="a path containing wheels the seeder may also use beside bundled (can be set 1+ times)",
+ help="a path containing wheels to extend the internal wheel list (can be set 1+ times)",
default=[],
)
- for package in cls.packages:
+ for distribution, default in cls.distributions().items():
parser.add_argument(
- "--{}".format(package),
- dest=package,
+ "--{}".format(distribution),
+ dest=distribution,
metavar="version",
- help="{} version to install, bundle for bundled".format(package),
- default="latest",
+ help="version of {} to install as seed: embed, bundle or exact version".format(distribution),
+ default=default,
)
- for package in cls.packages:
+ for distribution in cls.distributions():
parser.add_argument(
- "--no-{}".format(package),
- dest="no_{}".format(package),
+ "--no-{}".format(distribution),
+ dest="no_{}".format(distribution),
action="store_true",
- help="do not install {}".format(package),
+ help="do not install {}".format(distribution),
default=False,
)
+ parser.add_argument(
+ "--no-periodic-update",
+ dest="no_periodic_update",
+ action="store_true",
+ help="disable the periodic (once every 14 days) update of the embedded wheels",
+ default=not PERIODIC_UPDATE_ON_BY_DEFAULT,
+ )
def __unicode__(self):
result = self.__class__.__name__
@@ -91,11 +105,11 @@ class BaseEmbed(Seeder):
if self.extra_search_dir:
result += "extra_search_dir={},".format(", ".join(ensure_text(str(i)) for i in self.extra_search_dir))
result += "download={},".format(self.download)
- for package in self.packages:
- if getattr(self, "no_{}".format(package)):
+ for distribution in self.distributions():
+ if getattr(self, "no_{}".format(distribution)):
continue
result += " {}{},".format(
- package, "={}".format(getattr(self, "{}_version".format(package), None) or "latest"),
+ distribution, "={}".format(getattr(self, "{}_version".format(distribution), None) or "latest"),
)
return result[:-1] + ")"
diff --git a/src/virtualenv/seed/embed/pip_invoke.py b/src/virtualenv/seed/embed/pip_invoke.py
index 25be493..372e140 100644
--- a/src/virtualenv/seed/embed/pip_invoke.py
+++ b/src/virtualenv/seed/embed/pip_invoke.py
@@ -4,16 +4,10 @@ import logging
from contextlib import contextmanager
from virtualenv.discovery.cached_py_info import LogCmd
-from virtualenv.info import PY3
from virtualenv.seed.embed.base_embed import BaseEmbed
-from virtualenv.seed.embed.wheels.acquire import get_bundled_wheel, pip_wheel_env_run
from virtualenv.util.subprocess import Popen
-from virtualenv.util.zipapp import ensure_file_on_disk
-if PY3:
- from contextlib import ExitStack
-else:
- from contextlib2 import ExitStack
+from ..wheels import Version, get_wheel, pip_wheel_env_run
class PipInvoke(BaseEmbed):
@@ -23,9 +17,10 @@ class PipInvoke(BaseEmbed):
def run(self, creator):
if not self.enabled:
return
- with self.get_pip_install_cmd(creator.exe, creator.interpreter.version_release_str) as cmd:
- with pip_wheel_env_run(creator.interpreter.version_release_str, self.app_data) as env:
- self._execute(cmd, env)
+ for_py_version = creator.interpreter.version_release_str
+ with self.get_pip_install_cmd(creator.exe, for_py_version) as cmd:
+ env = pip_wheel_env_run(self.extra_search_dir, self.app_data)
+ self._execute(cmd, env)
@staticmethod
def _execute(cmd, env):
@@ -37,18 +32,25 @@ class PipInvoke(BaseEmbed):
return process
@contextmanager
- def get_pip_install_cmd(self, exe, version):
- cmd = [str(exe), "-m", "pip", "-q", "install", "--only-binary", ":all:"]
+ def get_pip_install_cmd(self, exe, for_py_version):
+ cmd = [str(exe), "-m", "pip", "-q", "install", "--only-binary", ":all:", "--disable-pip-version-check"]
if not self.download:
cmd.append("--no-index")
- pkg_versions = self.package_version()
- for key, ver in pkg_versions.items():
- cmd.append("{}{}".format(key, "=={}".format(ver) if ver is not None else ""))
- with ExitStack() as stack:
- folders = set()
- for context in (ensure_file_on_disk(get_bundled_wheel(p, version), self.app_data) for p in pkg_versions):
- folders.add(stack.enter_context(context).parent)
- folders.update(set(self.extra_search_dir))
- for folder in folders:
- cmd.extend(["--find-links", str(folder)])
- yield cmd
+ folders = set()
+ for dist, version in self.distribution_to_versions().items():
+ wheel = get_wheel(
+ distribution=dist,
+ version=version,
+ for_py_version=for_py_version,
+ search_dirs=self.extra_search_dir,
+ download=False,
+ app_data=self.app_data,
+ do_periodic_update=self.periodic_update,
+ )
+ if wheel is None:
+ raise RuntimeError("could not get wheel for distribution {}".format(dist))
+ folders.add(str(wheel.path.parent))
+ cmd.append(Version.as_pip_req(dist, wheel.version))
+ for folder in sorted(folders):
+ cmd.extend(["--find-links", str(folder)])
+ yield cmd
diff --git a/src/virtualenv/seed/via_app_data/__init__.py b/src/virtualenv/seed/embed/via_app_data/__init__.py
index e69de29..e69de29 100644
--- a/src/virtualenv/seed/via_app_data/__init__.py
+++ b/src/virtualenv/seed/embed/via_app_data/__init__.py
diff --git a/src/virtualenv/seed/via_app_data/pip_install/__init__.py b/src/virtualenv/seed/embed/via_app_data/pip_install/__init__.py
index e69de29..e69de29 100644
--- a/src/virtualenv/seed/via_app_data/pip_install/__init__.py
+++ b/src/virtualenv/seed/embed/via_app_data/pip_install/__init__.py
diff --git a/src/virtualenv/seed/via_app_data/pip_install/base.py b/src/virtualenv/seed/embed/via_app_data/pip_install/base.py
index f7f29ca..f382bda 100644
--- a/src/virtualenv/seed/via_app_data/pip_install/base.py
+++ b/src/virtualenv/seed/embed/via_app_data/pip_install/base.py
@@ -54,11 +54,11 @@ class PipInstall(object):
def build_image(self):
# 1. first extract the wheel
- logging.debug("build install image to %s of %s", self._image_dir, self._wheel.name)
+ logging.debug("build install image for %s to %s", self._wheel.name, self._image_dir)
with zipfile.ZipFile(str(self._wheel)) as zip_ref:
zip_ref.extractall(str(self._image_dir))
self._extracted = True
- # 2. now add additional files not present in the package
+ # 2. now add additional files not present in the distribution
new_files = self._generate_new_files()
# 3. finally fix the records file
self._fix_records(new_files)
diff --git a/src/virtualenv/seed/via_app_data/pip_install/copy.py b/src/virtualenv/seed/embed/via_app_data/pip_install/copy.py
index 29d0bc8..29d0bc8 100644
--- a/src/virtualenv/seed/via_app_data/pip_install/copy.py
+++ b/src/virtualenv/seed/embed/via_app_data/pip_install/copy.py
diff --git a/src/virtualenv/seed/via_app_data/pip_install/symlink.py b/src/virtualenv/seed/embed/via_app_data/pip_install/symlink.py
index f958b65..f958b65 100644
--- a/src/virtualenv/seed/via_app_data/pip_install/symlink.py
+++ b/src/virtualenv/seed/embed/via_app_data/pip_install/symlink.py
diff --git a/src/virtualenv/seed/embed/via_app_data/via_app_data.py b/src/virtualenv/seed/embed/via_app_data/via_app_data.py
new file mode 100644
index 0000000..779ee18
--- /dev/null
+++ b/src/virtualenv/seed/embed/via_app_data/via_app_data.py
@@ -0,0 +1,127 @@
+"""Bootstrap"""
+from __future__ import absolute_import, unicode_literals
+
+import logging
+from contextlib import contextmanager
+from subprocess import CalledProcessError
+from threading import Lock, Thread
+
+import six
+
+from virtualenv.info import fs_supports_symlink
+from virtualenv.seed.embed.base_embed import BaseEmbed
+from virtualenv.seed.wheels import get_wheel
+from virtualenv.util.path import Path
+
+from .pip_install.copy import CopyPipInstall
+from .pip_install.symlink import SymlinkPipInstall
+
+
+class FromAppData(BaseEmbed):
+ def __init__(self, options):
+ super(FromAppData, self).__init__(options)
+ self.symlinks = options.symlink_app_data
+
+ @classmethod
+ def add_parser_arguments(cls, parser, interpreter, app_data):
+ super(FromAppData, cls).add_parser_arguments(parser, interpreter, app_data)
+ can_symlink = app_data.transient is False and fs_supports_symlink()
+ parser.add_argument(
+ "--symlink-app-data",
+ dest="symlink_app_data",
+ action="store_true" if can_symlink else "store_false",
+ help="{} symlink the python packages from the app-data folder (requires seed pip>=19.3)".format(
+ "" if can_symlink else "not supported - ",
+ ),
+ default=False,
+ )
+
+ def run(self, creator):
+ if not self.enabled:
+ return
+ with self._get_seed_wheels(creator) as name_to_whl:
+ pip_version = name_to_whl["pip"].version_tuple if "pip" in name_to_whl else None
+ installer_class = self.installer_class(pip_version)
+
+ def _install(name, wheel):
+ logging.debug("install %s from wheel %s via %s", name, wheel, installer_class.__name__)
+ key = Path(installer_class.__name__) / wheel.path.stem
+ wheel_img = self.app_data.wheel_image(creator.interpreter.version_release_str, key)
+ installer = installer_class(wheel.path, creator, wheel_img)
+ if not installer.has_image():
+ installer.build_image()
+ installer.install(creator.interpreter.version_info)
+
+ threads = list(Thread(target=_install, args=(n, w)) for n, w in name_to_whl.items())
+ for thread in threads:
+ thread.start()
+ for thread in threads:
+ thread.join()
+
+ @contextmanager
+ def _get_seed_wheels(self, creator):
+ name_to_whl, lock, fail = {}, Lock(), {}
+
+ def _get(distribution, version):
+ for_py_version = creator.interpreter.version_release_str
+ failure, result = None, None
+ # fallback to download in case the exact version is not available
+ for download in [True] if self.download else [False, True]:
+ failure = None
+ try:
+ result = get_wheel(
+ distribution=distribution,
+ version=version,
+ for_py_version=for_py_version,
+ search_dirs=self.extra_search_dir,
+ download=download,
+ app_data=self.app_data,
+ do_periodic_update=self.periodic_update,
+ )
+ if result is not None:
+ break
+ except Exception as exception: # noqa
+ logging.exception("fail")
+ failure = exception
+ if failure:
+ if isinstance(failure, CalledProcessError):
+ msg = "failed to download {}".format(distribution)
+ if version is not None:
+ msg += " version {}".format(version)
+ msg += ", pip download exit code {}".format(failure.returncode)
+ output = failure.output if six.PY2 else (failure.output + failure.stderr)
+ if output:
+ msg += "\n"
+ msg += output
+ else:
+ msg = repr(failure)
+ logging.error(msg)
+ with lock:
+ fail[distribution] = version
+ else:
+ with lock:
+ name_to_whl[distribution] = result
+
+ threads = list(
+ Thread(target=_get, args=(distribution, version))
+ for distribution, version in self.distribution_to_versions().items()
+ )
+ for thread in threads:
+ thread.start()
+ for thread in threads:
+ thread.join()
+ if fail:
+ raise RuntimeError("seed failed due to failing to download wheels {}".format(", ".join(fail.keys())))
+ yield name_to_whl
+
+ def installer_class(self, pip_version_tuple):
+ if self.symlinks and pip_version_tuple:
+ # symlink support requires pip 19.3+
+ if pip_version_tuple >= (19, 3):
+ return SymlinkPipInstall
+ return CopyPipInstall
+
+ def __unicode__(self):
+ base = super(FromAppData, self).__unicode__()
+ msg = ", via={}, app_data_dir={}".format("symlink" if self.symlinks else "copy", self.app_data)
+ return base[:-1] + msg + base[-1]
diff --git a/src/virtualenv/seed/embed/wheels/acquire.py b/src/virtualenv/seed/embed/wheels/acquire.py
deleted file mode 100644
index 91b630d..0000000
--- a/src/virtualenv/seed/embed/wheels/acquire.py
+++ /dev/null
@@ -1,178 +0,0 @@
-"""Bootstrap"""
-from __future__ import absolute_import, unicode_literals
-
-import logging
-import os
-import sys
-from collections import defaultdict
-from contextlib import contextmanager
-from copy import copy
-from shutil import copy2
-from zipfile import ZipFile
-
-from virtualenv.info import IS_ZIPAPP
-from virtualenv.util.path import Path
-from virtualenv.util.six import ensure_str, ensure_text
-from virtualenv.util.subprocess import Popen, subprocess
-from virtualenv.util.zipapp import ensure_file_on_disk
-
-from . import BUNDLE_SUPPORT, MAX
-
-BUNDLE_FOLDER = Path(os.path.abspath(__file__)).parent
-
-
-class WheelDownloadFail(ValueError):
- def __init__(self, packages, for_py_version, exit_code, out, err):
- self.packages = packages
- self.for_py_version = for_py_version
- self.exit_code = exit_code
- self.out = out.strip()
- self.err = err.strip()
-
-
-def get_wheels(for_py_version, wheel_cache_dir, extra_search_dir, packages, app_data, download):
- # not all wheels are compatible with all python versions, so we need to py version qualify it
- processed = copy(packages)
- # 1. acquire from bundle
- acquire_from_bundle(processed, for_py_version, wheel_cache_dir)
- # 2. acquire from extra search dir
- acquire_from_dir(processed, for_py_version, wheel_cache_dir, extra_search_dir)
- # 3. download from the internet
- if download and processed:
- download_wheel(processed, for_py_version, wheel_cache_dir, app_data)
-
- # in the end just get the wheels
- wheels = _get_wheels(wheel_cache_dir, packages)
- return {p: next(iter(ver_to_files))[1] for p, ver_to_files in wheels.items()}
-
-
-def acquire_from_bundle(packages, for_py_version, to_folder):
- for pkg, version in list(packages.items()):
- bundle = get_bundled_wheel(pkg, for_py_version)
- if bundle is not None:
- pkg_version = bundle.stem.split("-")[1]
- exact_version_match = version == pkg_version
- if exact_version_match:
- del packages[pkg]
- if version is None or exact_version_match:
- bundled_wheel_file = to_folder / bundle.name
- if not bundled_wheel_file.exists():
- logging.debug("get bundled wheel %s", bundle)
- if IS_ZIPAPP:
- from virtualenv.util.zipapp import extract
-
- extract(bundle, bundled_wheel_file)
- else:
- copy2(str(bundle), str(bundled_wheel_file))
-
-
-def get_bundled_wheel(package, version_release):
- return BUNDLE_FOLDER / (BUNDLE_SUPPORT.get(version_release, {}) or BUNDLE_SUPPORT[MAX]).get(package)
-
-
-def acquire_from_dir(packages, for_py_version, to_folder, extra_search_dir):
- if not packages:
- return
- for search_dir in extra_search_dir:
- wheels = _get_wheels(search_dir, packages)
- for pkg, ver_wheels in wheels.items():
- stop = False
- for _, filename in ver_wheels:
- dest = to_folder / filename.name
- if not dest.exists():
- if wheel_support_py(filename, for_py_version):
- logging.debug("get extra search dir wheel %s", filename)
- copy2(str(filename), str(dest))
- stop = True
- else:
- stop = True
- if stop and packages[pkg] is not None:
- del packages[pkg]
- break
-
-
-def wheel_support_py(filename, py_version):
- name = "{}.dist-info/METADATA".format("-".join(filename.stem.split("-")[0:2]))
- with ZipFile(ensure_text(str(filename)), "r") as zip_file:
- metadata = zip_file.read(name).decode("utf-8")
- marker = "Requires-Python:"
- requires = next((i[len(marker) :] for i in metadata.splitlines() if i.startswith(marker)), None)
- if requires is None: # if it does not specify a python requires the assumption is compatible
- return True
- py_version_int = tuple(int(i) for i in py_version.split("."))
- for require in (i.strip() for i in requires.split(",")):
- # https://www.python.org/dev/peps/pep-0345/#version-specifiers
- for operator, check in [
- ("!=", lambda v: py_version_int != v),
- ("==", lambda v: py_version_int == v),
- ("<=", lambda v: py_version_int <= v),
- (">=", lambda v: py_version_int >= v),
- ("<", lambda v: py_version_int < v),
- (">", lambda v: py_version_int > v),
- ]:
- if require.startswith(operator):
- ver_str = require[len(operator) :].strip()
- version = tuple((int(i) if i != "*" else None) for i in ver_str.split("."))[0:2]
- if not check(version):
- return False
- break
- return True
-
-
-def _get_wheels(from_folder, packages):
- wheels = defaultdict(list)
- for filename in from_folder.iterdir():
- if filename.suffix == ".whl":
- data = filename.stem.split("-")
- if len(data) >= 2:
- pkg, version = data[0:2]
- if pkg in packages:
- pkg_version = packages[pkg]
- if pkg_version is None or pkg_version == version:
- wheels[pkg].append((version, filename))
- for versions in wheels.values():
- versions.sort(
- key=lambda a: tuple(int(i) if i.isdigit() else i for i in a[0].split(".")), reverse=True,
- )
- return wheels
-
-
-def download_wheel(packages, for_py_version, to_folder, app_data):
- to_download = list(p if v is None else "{}=={}".format(p, v) for p, v in packages.items())
- logging.debug("download wheels %s", to_download)
- cmd = [
- sys.executable,
- "-m",
- "pip",
- "download",
- "--disable-pip-version-check",
- "--only-binary=:all:",
- "--no-deps",
- "--python-version",
- for_py_version,
- "-d",
- str(to_folder),
- ]
- cmd.extend(to_download)
- # pip has no interface in python - must be a new sub-process
-
- with pip_wheel_env_run("{}.{}".format(*sys.version_info[0:2]), app_data) as env:
- process = Popen(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
- out, err = process.communicate()
- if process.returncode != 0:
- raise WheelDownloadFail(packages, for_py_version, process.returncode, out, err)
-
-
-@contextmanager
-def pip_wheel_env_run(version, app_data):
- env = os.environ.copy()
- env.update(
- {
- ensure_str(k): str(v) # python 2 requires these to be string only (non-unicode)
- for k, v in {"PIP_USE_WHEEL": "1", "PIP_USER": "0", "PIP_NO_INPUT": "1"}.items()
- },
- )
- with ensure_file_on_disk(get_bundled_wheel("pip", version), app_data) as pip_wheel_path:
- # put the bundled wheel onto the path, and use it to do the bootstrap operation
- env[str("PYTHONPATH")] = str(pip_wheel_path)
- yield env
diff --git a/src/virtualenv/seed/via_app_data/via_app_data.py b/src/virtualenv/seed/via_app_data/via_app_data.py
deleted file mode 100644
index de3757d..0000000
--- a/src/virtualenv/seed/via_app_data/via_app_data.py
+++ /dev/null
@@ -1,128 +0,0 @@
-"""Bootstrap"""
-from __future__ import absolute_import, unicode_literals
-
-import logging
-from contextlib import contextmanager
-from functools import partial
-from threading import Lock, Thread
-
-from virtualenv.info import fs_supports_symlink
-from virtualenv.seed.embed.base_embed import BaseEmbed
-from virtualenv.seed.embed.wheels.acquire import WheelDownloadFail, get_wheels
-from virtualenv.util.path import safe_delete
-
-from .pip_install.copy import CopyPipInstall
-from .pip_install.symlink import SymlinkPipInstall
-
-
-class FromAppData(BaseEmbed):
- def __init__(self, options):
- super(FromAppData, self).__init__(options)
- self.symlinks = options.symlink_app_data
- self.base_cache = self.app_data / "seed-app-data" / "v1.0.1"
-
- @classmethod
- def add_parser_arguments(cls, parser, interpreter, app_data):
- super(FromAppData, cls).add_parser_arguments(parser, interpreter, app_data)
- can_symlink = app_data.transient is False and fs_supports_symlink()
- parser.add_argument(
- "--symlink-app-data",
- dest="symlink_app_data",
- action="store_true" if can_symlink else "store_false",
- help="{} symlink the python packages from the app-data folder (requires seed pip>=19.3)".format(
- "" if can_symlink else "not supported - ",
- ),
- default=False,
- )
-
- def run(self, creator):
- if not self.enabled:
- return
- base_cache = self.base_cache / creator.interpreter.version_release_str
- with self._get_seed_wheels(creator, base_cache) as name_to_whl:
- pip_version = name_to_whl["pip"].stem.split("-")[1] if "pip" in name_to_whl else None
- installer_class = self.installer_class(pip_version)
-
- def _install(name, wheel):
- logging.debug("install %s from wheel %s via %s", name, wheel, installer_class.__name__)
- image_folder = base_cache.path / "image" / installer_class.__name__ / wheel.stem
- installer = installer_class(wheel, creator, image_folder)
- if not installer.has_image():
- installer.build_image()
- installer.install(creator.interpreter.version_info)
-
- threads = list(Thread(target=_install, args=(n, w)) for n, w in name_to_whl.items())
- for thread in threads:
- thread.start()
- for thread in threads:
- thread.join()
-
- @contextmanager
- def _get_seed_wheels(self, creator, base_cache):
- with base_cache.lock_for_key("wheels"):
- wheels_to = base_cache.path / "wheels"
- if wheels_to.exists():
- safe_delete(wheels_to)
- wheels_to.mkdir(parents=True, exist_ok=True)
- name_to_whl, lock, fail = {}, Lock(), {}
-
- def _get(package, version):
- wheel_loader = partial(
- get_wheels,
- creator.interpreter.version_release_str,
- wheels_to,
- self.extra_search_dir,
- {package: version},
- self.app_data,
- )
- failure, result = None, None
- # fallback to download in case the exact version is not available
- for download in [True] if self.download else [False, True]:
- failure = None
- try:
- result = wheel_loader(download)
- if result:
- break
- except Exception as exception:
- failure = exception
- if failure:
- if isinstance(failure, WheelDownloadFail):
- msg = "failed to download {}".format(package)
- if version is not None:
- msg += " version {}".format(version)
- msg += ", pip download exit code {}".format(failure.exit_code)
- output = failure.out + failure.err
- if output:
- msg += "\n"
- msg += output
- else:
- msg = repr(failure)
- logging.error(msg)
- with lock:
- fail[package] = version
- else:
- with lock:
- name_to_whl.update(result)
-
- package_versions = self.package_version()
- threads = list(Thread(target=_get, args=(pkg, v)) for pkg, v in package_versions.items())
- for thread in threads:
- thread.start()
- for thread in threads:
- thread.join()
- if fail:
- raise RuntimeError("seed failed due to failing to download wheels {}".format(", ".join(fail.keys())))
- yield name_to_whl
-
- def installer_class(self, pip_version):
- if self.symlinks and pip_version:
- # symlink support requires pip 19.3+
- pip_version_int = tuple(int(i) for i in pip_version.split(".")[0:2])
- if pip_version_int >= (19, 3):
- return SymlinkPipInstall
- return CopyPipInstall
-
- def __unicode__(self):
- base = super(FromAppData, self).__unicode__()
- msg = ", via={}, app_data_dir={}".format("symlink" if self.symlinks else "copy", self.base_cache.path)
- return base[:-1] + msg + base[-1]
diff --git a/src/virtualenv/seed/wheels/__init__.py b/src/virtualenv/seed/wheels/__init__.py
new file mode 100644
index 0000000..dbffe2e
--- /dev/null
+++ b/src/virtualenv/seed/wheels/__init__.py
@@ -0,0 +1,11 @@
+from __future__ import absolute_import, unicode_literals
+
+from .acquire import get_wheel, pip_wheel_env_run
+from .util import Version, Wheel
+
+__all__ = (
+ "get_wheel",
+ "pip_wheel_env_run",
+ "Version",
+ "Wheel",
+)
diff --git a/src/virtualenv/seed/wheels/acquire.py b/src/virtualenv/seed/wheels/acquire.py
new file mode 100644
index 0000000..8c88725
--- /dev/null
+++ b/src/virtualenv/seed/wheels/acquire.py
@@ -0,0 +1,114 @@
+"""Bootstrap"""
+from __future__ import absolute_import, unicode_literals
+
+import logging
+import os
+import sys
+from operator import eq, lt
+
+import six
+
+from virtualenv.util.path import Path
+from virtualenv.util.six import ensure_str
+from virtualenv.util.subprocess import Popen, subprocess
+
+from .bundle import from_bundle
+from .util import Version, Wheel, discover_wheels
+
+
+def get_wheel(distribution, version, for_py_version, search_dirs, download, app_data, do_periodic_update):
+ """
+ Get a wheel with the given distribution-version-for_py_version trio, by using the extra search dir + download
+ """
+ # not all wheels are compatible with all python versions, so we need to py version qualify it
+ # 1. acquire from bundle
+ wheel = from_bundle(distribution, version, for_py_version, search_dirs, app_data, do_periodic_update)
+
+ # 2. download from the internet
+ if version not in Version.non_version and download:
+ wheel = download_wheel(
+ distribution=distribution,
+ version_spec=Version.as_version_spec(version),
+ for_py_version=for_py_version,
+ search_dirs=search_dirs,
+ app_data=app_data,
+ to_folder=app_data.house,
+ )
+ return wheel
+
+
+def download_wheel(distribution, version_spec, for_py_version, search_dirs, app_data, to_folder):
+ to_download = "{}{}".format(distribution, version_spec or "")
+ logging.debug("download wheel %s", to_download)
+ cmd = [
+ sys.executable,
+ "-m",
+ "pip",
+ "download",
+ "--disable-pip-version-check",
+ "--only-binary=:all:",
+ "--no-deps",
+ "--python-version",
+ for_py_version,
+ "-d",
+ str(to_folder),
+ to_download,
+ ]
+ # pip has no interface in python - must be a new sub-process
+ env = pip_wheel_env_run(search_dirs, app_data)
+ process = Popen(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
+ out, err = process.communicate()
+ if process.returncode != 0:
+ kwargs = {"output": out}
+ if six.PY2:
+ kwargs["output"] += err
+ else:
+ kwargs["stderr"] = err
+ raise subprocess.CalledProcessError(process.returncode, cmd, **kwargs)
+ for line in out.splitlines():
+ line = line.lstrip()
+ for marker in ("Saved ", "File was already downloaded "):
+ if line.startswith(marker):
+ return Wheel(Path(line[len(marker) :]).absolute())
+ # if for some reason the output does not match fallback to latest version with that spec
+ return find_compatible_in_house(distribution, version_spec, for_py_version, to_folder)
+
+
+def find_compatible_in_house(distribution, version_spec, for_py_version, in_folder):
+ wheels = discover_wheels(in_folder, distribution, None, for_py_version)
+ start, end = 0, len(wheels)
+ if version_spec is not None:
+ if version_spec.startswith("<"):
+ from_pos, op = 1, lt
+ elif version_spec.startswith("=="):
+ from_pos, op = 2, eq
+ else:
+ raise ValueError(version_spec)
+ version = Wheel.as_version_tuple(version_spec[from_pos:])
+ start = next((at for at, w in enumerate(wheels) if op(w.version_tuple, version)), len(wheels))
+
+ return None if start == end else wheels[start]
+
+
+def pip_wheel_env_run(search_dirs, app_data):
+ for_py_version = "{}.{}".format(*sys.version_info[0:2])
+ env = os.environ.copy()
+ env.update(
+ {
+ ensure_str(k): str(v) # python 2 requires these to be string only (non-unicode)
+ for k, v in {"PIP_USE_WHEEL": "1", "PIP_USER": "0", "PIP_NO_INPUT": "1"}.items()
+ },
+ )
+ wheel = get_wheel(
+ distribution="pip",
+ version=None,
+ for_py_version=for_py_version,
+ search_dirs=search_dirs,
+ download=False,
+ app_data=app_data,
+ do_periodic_update=False,
+ )
+ if wheel is None:
+ raise RuntimeError("could not find the embedded pip")
+ env[str("PYTHONPATH")] = str(wheel.path)
+ return env
diff --git a/src/virtualenv/seed/wheels/bundle.py b/src/virtualenv/seed/wheels/bundle.py
new file mode 100644
index 0000000..6ac15f9
--- /dev/null
+++ b/src/virtualenv/seed/wheels/bundle.py
@@ -0,0 +1,51 @@
+from __future__ import absolute_import, unicode_literals
+
+from virtualenv.app_data import AppDataDiskFolder, TempAppData
+
+from ..wheels.embed import get_embed_wheel
+from .periodic_update import periodic_update
+from .util import Version, Wheel, discover_wheels
+
+
+def from_bundle(distribution, version, for_py_version, search_dirs, app_data, do_periodic_update):
+ """
+ Load the bundled wheel to a cache directory.
+ """
+ of_version = Version.of_version(version)
+ wheel = load_embed_wheel(app_data, distribution, for_py_version, of_version)
+
+ if version != Version.embed:
+ # 2. check if we have upgraded embed
+ if isinstance(app_data, AppDataDiskFolder) and not isinstance(app_data, TempAppData):
+ wheel = periodic_update(distribution, for_py_version, wheel, search_dirs, app_data, do_periodic_update)
+
+ # 3. acquire from extra search dir
+ found_wheel = from_dir(distribution, of_version, for_py_version, search_dirs)
+ if found_wheel is not None:
+ if wheel is None:
+ wheel = found_wheel
+ elif found_wheel.version_tuple > wheel.version_tuple:
+ wheel = found_wheel
+ return wheel
+
+
+def load_embed_wheel(app_data, distribution, for_py_version, version):
+ wheel = get_embed_wheel(distribution, for_py_version)
+ if wheel is not None:
+ version_match = version == wheel.version
+ if version is None or version_match:
+ with app_data.ensure_extracted(wheel.path, lambda: app_data.house) as wheel_path:
+ wheel = Wheel(wheel_path)
+ else: # if version does not match ignore
+ wheel = None
+ return wheel
+
+
+def from_dir(distribution, version, for_py_version, directories):
+ """
+ Load a compatible wheel from a given folder.
+ """
+ for folder in directories:
+ for wheel in discover_wheels(folder, distribution, version, for_py_version):
+ return wheel
+ return None
diff --git a/src/virtualenv/seed/embed/wheels/__init__.py b/src/virtualenv/seed/wheels/embed/__init__.py
index 28fd0c4..17860e0 100644
--- a/src/virtualenv/seed/embed/wheels/__init__.py
+++ b/src/virtualenv/seed/wheels/embed/__init__.py
@@ -1,6 +1,15 @@
from __future__ import absolute_import, unicode_literals
+from virtualenv.seed.wheels.util import Wheel
+from virtualenv.util.path import Path
+
+BUNDLE_FOLDER = Path(__file__).absolute().parent
BUNDLE_SUPPORT = {
+ "3.10": {
+ "pip": "pip-20.1.1-py2.py3-none-any.whl",
+ "setuptools": "setuptools-47.1.1-py3-none-any.whl",
+ "wheel": "wheel-0.34.2-py2.py3-none-any.whl",
+ },
"3.9": {
"pip": "pip-20.1.1-py2.py3-none-any.whl",
"setuptools": "setuptools-47.1.1-py3-none-any.whl",
@@ -37,4 +46,17 @@ BUNDLE_SUPPORT = {
"wheel": "wheel-0.34.2-py2.py3-none-any.whl",
},
}
-MAX = "3.9"
+MAX = "3.10"
+
+
+def get_embed_wheel(distribution, for_py_version):
+ path = BUNDLE_FOLDER / (BUNDLE_SUPPORT.get(for_py_version, {}) or BUNDLE_SUPPORT[MAX]).get(distribution)
+ return Wheel.from_path(path)
+
+
+__all__ = (
+ "get_embed_wheel",
+ "BUNDLE_SUPPORT",
+ "MAX",
+ "BUNDLE_FOLDER",
+)
diff --git a/src/virtualenv/seed/embed/wheels/pip-19.1.1-py2.py3-none-any.whl b/src/virtualenv/seed/wheels/embed/pip-19.1.1-py2.py3-none-any.whl
index 8476c11..8476c11 100644
--- a/src/virtualenv/seed/embed/wheels/pip-19.1.1-py2.py3-none-any.whl
+++ b/src/virtualenv/seed/wheels/embed/pip-19.1.1-py2.py3-none-any.whl
Binary files differ
diff --git a/src/virtualenv/seed/embed/wheels/pip-20.1.1-py2.py3-none-any.whl b/src/virtualenv/seed/wheels/embed/pip-20.1.1-py2.py3-none-any.whl
index ea1d0f7..ea1d0f7 100644
--- a/src/virtualenv/seed/embed/wheels/pip-20.1.1-py2.py3-none-any.whl
+++ b/src/virtualenv/seed/wheels/embed/pip-20.1.1-py2.py3-none-any.whl
Binary files differ
diff --git a/src/virtualenv/seed/embed/wheels/setuptools-43.0.0-py2.py3-none-any.whl b/src/virtualenv/seed/wheels/embed/setuptools-43.0.0-py2.py3-none-any.whl
index 733faa6..733faa6 100644
--- a/src/virtualenv/seed/embed/wheels/setuptools-43.0.0-py2.py3-none-any.whl
+++ b/src/virtualenv/seed/wheels/embed/setuptools-43.0.0-py2.py3-none-any.whl
Binary files differ
diff --git a/src/virtualenv/seed/embed/wheels/setuptools-44.1.1-py2.py3-none-any.whl b/src/virtualenv/seed/wheels/embed/setuptools-44.1.1-py2.py3-none-any.whl
index bf28513..bf28513 100644
--- a/src/virtualenv/seed/embed/wheels/setuptools-44.1.1-py2.py3-none-any.whl
+++ b/src/virtualenv/seed/wheels/embed/setuptools-44.1.1-py2.py3-none-any.whl
Binary files differ
diff --git a/src/virtualenv/seed/embed/wheels/setuptools-47.1.1-py3-none-any.whl b/src/virtualenv/seed/wheels/embed/setuptools-47.1.1-py3-none-any.whl
index e226baf..e226baf 100644
--- a/src/virtualenv/seed/embed/wheels/setuptools-47.1.1-py3-none-any.whl
+++ b/src/virtualenv/seed/wheels/embed/setuptools-47.1.1-py3-none-any.whl
Binary files differ
diff --git a/src/virtualenv/seed/embed/wheels/wheel-0.33.6-py2.py3-none-any.whl b/src/virtualenv/seed/wheels/embed/wheel-0.33.6-py2.py3-none-any.whl
index 2a71896..2a71896 100644
--- a/src/virtualenv/seed/embed/wheels/wheel-0.33.6-py2.py3-none-any.whl
+++ b/src/virtualenv/seed/wheels/embed/wheel-0.33.6-py2.py3-none-any.whl
Binary files differ
diff --git a/src/virtualenv/seed/embed/wheels/wheel-0.34.2-py2.py3-none-any.whl b/src/virtualenv/seed/wheels/embed/wheel-0.34.2-py2.py3-none-any.whl
index becbee8..becbee8 100644
--- a/src/virtualenv/seed/embed/wheels/wheel-0.34.2-py2.py3-none-any.whl
+++ b/src/virtualenv/seed/wheels/embed/wheel-0.34.2-py2.py3-none-any.whl
Binary files differ
diff --git a/src/virtualenv/seed/wheels/periodic_update.py b/src/virtualenv/seed/wheels/periodic_update.py
new file mode 100644
index 0000000..25270ad
--- /dev/null
+++ b/src/virtualenv/seed/wheels/periodic_update.py
@@ -0,0 +1,311 @@
+"""
+Periodically update bundled versions.
+"""
+
+from __future__ import absolute_import, unicode_literals
+
+import json
+import logging
+import os
+import subprocess
+import sys
+from datetime import datetime, timedelta
+from itertools import groupby
+from shutil import copy2
+from threading import Thread
+
+from six.moves.urllib.request import urlopen
+
+from virtualenv.app_data import AppDataDiskFolder
+from virtualenv.info import PY2
+from virtualenv.util.path import Path
+from virtualenv.util.subprocess import DETACHED_PROCESS, Popen
+
+from ..wheels.embed import BUNDLE_SUPPORT
+from ..wheels.util import Wheel
+
+if PY2:
+ # on Python 2 datetime.strptime throws the error below if the import did not trigger on main thread
+ # Failed to import _strptime because the import lock is held by
+ try:
+ import _strptime # noqa
+ except ImportError: # pragma: no cov
+ pass # pragma: no cov
+
+
+def periodic_update(distribution, for_py_version, wheel, search_dirs, app_data, do_periodic_update):
+ if do_periodic_update:
+ handle_auto_update(distribution, for_py_version, wheel, search_dirs, app_data)
+
+ now = datetime.now()
+
+ u_log = UpdateLog.from_app_data(app_data, distribution, for_py_version)
+ u_log_older_than_hour = now - u_log.completed > timedelta(hours=1) if u_log.completed is not None else False
+ for _, group in groupby(u_log.versions, key=lambda v: v.wheel.version_tuple[0:2]):
+ version = next(group) # use only latest patch version per minor, earlier assumed to be buggy
+ if wheel is not None and Path(version.filename).name == wheel.name:
+ break
+ if u_log.periodic is False or (u_log_older_than_hour and version.use(now)):
+ updated_wheel = Wheel(app_data.house / version.filename)
+ logging.debug("using %supdated wheel %s", "periodically " if updated_wheel else "", updated_wheel)
+ wheel = updated_wheel
+ break
+
+ return wheel
+
+
+def handle_auto_update(distribution, for_py_version, wheel, search_dirs, app_data):
+ embed_update_log = app_data.embed_update_log(distribution, for_py_version)
+ u_log = UpdateLog.from_dict(embed_update_log.read())
+ if u_log.needs_update:
+ u_log.periodic = True
+ u_log.started = datetime.now()
+ embed_update_log.write(u_log.to_dict())
+ trigger_update(distribution, for_py_version, wheel, search_dirs, app_data, periodic=True)
+
+
+DATETIME_FMT = "%Y-%m-%dT%H:%M:%S.%fZ"
+
+
+def dump_datetime(value):
+ return None if value is None else value.strftime(DATETIME_FMT)
+
+
+def load_datetime(value):
+ return None if value is None else datetime.strptime(value, DATETIME_FMT)
+
+
+class NewVersion(object):
+ def __init__(self, filename, found_date, release_date):
+ self.filename = filename
+ self.found_date = found_date
+ self.release_date = release_date
+
+ @classmethod
+ def from_dict(cls, dictionary):
+ return cls(
+ filename=dictionary["filename"],
+ found_date=load_datetime(dictionary["found_date"]),
+ release_date=load_datetime(dictionary["release_date"]),
+ )
+
+ def to_dict(self):
+ return {
+ "filename": self.filename,
+ "release_date": dump_datetime(self.release_date),
+ "found_date": dump_datetime(self.found_date),
+ }
+
+ def use(self, now):
+ compare_from = self.release_date or self.found_date
+ return now - compare_from >= timedelta(days=28)
+
+ def __repr__(self):
+ return "{}(filename={}), found_date={}, release_date={})".format(
+ self.__class__.__name__, self.filename, self.found_date, self.release_date,
+ )
+
+ def __eq__(self, other):
+ return type(self) == type(other) and all(
+ getattr(self, k) == getattr(other, k) for k in ["filename", "release_date", "found_date"]
+ )
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ @property
+ def wheel(self):
+ return Wheel(Path(self.filename))
+
+
+class UpdateLog(object):
+ def __init__(self, started, completed, versions, periodic):
+ self.started = started
+ self.completed = completed
+ self.versions = versions
+ self.periodic = periodic
+
+ @classmethod
+ def from_dict(cls, dictionary):
+ if dictionary is None:
+ dictionary = {}
+ return cls(
+ load_datetime(dictionary.get("started")),
+ load_datetime(dictionary.get("completed")),
+ [NewVersion.from_dict(v) for v in dictionary.get("versions", [])],
+ dictionary.get("periodic"),
+ )
+
+ @classmethod
+ def from_app_data(cls, app_data, distribution, for_py_version):
+ raw_json = app_data.embed_update_log(distribution, for_py_version).read()
+ return cls.from_dict(raw_json)
+
+ def to_dict(self):
+ return {
+ "started": dump_datetime(self.started),
+ "completed": dump_datetime(self.completed),
+ "periodic": self.periodic,
+ "versions": [r.to_dict() for r in self.versions],
+ }
+
+ @property
+ def needs_update(self):
+ now = datetime.now()
+ if self.completed is None: # never completed
+ return self._check_start(now)
+ else:
+ if now - self.completed <= timedelta(days=14):
+ return False
+ return self._check_start(now)
+
+ def _check_start(self, now):
+ return self.started is None or now - self.started > timedelta(hours=1)
+
+
+def trigger_update(distribution, for_py_version, wheel, search_dirs, app_data, periodic):
+ wheel_path = None if wheel is None else str(wheel.path)
+ cmd = [
+ sys.executable,
+ "-c",
+ "from virtualenv.seed.wheels.periodic_update import do_update;"
+ "do_update({!r}, {!r}, {!r}, {!r}, {!r}, {!r})".format(
+ distribution, for_py_version, wheel_path, str(app_data), [str(p) for p in search_dirs], periodic,
+ ),
+ ]
+ debug = os.environ.get(str("_VIRTUALENV_PERIODIC_UPDATE_INLINE")) == str("1")
+ pipe = None if debug else subprocess.PIPE
+ kwargs = {"stdout": pipe, "stderr": pipe}
+ if not debug and sys.platform == "win32":
+ kwargs["creationflags"] = DETACHED_PROCESS
+ process = Popen(cmd, **kwargs)
+ logging.info(
+ "triggered periodic upgrade of %s%s (for python %s) via background process having PID %d",
+ distribution,
+ "" if wheel is None else "=={}".format(wheel.version),
+ for_py_version,
+ process.pid,
+ )
+ if debug:
+ process.communicate() # on purpose not called to make it a background process
+
+
+def do_update(distribution, for_py_version, embed_filename, app_data, search_dirs, periodic):
+
+ from virtualenv.seed.wheels import acquire
+
+ wheel_filename = None if embed_filename is None else Path(embed_filename)
+ app_data = AppDataDiskFolder(app_data) if isinstance(app_data, str) else app_data
+ search_dirs = [Path(p) if isinstance(p, str) else p for p in search_dirs]
+ wheelhouse = app_data.house
+ embed_update_log = app_data.embed_update_log(distribution, for_py_version)
+ u_log = UpdateLog.from_dict(embed_update_log.read())
+
+ now = datetime.now()
+
+ if wheel_filename is not None:
+ dest = wheelhouse / wheel_filename.name
+ if not dest.exists():
+ copy2(str(wheel_filename), str(wheelhouse))
+
+ last, versions = None, []
+ while last is None or not last.use(now):
+ download_time = datetime.now()
+ dest = acquire.download_wheel(
+ distribution=distribution,
+ version_spec=None if last is None else "<{}".format(Wheel(Path(last.filename)).version),
+ for_py_version=for_py_version,
+ search_dirs=search_dirs,
+ app_data=app_data,
+ to_folder=wheelhouse,
+ )
+ if dest is None or (u_log.versions and u_log.versions[0].filename == dest.name):
+ break
+ release_date = _get_release_date(dest.path)
+ last = NewVersion(filename=dest.path.name, release_date=release_date, found_date=download_time)
+ logging.info("detected %s in %s", last, datetime.now() - download_time)
+ versions.append(last)
+ u_log.periodic = periodic
+ if not u_log.periodic:
+ u_log.started = now
+ u_log.versions = versions + u_log.versions
+ u_log.completed = datetime.now()
+ embed_update_log.write(u_log.to_dict())
+ return versions
+
+
+def _get_release_date(dest):
+ wheel = Wheel(dest)
+ # the most accurate is to ask PyPi - e.g. https://pypi.org/pypi/pip/json,
+ # see https://warehouse.pypa.io/api-reference/json/ for more details
+ try:
+ with urlopen("https://pypi.org/pypi/{}/json".format(wheel.distribution)) as file_handler:
+ content = json.load(file_handler)
+ return datetime.strptime(content["releases"][wheel.version][0]["upload_time"], "%Y-%m-%dT%H:%M:%S")
+ except Exception: # noqa
+ return None
+
+
+def manual_upgrade(app_data):
+ threads = []
+
+ for for_py_version, distribution_to_package in BUNDLE_SUPPORT.items():
+ # load extra search dir for the given for_py
+ for distribution in distribution_to_package.keys():
+ thread = Thread(target=_run_manual_upgrade, args=(app_data, distribution, for_py_version))
+ thread.start()
+ threads.append(thread)
+
+ for thread in threads:
+ thread.join()
+
+
+def _run_manual_upgrade(app_data, distribution, for_py_version):
+ start = datetime.now()
+ from .bundle import from_bundle
+
+ current = from_bundle(
+ distribution=distribution,
+ version=None,
+ for_py_version=for_py_version,
+ search_dirs=[],
+ app_data=app_data,
+ do_periodic_update=False,
+ )
+ logging.warning(
+ "upgrade %s for python %s with current %s",
+ distribution,
+ for_py_version,
+ "" if current is None else current.name,
+ )
+ versions = do_update(
+ distribution=distribution,
+ for_py_version=for_py_version,
+ embed_filename=current.path,
+ app_data=app_data,
+ search_dirs=[],
+ periodic=False,
+ )
+ msg = "upgraded %s for python %s in %s {}".format(
+ "new entries found:\n%s" if versions else "no new versions found",
+ )
+ args = [
+ distribution,
+ for_py_version,
+ datetime.now() - start,
+ ]
+ if versions:
+ args.append("\n".join("\t{}".format(v) for v in versions))
+ logging.warning(msg, *args)
+
+
+__all__ = (
+ "periodic_update",
+ "do_update",
+ "manual_upgrade",
+ "NewVersion",
+ "UpdateLog",
+ "load_datetime",
+ "dump_datetime",
+ "trigger_update",
+)
diff --git a/src/virtualenv/seed/wheels/util.py b/src/virtualenv/seed/wheels/util.py
new file mode 100644
index 0000000..1240eb2
--- /dev/null
+++ b/src/virtualenv/seed/wheels/util.py
@@ -0,0 +1,116 @@
+from __future__ import absolute_import, unicode_literals
+
+from operator import attrgetter
+from zipfile import ZipFile
+
+from virtualenv.util.six import ensure_text
+
+
+class Wheel(object):
+ def __init__(self, path):
+ # https://www.python.org/dev/peps/pep-0427/#file-name-convention
+ # The wheel filename is {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl
+ self.path = path
+ self._parts = path.stem.split("-")
+
+ @classmethod
+ def from_path(cls, path):
+ if path is not None and path.suffix == ".whl" and len(path.stem.split("-")) >= 5:
+ return cls(path)
+ return None
+
+ @property
+ def distribution(self):
+ return self._parts[0]
+
+ @property
+ def version(self):
+ return self._parts[1]
+
+ @property
+ def version_tuple(self):
+ return self.as_version_tuple(self.version)
+
+ @staticmethod
+ def as_version_tuple(version):
+ result = []
+ for part in version.split(".")[0:3]:
+ try:
+ result.append(int(part))
+ except ValueError:
+ break
+ if not result:
+ raise ValueError(version)
+ return tuple(result)
+
+ @property
+ def name(self):
+ return self.path.name
+
+ def support_py(self, py_version):
+ name = "{}.dist-info/METADATA".format("-".join(self.path.stem.split("-")[0:2]))
+ with ZipFile(ensure_text(str(self.path)), "r") as zip_file:
+ metadata = zip_file.read(name).decode("utf-8")
+ marker = "Requires-Python:"
+ requires = next((i[len(marker) :] for i in metadata.splitlines() if i.startswith(marker)), None)
+ if requires is None: # if it does not specify a python requires the assumption is compatible
+ return True
+ py_version_int = tuple(int(i) for i in py_version.split("."))
+ for require in (i.strip() for i in requires.split(",")):
+ # https://www.python.org/dev/peps/pep-0345/#version-specifiers
+ for operator, check in [
+ ("!=", lambda v: py_version_int != v),
+ ("==", lambda v: py_version_int == v),
+ ("<=", lambda v: py_version_int <= v),
+ (">=", lambda v: py_version_int >= v),
+ ("<", lambda v: py_version_int < v),
+ (">", lambda v: py_version_int > v),
+ ]:
+ if require.startswith(operator):
+ ver_str = require[len(operator) :].strip()
+ version = tuple((int(i) if i != "*" else None) for i in ver_str.split("."))[0:2]
+ if not check(version):
+ return False
+ break
+ return True
+
+ def __repr__(self):
+ return "{}({})".format(self.__class__.__name__, self.path)
+
+ def __str__(self):
+ return str(self.path)
+
+
+def discover_wheels(from_folder, distribution, version, for_py_version):
+ wheels = []
+ for filename in from_folder.iterdir():
+ wheel = Wheel.from_path(filename)
+ if wheel and wheel.distribution == distribution:
+ if version is None or wheel.version == version:
+ if wheel.support_py(for_py_version):
+ wheels.append(wheel)
+ return sorted(wheels, key=attrgetter("version_tuple", "distribution"), reverse=True)
+
+
+class Version:
+ #: the version bundled with virtualenv
+ bundle = "bundle"
+ embed = "embed"
+ #: custom version handlers
+ non_version = (
+ bundle,
+ embed,
+ )
+
+ @staticmethod
+ def of_version(value):
+ return None if value in Version.non_version else value
+
+ @staticmethod
+ def as_pip_req(distribution, version):
+ return "{}{}".format(distribution, Version.as_version_spec(version))
+
+ @staticmethod
+ def as_version_spec(version):
+ of_version = Version.of_version(version)
+ return "" if of_version is None else "=={}".format(of_version)
diff --git a/src/virtualenv/error.py b/src/virtualenv/util/error.py
index ac5aa50..ac5aa50 100644
--- a/src/virtualenv/error.py
+++ b/src/virtualenv/util/error.py
diff --git a/src/virtualenv/util/lock.py b/src/virtualenv/util/lock.py
index 0c5e72f..eb7a78f 100644
--- a/src/virtualenv/util/lock.py
+++ b/src/virtualenv/util/lock.py
@@ -74,7 +74,7 @@ class ReentrantFileLock(object):
def __exit__(self, exc_type, exc_val, exc_tb):
self._release(self._lock)
- def _lock_file(self, lock):
+ def _lock_file(self, lock, no_block=False):
# multiple processes might be trying to get a first lock... so we cannot check if this directory exist without
# a lock, but that lock might then become expensive, and it's not clear where that lock should live.
# Instead here we just ignore if we fail to create the directory.
@@ -85,6 +85,8 @@ class ReentrantFileLock(object):
try:
lock.acquire(0.0001)
except Timeout:
+ if no_block:
+ raise
logging.debug("lock file %s present, will block until released", lock.lock_file)
lock.release() # release the acquire try from above
lock.acquire()
@@ -94,13 +96,19 @@ class ReentrantFileLock(object):
lock.release()
@contextmanager
- def lock_for_key(self, name):
+ def lock_for_key(self, name, no_block=False):
lock = self._create_lock(name)
try:
try:
- self._lock_file(lock)
+ self._lock_file(lock, no_block)
yield
finally:
self._release(lock)
finally:
self._del_lock(lock)
+
+
+__all__ = (
+ "Timeout",
+ "ReentrantFileLock",
+)
diff --git a/src/virtualenv/util/path/_pathlib/via_os_path.py b/src/virtualenv/util/path/_pathlib/via_os_path.py
index 3afbe35..d11aeaa 100644
--- a/src/virtualenv/util/path/_pathlib/via_os_path.py
+++ b/src/virtualenv/util/path/_pathlib/via_os_path.py
@@ -74,8 +74,11 @@ class Path(object):
return os.path.isdir(self._path)
def mkdir(self, parents=True, exist_ok=True):
- if not self.exists() and exist_ok:
+ try:
os.makedirs(self._path)
+ except OSError:
+ if not exist_ok:
+ raise
def read_text(self, encoding="utf-8"):
return self.read_bytes().decode(encoding)
@@ -135,5 +138,8 @@ class Path(object):
def chmod(self, mode):
os.chmod(self._path, mode)
+ def absolute(self):
+ return Path(os.path.abspath(self._path))
+
__all__ = ("Path",)
diff --git a/src/virtualenv/util/subprocess/__init__.py b/src/virtualenv/util/subprocess/__init__.py
index 6cb0a28..22006da 100644
--- a/src/virtualenv/util/subprocess/__init__.py
+++ b/src/virtualenv/util/subprocess/__init__.py
@@ -13,6 +13,9 @@ else:
Popen = subprocess.Popen
+DETACHED_PROCESS = 0x00000008
+
+
def run_cmd(cmd):
try:
process = Popen(
@@ -29,4 +32,5 @@ __all__ = (
"subprocess",
"Popen",
"run_cmd",
+ "DETACHED_PROCESS",
)
diff --git a/src/virtualenv/util/zipapp.py b/src/virtualenv/util/zipapp.py
index 36cee5e..85d9294 100644
--- a/src/virtualenv/util/zipapp.py
+++ b/src/virtualenv/util/zipapp.py
@@ -3,13 +3,9 @@ from __future__ import absolute_import, unicode_literals
import logging
import os
import zipfile
-from contextlib import contextmanager
-from tempfile import TemporaryFile
-from virtualenv.info import IS_WIN, IS_ZIPAPP, ROOT
-from virtualenv.util.path import Path
+from virtualenv.info import IS_WIN, ROOT
from virtualenv.util.six import ensure_text
-from virtualenv.version import __version__
def read(full_path):
@@ -35,22 +31,3 @@ def _get_path_within_zip(full_path):
# paths are always UNIX separators, even on Windows, though __file__ still follows platform default
sub_file = sub_file.replace(os.sep, "/")
return sub_file
-
-
-@contextmanager
-def ensure_file_on_disk(path, app_data):
- if IS_ZIPAPP:
- if app_data is None:
- with TemporaryFile() as temp_file:
- dest = Path(temp_file.name)
- extract(path, dest)
- yield Path(dest)
- else:
- base = app_data / "zipapp" / "extract" / __version__
- with base.lock_for_key(path.name):
- dest = base.path / path.name
- if not dest.exists():
- extract(path, dest)
- yield dest
- else:
- yield path
diff --git a/tasks/upgrade_wheels.py b/tasks/upgrade_wheels.py
index 8de3eea..fe0010a 100644
--- a/tasks/upgrade_wheels.py
+++ b/tasks/upgrade_wheels.py
@@ -10,18 +10,31 @@ import sys
from collections import OrderedDict, defaultdict
from pathlib import Path
from tempfile import TemporaryDirectory
+from textwrap import dedent
from threading import Thread
STRICT = "UPGRADE_ADVISORY" not in os.environ
BUNDLED = ["pip", "setuptools", "wheel"]
-SUPPORT = list(reversed([(2, 7)] + [(3, i) for i in range(4, 10)]))
-DEST = Path(__file__).resolve().parents[1] / "src" / "virtualenv" / "seed" / "embed" / "wheels"
+SUPPORT = list(reversed([(2, 7)] + [(3, i) for i in range(4, 11)]))
+DEST = Path(__file__).resolve().parents[1] / "src" / "virtualenv" / "seed" / "wheels" / "embed"
def download(ver, dest, package):
subprocess.call(
- [sys.executable, "-m", "pip", "download", "--only-binary=:all:", "--python-version", ver, "-d", dest, package],
+ [
+ sys.executable,
+ "-m",
+ "pip",
+ "--disable-pip-version-check",
+ "download",
+ "--only-binary=:all:",
+ "--python-version",
+ ver,
+ "-d",
+ dest,
+ package,
+ ],
)
@@ -72,12 +85,37 @@ def run():
support_table[version].append(package)
support_table = {k: OrderedDict((i.split("-")[0], i) for i in v) for k, v in support_table.items()}
- msg = "from __future__ import absolute_import, unicode_literals; BUNDLE_SUPPORT = {{ {} }}; MAX = {!r}".format(
- ",".join(
- "{!r}: {{ {} }}".format(v, ",".join("{!r}: {!r}".format(p, f) for p, f in l.items()))
- for v, l in support_table.items()
+ msg = dedent(
+ """
+ from __future__ import absolute_import, unicode_literals
+
+ from virtualenv.seed.wheels.util import Wheel
+ from virtualenv.util.path import Path
+
+ BUNDLE_FOLDER = Path(__file__).absolute().parent
+ BUNDLE_SUPPORT = {{ {0} }}
+ MAX = {1}
+
+
+ def get_embed_wheel(distribution, for_py_version):
+ path = BUNDLE_FOLDER / (BUNDLE_SUPPORT.get(for_py_version, {{}}) or BUNDLE_SUPPORT[MAX]).get(distribution)
+ return Wheel.from_path(path)
+
+
+ __all__ = (
+ "get_embed_wheel",
+ "BUNDLE_SUPPORT",
+ "MAX",
+ "BUNDLE_FOLDER",
+ )
+
+ """.format(
+ ",".join(
+ "{!r}: {{ {} }}".format(v, ",".join("{!r}: {!r}".format(p, f) for p, f in l.items()))
+ for v, l in support_table.items()
+ ),
+ repr(next(iter(support_table.keys()))),
),
- next(iter(support_table.keys())),
)
dest_target = DEST / "__init__.py"
dest_target.write_text(msg)
diff --git a/tests/conftest.py b/tests/conftest.py
index a92b893..97e109e 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -12,11 +12,11 @@ import coverage
import pytest
import six
+from virtualenv.app_data import AppDataDiskFolder
from virtualenv.discovery.builtin import get_interpreter
from virtualenv.discovery.py_info import PythonInfo
from virtualenv.info import IS_PYPY, IS_WIN, fs_supports_symlink
from virtualenv.report import LOGGER
-from virtualenv.run.app_data import AppData
from virtualenv.util.path import Path
from virtualenv.util.six import ensure_str, ensure_text
@@ -145,7 +145,12 @@ def check_os_environ_stable():
if k.startswith(str("VIRTUALENV_")) or str("VIRTUAL_ENV") in k or k.startswith(str("TOX_"))
}
cleaned = {k: os.environ[k] for k, v in os.environ.items()}
- os.environ[str("VIRTUALENV_NO_DOWNLOAD")] = str("1")
+ override = {
+ "VIRTUALENV_NO_PERIODIC_UPDATE": "1",
+ "VIRTUALENV_NO_DOWNLOAD": "1",
+ }
+ for key, value in override.items():
+ os.environ[str(key)] = str(value)
is_exception = False
try:
yield
@@ -154,7 +159,8 @@ def check_os_environ_stable():
raise
finally:
try:
- del os.environ[str("VIRTUALENV_NO_DOWNLOAD")]
+ for key in override.keys():
+ del os.environ[str(key)]
if is_exception is False:
new = os.environ
extra = {k: new[k] for k in set(new) - set(old)}
@@ -291,9 +297,9 @@ def current_fastest(current_creators):
@pytest.fixture(scope="session")
def session_app_data(tmp_path_factory):
- app_data = AppData(folder=str(tmp_path_factory.mktemp("session-app-data")))
- with change_env_var(str("VIRTUALENV_OVERRIDE_APP_DATA"), str(app_data.folder.path)):
- yield app_data.folder
+ app_data = AppDataDiskFolder(folder=str(tmp_path_factory.mktemp("session-app-data")))
+ with change_env_var(str("VIRTUALENV_OVERRIDE_APP_DATA"), str(app_data.lock.path)):
+ yield app_data
@contextmanager
@@ -331,3 +337,8 @@ def cross_python(is_inside_ci, session_app_data):
raise RuntimeError(msg)
pytest.skip(msg=msg)
yield interpreter
+
+
+@pytest.fixture(scope="session")
+def for_py_version():
+ return "{}.{}".format(*sys.version_info[0:2])
diff --git a/tests/integration/test_zipapp.py b/tests/integration/test_zipapp.py
index 605496e..e519884 100644
--- a/tests/integration/test_zipapp.py
+++ b/tests/integration/test_zipapp.py
@@ -38,6 +38,7 @@ def zipapp_build_env(tmp_path_factory):
"",
str(create_env_path),
"--no-download",
+ "--no-periodic-update",
],
)
exe = str(session.creator.exe)
@@ -70,7 +71,7 @@ def zipapp(zipapp_build_env, tmp_path_factory):
@pytest.fixture(scope="session")
def zipapp_test_env(tmp_path_factory):
base_path = tmp_path_factory.mktemp("zipapp-test")
- session = cli_run(["-v", "--activators", "", "--without-pip", str(base_path / "env")])
+ session = cli_run(["-v", "--activators", "", "--without-pip", str(base_path / "env"), "--no-periodic-update"])
yield session.creator.exe
shutil.rmtree(str(base_path))
diff --git a/tests/unit/activation/conftest.py b/tests/unit/activation/conftest.py
index 826ea3d..98c3fda 100644
--- a/tests/unit/activation/conftest.py
+++ b/tests/unit/activation/conftest.py
@@ -74,7 +74,8 @@ class ActivationTester(object):
_raw, _ = process.communicate()
raw = _raw.decode("utf-8")
except subprocess.CalledProcessError as exception:
- assert not exception.returncode, ensure_text(exception.output)
+ output = ensure_text((exception.output + exception.stderr) if six.PY3 else exception.output)
+ assert not exception.returncode, output
return
out = re.sub(r"pydev debugger: process \d+ is connecting\n\n", "", raw, re.M).strip().splitlines()
@@ -208,7 +209,7 @@ def raise_on_non_source_class():
@pytest.fixture(scope="session", params=[True, False], ids=["with_prompt", "no_prompt"])
def activation_python(request, tmp_path_factory, special_char_name, current_fastest):
dest = os.path.join(ensure_text(str(tmp_path_factory.mktemp("activation-tester-env"))), special_char_name)
- cmd = ["--without-pip", dest, "--creator", current_fastest, "-vv"]
+ cmd = ["--without-pip", dest, "--creator", current_fastest, "-vv", "--no-periodic-update"]
if request.param:
cmd += ["--prompt", special_char_name]
session = cli_run(cmd)
diff --git a/tests/unit/activation/test_activate_this.py b/tests/unit/activation/test_activate_this.py
index 9446a4c..53f4b3f 100644
--- a/tests/unit/activation/test_activate_this.py
+++ b/tests/unit/activation/test_activate_this.py
@@ -10,7 +10,7 @@ def test_python_activator_cross(session_app_data, cross_python, special_name_dir
"-p",
str(cross_python.executable),
"--app-data",
- str(session_app_data.path),
+ str(session_app_data.lock.path),
"--without-pip",
"--activators",
"",
diff --git a/tests/unit/activation/test_xonsh.py b/tests/unit/activation/test_xonsh.py
index 3945d7c..60df8a4 100644
--- a/tests/unit/activation/test_xonsh.py
+++ b/tests/unit/activation/test_xonsh.py
@@ -3,6 +3,7 @@ from __future__ import absolute_import, unicode_literals
import sys
import pytest
+from flaky import flaky
from virtualenv.activation import XonshActivator
from virtualenv.info import IS_PYPY, PY3
@@ -13,6 +14,7 @@ from virtualenv.info import IS_PYPY, PY3
(sys.platform == "win32" and IS_PYPY and PY3) or sys.version_info[0:2] == (3, 9),
reason="xonsh on Windows blocks indefinitely and is not stable yet on 3.9",
)
+@flaky(max_runs=2, min_passes=1)
def test_xonsh(activation_tester_class, activation_tester):
class Xonsh(activation_tester_class):
def __init__(self, session):
diff --git a/tests/unit/config/test___main__.py b/tests/unit/config/test___main__.py
index 91fef27..3aad475 100644
--- a/tests/unit/config/test___main__.py
+++ b/tests/unit/config/test___main__.py
@@ -1,7 +1,12 @@
from __future__ import absolute_import, unicode_literals
+import re
import sys
+import pytest
+
+from virtualenv.__main__ import run_with_catch
+from virtualenv.util.error import ProcessCallFailed
from virtualenv.util.subprocess import Popen, subprocess
@@ -10,3 +15,68 @@ def test_main():
out, _ = process.communicate()
assert not process.returncode
assert out
+
+
+@pytest.fixture()
+def raise_on_session_done(mocker):
+ def _func(exception):
+ from virtualenv.run import session_via_cli
+
+ prev_session = session_via_cli
+
+ def _session_via_cli(args, options=None):
+ prev_session(args, options)
+ raise exception
+
+ mocker.patch("virtualenv.run.session_via_cli", side_effect=_session_via_cli)
+
+ return _func
+
+
+def test_fail_no_traceback(raise_on_session_done, tmp_path, capsys):
+ raise_on_session_done(ProcessCallFailed(code=2, out="out\n", err="err\n", cmd=["something"]))
+ with pytest.raises(SystemExit) as context:
+ run_with_catch([str(tmp_path)])
+ assert context.value.code == 2
+ out, err = capsys.readouterr()
+ assert out == "subprocess call failed for [{}] with code 2\nout\nSystemExit: 2\n".format(repr("something"))
+ assert err == "err\n"
+
+
+def test_fail_with_traceback(raise_on_session_done, tmp_path, capsys):
+ raise_on_session_done(TypeError("something bad"))
+
+ with pytest.raises(TypeError, match="something bad"):
+ run_with_catch([str(tmp_path), "--with-traceback"])
+ out, err = capsys.readouterr()
+ assert out == ""
+ assert err == ""
+
+
+def test_session_report_full(session_app_data, tmp_path, capsys):
+ run_with_catch([str(tmp_path)])
+ out, err = capsys.readouterr()
+ assert err == ""
+ lines = out.splitlines()
+ regexes = [
+ r"created virtual environment .* in \d+ms",
+ r" creator .*",
+ r" seeder .*",
+ r" added seed packages: .*pip==.*, setuptools==.*, wheel==.*",
+ r" activators .*",
+ ]
+ for line, regex in zip(lines, regexes):
+ assert re.match(regex, line), line
+
+
+def test_session_report_minimal(session_app_data, tmp_path, capsys):
+ run_with_catch([str(tmp_path), "--activators", "", "--without-pip"])
+ out, err = capsys.readouterr()
+ assert err == ""
+ lines = out.splitlines()
+ regexes = [
+ r"created virtual environment .* in \d+ms",
+ r" creator .*",
+ ]
+ for line, regex in zip(lines, regexes):
+ assert re.match(regex, line), line
diff --git a/tests/unit/create/conftest.py b/tests/unit/create/conftest.py
index 1b9be23..f51dcc2 100644
--- a/tests/unit/create/conftest.py
+++ b/tests/unit/create/conftest.py
@@ -30,7 +30,7 @@ def venv(tmp_path_factory, session_app_data):
if CURRENT.is_venv:
return sys.executable
elif CURRENT.version_info.major == 3:
- root_python = root(tmp_path_factory)
+ root_python = root(tmp_path_factory, session_app_data)
dest = tmp_path_factory.mktemp("venv")
process = Popen([str(root_python), "-m", "venv", "--without-pip", str(dest)])
process.communicate()
@@ -45,7 +45,7 @@ def old_virtualenv(tmp_path_factory, session_app_data):
return CURRENT.executable
else:
env_for_old_virtualenv = tmp_path_factory.mktemp("env-for-old-virtualenv")
- result = cli_run(["--no-download", "--activators", "", str(env_for_old_virtualenv)])
+ result = cli_run(["--no-download", "--activators", "", str(env_for_old_virtualenv), "--no-periodic-update"])
# noinspection PyBroadException
try:
process = Popen(
diff --git a/tests/unit/create/test_creator.py b/tests/unit/create/test_creator.py
index be1f21f..f928088 100644
--- a/tests/unit/create/test_creator.py
+++ b/tests/unit/create/test_creator.py
@@ -20,12 +20,12 @@ import pytest
from virtualenv.__main__ import run, run_with_catch
from virtualenv.create.creator import DEBUG_SCRIPT, Creator, get_env_debug_info
+from virtualenv.create.pyenv_cfg import PyEnvCfg
from virtualenv.create.via_global_ref.builtin.cpython.cpython2 import CPython2PosixBase
from virtualenv.create.via_global_ref.builtin.cpython.cpython3 import CPython3Posix
from virtualenv.create.via_global_ref.builtin.python2.python2 import Python2
from virtualenv.discovery.py_info import PythonInfo
from virtualenv.info import IS_PYPY, IS_WIN, PY2, PY3, fs_is_case_sensitive
-from virtualenv.pyenv_cfg import PyEnvCfg
from virtualenv.run import cli_run, session_via_cli
from virtualenv.util.path import Path
from virtualenv.util.six import ensure_str, ensure_text
@@ -48,7 +48,7 @@ def _non_success_exit_code(capsys, target):
run_with_catch(args=[target])
assert context.value.code != 0
out, err = capsys.readouterr()
- assert not out, out
+ assert "SystemExit: " in out
return err
@@ -384,7 +384,7 @@ def test_create_long_path(current_fastest, tmp_path):
@pytest.mark.parametrize("creator", sorted(set(PythonInfo.current_system().creators().key_to_class) - {"builtin"}))
-def test_create_distutils_cfg(creator, tmp_path, monkeypatch):
+def test_create_distutils_cfg(creator, tmp_path, monkeypatch, session_app_data):
result = cli_run([ensure_text(str(tmp_path / "venv")), "--activators", "", "--creator", creator])
app = Path(__file__).parent / "console_app"
diff --git a/tests/unit/seed/greet/greet2.c b/tests/unit/create/via_global_ref/greet/greet2.c
index 7dc421c..7dc421c 100644
--- a/tests/unit/seed/greet/greet2.c
+++ b/tests/unit/create/via_global_ref/greet/greet2.c
diff --git a/tests/unit/seed/greet/greet3.c b/tests/unit/create/via_global_ref/greet/greet3.c
index 3ec017d..3ec017d 100644
--- a/tests/unit/seed/greet/greet3.c
+++ b/tests/unit/create/via_global_ref/greet/greet3.c
diff --git a/tests/unit/seed/greet/setup.py b/tests/unit/create/via_global_ref/greet/setup.py
index 3c88fc8..3c88fc8 100644
--- a/tests/unit/seed/greet/setup.py
+++ b/tests/unit/create/via_global_ref/greet/setup.py
diff --git a/tests/unit/seed/test_extra_install.py b/tests/unit/create/via_global_ref/test_build_c_ext.py
index 0086cd1..0086cd1 100644
--- a/tests/unit/seed/test_extra_install.py
+++ b/tests/unit/create/via_global_ref/test_build_c_ext.py
diff --git a/tests/unit/seed/test_base_embed.py b/tests/unit/seed/embed/test_base_embed.py
index df4778c..df4778c 100644
--- a/tests/unit/seed/test_base_embed.py
+++ b/tests/unit/seed/embed/test_base_embed.py
diff --git a/tests/unit/seed/test_boostrap_link_via_app_data.py b/tests/unit/seed/embed/test_boostrap_link_via_app_data.py
index 8f4704f..a48edd4 100644
--- a/tests/unit/seed/test_boostrap_link_via_app_data.py
+++ b/tests/unit/seed/embed/test_boostrap_link_via_app_data.py
@@ -9,8 +9,7 @@ import pytest
from virtualenv.discovery.py_info import PythonInfo
from virtualenv.info import fs_supports_symlink
from virtualenv.run import cli_run
-from virtualenv.seed.embed.wheels import BUNDLE_SUPPORT
-from virtualenv.seed.embed.wheels.acquire import BUNDLE_FOLDER
+from virtualenv.seed.wheels.embed import BUNDLE_FOLDER, BUNDLE_SUPPORT
from virtualenv.util.six import ensure_text
from virtualenv.util.subprocess import Popen
@@ -107,7 +106,7 @@ def test_seed_link_via_app_data(tmp_path, coverage_env, current_fastest, copies)
@pytest.fixture()
-def read_only_folder(temp_app_data):
+def read_only_app_data(temp_app_data):
temp_app_data.mkdir()
try:
os.chmod(str(temp_app_data), S_IREAD | S_IRGRP | S_IROTH)
@@ -117,7 +116,7 @@ def read_only_folder(temp_app_data):
@pytest.mark.skipif(sys.platform == "win32", reason="Windows only applies R/O to files")
-def test_base_bootstrap_link_via_app_data_not_writable(tmp_path, current_fastest, read_only_folder, monkeypatch):
+def test_base_bootstrap_link_via_app_data_not_writable(tmp_path, current_fastest, read_only_app_data, monkeypatch):
dest = tmp_path / "venv"
result = cli_run(["--seeder", "app-data", "--creator", current_fastest, "--reset-app-data", "-vv", str(dest)])
assert result
diff --git a/tests/unit/seed/embed/test_pip_invoke.py b/tests/unit/seed/embed/test_pip_invoke.py
new file mode 100644
index 0000000..4c478df
--- /dev/null
+++ b/tests/unit/seed/embed/test_pip_invoke.py
@@ -0,0 +1,89 @@
+from __future__ import absolute_import, unicode_literals
+
+import itertools
+import sys
+from shutil import copy2
+
+import pytest
+
+from virtualenv.run import cli_run
+from virtualenv.seed.embed.pip_invoke import PipInvoke
+from virtualenv.seed.wheels.bundle import load_embed_wheel
+from virtualenv.seed.wheels.embed import BUNDLE_FOLDER, BUNDLE_SUPPORT
+
+
+@pytest.mark.slow
+@pytest.mark.parametrize("no", ["pip", "setuptools", "wheel", ""])
+def test_base_bootstrap_via_pip_invoke(tmp_path, coverage_env, mocker, current_fastest, no):
+ extra_search_dir = tmp_path / "extra"
+ extra_search_dir.mkdir()
+ for_py_version = "{}.{}".format(*sys.version_info[0:2])
+ new = BUNDLE_SUPPORT[for_py_version]
+ for wheel_filename in BUNDLE_SUPPORT[for_py_version].values():
+ copy2(str(BUNDLE_FOLDER / wheel_filename), str(extra_search_dir))
+
+ def _load_embed_wheel(app_data, distribution, for_py_version, version):
+ return load_embed_wheel(app_data, distribution, old_ver, version)
+
+ old_ver = "3.4"
+ old = BUNDLE_SUPPORT[old_ver]
+ mocker.patch("virtualenv.seed.wheels.bundle.load_embed_wheel", side_effect=_load_embed_wheel)
+
+ def _execute(cmd, env):
+ expected = set()
+ for distribution, with_version in versions.items():
+ if distribution == no:
+ continue
+ if with_version == "embed":
+ expected.add(BUNDLE_FOLDER)
+ elif old[dist] != new[dist]:
+ expected.add(extra_search_dir)
+ expected_list = list(
+ itertools.chain.from_iterable(["--find-links", str(e)] for e in sorted(expected, key=lambda x: str(x))),
+ )
+ found = cmd[-len(expected_list) :]
+ assert "--no-index" not in cmd
+ cmd.append("--no-index")
+ assert found == expected_list
+ return original(cmd, env)
+
+ original = PipInvoke._execute
+ run = mocker.patch.object(PipInvoke, "_execute", side_effect=_execute)
+ versions = {"pip": "embed", "setuptools": "bundle", "wheel": new["wheel"].split("-")[1]}
+
+ create_cmd = [
+ "--seeder",
+ "pip",
+ str(tmp_path / "env"),
+ "--download",
+ "--creator",
+ current_fastest,
+ "--extra-search-dir",
+ str(extra_search_dir),
+ "--app-data",
+ str(tmp_path / "app-data"),
+ ]
+ for dist, version in versions.items():
+ create_cmd.extend(["--{}".format(dist), version])
+ if no:
+ create_cmd.append("--no-{}".format(no))
+ result = cli_run(create_cmd)
+ coverage_env()
+
+ assert result
+ assert run.call_count == 1
+
+ site_package = result.creator.purelib
+ pip = site_package / "pip"
+ setuptools = site_package / "setuptools"
+ wheel = site_package / "wheel"
+ files_post_first_create = list(site_package.iterdir())
+
+ if no:
+ no_file = locals()[no]
+ assert no not in files_post_first_create
+
+ for key in ("pip", "setuptools", "wheel"):
+ if key == no:
+ continue
+ assert locals()[key] in files_post_first_create
diff --git a/tests/unit/seed/embed/wheels/test_acquire.py b/tests/unit/seed/embed/wheels/test_acquire.py
deleted file mode 100644
index 49f7433..0000000
--- a/tests/unit/seed/embed/wheels/test_acquire.py
+++ /dev/null
@@ -1,11 +0,0 @@
-from virtualenv.seed.embed.wheels.acquire import get_bundled_wheel, wheel_support_py
-
-
-def test_wheel_support_no_python_requires(mocker):
- wheel = get_bundled_wheel(package="setuptools", version_release=None)
- zip_mock = mocker.MagicMock()
- mocker.patch("virtualenv.seed.embed.wheels.acquire.ZipFile", new=zip_mock)
- zip_mock.return_value.__enter__.return_value.read = lambda name: b""
-
- supports = wheel_support_py(wheel, "3.8")
- assert supports is True
diff --git a/tests/unit/seed/test_pip_invoke.py b/tests/unit/seed/test_pip_invoke.py
deleted file mode 100644
index 65314aa..0000000
--- a/tests/unit/seed/test_pip_invoke.py
+++ /dev/null
@@ -1,63 +0,0 @@
-from __future__ import absolute_import, unicode_literals
-
-import pytest
-
-from virtualenv.discovery.py_info import PythonInfo
-from virtualenv.run import cli_run
-from virtualenv.seed.embed.pip_invoke import PipInvoke
-from virtualenv.seed.embed.wheels import BUNDLE_SUPPORT, MAX
-from virtualenv.seed.embed.wheels.acquire import BUNDLE_FOLDER
-
-
-@pytest.mark.slow
-@pytest.mark.parametrize("no", ["pip", "setuptools", "wheel", ""])
-def test_base_bootstrap_via_pip_invoke(tmp_path, coverage_env, mocker, current_fastest, no):
- bundle_ver = BUNDLE_SUPPORT.get(PythonInfo.current_system().version_release_str) or BUNDLE_SUPPORT.get(MAX)
-
- extra_search_dir = tmp_path / "extra"
- extra_search_dir.mkdir()
-
- original = PipInvoke._execute
-
- def _execute(cmd, env):
- assert set(cmd[-4:]) == {"--find-links", str(extra_search_dir), str(BUNDLE_FOLDER)}
- return original(cmd, env)
-
- run = mocker.patch.object(PipInvoke, "_execute", side_effect=_execute)
-
- create_cmd = [
- "--seeder",
- "pip",
- str(tmp_path / "env"),
- "--download",
- "--pip",
- bundle_ver["pip"].split("-")[1],
- "--setuptools",
- bundle_ver["setuptools"].split("-")[1],
- "--creator",
- current_fastest,
- "--extra-search-dir",
- str(extra_search_dir),
- ]
- if no:
- create_cmd.append("--no-{}".format(no))
- result = cli_run(create_cmd)
- coverage_env()
-
- assert result
- assert run.call_count == 1
-
- site_package = result.creator.purelib
- pip = site_package / "pip"
- setuptools = site_package / "setuptools"
- wheel = site_package / "wheel"
- files_post_first_create = list(site_package.iterdir())
-
- if no:
- no_file = locals()[no]
- assert no not in files_post_first_create
-
- for key in ("pip", "setuptools", "wheel"):
- if key == no:
- continue
- assert locals()[key] in files_post_first_create
diff --git a/tests/unit/seed/wheels/test_acquire.py b/tests/unit/seed/wheels/test_acquire.py
new file mode 100644
index 0000000..944ed4c
--- /dev/null
+++ b/tests/unit/seed/wheels/test_acquire.py
@@ -0,0 +1,66 @@
+from __future__ import absolute_import, unicode_literals
+
+import sys
+from subprocess import CalledProcessError
+
+import pytest
+
+from virtualenv.info import PY2
+from virtualenv.seed.wheels.acquire import download_wheel, pip_wheel_env_run
+from virtualenv.seed.wheels.embed import BUNDLE_FOLDER, get_embed_wheel
+from virtualenv.seed.wheels.util import discover_wheels
+
+
+def test_pip_wheel_env_run_could_not_find(session_app_data, mocker):
+ mocker.patch("virtualenv.seed.wheels.acquire.from_bundle", return_value=None)
+ with pytest.raises(RuntimeError, match="could not find the embedded pip"):
+ pip_wheel_env_run([], session_app_data)
+
+
+def test_download_wheel_bad_output(mocker, for_py_version, session_app_data):
+ """if the download contains no match for what wheel was downloaded, pick one that matches from target"""
+ distribution = "setuptools"
+ p_open = mocker.MagicMock()
+ mocker.patch("virtualenv.seed.wheels.acquire.Popen", return_value=p_open)
+ p_open.communicate.return_value = "", ""
+ p_open.returncode = 0
+
+ embed = get_embed_wheel(distribution, for_py_version)
+ as_path = mocker.MagicMock()
+ available = discover_wheels(BUNDLE_FOLDER, "setuptools", None, for_py_version)
+ as_path.iterdir.return_value = [i.path for i in available]
+
+ result = download_wheel(distribution, "=={}".format(embed.version), for_py_version, [], session_app_data, as_path)
+ assert result.path == embed.path
+
+
+def test_download_fails(mocker, for_py_version, session_app_data):
+ p_open = mocker.MagicMock()
+ mocker.patch("virtualenv.seed.wheels.acquire.Popen", return_value=p_open)
+ p_open.communicate.return_value = "out", "err"
+ p_open.returncode = 1
+
+ as_path = mocker.MagicMock()
+ with pytest.raises(CalledProcessError) as context:
+ download_wheel("pip", "==1", for_py_version, [], session_app_data, as_path),
+ exc = context.value
+ if PY2:
+ assert exc.output == "outerr"
+ else:
+ assert exc.output == "out"
+ assert exc.stderr == "err"
+ assert exc.returncode == 1
+ assert [
+ sys.executable,
+ "-m",
+ "pip",
+ "download",
+ "--disable-pip-version-check",
+ "--only-binary=:all:",
+ "--no-deps",
+ "--python-version",
+ for_py_version,
+ "-d",
+ str(as_path),
+ "pip==1",
+ ] == exc.cmd
diff --git a/tests/unit/seed/wheels/test_acquire_find_wheel.py b/tests/unit/seed/wheels/test_acquire_find_wheel.py
new file mode 100644
index 0000000..18c4692
--- /dev/null
+++ b/tests/unit/seed/wheels/test_acquire_find_wheel.py
@@ -0,0 +1,30 @@
+from __future__ import absolute_import, unicode_literals
+
+import pytest
+
+from virtualenv.seed.wheels.acquire import find_compatible_in_house
+from virtualenv.seed.wheels.embed import BUNDLE_FOLDER, MAX, get_embed_wheel
+
+
+def test_find_latest(for_py_version):
+ result = find_compatible_in_house("setuptools", None, for_py_version, BUNDLE_FOLDER)
+ expected = get_embed_wheel("setuptools", for_py_version)
+ assert result.path == expected.path
+
+
+def test_find_exact(for_py_version):
+ expected = get_embed_wheel("setuptools", for_py_version)
+ result = find_compatible_in_house("setuptools", "=={}".format(expected.version), for_py_version, BUNDLE_FOLDER)
+ assert result.path == expected.path
+
+
+def test_find_less_than(for_py_version):
+ latest = get_embed_wheel("setuptools", MAX)
+ result = find_compatible_in_house("setuptools", "<{}".format(latest.version), MAX, BUNDLE_FOLDER)
+ assert result is not None
+ assert result.path != latest.path
+
+
+def test_find_bad_spec(for_py_version):
+ with pytest.raises(ValueError, match="bad"):
+ find_compatible_in_house("setuptools", "bad", MAX, BUNDLE_FOLDER)
diff --git a/tests/unit/seed/wheels/test_periodic_update.py b/tests/unit/seed/wheels/test_periodic_update.py
new file mode 100644
index 0000000..3d9507e
--- /dev/null
+++ b/tests/unit/seed/wheels/test_periodic_update.py
@@ -0,0 +1,351 @@
+from __future__ import absolute_import, unicode_literals
+
+import json
+import subprocess
+import sys
+from contextlib import contextmanager
+from datetime import datetime, timedelta
+
+import pytest
+from six import StringIO
+from six.moves import zip_longest
+
+from virtualenv import cli_run
+from virtualenv.app_data import AppDataDiskFolder
+from virtualenv.seed.wheels import Wheel
+from virtualenv.seed.wheels.embed import BUNDLE_SUPPORT, get_embed_wheel
+from virtualenv.seed.wheels.periodic_update import (
+ NewVersion,
+ UpdateLog,
+ do_update,
+ dump_datetime,
+ load_datetime,
+ manual_upgrade,
+ periodic_update,
+ trigger_update,
+)
+from virtualenv.util.path import Path
+from virtualenv.util.subprocess import DETACHED_PROCESS
+
+
+def test_manual_upgrade(session_app_data, caplog, mocker, for_py_version):
+ wheel = get_embed_wheel("pip", for_py_version)
+ new_version = NewVersion(wheel.path, datetime.now(), datetime.now() - timedelta(days=20))
+
+ def _do_update(distribution, for_py_version, embed_filename, app_data, search_dirs, periodic): # noqa
+ if distribution == "pip":
+ return [new_version]
+ return []
+
+ do_update = mocker.patch("virtualenv.seed.wheels.periodic_update.do_update", side_effect=_do_update)
+ manual_upgrade(session_app_data)
+
+ assert "upgrade pip" in caplog.text
+ assert "upgraded pip" in caplog.text
+ assert " new entries found:\n\tNewVersion" in caplog.text
+ assert " no new versions found" in caplog.text
+ assert do_update.call_count == 3 * len(BUNDLE_SUPPORT)
+
+
+def test_pick_periodic_update(tmp_path, session_app_data, mocker, for_py_version):
+ embed, current = get_embed_wheel("setuptools", "3.4"), get_embed_wheel("setuptools", for_py_version)
+ mocker.patch("virtualenv.seed.wheels.bundle.load_embed_wheel", return_value=embed)
+ completed = datetime.now() - timedelta(days=29)
+ u_log = UpdateLog(
+ started=datetime.now() - timedelta(days=30),
+ completed=completed,
+ versions=[NewVersion(filename=current.path, found_date=completed, release_date=completed)],
+ periodic=True,
+ )
+ read_dict = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict())
+
+ result = cli_run([str(tmp_path), "--activators", "", "--no-periodic-update", "--no-wheel", "--no-pip"])
+
+ assert read_dict.call_count == 1
+ installed = list(i.name for i in result.creator.purelib.iterdir() if i.suffix == ".dist-info")
+ assert "setuptools-{}.dist-info".format(current.version) in installed
+
+
+def test_periodic_update_stops_at_current(mocker, session_app_data, for_py_version):
+ current = get_embed_wheel("setuptools", for_py_version)
+
+ now, completed = datetime.now(), datetime.now() - timedelta(days=29)
+ u_log = UpdateLog(
+ started=completed,
+ completed=completed,
+ versions=[
+ NewVersion(wheel_path(current, (1,)), completed, now - timedelta(days=1)),
+ NewVersion(filename=current.path, found_date=completed, release_date=now - timedelta(days=2)),
+ NewVersion(wheel_path(current, (-1,)), completed, now - timedelta(days=30)),
+ ],
+ periodic=True,
+ )
+ mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict())
+
+ result = periodic_update("setuptools", for_py_version, current, [], session_app_data, False)
+ assert result.path == current.path
+
+
+def test_periodic_update_latest_per_patch(mocker, session_app_data, for_py_version):
+ current = get_embed_wheel("setuptools", for_py_version)
+ now, completed = datetime.now(), datetime.now() - timedelta(days=29)
+ u_log = UpdateLog(
+ started=completed,
+ completed=completed,
+ versions=[
+ NewVersion(wheel_path(current, (0, 1, 2)), completed, now - timedelta(days=1)),
+ NewVersion(wheel_path(current, (0, 1, 1)), completed, now - timedelta(days=30)),
+ NewVersion(filename=str(current.path), found_date=completed, release_date=now - timedelta(days=2)),
+ ],
+ periodic=True,
+ )
+ mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict())
+
+ result = periodic_update("setuptools", for_py_version, current, [], session_app_data, False)
+ assert result.path == current.path
+
+
+def wheel_path(wheel, of):
+ new_version = ".".join(str(i) for i in (tuple(sum(x) for x in zip_longest(wheel.version_tuple, of, fillvalue=0))))
+ new_name = wheel.name.replace(wheel.version, new_version)
+ return str(wheel.path.parent / new_name)
+
+
+_UP_NOW = datetime.now()
+_UPDATE_SKIP = {
+ "started_just_now_no_complete": UpdateLog(started=_UP_NOW, completed=None, versions=[], periodic=True),
+ "started_1_hour_no_complete": UpdateLog(
+ started=_UP_NOW - timedelta(hours=1), completed=None, versions=[], periodic=True,
+ ),
+ "completed_under_two_weeks": UpdateLog(
+ started=None, completed=_UP_NOW - timedelta(days=14), versions=[], periodic=True,
+ ),
+ "started_just_now_completed_two_weeks": UpdateLog(
+ started=_UP_NOW, completed=_UP_NOW - timedelta(days=14, seconds=1), versions=[], periodic=True,
+ ),
+ "started_1_hour_completed_two_weeks": UpdateLog(
+ started=_UP_NOW - timedelta(hours=1),
+ completed=_UP_NOW - timedelta(days=14, seconds=1),
+ versions=[],
+ periodic=True,
+ ),
+}
+
+
+@pytest.mark.parametrize("u_log", _UPDATE_SKIP.values(), ids=_UPDATE_SKIP.keys())
+def test_periodic_update_skip(u_log, mocker, for_py_version, session_app_data, freezer):
+ freezer.move_to(_UP_NOW)
+ mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict())
+ mocker.patch("virtualenv.seed.wheels.periodic_update.trigger_update", side_effect=RuntimeError)
+
+ result = periodic_update("setuptools", for_py_version, None, [], session_app_data, True)
+ assert result is None
+
+
+_UPDATE_YES = {
+ "never_started": UpdateLog(started=None, completed=None, versions=[], periodic=False),
+ "started_1_hour": UpdateLog(
+ started=_UP_NOW - timedelta(hours=1, microseconds=1), completed=None, versions=[], periodic=False,
+ ),
+ "completed_two_week": UpdateLog(
+ started=_UP_NOW - timedelta(days=14, microseconds=2),
+ completed=_UP_NOW - timedelta(days=14, microseconds=1),
+ versions=[],
+ periodic=False,
+ ),
+}
+
+
+@pytest.mark.parametrize("u_log", _UPDATE_YES.values(), ids=_UPDATE_YES.keys())
+def test_periodic_update_trigger(u_log, mocker, for_py_version, session_app_data, freezer):
+ freezer.move_to(_UP_NOW)
+ mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict())
+ write = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.write")
+ trigger_update_ = mocker.patch("virtualenv.seed.wheels.periodic_update.trigger_update")
+
+ result = periodic_update("setuptools", for_py_version, None, [], session_app_data, True)
+
+ assert result is None
+ assert trigger_update_.call_count
+ assert write.call_count == 1
+ wrote_json = write.call_args[0][0]
+ assert wrote_json["periodic"] is True
+ assert load_datetime(wrote_json["started"]) == _UP_NOW
+
+
+def test_trigger_update_no_debug(for_py_version, session_app_data, tmp_path, mocker, monkeypatch):
+ monkeypatch.delenv(str("_VIRTUALENV_PERIODIC_UPDATE_INLINE"), raising=False)
+ current = get_embed_wheel("setuptools", for_py_version)
+ process = mocker.MagicMock()
+ process.communicate.return_value = None, None
+ Popen = mocker.patch("virtualenv.seed.wheels.periodic_update.Popen", return_value=process)
+
+ trigger_update("setuptools", for_py_version, current, [tmp_path / "a", tmp_path / "b"], session_app_data, True)
+
+ assert Popen.call_count == 1
+ args, kwargs = Popen.call_args
+ cmd = (
+ "from virtualenv.seed.wheels.periodic_update import do_update;"
+ "do_update({!r}, {!r}, {!r}, {!r}, [{!r}, {!r}], True)".format(
+ "setuptools",
+ for_py_version,
+ str(current.path),
+ str(session_app_data),
+ str(tmp_path / "a"),
+ str(tmp_path / "b"),
+ )
+ )
+ assert args == ([sys.executable, "-c", cmd],)
+ expected = {"stdout": subprocess.PIPE, "stderr": subprocess.PIPE}
+ if sys.platform == "win32":
+ expected["creationflags"] = DETACHED_PROCESS
+ assert kwargs == expected
+ assert process.communicate.call_count == 0
+
+
+def test_trigger_update_debug(for_py_version, session_app_data, tmp_path, mocker, monkeypatch):
+ monkeypatch.setenv(str("_VIRTUALENV_PERIODIC_UPDATE_INLINE"), str("1"))
+ current = get_embed_wheel("pip", for_py_version)
+
+ process = mocker.MagicMock()
+ process.communicate.return_value = None, None
+ Popen = mocker.patch("virtualenv.seed.wheels.periodic_update.Popen", return_value=process)
+
+ trigger_update("pip", for_py_version, current, [tmp_path / "a", tmp_path / "b"], session_app_data, False)
+
+ assert Popen.call_count == 1
+ args, kwargs = Popen.call_args
+ cmd = (
+ "from virtualenv.seed.wheels.periodic_update import do_update;"
+ "do_update({!r}, {!r}, {!r}, {!r}, [{!r}, {!r}], False)".format(
+ "pip", for_py_version, str(current.path), str(session_app_data), str(tmp_path / "a"), str(tmp_path / "b"),
+ )
+ )
+ assert args == ([sys.executable, "-c", cmd],)
+ expected = {"stdout": None, "stderr": None}
+ assert kwargs == expected
+ assert process.communicate.call_count == 1
+
+
+def test_do_update_first(tmp_path, mocker, freezer):
+ freezer.move_to(_UP_NOW)
+ wheel = get_embed_wheel("pip", "3.9")
+ app_data_outer = AppDataDiskFolder(str(tmp_path / "app"))
+ extra = tmp_path / "extra"
+ extra.mkdir()
+
+ pip_version_remote = [
+ (wheel_path(wheel, (1, 0, 0)), None),
+ (wheel_path(wheel, (0, 1, 0)), _UP_NOW - timedelta(days=1)),
+ (wheel_path(wheel, (0, 0, 1)), _UP_NOW - timedelta(days=2)),
+ (wheel.path, _UP_NOW - timedelta(days=3)),
+ (wheel_path(wheel, (-1, 0, 0)), _UP_NOW - timedelta(days=30)),
+ ]
+ download_wheels = (Wheel(Path(i[0])) for i in pip_version_remote)
+
+ def _download_wheel(distribution, version_spec, for_py_version, search_dirs, app_data, to_folder):
+ assert distribution == "pip"
+ assert for_py_version == "3.9"
+ assert [str(i) for i in search_dirs] == [str(extra)]
+ assert isinstance(app_data, AppDataDiskFolder)
+ assert to_folder == app_data_outer.house
+ return next(download_wheels)
+
+ download_wheel = mocker.patch("virtualenv.seed.wheels.acquire.download_wheel", side_effect=_download_wheel)
+ releases = {
+ Wheel(Path(wheel)).version: [
+ {"upload_time": datetime.strftime(release_date, "%Y-%m-%dT%H:%M:%S") if release_date is not None else None},
+ ]
+ for wheel, release_date in pip_version_remote
+ }
+ pypi_release = json.dumps({"releases": releases})
+
+ @contextmanager
+ def _release(of):
+ assert of == "https://pypi.org/pypi/pip/json"
+ yield StringIO(pypi_release)
+
+ url_o = mocker.patch("virtualenv.seed.wheels.periodic_update.urlopen", side_effect=_release)
+
+ last_update = _UP_NOW - timedelta(days=14)
+ u_log = UpdateLog(started=last_update, completed=last_update, versions=[], periodic=True)
+ read_dict = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict())
+ write = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.write")
+
+ versions = do_update("pip", "3.9", str(pip_version_remote[-1][0]), str(app_data_outer), [str(extra)], True)
+
+ assert download_wheel.call_count == len(pip_version_remote)
+ assert url_o.call_count == len(pip_version_remote)
+
+ expected = [
+ NewVersion(Path(wheel).name, _UP_NOW, None if release is None else release.replace(microsecond=0))
+ for wheel, release in pip_version_remote
+ ]
+ assert versions == expected
+
+ assert read_dict.call_count == 1
+ assert write.call_count == 1
+ wrote_json = write.call_args[0][0]
+ assert wrote_json == {
+ "started": dump_datetime(last_update),
+ "completed": dump_datetime(_UP_NOW),
+ "periodic": True,
+ "versions": [e.to_dict() for e in expected],
+ }
+
+
+def test_do_update_skip_already_done(tmp_path, mocker, freezer):
+ freezer.move_to(_UP_NOW + timedelta(hours=1))
+ wheel = get_embed_wheel("pip", "3.9")
+ app_data_outer = AppDataDiskFolder(str(tmp_path / "app"))
+ extra = tmp_path / "extra"
+ extra.mkdir()
+
+ def _download_wheel(distribution, version_spec, for_py_version, search_dirs, app_data, to_folder): # noqa
+ return wheel.path
+
+ download_wheel = mocker.patch("virtualenv.seed.wheels.acquire.download_wheel", side_effect=_download_wheel)
+ url_o = mocker.patch("virtualenv.seed.wheels.periodic_update.urlopen", side_effect=RuntimeError)
+
+ released = _UP_NOW - timedelta(days=30)
+ u_log = UpdateLog(
+ started=_UP_NOW - timedelta(days=31),
+ completed=released,
+ versions=[NewVersion(filename=wheel.path.name, found_date=released, release_date=released)],
+ periodic=True,
+ )
+ read_dict = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict())
+ write = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.write")
+
+ versions = do_update("pip", "3.9", str(wheel.path), str(app_data_outer), [str(extra)], False)
+
+ assert download_wheel.call_count == 1
+ assert read_dict.call_count == 1
+ assert not url_o.call_count
+ assert versions == []
+
+ assert write.call_count == 1
+ wrote_json = write.call_args[0][0]
+ assert wrote_json == {
+ "started": dump_datetime(_UP_NOW + timedelta(hours=1)),
+ "completed": dump_datetime(_UP_NOW + timedelta(hours=1)),
+ "periodic": False,
+ "versions": [
+ {
+ "filename": wheel.path.name,
+ "release_date": dump_datetime(released),
+ "found_date": dump_datetime(released),
+ },
+ ],
+ }
+
+
+def test_new_version_eq():
+ value = NewVersion("a", datetime.now(), datetime.now())
+ assert value == value
+
+
+def test_new_version_ne():
+ assert NewVersion("a", datetime.now(), datetime.now()) != NewVersion(
+ "a", datetime.now(), datetime.now() + timedelta(hours=1),
+ )
diff --git a/tests/unit/seed/wheels/test_wheels_util.py b/tests/unit/seed/wheels/test_wheels_util.py
new file mode 100644
index 0000000..e487797
--- /dev/null
+++ b/tests/unit/seed/wheels/test_wheels_util.py
@@ -0,0 +1,31 @@
+from __future__ import absolute_import, unicode_literals
+
+import pytest
+
+from virtualenv.seed.wheels.embed import MAX, get_embed_wheel
+from virtualenv.seed.wheels.util import Wheel
+
+
+def test_wheel_support_no_python_requires(mocker):
+ wheel = get_embed_wheel("setuptools", for_py_version=None)
+ zip_mock = mocker.MagicMock()
+ mocker.patch("virtualenv.seed.wheels.util.ZipFile", new=zip_mock)
+ zip_mock.return_value.__enter__.return_value.read = lambda name: b""
+
+ supports = wheel.support_py("3.8")
+ assert supports is True
+
+
+def test_bad_as_version_tuple():
+ with pytest.raises(ValueError, match="bad"):
+ Wheel.as_version_tuple("bad")
+
+
+def test_wheel_not_support():
+ wheel = get_embed_wheel("setuptools", MAX)
+ assert wheel.support_py("3.3") is False
+
+
+def test_wheel_repr():
+ wheel = get_embed_wheel("setuptools", MAX)
+ assert str(wheel.path) in repr(wheel)
diff --git a/tox.ini b/tox.ini
index 1570e68..0987acb 100644
--- a/tox.ini
+++ b/tox.ini
@@ -25,7 +25,7 @@ setenv =
COVERAGE_PROCESS_START = {toxinidir}/.coveragerc
_COVERAGE_SRC = {envsitepackagesdir}/virtualenv
PYTHONIOENCODING=utf-8
- {py34,py27,pypy}: PYTHONWARNINGS=ignore:DEPRECATION::pip._internal.cli.base_command
+ {py34,py27,pypy, upgrade}: PYTHONWARNINGS=ignore:DEPRECATION::pip._internal.cli.base_command
{pypy,py27}: PYTEST_XDIST = 0
passenv = https_proxy http_proxy no_proxy HOME PYTEST_* PIP_* CI_RUN TERM
extras = testing
@@ -37,7 +37,7 @@ commands =
tests {posargs:--int --timeout 600 -n {env:PYTEST_XDIST:auto}}
python -m coverage combine
- python -m coverage report
+ python -m coverage report --skip-covered --show-missing
python -m coverage xml -o {toxworkdir}/coverage.{envname}.xml
python -m coverage html -d {envtmpdir}/htmlcov
@@ -55,7 +55,7 @@ setenv =
COVERAGE_FILE={toxworkdir}/.coverage
commands =
python -m coverage combine
- python -m coverage report --show-missing
+ python -m coverage report --skip-covered --show-missing
python -m coverage xml -o {toxworkdir}/coverage.xml
python -m coverage html -d {toxworkdir}/htmlcov
python -m diff_cover.diff_cover_tool --compare-branch {env:DIFF_AGAINST:origin/master} {toxworkdir}/coverage.xml