feature: store version update source in embed JSON file (#2273)

7 files changed, 216 insertions, 41 deletions

diff --git a/docs/changelog/2265.bugfix.rst b/docs/changelog/2265.bugfix.rst
new file mode 100644
index 0000000..bc90ccc
--- /dev/null
+++ b/docs/changelog/2265.bugfix.rst
@@ -0,0 +1,3 @@
+Try using previous updates of ``pip``, ``setuptools`` & ``wheel``
+when inside an update grace period rather than always falling back
+to embedded wheels - by :user:`mayeut`.
diff --git a/docs/changelog/2266.bugfix.rst b/docs/changelog/2266.bugfix.rst
new file mode 100644
index 0000000..26ff3ad
--- /dev/null
+++ b/docs/changelog/2266.bugfix.rst
@@ -0,0 +1,2 @@
+New patch versions of ``pip``, ``setuptools`` & ``wheel`` are now
+returned in the expected timeframe. - by :user:`mayeut`.
diff --git a/docs/changelog/2267.bugfix.rst b/docs/changelog/2267.bugfix.rst
new file mode 100644
index 0000000..05af3cd
--- /dev/null
+++ b/docs/changelog/2267.bugfix.rst
@@ -0,0 +1,2 @@
+Manual upgrades of ``pip``, ``setuptools`` & ``wheel`` are
+not discarded by a periodic update - by :user:`mayeut`.
diff --git a/src/virtualenv/app_data/via_disk_folder.py b/src/virtualenv/app_data/via_disk_folder.py
index 257a85f..265db1b 100644
--- a/src/virtualenv/app_data/via_disk_folder.py
+++ b/src/virtualenv/app_data/via_disk_folder.py
@@ -14,7 +14,7 @@ virtualenv-app-data
 │       │           └── <install class> -> CopyPipInstall / SymlinkPipInstall
 │       │               └── <wheel name> -> pip-20.1.1-py2.py3-none-any
 │       └── embed
-│           └── 1
+│           └── 2 -> json format versioning
 │               └── *.json -> for every distribution contains data about newer embed versions and releases
 └─── unzip <in zip app we cannot refer to some internal files, so first extract them>
      └── <virtualenv version>
@@ -101,7 +101,7 @@ class AppDataDiskFolder(AppData):
                             filename.unlink()
 
     def embed_update_log(self, distribution, for_py_version):
-        return EmbedDistributionUpdateStoreDisk(self.lock / "wheel" / for_py_version / "embed" / "1", distribution)
+        return EmbedDistributionUpdateStoreDisk(self.lock / "wheel" / for_py_version / "embed" / "2", distribution)
 
     @property
     def house(self):
diff --git a/src/virtualenv/seed/wheels/periodic_update.py b/src/virtualenv/seed/wheels/periodic_update.py
index 3ea764e..1a6cbe8 100644
--- a/src/virtualenv/seed/wheels/periodic_update.py
+++ b/src/virtualenv/seed/wheels/periodic_update.py
@@ -36,6 +36,12 @@ if PY2:
         pass  # pragma: no cov
 
 
+GRACE_PERIOD_CI = timedelta(hours=1)  # prevent version switch in the middle of a CI run
+GRACE_PERIOD_MINOR = timedelta(days=28)
+UPDATE_PERIOD = timedelta(days=14)
+UPDATE_ABORTED_DELAY = timedelta(hours=1)
+
+
 def periodic_update(distribution, of_version, for_py_version, wheel, search_dirs, app_data, do_periodic_update, env):
     if do_periodic_update:
         handle_auto_update(distribution, for_py_version, wheel, search_dirs, app_data, env)
@@ -48,20 +54,20 @@ def periodic_update(distribution, of_version, for_py_version, wheel, search_dirs
         return updated_wheel
 
     u_log = UpdateLog.from_app_data(app_data, distribution, for_py_version)
-    u_log_older_than_hour = now - u_log.completed > timedelta(hours=1) if u_log.completed is not None else False
     if of_version is None:
         for _, group in groupby(u_log.versions, key=lambda v: v.wheel.version_tuple[0:2]):
-            version = next(group)  # use only latest patch version per minor, earlier assumed to be buggy
-            if wheel is not None and Path(version.filename).name == wheel.name:
-                break
-            if u_log.periodic is False or (u_log_older_than_hour and version.use(now)):
-                wheel = _update_wheel(version)
-                break
-    elif u_log.periodic is False or u_log_older_than_hour:
+            # use only latest patch version per minor, earlier assumed to be buggy
+            all_patches = list(group)
+            ignore_grace_period_minor = any(version for version in all_patches if version.use(now))
+            for version in all_patches:
+                if wheel is not None and Path(version.filename).name == wheel.name:
+                    return wheel
+                if version.use(now, ignore_grace_period_minor):
+                    return _update_wheel(version)
+    else:
         for version in u_log.versions:
             if version.wheel.version == of_version:
-                wheel = _update_wheel(version)
-                break
+                return _update_wheel(version)
 
     return wheel
 
@@ -88,10 +94,11 @@ def load_datetime(value):
 
 
 class NewVersion(object):
-    def __init__(self, filename, found_date, release_date):
+    def __init__(self, filename, found_date, release_date, source):
         self.filename = filename
         self.found_date = found_date
         self.release_date = release_date
+        self.source = source
 
     @classmethod
     def from_dict(cls, dictionary):
@@ -99,6 +106,7 @@ class NewVersion(object):
             filename=dictionary["filename"],
             found_date=load_datetime(dictionary["found_date"]),
             release_date=load_datetime(dictionary["release_date"]),
+            source=dictionary["source"],
         )
 
     def to_dict(self):
@@ -106,23 +114,32 @@ class NewVersion(object):
             "filename": self.filename,
             "release_date": dump_datetime(self.release_date),
             "found_date": dump_datetime(self.found_date),
+            "source": self.source,
         }
 
-    def use(self, now):
-        compare_from = self.release_date or self.found_date
-        return now - compare_from >= timedelta(days=28)
+    def use(self, now, ignore_grace_period_minor=False, ignore_grace_period_ci=False):
+        if self.source == "manual":
+            return True
+        elif self.source == "periodic":
+            if self.found_date < now - GRACE_PERIOD_CI or ignore_grace_period_ci:
+                if not ignore_grace_period_minor:
+                    compare_from = self.release_date or self.found_date
+                    return now - compare_from >= GRACE_PERIOD_MINOR
+                return True
+        return False
 
     def __repr__(self):
-        return "{}(filename={}), found_date={}, release_date={})".format(
+        return "{}(filename={}), found_date={}, release_date={}, source={})".format(
             self.__class__.__name__,
             self.filename,
             self.found_date,
             self.release_date,
+            self.source,
         )
 
     def __eq__(self, other):
         return type(self) == type(other) and all(
-            getattr(self, k) == getattr(other, k) for k in ["filename", "release_date", "found_date"]
+            getattr(self, k) == getattr(other, k) for k in ["filename", "release_date", "found_date", "source"]
         )
 
     def __ne__(self, other):
@@ -170,12 +187,12 @@ class UpdateLog(object):
         if self.completed is None:  # never completed
             return self._check_start(now)
         else:
-            if now - self.completed <= timedelta(days=14):
+            if now - self.completed <= UPDATE_PERIOD:
                 return False
             return self._check_start(now)
 
     def _check_start(self, now):
-        return self.started is None or now - self.started > timedelta(hours=1)
+        return self.started is None or now - self.started > UPDATE_ABORTED_DELAY
 
 
 def trigger_update(distribution, for_py_version, wheel, search_dirs, app_data, env, periodic):
@@ -231,12 +248,24 @@ def _run_do_update(app_data, distribution, embed_filename, for_py_version, perio
     embed_update_log = app_data.embed_update_log(distribution, for_py_version)
     u_log = UpdateLog.from_dict(embed_update_log.read())
     now = datetime.now()
+    if periodic:
+        source = "periodic"
+        # mark everything not updated manually as source "periodic"
+        for version in u_log.versions:
+            if version.source != "manual":
+                version.source = source
+    else:
+        source = "manual"
+        # mark everything as source "manual"
+        for version in u_log.versions:
+            version.source = source
+
     if wheel_filename is not None:
         dest = wheelhouse / wheel_filename.name
         if not dest.exists():
             copy2(str(wheel_filename), str(wheelhouse))
     last, last_version, versions = None, None, []
-    while last is None or not last.use(now):
+    while last is None or not last.use(now, ignore_grace_period_ci=True):
         download_time = datetime.now()
         dest = acquire.download_wheel(
             distribution=distribution,
@@ -250,7 +279,7 @@ def _run_do_update(app_data, distribution, embed_filename, for_py_version, perio
         if dest is None or (u_log.versions and u_log.versions[0].filename == dest.name):
             break
         release_date = release_date_for_wheel_path(dest.path)
-        last = NewVersion(filename=dest.path.name, release_date=release_date, found_date=download_time)
+        last = NewVersion(filename=dest.path.name, release_date=release_date, found_date=download_time, source=source)
         logging.info("detected %s in %s", last, datetime.now() - download_time)
         versions.append(last)
         last_wheel = Wheel(Path(last.filename))
diff --git a/tests/unit/seed/wheels/test_bundle.py b/tests/unit/seed/wheels/test_bundle.py
index 16ea2fd..767a2b4 100644
--- a/tests/unit/seed/wheels/test_bundle.py
+++ b/tests/unit/seed/wheels/test_bundle.py
@@ -1,12 +1,14 @@
 from __future__ import absolute_import, unicode_literals
 
 import os
+from datetime import datetime
 
 import pytest
 
 from virtualenv.app_data import AppDataDiskFolder
 from virtualenv.seed.wheels.bundle import from_bundle
 from virtualenv.seed.wheels.embed import get_embed_wheel
+from virtualenv.seed.wheels.periodic_update import dump_datetime
 from virtualenv.seed.wheels.util import Version, Wheel
 from virtualenv.util.path import Path
 
@@ -23,17 +25,19 @@ def next_pip_wheel(for_py_version):
 @pytest.fixture(scope="module")
 def app_data(tmp_path_factory, for_py_version, next_pip_wheel):
     temp_folder = tmp_path_factory.mktemp("module-app-data")
+    now = dump_datetime(datetime.now())
     app_data_ = AppDataDiskFolder(str(temp_folder))
     app_data_.embed_update_log("pip", for_py_version).write(
         {
-            "completed": "2000-01-01T00:00:00.000000Z",
+            "completed": now,
             "periodic": True,
-            "started": "2000-01-01T00:00:00.000000Z",
+            "started": now,
             "versions": [
                 {
                     "filename": next_pip_wheel.name,
                     "found_date": "2000-01-01T00:00:00.000000Z",
                     "release_date": "2000-01-01T00:00:00.000000Z",
+                    "source": "periodic",
                 }
             ],
         }
diff --git a/tests/unit/seed/wheels/test_periodic_update.py b/tests/unit/seed/wheels/test_periodic_update.py
index 9b32770..e35dfed 100644
--- a/tests/unit/seed/wheels/test_periodic_update.py
+++ b/tests/unit/seed/wheels/test_periodic_update.py
@@ -42,7 +42,7 @@ def clear_pypi_info_cache():
 
 def test_manual_upgrade(session_app_data, caplog, mocker, for_py_version):
     wheel = get_embed_wheel("pip", for_py_version)
-    new_version = NewVersion(wheel.path, datetime.now(), datetime.now() - timedelta(days=20))
+    new_version = NewVersion(wheel.path, datetime.now(), datetime.now() - timedelta(days=20), "manual")
 
     def _do_update(distribution, for_py_version, embed_filename, app_data, search_dirs, periodic):  # noqa
         if distribution == "pip":
@@ -73,7 +73,7 @@ def test_pick_periodic_update(tmp_path, session_app_data, mocker, for_py_version
     u_log = UpdateLog(
         started=datetime.now() - timedelta(days=30),
         completed=completed,
-        versions=[NewVersion(filename=current.path, found_date=completed, release_date=completed)],
+        versions=[NewVersion(filename=current.path, found_date=completed, release_date=completed, source="periodic")],
         periodic=True,
     )
     read_dict = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict())
@@ -93,9 +93,9 @@ def test_periodic_update_stops_at_current(mocker, session_app_data, for_py_versi
         started=completed,
         completed=completed,
         versions=[
-            NewVersion(wheel_path(current, (1,)), completed, now - timedelta(days=1)),
-            NewVersion(filename=current.path, found_date=completed, release_date=now - timedelta(days=2)),
-            NewVersion(wheel_path(current, (-1,)), completed, now - timedelta(days=30)),
+            NewVersion(wheel_path(current, (1,)), completed, now - timedelta(days=1), "periodic"),
+            NewVersion(current.path, completed, now - timedelta(days=2), "periodic"),
+            NewVersion(wheel_path(current, (-1,)), completed, now - timedelta(days=30), "periodic"),
         ],
         periodic=True,
     )
@@ -107,21 +107,67 @@ def test_periodic_update_stops_at_current(mocker, session_app_data, for_py_versi
 
 def test_periodic_update_latest_per_patch(mocker, session_app_data, for_py_version):
     current = get_embed_wheel("setuptools", for_py_version)
-    now, completed = datetime.now(), datetime.now() - timedelta(days=29)
+    expected_path = wheel_path(current, (0, 1, 2))
+    now = datetime.now()
+    completed = now - timedelta(hours=2)
     u_log = UpdateLog(
         started=completed,
         completed=completed,
+        periodic=True,
         versions=[
-            NewVersion(wheel_path(current, (0, 1, 2)), completed, now - timedelta(days=1)),
-            NewVersion(wheel_path(current, (0, 1, 1)), completed, now - timedelta(days=30)),
-            NewVersion(filename=str(current.path), found_date=completed, release_date=now - timedelta(days=2)),
+            NewVersion(expected_path, completed, now - timedelta(days=1), "periodic"),
+            NewVersion(wheel_path(current, (0, 1, 1)), completed, now - timedelta(days=30), "periodic"),
+            NewVersion(str(current.path), completed, now - timedelta(days=31), "periodic"),
         ],
+    )
+    mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict())
+
+    result = periodic_update("setuptools", None, for_py_version, current, [], session_app_data, False, os.environ)
+    assert str(result.path) == expected_path
+
+
+def test_periodic_update_latest_per_patch_prev_is_manual(mocker, session_app_data, for_py_version):
+    current = get_embed_wheel("setuptools", for_py_version)
+    expected_path = wheel_path(current, (0, 1, 2))
+    now = datetime.now()
+    completed = now - timedelta(hours=2)
+    u_log = UpdateLog(
+        started=completed,
+        completed=completed,
         periodic=True,
+        versions=[
+            NewVersion(expected_path, completed, completed, "periodic"),
+            NewVersion(wheel_path(current, (0, 1, 1)), completed, now - timedelta(days=10), "manual"),
+            NewVersion(wheel_path(current, (0, 1, 0)), completed, now - timedelta(days=11), "periodic"),
+            NewVersion(str(current.path), completed, now - timedelta(days=12), "manual"),
+        ],
     )
     mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict())
 
     result = periodic_update("setuptools", None, for_py_version, current, [], session_app_data, False, os.environ)
-    assert result.path == current.path
+    assert str(result.path) == expected_path
+
+
+def test_manual_update_honored(mocker, session_app_data, for_py_version):
+    current = get_embed_wheel("setuptools", for_py_version)
+    expected_path = wheel_path(current, (0, 1, 1))
+    now = datetime.now()
+    completed = now
+    u_log = UpdateLog(
+        started=completed,
+        completed=completed,
+        periodic=True,
+        versions=[
+            NewVersion(wheel_path(current, (0, 1, 2)), completed, completed, "periodic"),
+            NewVersion(expected_path, completed, now - timedelta(days=10), "manual"),
+            NewVersion(wheel_path(current, (0, 1, 0)), completed, now - timedelta(days=11), "periodic"),
+            NewVersion(str(current.path), completed, now - timedelta(days=12), "manual"),
+        ],
+    )
+    mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict())
+
+    result = periodic_update("setuptools", None, for_py_version, current, [], session_app_data, False, os.environ)
+    assert str(result.path) == expected_path
 
 
 def wheel_path(wheel, of):
@@ -338,7 +384,7 @@ def test_do_update_first(tmp_path, mocker, freezer):
     assert copy.call_count == 1
 
     expected = [
-        NewVersion(Path(wheel).name, _UP_NOW, None if release is None else release.replace(microsecond=0))
+        NewVersion(Path(wheel).name, _UP_NOW, None if release is None else release.replace(microsecond=0), "periodic")
         for wheel, release in pip_version_remote
     ]
     assert versions == expected
@@ -371,7 +417,7 @@ def test_do_update_skip_already_done(tmp_path, mocker, freezer):
     u_log = UpdateLog(
         started=_UP_NOW - timedelta(days=31),
         completed=released,
-        versions=[NewVersion(filename=wheel.path.name, found_date=released, release_date=released)],
+        versions=[NewVersion(filename=wheel.path.name, found_date=released, release_date=released, source="periodic")],
         periodic=True,
     )
     read_dict = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict())
@@ -395,21 +441,20 @@ def test_do_update_skip_already_done(tmp_path, mocker, freezer):
                 "filename": wheel.path.name,
                 "release_date": dump_datetime(released),
                 "found_date": dump_datetime(released),
+                "source": "manual",  # changed from "periodic" to "manual"
             },
         ],
     }
 
 
 def test_new_version_eq():
-    value = NewVersion("a", datetime.now(), datetime.now())
+    value = NewVersion("a", datetime.now(), datetime.now(), "periodic")
     assert value == value
 
 
 def test_new_version_ne():
-    assert NewVersion("a", datetime.now(), datetime.now()) != NewVersion(
-        "a",
-        datetime.now(),
-        datetime.now() + timedelta(hours=1),
+    assert NewVersion("a", datetime.now(), datetime.now(), "periodic") != NewVersion(
+        "a", datetime.now(), datetime.now() + timedelta(hours=1), "manual"
     )
 
 
@@ -472,3 +517,93 @@ def test_download_stop_with_embed(tmp_path, mocker, freezer):
 
     assert read_dict.call_count == 1
     assert write.call_count == 1
+
+
+def test_download_periodic_stop_at_first_usable(tmp_path, mocker, freezer):
+    freezer.move_to(_UP_NOW)
+    wheel = get_embed_wheel("pip", "3.9")
+    app_data_outer = AppDataDiskFolder(str(tmp_path / "app"))
+    pip_version_remote = [wheel_path(wheel, (0, 1, 1)), wheel_path(wheel, (0, 1, 0))]
+    rel_date_remote = [_UP_NOW - timedelta(days=1), _UP_NOW - timedelta(days=30)]
+    at = {"download": 0, "release_date": 0}
+
+    def download():
+        while True:
+            path = pip_version_remote[at["download"]]
+            at["download"] += 1
+            yield Wheel(Path(path))
+
+    download_gen = download()
+    download_wheel = mocker.patch(
+        "virtualenv.seed.wheels.acquire.download_wheel", side_effect=lambda *a, **k: next(download_gen)
+    )
+
+    def rel_date():
+        while True:
+            value = rel_date_remote[at["release_date"]]
+            at["release_date"] += 1
+            yield value
+
+    rel_date_gen = rel_date()
+    release_date = mocker.patch(
+        "virtualenv.seed.wheels.periodic_update.release_date_for_wheel_path",
+        side_effect=lambda *a, **k: next(rel_date_gen),
+    )
+
+    last_update = _UP_NOW - timedelta(days=14)
+    u_log = UpdateLog(started=last_update, completed=last_update, versions=[], periodic=True)
+    read_dict = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict())
+    write = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.write")
+
+    do_update("pip", "3.9", str(wheel.path), str(app_data_outer), [], True)
+
+    assert download_wheel.call_count == 2
+    assert release_date.call_count == 2
+
+    assert read_dict.call_count == 1
+    assert write.call_count == 1
+
+
+def test_download_manual_stop_at_first_usable(tmp_path, mocker, freezer):
+    freezer.move_to(_UP_NOW)
+    wheel = get_embed_wheel("pip", "3.9")
+    app_data_outer = AppDataDiskFolder(str(tmp_path / "app"))
+    pip_version_remote = [wheel_path(wheel, (0, 1, 1))]
+    rel_date_remote = [_UP_NOW + timedelta(hours=1)]
+    at = {"download": 0, "release_date": 0}
+
+    def download():
+        while True:
+            path = pip_version_remote[at["download"]]
+            at["download"] += 1
+            yield Wheel(Path(path))
+
+    download_gen = download()
+    download_wheel = mocker.patch(
+        "virtualenv.seed.wheels.acquire.download_wheel", side_effect=lambda *a, **k: next(download_gen)
+    )
+
+    def rel_date():
+        while True:
+            value = rel_date_remote[at["release_date"]]
+            at["release_date"] += 1
+            yield value
+
+    rel_date_gen = rel_date()
+    release_date = mocker.patch(
+        "virtualenv.seed.wheels.periodic_update.release_date_for_wheel_path",
+        side_effect=lambda *a, **k: next(rel_date_gen),
+    )
+
+    last_update = _UP_NOW - timedelta(days=14)
+    u_log = UpdateLog(started=last_update, completed=last_update, versions=[], periodic=True)
+    read_dict = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict())
+    write = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.write")
+
+    do_update("pip", "3.9", str(wheel.path), str(app_data_outer), [], False)
+
+    assert download_wheel.call_count == 1
+    assert release_date.call_count == 1
+
+    assert read_dict.call_count == 1
+    assert write.call_count == 1