From dcc71f773576c19a3658735879893515b056ece5 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 26 Jul 2020 10:35:02 -0400 Subject: Rename _distutils_importer to _distutils_hack, as it supplies more than just an importer. --- _distutils_hack/__init__.py | 45 +++++++++++++++++++++++++++++++++++++ _distutils_hack/install.py | 5 +++++ _distutils_hack/override.py | 54 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+) create mode 100644 _distutils_hack/__init__.py create mode 100644 _distutils_hack/install.py create mode 100644 _distutils_hack/override.py (limited to '_distutils_hack') diff --git a/_distutils_hack/__init__.py b/_distutils_hack/__init__.py new file mode 100644 index 00000000..3ad70100 --- /dev/null +++ b/_distutils_hack/__init__.py @@ -0,0 +1,45 @@ +import sys +import os + + +def enabled(): + """ + Allow selection of distutils by environment variable. + """ + which = os.environ.get('SETUPTOOLS_USE_DISTUTILS', 'stdlib') + return which == 'local' + + +class DistutilsMetaFinder: + def find_spec(self, fullname, path, target=None): + if path is not None or fullname != "distutils": + return None + + return self.get_distutils_spec() + + def get_distutils_spec(self): + import importlib.util + + class DistutilsLoader(importlib.util.abc.Loader): + + def create_module(self, spec): + return importlib.import_module('._distutils', 'setuptools') + + def exec_module(self, module): + pass + + return importlib.util.spec_from_loader('distutils', DistutilsLoader()) + + +DISTUTILS_FINDER = DistutilsMetaFinder() + + +def add_shim(): + sys.meta_path.insert(0, DISTUTILS_FINDER) + + +def remove_shim(): + try: + sys.path.remove(DISTUTILS_FINDER) + except ValueError: + pass diff --git a/_distutils_hack/install.py b/_distutils_hack/install.py new file mode 100644 index 00000000..73f13b29 --- /dev/null +++ b/_distutils_hack/install.py @@ -0,0 +1,5 @@ +from . import enabled, add_shim + + +if enabled(): + add_shim() diff --git a/_distutils_hack/override.py b/_distutils_hack/override.py new file mode 100644 index 00000000..523139bb --- /dev/null +++ b/_distutils_hack/override.py @@ -0,0 +1,54 @@ +""" +Ensure that the local copy of distutils is preferred over stdlib. + +See https://github.com/pypa/setuptools/issues/417#issuecomment-392298401 +for more motivation. +""" + +import sys +import re +import importlib +import warnings + +from . import enabled + + +is_pypy = '__pypy__' in sys.builtin_module_names + + +def warn_distutils_present(): + if 'distutils' not in sys.modules: + return + if is_pypy and sys.version_info < (3, 7): + # PyPy for 3.6 unconditionally imports distutils, so bypass the warning + # https://foss.heptapod.net/pypy/pypy/-/blob/be829135bc0d758997b3566062999ee8b23872b4/lib-python/3/site.py#L250 + return + warnings.warn( + "Distutils was imported before Setuptools. This usage is discouraged " + "and may exhibit undesirable behaviors or errors. Please use " + "Setuptools' objects directly or at least import Setuptools first.") + + +def clear_distutils(): + if 'distutils' not in sys.modules: + return + warnings.warn("Setuptools is replacing distutils.") + mods = [name for name in sys.modules if re.match(r'distutils\b', name)] + for name in mods: + del sys.modules[name] + + +def ensure_local_distutils(): + clear_distutils() + distutils = importlib.import_module('setuptools._distutils') + distutils.__name__ = 'distutils' + sys.modules['distutils'] = distutils + + # sanity check that submodules load as expected + core = importlib.import_module('distutils.core') + assert '_distutils' in core.__file__, core.__file__ + + +warn_distutils_present() +if enabled(): + ensure_local_distutils() -- cgit v1.2.1 From 1251a231ad75fa649da700645690eb3c0a348f08 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 26 Jul 2020 10:53:31 -0400 Subject: Replace install behavior on import with direct invocation (now that 'enabled' logic is duplicated in pth file). --- _distutils_hack/install.py | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 _distutils_hack/install.py (limited to '_distutils_hack') diff --git a/_distutils_hack/install.py b/_distutils_hack/install.py deleted file mode 100644 index 73f13b29..00000000 --- a/_distutils_hack/install.py +++ /dev/null @@ -1,5 +0,0 @@ -from . import enabled, add_shim - - -if enabled(): - add_shim() -- cgit v1.2.1 From 6c6d69e0213c4012caa36f0087f1fe54bac62c89 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 26 Jul 2020 10:59:11 -0400 Subject: Move all but a small shim in override into _distutils_hack --- _distutils_hack/__init__.py | 51 +++++++++++++++++++++++++++++++++++++++++ _distutils_hack/override.py | 55 +-------------------------------------------- 2 files changed, 52 insertions(+), 54 deletions(-) (limited to '_distutils_hack') diff --git a/_distutils_hack/__init__.py b/_distutils_hack/__init__.py index 3ad70100..a8638344 100644 --- a/_distutils_hack/__init__.py +++ b/_distutils_hack/__init__.py @@ -1,5 +1,11 @@ import sys import os +import re +import importlib +import warnings + + +is_pypy = '__pypy__' in sys.builtin_module_names def enabled(): @@ -10,6 +16,51 @@ def enabled(): return which == 'local' +def warn_distutils_present(): + if 'distutils' not in sys.modules: + return + if is_pypy and sys.version_info < (3, 7): + # PyPy for 3.6 unconditionally imports distutils, so bypass the warning + # https://foss.heptapod.net/pypy/pypy/-/blob/be829135bc0d758997b3566062999ee8b23872b4/lib-python/3/site.py#L250 + return + warnings.warn( + "Distutils was imported before Setuptools. This usage is discouraged " + "and may exhibit undesirable behaviors or errors. Please use " + "Setuptools' objects directly or at least import Setuptools first.") + + +def clear_distutils(): + if 'distutils' not in sys.modules: + return + warnings.warn("Setuptools is replacing distutils.") + mods = [name for name in sys.modules if re.match(r'distutils\b', name)] + for name in mods: + del sys.modules[name] + + +def ensure_local_distutils(): + clear_distutils() + distutils = importlib.import_module('setuptools._distutils') + distutils.__name__ = 'distutils' + sys.modules['distutils'] = distutils + + # sanity check that submodules load as expected + core = importlib.import_module('distutils.core') + assert '_distutils' in core.__file__, core.__file__ + + +def do_override(): + """ + Ensure that the local copy of distutils is preferred over stdlib. + + See https://github.com/pypa/setuptools/issues/417#issuecomment-392298401 + for more motivation. + """ + warn_distutils_present() + if enabled(): + ensure_local_distutils() + + class DistutilsMetaFinder: def find_spec(self, fullname, path, target=None): if path is not None or fullname != "distutils": diff --git a/_distutils_hack/override.py b/_distutils_hack/override.py index 523139bb..2cc433a4 100644 --- a/_distutils_hack/override.py +++ b/_distutils_hack/override.py @@ -1,54 +1 @@ -""" -Ensure that the local copy of distutils is preferred over stdlib. - -See https://github.com/pypa/setuptools/issues/417#issuecomment-392298401 -for more motivation. -""" - -import sys -import re -import importlib -import warnings - -from . import enabled - - -is_pypy = '__pypy__' in sys.builtin_module_names - - -def warn_distutils_present(): - if 'distutils' not in sys.modules: - return - if is_pypy and sys.version_info < (3, 7): - # PyPy for 3.6 unconditionally imports distutils, so bypass the warning - # https://foss.heptapod.net/pypy/pypy/-/blob/be829135bc0d758997b3566062999ee8b23872b4/lib-python/3/site.py#L250 - return - warnings.warn( - "Distutils was imported before Setuptools. This usage is discouraged " - "and may exhibit undesirable behaviors or errors. Please use " - "Setuptools' objects directly or at least import Setuptools first.") - - -def clear_distutils(): - if 'distutils' not in sys.modules: - return - warnings.warn("Setuptools is replacing distutils.") - mods = [name for name in sys.modules if re.match(r'distutils\b', name)] - for name in mods: - del sys.modules[name] - - -def ensure_local_distutils(): - clear_distutils() - distutils = importlib.import_module('setuptools._distutils') - distutils.__name__ = 'distutils' - sys.modules['distutils'] = distutils - - # sanity check that submodules load as expected - core = importlib.import_module('distutils.core') - assert '_distutils' in core.__file__, core.__file__ - - -warn_distutils_present() -if enabled(): - ensure_local_distutils() +__import__('_distutils_hack').do_override() -- cgit v1.2.1 From 8db806d30d7591828528ac937e8f3b334e957ed3 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 26 Jul 2020 20:19:45 -0400 Subject: remove shim should by symmetric to add_shim --- _distutils_hack/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to '_distutils_hack') diff --git a/_distutils_hack/__init__.py b/_distutils_hack/__init__.py index a8638344..a10af2cc 100644 --- a/_distutils_hack/__init__.py +++ b/_distutils_hack/__init__.py @@ -91,6 +91,6 @@ def add_shim(): def remove_shim(): try: - sys.path.remove(DISTUTILS_FINDER) + sys.meta_path.remove(DISTUTILS_FINDER) except ValueError: pass -- cgit v1.2.1 From 6b70fb201d6a81448de6ca6f71d7091b9a26096c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 8 Aug 2020 13:38:56 -0400 Subject: Restore location of 'enabled' --- _distutils_hack/__init__.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to '_distutils_hack') diff --git a/_distutils_hack/__init__.py b/_distutils_hack/__init__.py index a10af2cc..71fa7ce1 100644 --- a/_distutils_hack/__init__.py +++ b/_distutils_hack/__init__.py @@ -8,14 +8,6 @@ import warnings is_pypy = '__pypy__' in sys.builtin_module_names -def enabled(): - """ - Allow selection of distutils by environment variable. - """ - which = os.environ.get('SETUPTOOLS_USE_DISTUTILS', 'stdlib') - return which == 'local' - - def warn_distutils_present(): if 'distutils' not in sys.modules: return @@ -38,6 +30,14 @@ def clear_distutils(): del sys.modules[name] +def enabled(): + """ + Allow selection of distutils by environment variable. + """ + which = os.environ.get('SETUPTOOLS_USE_DISTUTILS', 'stdlib') + return which == 'local' + + def ensure_local_distutils(): clear_distutils() distutils = importlib.import_module('setuptools._distutils') -- cgit v1.2.1 From 1410a8e6fda5fb8cd78f915e1ffd68f6c7f35af3 Mon Sep 17 00:00:00 2001 From: Paul Ganssle Date: Mon, 10 Aug 2020 09:18:46 -0400 Subject: Fix issue with distutils warning --- _distutils_hack/__init__.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to '_distutils_hack') diff --git a/_distutils_hack/__init__.py b/_distutils_hack/__init__.py index 71fa7ce1..1e7b294b 100644 --- a/_distutils_hack/__init__.py +++ b/_distutils_hack/__init__.py @@ -16,9 +16,12 @@ def warn_distutils_present(): # https://foss.heptapod.net/pypy/pypy/-/blob/be829135bc0d758997b3566062999ee8b23872b4/lib-python/3/site.py#L250 return warnings.warn( - "Distutils was imported before Setuptools. This usage is discouraged " - "and may exhibit undesirable behaviors or errors. Please use " - "Setuptools' objects directly or at least import Setuptools first.") + "Distutils was imported before Setuptools, but importing Setuptools " + "also replaces the `distutils` module in `sys.modules`. This may lead " + "to undesirable behaviors or errors. To avoid these issues, avoid " + "using distutils directly, ensure that setuptools is installed in the " + "traditional way (e.g. not an editable install), and/or make sure that " + "setuptools is always imported before distutils.") def clear_distutils(): @@ -56,8 +59,8 @@ def do_override(): See https://github.com/pypa/setuptools/issues/417#issuecomment-392298401 for more motivation. """ - warn_distutils_present() if enabled(): + warn_distutils_present() ensure_local_distutils() -- cgit v1.2.1 From be7a0a31a116f6df9bef616ef1adef72b9d0d48d Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 14 Aug 2020 19:32:18 -0400 Subject: Bypass .pth loader when distutils is loaded from pip. Workaround for pypa/pip#8761. --- _distutils_hack/__init__.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to '_distutils_hack') diff --git a/_distutils_hack/__init__.py b/_distutils_hack/__init__.py index 814ee97e..2d358c37 100644 --- a/_distutils_hack/__init__.py +++ b/_distutils_hack/__init__.py @@ -3,6 +3,7 @@ import os import re import importlib import warnings +import inspect is_pypy = '__pypy__' in sys.builtin_module_names @@ -66,7 +67,7 @@ def do_override(): class DistutilsMetaFinder: def find_spec(self, fullname, path, target=None): - if path is not None or fullname != "distutils": + if path is not None or fullname != "distutils" or self._bypass(): return None return self.get_distutils_spec() @@ -84,6 +85,16 @@ class DistutilsMetaFinder: return importlib.util.spec_from_loader('distutils', DistutilsLoader()) + def _bypass(self): + """ + Suppress the import of distutils from setuptools when running under pip. + See pypa/pip#8761 for rationale. + """ + return any( + level.frame.f_globals['__name__'].startswith('pip.') + for level in inspect.stack(context=False) + ) + DISTUTILS_FINDER = DistutilsMetaFinder() -- cgit v1.2.1 From 77eeb77f48484a6e227275a9a8cc7d4c322596f7 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 15 Aug 2020 14:39:25 -0400 Subject: When pip is imported, unload any existing distutils and disable the DistutilsMetaFinder. --- _distutils_hack/__init__.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to '_distutils_hack') diff --git a/_distutils_hack/__init__.py b/_distutils_hack/__init__.py index 2d358c37..bdb365d5 100644 --- a/_distutils_hack/__init__.py +++ b/_distutils_hack/__init__.py @@ -3,7 +3,6 @@ import os import re import importlib import warnings -import inspect is_pypy = '__pypy__' in sys.builtin_module_names @@ -67,7 +66,9 @@ def do_override(): class DistutilsMetaFinder: def find_spec(self, fullname, path, target=None): - if path is not None or fullname != "distutils" or self._bypass(): + self._disable_for_pip(fullname, path) + + if path is not None or fullname != "distutils": return None return self.get_distutils_spec() @@ -85,15 +86,17 @@ class DistutilsMetaFinder: return importlib.util.spec_from_loader('distutils', DistutilsLoader()) - def _bypass(self): + def _disable_for_pip(self, fullname, path): """ - Suppress the import of distutils from setuptools when running under pip. + Ensure stdlib distutils when running under pip. See pypa/pip#8761 for rationale. """ - return any( - level.frame.f_globals['__name__'].startswith('pip.') - for level in inspect.stack(context=False) - ) + if path is not None or fullname != "pip": + return + + # pip is being imported the first time. + clear_distutils() + self.get_distutils_spec = lambda: None DISTUTILS_FINDER = DistutilsMetaFinder() -- cgit v1.2.1 From 3d404fd3268b0fb7d84916779ca2e282f4586396 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 15 Aug 2020 14:59:37 -0400 Subject: Refactor to use lookups and consolidate behaviors. --- _distutils_hack/__init__.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) (limited to '_distutils_hack') diff --git a/_distutils_hack/__init__.py b/_distutils_hack/__init__.py index bdb365d5..074bd5e9 100644 --- a/_distutils_hack/__init__.py +++ b/_distutils_hack/__init__.py @@ -66,14 +66,14 @@ def do_override(): class DistutilsMetaFinder: def find_spec(self, fullname, path, target=None): - self._disable_for_pip(fullname, path) - - if path is not None or fullname != "distutils": - return None + if path is not None: + return - return self.get_distutils_spec() + method_name = 'spec_for_{fullname}'.format(**locals()) + method = getattr(self, method_name, lambda: None) + return method() - def get_distutils_spec(self): + def spec_for_distutils(self): import importlib.util class DistutilsLoader(importlib.util.abc.Loader): @@ -86,17 +86,13 @@ class DistutilsMetaFinder: return importlib.util.spec_from_loader('distutils', DistutilsLoader()) - def _disable_for_pip(self, fullname, path): + def spec_for_pip(self): """ Ensure stdlib distutils when running under pip. See pypa/pip#8761 for rationale. """ - if path is not None or fullname != "pip": - return - - # pip is being imported the first time. clear_distutils() - self.get_distutils_spec = lambda: None + self.spec_for_distutils = lambda: None DISTUTILS_FINDER = DistutilsMetaFinder() -- cgit v1.2.1 From df8c341bacb103e085d6714bf9778f008a56cb04 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 31 Aug 2020 18:16:41 +0200 Subject: Add Python 3.10 support to _distutils_hack Get the 'Loader' abstract class from importlib.abc rather than importlib.util.abc (alias removed in Python 3.10). --- _distutils_hack/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to '_distutils_hack') diff --git a/_distutils_hack/__init__.py b/_distutils_hack/__init__.py index 074bd5e9..b8410e1f 100644 --- a/_distutils_hack/__init__.py +++ b/_distutils_hack/__init__.py @@ -74,9 +74,10 @@ class DistutilsMetaFinder: return method() def spec_for_distutils(self): + import importlib.abc import importlib.util - class DistutilsLoader(importlib.util.abc.Loader): + class DistutilsLoader(importlib.abc.Loader): def create_module(self, spec): return importlib.import_module('._distutils', 'setuptools') -- cgit v1.2.1 From a3319a388913facb5b9236c938ba9adabd5b9fdd Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 1 Sep 2020 16:58:40 -0400 Subject: In distutils hack, use absolute import rather than relative to avoid bpo-30876. Fixes #2352. --- _distutils_hack/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to '_distutils_hack') diff --git a/_distutils_hack/__init__.py b/_distutils_hack/__init__.py index b8410e1f..2bc6df7a 100644 --- a/_distutils_hack/__init__.py +++ b/_distutils_hack/__init__.py @@ -80,7 +80,7 @@ class DistutilsMetaFinder: class DistutilsLoader(importlib.abc.Loader): def create_module(self, spec): - return importlib.import_module('._distutils', 'setuptools') + return importlib.import_module('setuptools._distutils') def exec_module(self, module): pass -- cgit v1.2.1 From c0085e87c69629ec3cf24dfce887c0fb93fa80dd Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 2 Sep 2020 17:41:38 -0400 Subject: Make stdlib distutils the default again. Stop the burning. Ref #2350 and others. --- _distutils_hack/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to '_distutils_hack') diff --git a/_distutils_hack/__init__.py b/_distutils_hack/__init__.py index 2bc6df7a..3325817b 100644 --- a/_distutils_hack/__init__.py +++ b/_distutils_hack/__init__.py @@ -37,7 +37,7 @@ def enabled(): """ Allow selection of distutils by environment variable. """ - which = os.environ.get('SETUPTOOLS_USE_DISTUTILS', 'local') + which = os.environ.get('SETUPTOOLS_USE_DISTUTILS', 'stdlib') return which == 'local' -- cgit v1.2.1 From a60ba76395353974f5bb51ad1c117d7239ed5032 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 4 Sep 2020 08:59:40 -0400 Subject: =?UTF-8?q?=F0=9F=91=B9=20Feed=20the=20hobgoblins=20(delint).?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- _distutils_hack/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to '_distutils_hack') diff --git a/_distutils_hack/__init__.py b/_distutils_hack/__init__.py index 3325817b..504c4ac0 100644 --- a/_distutils_hack/__init__.py +++ b/_distutils_hack/__init__.py @@ -20,8 +20,8 @@ def warn_distutils_present(): "also replaces the `distutils` module in `sys.modules`. This may lead " "to undesirable behaviors or errors. To avoid these issues, avoid " "using distutils directly, ensure that setuptools is installed in the " - "traditional way (e.g. not an editable install), and/or make sure that " - "setuptools is always imported before distutils.") + "traditional way (e.g. not an editable install), and/or make sure " + "that setuptools is always imported before distutils.") def clear_distutils(): -- cgit v1.2.1 From 7c07fec3f02093bf048a1192473ab0736211e47e Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 4 Sep 2020 09:29:21 -0400 Subject: When pip is imported as part of a build, leave distutils patched. Fixes #2355. --- _distutils_hack/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to '_distutils_hack') diff --git a/_distutils_hack/__init__.py b/_distutils_hack/__init__.py index 504c4ac0..c31edfed 100644 --- a/_distutils_hack/__init__.py +++ b/_distutils_hack/__init__.py @@ -92,9 +92,22 @@ class DistutilsMetaFinder: Ensure stdlib distutils when running under pip. See pypa/pip#8761 for rationale. """ + if self.pip_imported_during_build(): + return clear_distutils() self.spec_for_distutils = lambda: None + @staticmethod + def pip_imported_during_build(): + """ + Detect if pip is being imported in a build script. Ref #2355. + """ + import traceback + return any( + frame.f_globals['__file__'].endswith('setup.py') + for frame, line in traceback.walk_stack(None) + ) + DISTUTILS_FINDER = DistutilsMetaFinder() -- cgit v1.2.1