From be3778e796424146e53b93a032a0a6a39979d9ff Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Fri, 4 Feb 2022 15:19:11 -0500 Subject: include pep660 proof of concept --- setuptools/command/editable_wheel.py | 157 +++++++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 setuptools/command/editable_wheel.py (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py new file mode 100644 index 00000000..c827efa3 --- /dev/null +++ b/setuptools/command/editable_wheel.py @@ -0,0 +1,157 @@ +""" +Create a wheel that, when installed, will make the source package 'editable' +(add it to the interpreter's path, including metadata) per PEP 660. Replaces +'setup.py develop'. Based on the setuptools develop command. +""" + +# TODO doesn't behave when called outside the hook + +import os +import time +from pathlib import Path + +from distutils.core import Command +from distutils.errors import DistutilsError + +import pkg_resources + +SOURCE_EPOCH_ZIP = 499162860 + + +class editable_wheel(Command): + """Build 'editable' wheel for development""" + + description = "create a PEP 660 'editable' wheel" + + user_options = [ + ("dist-dir=", "d", "directory to put final built distributions in"), + ] + + boolean_options = [] + + def run(self): + self.build_editable_wheel() + + def initialize_options(self): + self.dist_dir = None + + def finalize_options(self): + # is this part of the 'develop' command needed? + ei = self.get_finalized_command("egg_info") + if ei.broken_egg_info: + template = "Please rename %r to %r before using 'develop'" + args = ei.egg_info, ei.broken_egg_info + raise DistutilsError(template % args) + self.args = [ei.egg_name] + + # the .pth file should point to target + self.egg_base = ei.egg_base + self.target = pkg_resources.normalize_path(self.egg_base) + self.dist_info_dir = Path( + (ei.egg_info[: -len(".egg-info")] + ".dist-info").rpartition("/")[-1] + ) + + def build_editable_wheel(self): + if getattr(self.distribution, "use_2to3", False): + raise NotImplementedError("2to3 not supported") + + di = self.get_finalized_command("dist_info") + di.egg_base = self.dist_dir + di.finalize_options() + self.run_command("dist_info") + + # Build extensions in-place + self.reinitialize_command("build_ext", inplace=1) + self.run_command("build_ext") + + # now build the wheel + # with the dist-info directory and .pth from 'editables' library + # ... + + import zipfile + import editables # could we use 'develop' command's .pth file + + project = editables.EditableProject( + self.distribution.metadata.name, self.target + ) + project.add_to_path(self.target) + + dist_dir = Path(self.dist_dir) + dist_info_dir = self.dist_info_dir + fullname = self.distribution.metadata.get_fullname() + # superfluous 'ed' tag is only a hint to the user, + # and guarantees we can't overwrite the normal wheel + wheel_name = f"{fullname}-ed.py3-none-any.whl" + wheel_path = dist_dir / wheel_name + + wheelmeta_builder(dist_dir / dist_info_dir / "WHEEL") + + if wheel_path.exists(): + wheel_path.unlink() + + with zipfile.ZipFile( + wheel_path, "a", compression=zipfile.ZIP_DEFLATED + ) as archive: + + # copy .pth file + for f, data in project.files(): + archive.writestr( + zipfile.ZipInfo(f, time.gmtime(SOURCE_EPOCH_ZIP)[:6]), data + ) + + # copy .dist-info directory + for f in sorted(os.listdir(dist_dir / dist_info_dir)): + with (dist_dir / dist_info_dir / f).open() as metadata: + archive.writestr( + zipfile.ZipInfo( + str(dist_info_dir / f), time.gmtime(SOURCE_EPOCH_ZIP)[:6] + ), + metadata.read(), + ) + + add_manifest(archive, dist_info_dir) + + +import base64 + + +def urlsafe_b64encode(data): + """urlsafe_b64encode without padding""" + return base64.urlsafe_b64encode(data).rstrip(b"=") + + +# standalone wheel helpers based on enscons +def add_manifest(archive, dist_info_dir): + """ + Add the wheel manifest. + """ + import hashlib + import zipfile + + lines = [] + for f in archive.namelist(): + data = archive.read(f) + size = len(data) + digest = hashlib.sha256(data).digest() + digest = "sha256=" + (urlsafe_b64encode(digest).decode("ascii")) + lines.append("%s,%s,%s" % (f.replace(",", ",,"), digest, size)) + + record_path = dist_info_dir / "RECORD" + lines.append(str(record_path) + ",,") + RECORD = "\n".join(lines) + archive.writestr( + zipfile.ZipInfo(str(record_path), time.gmtime(SOURCE_EPOCH_ZIP)[:6]), RECORD + ) + archive.close() + + +def wheelmeta_builder(target): + with open(target, "w+") as f: + f.write( + """Wheel-Version: 1.0 +Generator: setuptools_pep660 (0.1) +Root-Is-Purelib: false +Tag: py3-none-any +Tag: ed-none-any +""" + ) -- cgit v1.2.1 From ff3447a694f3b08dae8bd5268e64aa43f05a47a9 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 6 Feb 2022 11:20:38 -0500 Subject: Migrate remainder of 'iter_entry_points' to importlib_metadata. --- setuptools/command/egg_info.py | 11 ++++++----- setuptools/command/sdist.py | 4 ++-- setuptools/command/upload_docs.py | 10 ++++++---- 3 files changed, 14 insertions(+), 11 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py index 379f9398..d0e73002 100644 --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py @@ -17,6 +17,8 @@ import warnings import time import collections +from .._importlib import metadata + from setuptools import Command from setuptools.command.sdist import sdist from setuptools.command.sdist import walk_revctrl @@ -24,7 +26,7 @@ from setuptools.command.setopt import edit_config from setuptools.command import bdist_egg from pkg_resources import ( Requirement, safe_name, parse_version, - safe_version, yield_lines, EntryPoint, iter_entry_points, to_filename) + safe_version, yield_lines, EntryPoint, to_filename) import setuptools.unicode_utils as unicode_utils from setuptools.glob import glob @@ -281,10 +283,9 @@ class egg_info(InfoCommon, Command): def run(self): self.mkpath(self.egg_info) os.utime(self.egg_info, None) - installer = self.distribution.fetch_build_egg - for ep in iter_entry_points('egg_info.writers'): - ep.require(installer=installer) - writer = ep.resolve() + for ep in metadata.entry_points(group='egg_info.writers'): + self.distribution._install_dependencies(ep) + writer = ep.load() writer(self, ep.name, os.path.join(self.egg_info, ep.name)) # Get rid of native_libs.txt if it was put there by older bdist_egg diff --git a/setuptools/command/sdist.py b/setuptools/command/sdist.py index 0285b690..0ffeacf3 100644 --- a/setuptools/command/sdist.py +++ b/setuptools/command/sdist.py @@ -7,14 +7,14 @@ import contextlib from .py36compat import sdist_add_defaults -import pkg_resources +from .._importlib import metadata _default_revctrl = list def walk_revctrl(dirname=''): """Find all files under revision control""" - for ep in pkg_resources.iter_entry_points('setuptools.file_finders'): + for ep in metadata.entry_points(group='setuptools.file_finders'): for item in ep.load()(dirname): yield item diff --git a/setuptools/command/upload_docs.py b/setuptools/command/upload_docs.py index 845bff44..f429f568 100644 --- a/setuptools/command/upload_docs.py +++ b/setuptools/command/upload_docs.py @@ -18,7 +18,8 @@ import functools import http.client import urllib.parse -from pkg_resources import iter_entry_points +from .._importlib import metadata + from .upload import upload @@ -43,9 +44,10 @@ class upload_docs(upload): boolean_options = upload.boolean_options def has_sphinx(self): - if self.upload_dir is None: - for ep in iter_entry_points('distutils.commands', 'build_sphinx'): - return True + return bool( + self.upload_dir is None + and metadata.entry_points(group='distutils.commands', name='build_sphinx') + ) sub_commands = [('build_sphinx', has_sphinx)] -- cgit v1.2.1 From 867147f45c2b929f32b364284448b9d08c397dcb Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 6 Feb 2022 11:28:02 -0500 Subject: Avoid dual-use variable. --- setuptools/command/egg_info.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py index d0e73002..17955207 100644 --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py @@ -721,13 +721,13 @@ def write_entries(cmd, basename, filename): if isinstance(ep, str) or ep is None: data = ep elif ep is not None: - data = [] + lines = [] for section, contents in sorted(ep.items()): if not isinstance(contents, str): contents = EntryPoint.parse_group(section, contents) contents = '\n'.join(sorted(map(str, contents.values()))) - data.append('[%s]\n%s\n\n' % (section, contents)) - data = ''.join(data) + lines.append('[%s]\n%s\n\n' % (section, contents)) + data = ''.join(lines) cmd.write_or_delete_file('entry points', filename, data, True) -- cgit v1.2.1 From 282f2120979d0d97ae52feb557a19c094e548c87 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 6 Feb 2022 11:29:10 -0500 Subject: Remove duplicate check on ep is None. --- setuptools/command/egg_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py index 17955207..439fe213 100644 --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py @@ -720,7 +720,7 @@ def write_entries(cmd, basename, filename): if isinstance(ep, str) or ep is None: data = ep - elif ep is not None: + else: lines = [] for section, contents in sorted(ep.items()): if not isinstance(contents, str): -- cgit v1.2.1 From c5f7e3b19c153712f4e77e3c71ce5a7ba9668bc6 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 6 Feb 2022 11:41:00 -0500 Subject: Refactor to construct data in a single expression and extract 'to_str'. --- setuptools/command/egg_info.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py index 439fe213..473b7aa4 100644 --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py @@ -721,13 +721,15 @@ def write_entries(cmd, basename, filename): if isinstance(ep, str) or ep is None: data = ep else: - lines = [] - for section, contents in sorted(ep.items()): - if not isinstance(contents, str): - contents = EntryPoint.parse_group(section, contents) - contents = '\n'.join(sorted(map(str, contents.values()))) - lines.append('[%s]\n%s\n\n' % (section, contents)) - data = ''.join(lines) + def to_str(contents): + if isinstance(contents, str): + return contents + eps = EntryPoint.parse_group('anything', contents) + return '\n'.join(sorted(map(str, eps.values()))) + data = ''.join( + f'[{section}]\n{to_str(contents)}\n\n' + for section, contents in sorted(ep.items()) + ) cmd.write_or_delete_file('entry points', filename, data, True) -- cgit v1.2.1 From d47d35616920f2f373cc6afbdaf4f30f3faca90f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 6 Feb 2022 11:52:27 -0500 Subject: Refactor to extract entry_points_definition generation. --- setuptools/command/egg_info.py | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py index 473b7aa4..b98b84d4 100644 --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py @@ -715,23 +715,30 @@ def write_arg(cmd, basename, filename, force=False): cmd.write_or_delete_file(argname, filename, value, force) -def write_entries(cmd, basename, filename): - ep = cmd.distribution.entry_points - - if isinstance(ep, str) or ep is None: - data = ep - else: - def to_str(contents): - if isinstance(contents, str): - return contents - eps = EntryPoint.parse_group('anything', contents) - return '\n'.join(sorted(map(str, eps.values()))) - data = ''.join( - f'[{section}]\n{to_str(contents)}\n\n' - for section, contents in sorted(ep.items()) - ) +@functools.singledispatch +def entry_points_definition(eps): + """ + Given a Distribution.entry_points, produce a multiline + string definition of those entry points. + """ + def to_str(contents): + if isinstance(contents, str): + return contents + parsed = EntryPoint.parse_group('anything', contents) + return '\n'.join(sorted(map(str, parsed.values()))) + return ''.join( + f'[{section}]\n{to_str(contents)}\n\n' + for section, contents in sorted(eps.items()) + ) - cmd.write_or_delete_file('entry points', filename, data, True) + +entry_points_definition.register(type(None), lambda x: x) +entry_points_definition.register(str, lambda x: x) + + +def write_entries(cmd, basename, filename): + defn = entry_points_definition(cmd.distribution.entry_points) + cmd.write_or_delete_file('entry points', filename, defn, True) def get_pkg_info_revision(): -- cgit v1.2.1 From 161ff0ff6f679967d323e9fd461eff312d0f12e6 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 6 Feb 2022 12:00:14 -0500 Subject: Extract function for converting entry points to a string. --- setuptools/command/egg_info.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py index b98b84d4..afab5cd6 100644 --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py @@ -715,19 +715,27 @@ def write_arg(cmd, basename, filename, force=False): cmd.write_or_delete_file(argname, filename, value, force) +@functools.singledispatch +def entry_point_definition_to_str(value): + """ + Given a value of an entry point or series of entry points, + return each entry point on a single line. + """ + parsed = EntryPoint.parse_group('anything', value) + return '\n'.join(sorted(map(str, parsed.values()))) + + +entry_point_definition_to_str.register(str, lambda x: x) + + @functools.singledispatch def entry_points_definition(eps): """ Given a Distribution.entry_points, produce a multiline string definition of those entry points. """ - def to_str(contents): - if isinstance(contents, str): - return contents - parsed = EntryPoint.parse_group('anything', contents) - return '\n'.join(sorted(map(str, parsed.values()))) return ''.join( - f'[{section}]\n{to_str(contents)}\n\n' + f'[{section}]\n{entry_point_definition_to_str(contents)}\n\n' for section, contents in sorted(eps.items()) ) -- cgit v1.2.1 From b257d137ae2bcf1ef2e188f20e60f3ca5770e090 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 6 Feb 2022 12:43:29 -0500 Subject: In egg_info, port use of pkg_resources.EntryPoint to importlib.metadata --- setuptools/command/egg_info.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py index afab5cd6..2ed58eef 100644 --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py @@ -16,8 +16,10 @@ import io import warnings import time import collections +import operator from .._importlib import metadata +from .._itertools import ensure_unique from setuptools import Command from setuptools.command.sdist import sdist @@ -26,7 +28,7 @@ from setuptools.command.setopt import edit_config from setuptools.command import bdist_egg from pkg_resources import ( Requirement, safe_name, parse_version, - safe_version, yield_lines, EntryPoint, to_filename) + safe_version, yield_lines, to_filename) import setuptools.unicode_utils as unicode_utils from setuptools.glob import glob @@ -721,8 +723,14 @@ def entry_point_definition_to_str(value): Given a value of an entry point or series of entry points, return each entry point on a single line. """ - parsed = EntryPoint.parse_group('anything', value) - return '\n'.join(sorted(map(str, parsed.values()))) + # normalize to a single sequence of lines + lines = yield_lines(value) + parsed = metadata.EntryPoints._from_text('[x]\n' + '\n'.join(lines)) + valid = ensure_unique(parsed, key=operator.attrgetter('name')) + + def ep_to_str(ep): + return f'{ep.name} = {ep.value}' + return '\n'.join(sorted(map(ep_to_str, valid))) entry_point_definition_to_str.register(str, lambda x: x) -- cgit v1.2.1 From abf002112b77c26102a60116a0336ad2e4f56611 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 6 Feb 2022 13:31:44 -0500 Subject: In test command, rely on metadata.EntryPoint for loading the value. --- setuptools/command/test.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/test.py b/setuptools/command/test.py index 4a389e4d..652f3e4a 100644 --- a/setuptools/command/test.py +++ b/setuptools/command/test.py @@ -16,10 +16,11 @@ from pkg_resources import ( evaluate_marker, add_activation_listener, require, - EntryPoint, ) +from .._importlib import metadata from setuptools import Command from setuptools.extern.more_itertools import unique_everseen +from setuptools.extern.jaraco.functools import pass_none class ScanningLoader(TestLoader): @@ -241,12 +242,10 @@ class test(Command): return ['unittest'] + self.test_args @staticmethod + @pass_none def _resolve_as_ep(val): """ Load the indicated attribute value, called, as a as if it were specified as an entry point. """ - if val is None: - return - parsed = EntryPoint.parse("x=" + val) - return parsed.resolve()() + return metadata.EntryPoint(value=val, name=None, group=None).load()() -- cgit v1.2.1 From 67b25e3986aef5ac04b57be1a5c569e18f95a3d1 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 6 Feb 2022 14:30:08 -0500 Subject: Extract module for entry point management. --- setuptools/command/egg_info.py | 40 ++-------------------------------------- 1 file changed, 2 insertions(+), 38 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py index 2ed58eef..2e8ca4b7 100644 --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py @@ -16,10 +16,9 @@ import io import warnings import time import collections -import operator from .._importlib import metadata -from .._itertools import ensure_unique +from .. import _entry_points from setuptools import Command from setuptools.command.sdist import sdist @@ -717,43 +716,8 @@ def write_arg(cmd, basename, filename, force=False): cmd.write_or_delete_file(argname, filename, value, force) -@functools.singledispatch -def entry_point_definition_to_str(value): - """ - Given a value of an entry point or series of entry points, - return each entry point on a single line. - """ - # normalize to a single sequence of lines - lines = yield_lines(value) - parsed = metadata.EntryPoints._from_text('[x]\n' + '\n'.join(lines)) - valid = ensure_unique(parsed, key=operator.attrgetter('name')) - - def ep_to_str(ep): - return f'{ep.name} = {ep.value}' - return '\n'.join(sorted(map(ep_to_str, valid))) - - -entry_point_definition_to_str.register(str, lambda x: x) - - -@functools.singledispatch -def entry_points_definition(eps): - """ - Given a Distribution.entry_points, produce a multiline - string definition of those entry points. - """ - return ''.join( - f'[{section}]\n{entry_point_definition_to_str(contents)}\n\n' - for section, contents in sorted(eps.items()) - ) - - -entry_points_definition.register(type(None), lambda x: x) -entry_points_definition.register(str, lambda x: x) - - def write_entries(cmd, basename, filename): - defn = entry_points_definition(cmd.distribution.entry_points) + defn = _entry_points.render(cmd.distribution.entry_points) cmd.write_or_delete_file('entry points', filename, defn, True) -- cgit v1.2.1 From ebdaa76c3c6c55d5cffd1a80903484d80cf146c6 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 6 Feb 2022 15:22:38 -0500 Subject: Refactor _entry_points to separate loading from rendering. Explicitly validate and restore validation of entry points that don't match the pattern. --- setuptools/command/egg_info.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py index 2e8ca4b7..8af018f4 100644 --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py @@ -717,7 +717,8 @@ def write_arg(cmd, basename, filename, force=False): def write_entries(cmd, basename, filename): - defn = _entry_points.render(cmd.distribution.entry_points) + eps = _entry_points.load(cmd.distribution.entry_points) + defn = _entry_points.render(eps) cmd.write_or_delete_file('entry points', filename, defn, True) -- cgit v1.2.1 From 740c3b13427aac1b353c0ad6f776d4c6f2655957 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 6 Feb 2022 15:43:03 -0500 Subject: Prefer jaraco.text for yield_lines. --- setuptools/command/easy_install.py | 4 +++- setuptools/command/egg_info.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index b1260dcd..5b73e6e9 100644 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -56,13 +56,15 @@ from setuptools.package_index import ( from setuptools.command import bdist_egg, egg_info from setuptools.wheel import Wheel from pkg_resources import ( - yield_lines, normalize_path, resource_string, + normalize_path, resource_string, get_distribution, find_distributions, Environment, Requirement, Distribution, PathMetadata, EggMetadata, WorkingSet, DistributionNotFound, VersionConflict, DEVELOP_DIST, ) import pkg_resources from .._path import ensure_directory +from ..extern.jaraco.text import yield_lines + # Turn on PEP440Warnings warnings.filterwarnings("default", category=pkg_resources.PEP440Warning) diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py index 8af018f4..63389654 100644 --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py @@ -27,11 +27,12 @@ from setuptools.command.setopt import edit_config from setuptools.command import bdist_egg from pkg_resources import ( Requirement, safe_name, parse_version, - safe_version, yield_lines, to_filename) + safe_version, to_filename) import setuptools.unicode_utils as unicode_utils from setuptools.glob import glob from setuptools.extern import packaging +from setuptools.extern.jaraco.text import yield_lines from setuptools import SetuptoolsDeprecationWarning -- cgit v1.2.1 From a5e663d83bee3ec96890a5f9b5d818c1fdd2d6bc Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 17 Feb 2022 19:36:43 -0500 Subject: Deprecated upload_docs command. Ref #2971 --- setuptools/command/upload_docs.py | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'setuptools/command') diff --git a/setuptools/command/upload_docs.py b/setuptools/command/upload_docs.py index f429f568..a5480005 100644 --- a/setuptools/command/upload_docs.py +++ b/setuptools/command/upload_docs.py @@ -17,8 +17,10 @@ import itertools import functools import http.client import urllib.parse +import warnings from .._importlib import metadata +from .. import SetuptoolsDeprecationWarning from .upload import upload @@ -89,6 +91,12 @@ class upload_docs(upload): zip_file.close() def run(self): + warnings.warn( + "upload_docs is deprecated and will be removed in a future " + "version. Use tools like httpie or curl instead.", + SetuptoolsDeprecationWarning, + ) + # Run sub commands for cmd_name in self.get_sub_commands(): self.run_command(cmd_name) -- cgit v1.2.1 From c44e416b44e5e7126f435a7c0b9adc9b88b85cbd Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Feb 2022 12:44:11 -0500 Subject: Prefer range().__contains__ for bounds check. --- setuptools/command/easy_install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index 5b73e6e9..07b45e59 100644 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -358,7 +358,7 @@ class easy_install(Command): if not isinstance(self.optimize, int): try: self.optimize = int(self.optimize) - if not (0 <= self.optimize <= 2): + if self.optimize not in range(3): raise ValueError except ValueError as e: raise DistutilsOptionError( -- cgit v1.2.1 From 66dcd5e54fd8fb1f9413b4fac04e073984ed0713 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Feb 2022 12:29:32 -0500 Subject: Use samefile from stdlib, supported on Windows since Python 3.2. --- setuptools/command/easy_install.py | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index 07b45e59..63403d19 100644 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -70,7 +70,7 @@ from ..extern.jaraco.text import yield_lines warnings.filterwarnings("default", category=pkg_resources.PEP440Warning) __all__ = [ - 'samefile', 'easy_install', 'PthDistributions', 'extract_wininst_cfg', + 'easy_install', 'PthDistributions', 'extract_wininst_cfg', 'get_exe_prefixes', ] @@ -79,22 +79,6 @@ def is_64bit(): return struct.calcsize("P") == 8 -def samefile(p1, p2): - """ - Determine if two paths reference the same file. - - Augments os.path.samefile to work on Windows and - suppresses errors if the path doesn't exist. - """ - both_exist = os.path.exists(p1) and os.path.exists(p2) - use_samefile = hasattr(os.path, 'samefile') and both_exist - if use_samefile: - return os.path.samefile(p1, p2) - norm_p1 = os.path.normpath(os.path.normcase(p1)) - norm_p2 = os.path.normpath(os.path.normcase(p2)) - return norm_p1 == norm_p2 - - def _to_bytes(s): return s.encode('utf8') @@ -928,7 +912,7 @@ class easy_install(Command): ensure_directory(destination) dist = self.egg_distribution(egg_path) - if not samefile(egg_path, destination): + if not os.path.samefile(egg_path, destination): if os.path.isdir(destination) and not os.path.islink(destination): dir_util.remove_tree(destination, dry_run=self.dry_run) elif os.path.exists(destination): -- cgit v1.2.1 From bbe8b50eccb5700c44bf793346dd09540bff97ee Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Feb 2022 12:48:11 -0500 Subject: Extract method to validate optimize parameter. --- setuptools/command/easy_install.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index 07b45e59..abf25eb9 100644 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -355,15 +355,7 @@ class easy_install(Command): if not self.no_find_links: self.package_index.add_find_links(self.find_links) self.set_undefined_options('install_lib', ('optimize', 'optimize')) - if not isinstance(self.optimize, int): - try: - self.optimize = int(self.optimize) - if self.optimize not in range(3): - raise ValueError - except ValueError as e: - raise DistutilsOptionError( - "--optimize must be 0, 1, or 2" - ) from e + self.optimize = self._validate_optimize(self.optimize) if self.editable and not self.build_directory: raise DistutilsArgError( @@ -375,6 +367,22 @@ class easy_install(Command): self.outputs = [] + @staticmethod + def _validate_optimize(value): + if isinstance(value, int): + return value + + try: + value = int(value) + if value not in range(3): + raise ValueError + except ValueError as e: + raise DistutilsOptionError( + "--optimize must be 0, 1, or 2" + ) from e + + return value + def _fix_install_dir_for_user_site(self): """ Fix the install_dir if "--user" was used. -- cgit v1.2.1 From 99f5ac503ab030c4622cbd8b5129e0880103a68f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Feb 2022 12:49:54 -0500 Subject: Remove 'isinstance(int)' check and just validate unconditionally. --- setuptools/command/easy_install.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index abf25eb9..e2a6543e 100644 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -369,9 +369,6 @@ class easy_install(Command): @staticmethod def _validate_optimize(value): - if isinstance(value, int): - return value - try: value = int(value) if value not in range(3): -- cgit v1.2.1 From d387ae78b3c6384cee30a441045e5b33f2a226b4 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Feb 2022 12:51:43 -0500 Subject: Move normpath into if block. --- setuptools/command/easy_install.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index e2a6543e..a526d705 100644 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -309,11 +309,9 @@ class easy_install(Command): self.script_dir = self.install_scripts # default --record from the install command self.set_undefined_options('install', ('record', 'record')) - # Should this be moved to the if statement below? It's not used - # elsewhere - normpath = map(normalize_path, sys.path) self.all_site_dirs = get_site_dirs() if self.site_dirs is not None: + normpath = map(normalize_path, sys.path) site_dirs = [ os.path.expanduser(s.strip()) for s in self.site_dirs.split(',') -- cgit v1.2.1 From 339c29920abdabdd9e6b5983ae711efb61b15d76 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Feb 2022 12:57:38 -0500 Subject: Extract method for processing site dirs --- setuptools/command/easy_install.py | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index a526d705..905bc627 100644 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -310,21 +310,8 @@ class easy_install(Command): # default --record from the install command self.set_undefined_options('install', ('record', 'record')) self.all_site_dirs = get_site_dirs() - if self.site_dirs is not None: - normpath = map(normalize_path, sys.path) - site_dirs = [ - os.path.expanduser(s.strip()) for s in - self.site_dirs.split(',') - ] - for d in site_dirs: - if not os.path.isdir(d): - log.warn("%s (in --site-dirs) does not exist", d) - elif normalize_path(d) not in normpath: - raise DistutilsOptionError( - d + " (in --site-dirs) is not on sys.path" - ) - else: - self.all_site_dirs.append(normalize_path(d)) + self.all_site_dirs.extend(self._process_site_dirs(self.site_dirs)) + if not self.editable: self.check_site_dir() self.index_url = self.index_url or "https://pypi.org/simple/" @@ -365,6 +352,26 @@ class easy_install(Command): self.outputs = [] + @staticmethod + def _process_site_dirs(site_dirs): + if site_dirs is None: + return + + normpath = map(normalize_path, sys.path) + site_dirs = [ + os.path.expanduser(s.strip()) for s in + site_dirs.split(',') + ] + for d in site_dirs: + if not os.path.isdir(d): + log.warn("%s (in --site-dirs) does not exist", d) + elif normalize_path(d) not in normpath: + raise DistutilsOptionError( + d + " (in --site-dirs) is not on sys.path" + ) + else: + yield normalize_path(d) + @staticmethod def _validate_optimize(value): try: -- cgit v1.2.1 From fb7b30d64eb1475a0f5692e015ac123834ff6c40 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sun, 27 Feb 2022 18:43:22 +0000 Subject: Check for file existence before using samefile --- setuptools/command/easy_install.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index 3aed8caa..80ff6347 100644 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -922,7 +922,8 @@ class easy_install(Command): ensure_directory(destination) dist = self.egg_distribution(egg_path) - if not os.path.samefile(egg_path, destination): + both_exist = os.path.exists(egg_path) and os.path.exists(destination) + if not (both_exist and os.path.samefile(egg_path, destination)): if os.path.isdir(destination) and not os.path.islink(destination): dir_util.remove_tree(destination, dry_run=self.dry_run) elif os.path.exists(destination): -- cgit v1.2.1 From 597ff8774e505803a565d9bebde2f8a48519b033 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sun, 27 Feb 2022 18:57:50 +0000 Subject: Just check for if destination file exists --- setuptools/command/easy_install.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index 80ff6347..6da39e73 100644 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -922,8 +922,9 @@ class easy_install(Command): ensure_directory(destination) dist = self.egg_distribution(egg_path) - both_exist = os.path.exists(egg_path) and os.path.exists(destination) - if not (both_exist and os.path.samefile(egg_path, destination)): + if not ( + os.path.exists(destination) and os.path.samefile(egg_path, destination) + ): if os.path.isdir(destination) and not os.path.islink(destination): dir_util.remove_tree(destination, dry_run=self.dry_run) elif os.path.exists(destination): -- cgit v1.2.1 From cb229fa27a86fc48bd40340eacbec60fe5aa609b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 27 Feb 2022 20:38:16 -0500 Subject: Use super throughout. --- setuptools/command/easy_install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index 6da39e73..107850a9 100644 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -1655,14 +1655,14 @@ class PthDistributions(Environment): if new_path: self.paths.append(dist.location) self.dirty = True - Environment.add(self, dist) + super().add(dist) def remove(self, dist): """Remove `dist` from the distribution map""" while dist.location in self.paths: self.paths.remove(dist.location) self.dirty = True - Environment.remove(self, dist) + super().remove(dist) def make_relative(self, path): npath, last = os.path.split(normalize_path(path)) -- cgit v1.2.1 From 6376ad10547315c15dfec719ff3f384e7a94dfc2 Mon Sep 17 00:00:00 2001 From: Andrey Bienkowski Date: Sun, 6 Mar 2022 00:43:07 +0300 Subject: XXX: Debugging --- setuptools/command/easy_install.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) (limited to 'setuptools/command') diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index 107850a9..318eac31 100644 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -221,6 +221,42 @@ class easy_install(Command): raise SystemExit() def finalize_options(self): # noqa: C901 # is too complex (25) # FIXME + print(sysconfig._INSTALL_SCHEMES) + print(f'{os.name}_user') + + def global_trace(frame, event, arg): + pass + + import sys + + sys.settrace(global_trace) + + XXX = [set()] + + def trace(frame, event, arg): + config_vars = getattr(self, 'config_vars', {}) + o = { + ('USER_BASE', site.USER_BASE), + ('USER_SITE', site.USER_SITE), + ('install_dir', self.install_dir), + ('install_userbase', self.install_userbase), + ('install_usersite', self.install_usersite), + ('install_purelib', self.install_purelib), + ('install_scripts', self.install_scripts), + ('userbase', config_vars.get('userbase')), + ('usersite', config_vars.get('usersite')), + } + if XXX[0] - o: + print('-', XXX[0] - o) + if o - XXX[0]: + print('+', o - XXX[0]) + XXX[0] = o + lines, start = inspect.getsourcelines(frame) + print(frame.f_lineno, lines[frame.f_lineno - start]) + + import inspect + inspect.currentframe().f_trace = trace + self.version and self._render_version() py_version = sys.version.split()[0] @@ -459,6 +495,7 @@ class easy_install(Command): instdir = normalize_path(self.install_dir) pth_file = os.path.join(instdir, 'easy-install.pth') + print('XXX', instdir, os.path.exists(instdir)) if not os.path.exists(instdir): try: os.makedirs(instdir) -- cgit v1.2.1 From 45340d00688ba29fc3492c52c88c47d14ce918e6 Mon Sep 17 00:00:00 2001 From: Andrey Bienkowski Date: Sun, 6 Mar 2022 07:58:24 +0300 Subject: Revert "XXX: Debugging" This reverts commit 6376ad10547315c15dfec719ff3f384e7a94dfc2. --- setuptools/command/easy_install.py | 37 ------------------------------------- 1 file changed, 37 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index 318eac31..107850a9 100644 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -221,42 +221,6 @@ class easy_install(Command): raise SystemExit() def finalize_options(self): # noqa: C901 # is too complex (25) # FIXME - print(sysconfig._INSTALL_SCHEMES) - print(f'{os.name}_user') - - def global_trace(frame, event, arg): - pass - - import sys - - sys.settrace(global_trace) - - XXX = [set()] - - def trace(frame, event, arg): - config_vars = getattr(self, 'config_vars', {}) - o = { - ('USER_BASE', site.USER_BASE), - ('USER_SITE', site.USER_SITE), - ('install_dir', self.install_dir), - ('install_userbase', self.install_userbase), - ('install_usersite', self.install_usersite), - ('install_purelib', self.install_purelib), - ('install_scripts', self.install_scripts), - ('userbase', config_vars.get('userbase')), - ('usersite', config_vars.get('usersite')), - } - if XXX[0] - o: - print('-', XXX[0] - o) - if o - XXX[0]: - print('+', o - XXX[0]) - XXX[0] = o - lines, start = inspect.getsourcelines(frame) - print(frame.f_lineno, lines[frame.f_lineno - start]) - - import inspect - inspect.currentframe().f_trace = trace - self.version and self._render_version() py_version = sys.version.split()[0] @@ -495,7 +459,6 @@ class easy_install(Command): instdir = normalize_path(self.install_dir) pth_file = os.path.join(instdir, 'easy-install.pth') - print('XXX', instdir, os.path.exists(instdir)) if not os.path.exists(instdir): try: os.makedirs(instdir) -- cgit v1.2.1 From b828c32cd49f2281156644fce55d3c40663081dd Mon Sep 17 00:00:00 2001 From: Andrey Bienkowski Date: Sat, 5 Mar 2022 15:20:42 +0300 Subject: Fix editable --user installs with build isolation https://github.com/pypa/setuptools/issues/3019 --- setuptools/command/easy_install.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index 107850a9..940c916f 100644 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -169,12 +169,8 @@ class easy_install(Command): self.install_data = None self.install_base = None self.install_platbase = None - if site.ENABLE_USER_SITE: - self.install_userbase = site.USER_BASE - self.install_usersite = site.USER_SITE - else: - self.install_userbase = None - self.install_usersite = None + self.install_userbase = site.USER_BASE + self.install_usersite = site.USER_SITE self.no_find_links = None # Options not specifiable via command line @@ -253,11 +249,9 @@ class easy_install(Command): getattr(sys, 'windir', '').replace('.', ''), ) - if site.ENABLE_USER_SITE: - self.config_vars['userbase'] = self.install_userbase - self.config_vars['usersite'] = self.install_usersite - - elif self.user: + self.config_vars['userbase'] = self.install_userbase + self.config_vars['usersite'] = self.install_usersite + if self.user and not site.ENABLE_USER_SITE: log.warn("WARNING: The user site-packages directory is disabled.") self._fix_install_dir_for_user_site() @@ -373,7 +367,7 @@ class easy_install(Command): """ Fix the install_dir if "--user" was used. """ - if not self.user or not site.ENABLE_USER_SITE: + if not self.user: return self.create_home_path() -- cgit v1.2.1 From a5658e826c1191eb1a40bff894fb625af7cccaa9 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 25 Mar 2022 22:47:31 +0000 Subject: Add test for setup.py install and dependencies --- setuptools/command/easy_install.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index 107850a9..77dcd25c 100644 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -298,7 +298,9 @@ class easy_install(Command): if not self.editable: self.check_site_dir() - self.index_url = self.index_url or "https://pypi.org/simple/" + default_index = os.getenv("__EASYINSTALL_INDEX", "https://pypi.org/simple/") + # ^ Private API for testing purposes only + self.index_url = self.index_url or default_index self.shadow_path = self.all_site_dirs[:] for path_item in self.install_dir, normalize_path(self.script_dir): if path_item not in self.shadow_path: -- cgit v1.2.1 From 93a24585683944a9369d8fd37a824c0bca345af4 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 25 Mar 2022 22:48:21 +0000 Subject: Make install consider dist.run_command is overwritten in v61.0.0 Starting in v61, setuptools.dist overwrites distutils.dist.run_command to add auto-discovery functionality on top of the original implementation. This change modifies the existing code in setuptools.command.install to consider that previous change when trying to decide if the install command was called directly from `setup.py` or not. --- setuptools/command/install.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/install.py b/setuptools/command/install.py index 35e54d20..55fdb124 100644 --- a/setuptools/command/install.py +++ b/setuptools/command/install.py @@ -91,14 +91,21 @@ class install(orig.install): msg = "For best results, pass -X:Frames to enable call stack." warnings.warn(msg) return True - res = inspect.getouterframes(run_frame)[2] - caller, = res[:1] - info = inspect.getframeinfo(caller) - caller_module = caller.f_globals.get('__name__', '') - return ( - caller_module == 'distutils.dist' - and info.function == 'run_commands' - ) + + frames = inspect.getouterframes(run_frame) + for frame in frames[2:4]: + caller, = frame[:1] + info = inspect.getframeinfo(caller) + caller_module = caller.f_globals.get('__name__', '') + + if caller_module == "setuptools.dist" and info.function == "run_command": + # Starting from v61.0.0 setuptools overwrites dist.run_command + continue + + return ( + caller_module == 'distutils.dist' + and info.function == 'run_commands' + ) def do_egg_install(self): -- cgit v1.2.1 From cc93191764ed8b5de21369eec53aba32e692389c Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Tue, 29 Mar 2022 03:03:34 +0100 Subject: Fix duplicated version tags in egg_info Previously egg_info was adding duplicated tags to the version string. This was happening because of the version normalization. When the version normalization was applied to the string the tag was modified, then later egg_info could no longer recognize it before applying. The fix for this problem was to normalize the tag string before applying. --- setuptools/command/egg_info.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py index 63389654..ea47e519 100644 --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py @@ -140,13 +140,18 @@ class InfoCommon: else version + self.vtags ) - def tags(self): + def _safe_tags(self, tags: str) -> str: + # To implement this we can rely on `safe_version` pretending to be version 0 + # followed by tags. Then we simply discard the starting 0 (fake version number) + return safe_version(f"0{tags}")[1:] + + def tags(self) -> str: version = '' if self.tag_build: version += self.tag_build if self.tag_date: version += time.strftime("-%Y%m%d") - return version + return self._safe_tags(version) vtags = property(tags) -- cgit v1.2.1 From cabdd37db15e306060c1b5edcaeb242c218152f8 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Tue, 29 Mar 2022 03:37:04 +0100 Subject: Restore tags in egg_info but change the idempotency check --- setuptools/command/egg_info.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py index ea47e519..c37ab81f 100644 --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py @@ -136,14 +136,19 @@ class InfoCommon: in which case the version string already contains all tags. """ return ( - version if self.vtags and version.endswith(self.vtags) + version if self.vtags and self._already_tagged(version) else version + self.vtags ) - def _safe_tags(self, tags: str) -> str: + def _already_tagged(self, version: str) -> bool: + # Depending on their format, tags may change with version normalization. + # So in addition the regular tags, we have to search for the normalized ones. + return version.endswith(self.vtags) or version.endswith(self._safe_tags()) + + def _safe_tags(self) -> str: # To implement this we can rely on `safe_version` pretending to be version 0 # followed by tags. Then we simply discard the starting 0 (fake version number) - return safe_version(f"0{tags}")[1:] + return safe_version(f"0{self.vtags}")[1:] def tags(self) -> str: version = '' @@ -151,7 +156,7 @@ class InfoCommon: version += self.tag_build if self.tag_date: version += time.strftime("-%Y%m%d") - return self._safe_tags(version) + return version vtags = property(tags) -- cgit v1.2.1 From 4621b08512ab5c682191c13bf8810d7c200d7e34 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Tue, 29 Mar 2022 03:38:32 +0100 Subject: Change dist_info naming to use the same convention as bdist_wheel --- setuptools/command/dist_info.py | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/dist_info.py b/setuptools/command/dist_info.py index c45258fa..8b8509f3 100644 --- a/setuptools/command/dist_info.py +++ b/setuptools/command/dist_info.py @@ -4,9 +4,13 @@ As defined in the wheel specification """ import os +import re +import warnings +from inspect import cleandoc from distutils.core import Command from distutils import log +from setuptools.extern import packaging class dist_info(Command): @@ -29,8 +33,36 @@ class dist_info(Command): egg_info.egg_base = self.egg_base egg_info.finalize_options() egg_info.run() - dist_info_dir = egg_info.egg_info[:-len('.egg-info')] + '.dist-info' + name = _safe(self.distribution.get_name()) + version = _version(self.distribution.get_version()) + base = self.egg_base or os.curdir + dist_info_dir = os.path.join(base, f"{name}-{version}.dist-info") log.info("creating '{}'".format(os.path.abspath(dist_info_dir))) bdist_wheel = self.get_finalized_command('bdist_wheel') bdist_wheel.egg2dist(egg_info.egg_info, dist_info_dir) + + +def _safe(component: str) -> str: + """Escape a component used to form a wheel name according to PEP 491""" + return re.sub(r"[^\w\d.]+", "_", component) + + +def _version(version: str) -> str: + """Convert an arbitrary string to a version string.""" + v = version.replace(' ', '.') + try: + return str(packaging.version.Version(v)).replace("-", "_") + except packaging.version.InvalidVersion: + msg = f"""!!\n\n + ################### + # Invalid version # + ################### + {version!r} is not valid according to PEP 440.\n + Please make sure specify a valid version for your package. + Also note that future releases of setuptools may halt the build process + if an invalid version is given. + \n\n!! + """ + warnings.warn(cleandoc(msg)) + return _safe(v).strip("_") -- cgit v1.2.1 From b517cfae6b11c15834aa7aaf439bc45894a238f4 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Thu, 7 Apr 2022 14:32:04 -0500 Subject: Add setuptools.command.build In order to override distutils.command.build on downstream projects it is good to have a setuptools specific command which allows downstream projects to avoid importing distutils. --- setuptools/command/build.py | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 setuptools/command/build.py (limited to 'setuptools/command') diff --git a/setuptools/command/build.py b/setuptools/command/build.py new file mode 100644 index 00000000..46b75559 --- /dev/null +++ b/setuptools/command/build.py @@ -0,0 +1,4 @@ +from distutils.command.build import build as _build + +class build(_build): + pass -- cgit v1.2.1 From 8a8aa3c861efb589a50dead84dc2dcb89cddcf43 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Mon, 11 Apr 2022 21:25:23 +0100 Subject: Fix flake8 error --- setuptools/command/build.py | 1 + 1 file changed, 1 insertion(+) (limited to 'setuptools/command') diff --git a/setuptools/command/build.py b/setuptools/command/build.py index 46b75559..6f0e560c 100644 --- a/setuptools/command/build.py +++ b/setuptools/command/build.py @@ -1,4 +1,5 @@ from distutils.command.build import build as _build + class build(_build): pass -- cgit v1.2.1 From 56a8b90551411024f792efe853c48de5e5097e59 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Thu, 5 May 2022 19:34:46 +0100 Subject: Warn about packages/modules included as package data --- setuptools/command/build_py.py | 56 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index c3fdc092..ba7b7259 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -8,6 +8,9 @@ import io import distutils.errors import itertools import stat +import warnings +from pathlib import Path +from setuptools._deprecation_warning import SetuptoolsDeprecationWarning from setuptools.extern.more_itertools import unique_everseen @@ -129,6 +132,7 @@ class build_py(orig.build_py): src_dirs[assert_relative(self.get_package_dir(package))] = package self.run_command('egg_info') + checker = _IncludePackageDataAbuse() ei_cmd = self.get_finalized_command('egg_info') for path in ei_cmd.filelist.files: d, f = os.path.split(assert_relative(path)) @@ -139,8 +143,13 @@ class build_py(orig.build_py): d, df = os.path.split(d) f = os.path.join(df, f) if d in src_dirs: - if path.endswith('.py') and f == oldf: - continue # it's a module, not data + if f == oldf: + if checker.is_module(f): + continue # it's a module, not data + else: + importable = checker.importable_item(src_dirs[d], f) + if importable: + checker.warn(importable) mf.setdefault(src_dirs[d], []).append(path) def get_data_files(self): @@ -240,3 +249,46 @@ def assert_relative(path): % path ) raise DistutilsSetupError(msg) + + +class _IncludePackageDataAbuse: + """Inform users that package or module is included as 'data file'""" + + MESSAGE = """\ + !!\n\n + ################################### + # Package/module would be ignored # + ################################### + Python recognizes {importable!r} as an importable package or module, however + it is included in the distribution as "data". + This behavior is likely to change in future versions of setuptools (and + therefore is considered deprecated). + + Please make sure that {importable!r} is recognized as a package/module by using + setuptools' `packages` configuration field or the proper package discovery methods. + + To find more information, look for "package discovery" and "data files" on + setuptools documentation page. + \n\n!! + """ + + def __init__(self): + self._already_warned = set() + + def is_module(self, file): + return file.endswith(".py") and file[:-len(".py")].isidentifier() + + def importable_item(self, pkg, file): + path = Path(file) + parents = path.parent.parts + module = [path.stem] if tuple(path.suffixes) == (".py",) else [] + parts = list(itertools.takewhile(str.isidentifier, [*parents, *module])) + if parts: + return ".".join([pkg, *parts]) + return None + + def warn(self, importable): + if importable not in self._already_warned: + msg = textwrap.dedent(self.MESSAGE).format(importable=importable) + warnings.warn(msg, SetuptoolsDeprecationWarning, stacklevel=2) + self._already_warned.add(importable) -- cgit v1.2.1 From 2b218927334c58a655fc285a5c241828d394cffe Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 6 May 2022 10:09:48 +0100 Subject: Simplify checks for abuse of include_package_data Previously, the checks would result in a warning per module additionally to the parent package. Now only one warning per parent package is issued. --- setuptools/command/build_py.py | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index ba7b7259..62f61e04 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -132,7 +132,7 @@ class build_py(orig.build_py): src_dirs[assert_relative(self.get_package_dir(package))] = package self.run_command('egg_info') - checker = _IncludePackageDataAbuse() + check = _IncludePackageDataAbuse() ei_cmd = self.get_finalized_command('egg_info') for path in ei_cmd.filelist.files: d, f = os.path.split(assert_relative(path)) @@ -144,12 +144,12 @@ class build_py(orig.build_py): f = os.path.join(df, f) if d in src_dirs: if f == oldf: - if checker.is_module(f): + if check.is_module(f): continue # it's a module, not data else: - importable = checker.importable_item(src_dirs[d], f) + importable = check.importable_subpackage(src_dirs[d], f) if importable: - checker.warn(importable) + check.warn(importable) mf.setdefault(src_dirs[d], []).append(path) def get_data_files(self): @@ -256,19 +256,19 @@ class _IncludePackageDataAbuse: MESSAGE = """\ !!\n\n - ################################### - # Package/module would be ignored # - ################################### - Python recognizes {importable!r} as an importable package or module, however - it is included in the distribution as "data". + ############################ + # Package would be ignored # + ############################ + Python recognizes {importable!r} as an importable package, however it is + included in the distribution as "data". This behavior is likely to change in future versions of setuptools (and therefore is considered deprecated). - Please make sure that {importable!r} is recognized as a package/module by using - setuptools' `packages` configuration field or the proper package discovery methods. + Please make sure that {importable!r} is included as a package by using + setuptools' `packages` configuration field or the proper discovery methods. - To find more information, look for "package discovery" and "data files" on - setuptools documentation page. + You can read more about "package discovery" and "data files" on setuptools + documentation page. \n\n!! """ @@ -278,13 +278,11 @@ class _IncludePackageDataAbuse: def is_module(self, file): return file.endswith(".py") and file[:-len(".py")].isidentifier() - def importable_item(self, pkg, file): - path = Path(file) - parents = path.parent.parts - module = [path.stem] if tuple(path.suffixes) == (".py",) else [] - parts = list(itertools.takewhile(str.isidentifier, [*parents, *module])) + def importable_subpackage(self, parent, file): + pkg = Path(file).parent + parts = list(itertools.takewhile(str.isidentifier, pkg.parts)) if parts: - return ".".join([pkg, *parts]) + return ".".join([parent, *parts]) return None def warn(self, importable): -- cgit v1.2.1 From ed6790fb0220686c648c578726c92664984bcaec Mon Sep 17 00:00:00 2001 From: nullableVoidPtr <30564701+nullableVoidPtr@users.noreply.github.com> Date: Sat, 7 May 2022 21:25:55 +0800 Subject: Allow recursive globs for package_data --- setuptools/command/build_py.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index c3fdc092..ac7cff95 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -1,3 +1,4 @@ +from functools import partial from glob import glob from distutils.util import convert_path import distutils.command.build_py as orig @@ -98,7 +99,7 @@ class build_py(orig.build_py): package, src_dir, ) - globs_expanded = map(glob, patterns) + globs_expanded = map(partial(glob, recursive=True), patterns) # flatten the expanded globs into an iterable of matches globs_matches = itertools.chain.from_iterable(globs_expanded) glob_files = filter(os.path.isfile, globs_matches) -- cgit v1.2.1 From 89173d2f8e88e9fc735806a237215f50df72e962 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Wed, 18 May 2022 11:33:49 +0100 Subject: Improve warning message for _IncludePackageDataAbuse --- setuptools/command/build_py.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index 91f47416..f3d43c56 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -256,6 +256,7 @@ class _IncludePackageDataAbuse: """Inform users that package or module is included as 'data file'""" MESSAGE = """\ + Installing {importable!r} as data is deprecated, please list it as `package`. !!\n\n ############################ # Package would be ignored # @@ -266,7 +267,9 @@ class _IncludePackageDataAbuse: therefore is considered deprecated). Please make sure that {importable!r} is included as a package by using - setuptools' `packages` configuration field or the proper discovery methods. + setuptools' `packages` configuration field or the proper discovery methods + (for example by using `find_namespace_packages(...)`/`find_namespace:` + instead of `find_packages(...)`/`find:`). You can read more about "package discovery" and "data files" on setuptools documentation page. -- cgit v1.2.1 From 81c944b60b81c9c2d25a38be050355d891101588 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Wed, 18 May 2022 11:59:49 +0100 Subject: Improve warning message for dist_info._version --- setuptools/command/dist_info.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/dist_info.py b/setuptools/command/dist_info.py index 8b8509f3..ca540ad1 100644 --- a/setuptools/command/dist_info.py +++ b/setuptools/command/dist_info.py @@ -54,7 +54,8 @@ def _version(version: str) -> str: try: return str(packaging.version.Version(v)).replace("-", "_") except packaging.version.InvalidVersion: - msg = f"""!!\n\n + msg = f"""Invalid version: {version!r}. + !!\n\n ################### # Invalid version # ################### -- cgit v1.2.1 From 3eba1cb02fba0d00ef65a653a3a729e5a524d771 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Wed, 18 May 2022 14:34:41 +0100 Subject: Update setuptools/command/build_py.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Michał Górny --- setuptools/command/build_py.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index f3d43c56..d2ccb514 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -256,7 +256,7 @@ class _IncludePackageDataAbuse: """Inform users that package or module is included as 'data file'""" MESSAGE = """\ - Installing {importable!r} as data is deprecated, please list it as `package`. + Installing {importable!r} as data is deprecated, please list it in `packages`. !!\n\n ############################ # Package would be ignored # -- cgit v1.2.1 From bb0eb4ebf8d289c6c37926bfadd44fe78548c01a Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sun, 5 Jun 2022 11:07:35 +0100 Subject: Improve warning message for package data abuse --- setuptools/command/build_py.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index d2ccb514..86847f07 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -261,13 +261,14 @@ class _IncludePackageDataAbuse: ############################ # Package would be ignored # ############################ - Python recognizes {importable!r} as an importable package, however it is - included in the distribution as "data". - This behavior is likely to change in future versions of setuptools (and - therefore is considered deprecated). + Python recognizes {importable!r} as an importable package, + but it is not listed in the `packages` configuration of setuptools. + Currently {importable!r} is only added to the distribution because it may + contain data files, but this behavior is likely to change in future + versions of setuptools (and therefore is considered deprecated). Please make sure that {importable!r} is included as a package by using - setuptools' `packages` configuration field or the proper discovery methods + the `packages` configuration field or the proper discovery methods (for example by using `find_namespace_packages(...)`/`find_namespace:` instead of `find_packages(...)`/`find:`). -- cgit v1.2.1 From e91918d69443eb7ef1e1d84dca277965ef048ade Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Thu, 9 Jun 2022 13:55:39 +0100 Subject: Change warning about namespace packages and data files This follows a suggestion given by users to improve clarity. --- setuptools/command/build_py.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index 86847f07..2fced3d6 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -263,9 +263,10 @@ class _IncludePackageDataAbuse: ############################ Python recognizes {importable!r} as an importable package, but it is not listed in the `packages` configuration of setuptools. - Currently {importable!r} is only added to the distribution because it may - contain data files, but this behavior is likely to change in future - versions of setuptools (and therefore is considered deprecated). + + {importable!r} has been automatically added to the distribution only + because it may contain data files, but this behavior is likely to change + in future versions of setuptools (and therefore is considered deprecated). Please make sure that {importable!r} is included as a package by using the `packages` configuration field or the proper discovery methods -- cgit v1.2.1 From 082c80e394a4825bc5ec31ffed1f5ab9caeb5d83 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Thu, 9 Jun 2022 11:36:15 -0500 Subject: separate subcommands for setuptools class Original author: Anderson Bravalheri --- setuptools/command/build.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/build.py b/setuptools/command/build.py index 6f0e560c..932de472 100644 --- a/setuptools/command/build.py +++ b/setuptools/command/build.py @@ -1,5 +1,24 @@ from distutils.command.build import build as _build +import warnings + +from setuptools import SetuptoolsDeprecationWarning + + +_ORIGINAL_SUBCOMMANDS = {"build_py", "build_clib", "build_ext", "build_scripts"} class build(_build): - pass + # copy to avoid sharing the object with parent class + sub_commands = _build.sub_commands[:] + + def run(self): + subcommands = {cmd[0] for cmd in _build.sub_commands} + if subcommands - _ORIGINAL_SUBCOMMANDS: + msg = """ + It seems that you are using `distutils.command.build.build` to add + new subcommands. Using `distutils` directly is considered deprecated, + please use `setuptools.command.build`. + """ + warnings.warns(msg, SetuptoolsDeprecationWarning) + self.sub_commands = _build.sub_commands + super().run() -- cgit v1.2.1 From 6cae4d5522b7608bbbbe53073714d3be0961a1ed Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Thu, 9 Jun 2022 19:03:49 +0100 Subject: Test that users are informed about subcommands in distutils --- setuptools/command/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/build.py b/setuptools/command/build.py index 932de472..7ab60ce6 100644 --- a/setuptools/command/build.py +++ b/setuptools/command/build.py @@ -19,6 +19,6 @@ class build(_build): new subcommands. Using `distutils` directly is considered deprecated, please use `setuptools.command.build`. """ - warnings.warns(msg, SetuptoolsDeprecationWarning) + warnings.warn(msg, SetuptoolsDeprecationWarning) self.sub_commands = _build.sub_commands super().run() -- cgit v1.2.1 From ddf7b953d1605374e6aa62ed4eb3760b3c5bcba0 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Mon, 13 Jun 2022 10:59:43 +0100 Subject: Improve warning message --- setuptools/command/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/build.py b/setuptools/command/build.py index 7ab60ce6..12a43622 100644 --- a/setuptools/command/build.py +++ b/setuptools/command/build.py @@ -15,7 +15,7 @@ class build(_build): subcommands = {cmd[0] for cmd in _build.sub_commands} if subcommands - _ORIGINAL_SUBCOMMANDS: msg = """ - It seems that you are using `distutils.command.build.build` to add + It seems that you are using `distutils.command.build` to add new subcommands. Using `distutils` directly is considered deprecated, please use `setuptools.command.build`. """ -- cgit v1.2.1 From c6106b7b5eac56f6f84174ede07aa221d5aa635d Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Wed, 13 Apr 2022 10:04:53 +0100 Subject: Add dist_info_dir param to dist_info command --- setuptools/command/dist_info.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/dist_info.py b/setuptools/command/dist_info.py index ca540ad1..5e38c96c 100644 --- a/setuptools/command/dist_info.py +++ b/setuptools/command/dist_info.py @@ -24,23 +24,24 @@ class dist_info(Command): def initialize_options(self): self.egg_base = None + self.dist_info_dir = None def finalize_options(self): - pass - - def run(self): egg_info = self.get_finalized_command('egg_info') egg_info.egg_base = self.egg_base egg_info.finalize_options() - egg_info.run() name = _safe(self.distribution.get_name()) - version = _version(self.distribution.get_version()) base = self.egg_base or os.curdir - dist_info_dir = os.path.join(base, f"{name}-{version}.dist-info") - log.info("creating '{}'".format(os.path.abspath(dist_info_dir))) + version = _version(self.distribution.get_version()) + self.dist_info_dir = os.path.join(base, f"{name}-{version}.dist-info") + self.egg_info = egg_info + self.egg_base = egg_info.egg_base + def run(self): + self.egg_info.run() + log.info("creating '{}'".format(os.path.abspath(self.dist_info_dir))) bdist_wheel = self.get_finalized_command('bdist_wheel') - bdist_wheel.egg2dist(egg_info.egg_info, dist_info_dir) + bdist_wheel.egg2dist(self.egg_info.egg_info, self.dist_info_dir) def _safe(component: str) -> str: -- cgit v1.2.1 From ad803aeaeb3aa36d615193e627219c813cae16b6 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sun, 3 Apr 2022 23:08:34 +0100 Subject: Change dist_info to better control output directory --- setuptools/command/dist_info.py | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/dist_info.py b/setuptools/command/dist_info.py index 5e38c96c..b948763d 100644 --- a/setuptools/command/dist_info.py +++ b/setuptools/command/dist_info.py @@ -11,6 +11,7 @@ from inspect import cleandoc from distutils.core import Command from distutils import log from setuptools.extern import packaging +from setuptools._deprecation_warning import SetuptoolsDeprecationWarning class dist_info(Command): @@ -19,29 +20,45 @@ class dist_info(Command): user_options = [ ('egg-base=', 'e', "directory containing .egg-info directories" - " (default: top of the source tree)"), + " (default: top of the source tree)" + " DEPRECATED: use --output-dir."), + ('output-dir=', 'o', "directory inside of which the .dist-info will be" + "created (default: top of the source tree)"), ] def initialize_options(self): self.egg_base = None + self.output_dir = None + self.name = None self.dist_info_dir = None def finalize_options(self): - egg_info = self.get_finalized_command('egg_info') - egg_info.egg_base = self.egg_base + if self.egg_base: + msg = "--egg-base is deprecated for dist_info command. Use --output-dir." + warnings.warn(msg, SetuptoolsDeprecationWarning) + self.output_dir = self.egg_base or self.output_dir + + dist = self.distribution + project_dir = dist.src_root or os.curdir + self.output_dir = self.output_dir or project_dir + + egg_info = self.reinitialize_command('egg_info') + egg_info.egg_base = self.output_dir egg_info.finalize_options() - name = _safe(self.distribution.get_name()) - base = self.egg_base or os.curdir - version = _version(self.distribution.get_version()) - self.dist_info_dir = os.path.join(base, f"{name}-{version}.dist-info") self.egg_info = egg_info - self.egg_base = egg_info.egg_base + + name = _safe(dist.get_name()) + version = _version(dist.get_version()) + self.name = f"{name}-{version}" + self.dist_info_dir = os.path.join(self.output_dir, f"{self.name}.dist-info") def run(self): self.egg_info.run() + egg_info_dir = self.egg_info.egg_info log.info("creating '{}'".format(os.path.abspath(self.dist_info_dir))) bdist_wheel = self.get_finalized_command('bdist_wheel') - bdist_wheel.egg2dist(self.egg_info.egg_info, self.dist_info_dir) + bdist_wheel.egg2dist(egg_info_dir, self.dist_info_dir) + assert os.path.exists(egg_info_dir) is False def _safe(component: str) -> str: -- cgit v1.2.1 From 1afb6c0613a5171e3e8caa6fc5b020e671930df7 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Mon, 4 Apr 2022 01:01:06 +0100 Subject: Make sure output_dir exists with dist_info --- setuptools/command/dist_info.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/dist_info.py b/setuptools/command/dist_info.py index b948763d..79647bc7 100644 --- a/setuptools/command/dist_info.py +++ b/setuptools/command/dist_info.py @@ -7,6 +7,7 @@ import os import re import warnings from inspect import cleandoc +from pathlib import Path from distutils.core import Command from distutils import log @@ -40,10 +41,11 @@ class dist_info(Command): dist = self.distribution project_dir = dist.src_root or os.curdir - self.output_dir = self.output_dir or project_dir + self.output_dir = Path(self.output_dir or project_dir) + self.output_dir.mkdir(parents=True, exist_ok=True) egg_info = self.reinitialize_command('egg_info') - egg_info.egg_base = self.output_dir + egg_info.egg_base = str(self.output_dir) egg_info.finalize_options() self.egg_info = egg_info -- cgit v1.2.1 From 0e6870b5256d36ebda9fbea8553df2e699dc373c Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Wed, 13 Apr 2022 10:00:49 +0100 Subject: Avoid creating dist_info_dir before the command runs --- setuptools/command/dist_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/dist_info.py b/setuptools/command/dist_info.py index 79647bc7..aa7af48c 100644 --- a/setuptools/command/dist_info.py +++ b/setuptools/command/dist_info.py @@ -42,7 +42,6 @@ class dist_info(Command): dist = self.distribution project_dir = dist.src_root or os.curdir self.output_dir = Path(self.output_dir or project_dir) - self.output_dir.mkdir(parents=True, exist_ok=True) egg_info = self.reinitialize_command('egg_info') egg_info.egg_base = str(self.output_dir) @@ -55,6 +54,7 @@ class dist_info(Command): self.dist_info_dir = os.path.join(self.output_dir, f"{self.name}.dist-info") def run(self): + self.output_dir.mkdir(parents=True, exist_ok=True) self.egg_info.run() egg_info_dir = self.egg_info.egg_info log.info("creating '{}'".format(os.path.abspath(self.dist_info_dir))) -- cgit v1.2.1 From 5bb97d0f8b833bec6f6d4c70561b68dc2c559168 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sun, 3 Apr 2022 21:02:15 +0100 Subject: Remove unecessary editable dependency --- setuptools/command/editable_wheel.py | 32 ++++++++------------------------ 1 file changed, 8 insertions(+), 24 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index c827efa3..94284d55 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -6,12 +6,14 @@ Create a wheel that, when installed, will make the source package 'editable' # TODO doesn't behave when called outside the hook +import base64 import os import time from pathlib import Path from distutils.core import Command from distutils.errors import DistutilsError +from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo import pkg_resources @@ -68,13 +70,7 @@ class editable_wheel(Command): # with the dist-info directory and .pth from 'editables' library # ... - import zipfile - import editables # could we use 'develop' command's .pth file - - project = editables.EditableProject( - self.distribution.metadata.name, self.target - ) - project.add_to_path(self.target) + mtime = time.gmtime(SOURCE_EPOCH_ZIP)[:6] dist_dir = Path(self.dist_dir) dist_info_dir = self.dist_info_dir @@ -89,32 +85,20 @@ class editable_wheel(Command): if wheel_path.exists(): wheel_path.unlink() - with zipfile.ZipFile( - wheel_path, "a", compression=zipfile.ZIP_DEFLATED - ) as archive: - + with ZipFile(wheel_path, "a", compression=ZIP_DEFLATED) as archive: # copy .pth file - for f, data in project.files(): - archive.writestr( - zipfile.ZipInfo(f, time.gmtime(SOURCE_EPOCH_ZIP)[:6]), data - ) + pth = ZipInfo(f"{fullname}_ed.pth", mtime) + archive.writestr(pth, f"{self.target}\n") # copy .dist-info directory for f in sorted(os.listdir(dist_dir / dist_info_dir)): with (dist_dir / dist_info_dir / f).open() as metadata: - archive.writestr( - zipfile.ZipInfo( - str(dist_info_dir / f), time.gmtime(SOURCE_EPOCH_ZIP)[:6] - ), - metadata.read(), - ) + info = ZipInfo(str(dist_info_dir / f), mtime) + archive.writestr(info, metadata.read()) add_manifest(archive, dist_info_dir) -import base64 - - def urlsafe_b64encode(data): """urlsafe_b64encode without padding""" return base64.urlsafe_b64encode(data).rstrip(b"=") -- cgit v1.2.1 From 3e9f4418d528228af7aba1a3bddbbf20b2327422 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Wed, 13 Apr 2022 10:03:37 +0100 Subject: Make sure setuptools itself can be installed with editable_wheel --- setuptools/command/editable_wheel.py | 48 ++++++++++++------------------------ 1 file changed, 16 insertions(+), 32 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index 94284d55..f862e6a0 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -12,13 +12,21 @@ import time from pathlib import Path from distutils.core import Command -from distutils.errors import DistutilsError from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo import pkg_resources +from setuptools import __version__ SOURCE_EPOCH_ZIP = 499162860 +WHEEL_FILE = f"""\ +Wheel-Version: 1.0 +Generator: setuptools ({__version__}) +Root-Is-Purelib: false +Tag: py3-none-any +Tag: ed-none-any +""" + class editable_wheel(Command): """Build 'editable' wheel for development""" @@ -38,20 +46,10 @@ class editable_wheel(Command): self.dist_dir = None def finalize_options(self): - # is this part of the 'develop' command needed? - ei = self.get_finalized_command("egg_info") - if ei.broken_egg_info: - template = "Please rename %r to %r before using 'develop'" - args = ei.egg_info, ei.broken_egg_info - raise DistutilsError(template % args) - self.args = [ei.egg_name] - - # the .pth file should point to target - self.egg_base = ei.egg_base + self.dist_info = self.get_finalized_command("dist_info") + self.egg_base = self.dist_info.egg_base + self.dist_info_dir = Path(self.dist_info.dist_info_dir) self.target = pkg_resources.normalize_path(self.egg_base) - self.dist_info_dir = Path( - (ei.egg_info[: -len(".egg-info")] + ".dist-info").rpartition("/")[-1] - ) def build_editable_wheel(self): if getattr(self.distribution, "use_2to3", False): @@ -66,10 +64,6 @@ class editable_wheel(Command): self.reinitialize_command("build_ext", inplace=1) self.run_command("build_ext") - # now build the wheel - # with the dist-info directory and .pth from 'editables' library - # ... - mtime = time.gmtime(SOURCE_EPOCH_ZIP)[:6] dist_dir = Path(self.dist_dir) @@ -80,8 +74,6 @@ class editable_wheel(Command): wheel_name = f"{fullname}-ed.py3-none-any.whl" wheel_path = dist_dir / wheel_name - wheelmeta_builder(dist_dir / dist_info_dir / "WHEEL") - if wheel_path.exists(): wheel_path.unlink() @@ -96,6 +88,10 @@ class editable_wheel(Command): info = ZipInfo(str(dist_info_dir / f), mtime) archive.writestr(info, metadata.read()) + # Add WHEEL file + info = ZipInfo(str(dist_info_dir / "WHEEL"), mtime) + archive.writestr(info, WHEEL_FILE) + add_manifest(archive, dist_info_dir) @@ -127,15 +123,3 @@ def add_manifest(archive, dist_info_dir): zipfile.ZipInfo(str(record_path), time.gmtime(SOURCE_EPOCH_ZIP)[:6]), RECORD ) archive.close() - - -def wheelmeta_builder(target): - with open(target, "w+") as f: - f.write( - """Wheel-Version: 1.0 -Generator: setuptools_pep660 (0.1) -Root-Is-Purelib: false -Tag: py3-none-any -Tag: ed-none-any -""" - ) -- cgit v1.2.1 From 5866b8563cc35d01d08053d8142f4c09255a07f3 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Mon, 4 Apr 2022 01:02:41 +0100 Subject: Rely on wheel and bdist_wheel for editable_wheel --- setuptools/command/editable_wheel.py | 151 ++++++++++++++--------------------- 1 file changed, 61 insertions(+), 90 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index f862e6a0..199dbd7c 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -4,28 +4,12 @@ Create a wheel that, when installed, will make the source package 'editable' 'setup.py develop'. Based on the setuptools develop command. """ -# TODO doesn't behave when called outside the hook - -import base64 import os -import time -from pathlib import Path - +import shutil +import sys from distutils.core import Command -from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo - -import pkg_resources -from setuptools import __version__ - -SOURCE_EPOCH_ZIP = 499162860 - -WHEEL_FILE = f"""\ -Wheel-Version: 1.0 -Generator: setuptools ({__version__}) -Root-Is-Purelib: false -Tag: py3-none-any -Tag: ed-none-any -""" +from pathlib import Path +from tempfile import TemporaryDirectory class editable_wheel(Command): @@ -35,91 +19,78 @@ class editable_wheel(Command): user_options = [ ("dist-dir=", "d", "directory to put final built distributions in"), + ("dist-info-dir=", "I", "path to a pre-build .dist-info directory"), ] boolean_options = [] - def run(self): - self.build_editable_wheel() - def initialize_options(self): self.dist_dir = None + self.dist_info_dir = None + self.project_dir = None def finalize_options(self): - self.dist_info = self.get_finalized_command("dist_info") - self.egg_base = self.dist_info.egg_base - self.dist_info_dir = Path(self.dist_info.dist_info_dir) - self.target = pkg_resources.normalize_path(self.egg_base) + dist = self.distribution + self.project_dir = dist.src_root or os.curdir + self.dist_dir = Path(self.dist_dir or os.path.join(self.project_dir, "dist")) + self.dist_dir.mkdir(exist_ok=True) + + @property + def target(self): + package_dir = self.distribution.package_dir or {} + return package_dir.get("") or self.project_dir - def build_editable_wheel(self): - if getattr(self.distribution, "use_2to3", False): - raise NotImplementedError("2to3 not supported") + def run(self): + self._ensure_dist_info() - di = self.get_finalized_command("dist_info") - di.egg_base = self.dist_dir - di.finalize_options() - self.run_command("dist_info") + # Add missing dist_info files + bdist_wheel = self.reinitialize_command("bdist_wheel") + bdist_wheel.write_wheelfile(self.dist_info_dir) # Build extensions in-place self.reinitialize_command("build_ext", inplace=1) self.run_command("build_ext") - mtime = time.gmtime(SOURCE_EPOCH_ZIP)[:6] - - dist_dir = Path(self.dist_dir) - dist_info_dir = self.dist_info_dir - fullname = self.distribution.metadata.get_fullname() - # superfluous 'ed' tag is only a hint to the user, - # and guarantees we can't overwrite the normal wheel - wheel_name = f"{fullname}-ed.py3-none-any.whl" - wheel_path = dist_dir / wheel_name - + self._create_wheel_file(bdist_wheel) + + def _ensure_dist_info(self): + if self.dist_info_dir is None: + dist_info = self.reinitialize_command("dist_info") + dist_info.output_dir = self.dist_dir + dist_info.finalize_options() + dist_info.run() + self.dist_info_dir = dist_info.dist_info_dir + else: + assert str(self.dist_info_dir).endswith(".dist-info") + assert list(Path(self.dist_info_dir).glob("*.dist-info/METADATA")) + + def _create_wheel_file(self, bdist_wheel): + from wheel.wheelfile import WheelFile + dist_info = self.get_finalized_command("dist_info") + tag = "-".join(bdist_wheel.get_tag()) + editable_name = dist_info.name + build_tag = "0.editable" # According to PEP 427 needs to start with digit + archive_name = f"{editable_name}-{build_tag}-{tag}.whl" + wheel_path = Path(self.dist_dir, archive_name) if wheel_path.exists(): wheel_path.unlink() - with ZipFile(wheel_path, "a", compression=ZIP_DEFLATED) as archive: - # copy .pth file - pth = ZipInfo(f"{fullname}_ed.pth", mtime) - archive.writestr(pth, f"{self.target}\n") - - # copy .dist-info directory - for f in sorted(os.listdir(dist_dir / dist_info_dir)): - with (dist_dir / dist_info_dir / f).open() as metadata: - info = ZipInfo(str(dist_info_dir / f), mtime) - archive.writestr(info, metadata.read()) - - # Add WHEEL file - info = ZipInfo(str(dist_info_dir / "WHEEL"), mtime) - archive.writestr(info, WHEEL_FILE) - - add_manifest(archive, dist_info_dir) - - -def urlsafe_b64encode(data): - """urlsafe_b64encode without padding""" - return base64.urlsafe_b64encode(data).rstrip(b"=") - - -# standalone wheel helpers based on enscons -def add_manifest(archive, dist_info_dir): - """ - Add the wheel manifest. - """ - import hashlib - import zipfile - - lines = [] - for f in archive.namelist(): - data = archive.read(f) - size = len(data) - digest = hashlib.sha256(data).digest() - digest = "sha256=" + (urlsafe_b64encode(digest).decode("ascii")) - lines.append("%s,%s,%s" % (f.replace(",", ",,"), digest, size)) - - record_path = dist_info_dir / "RECORD" - lines.append(str(record_path) + ",,") - RECORD = "\n".join(lines) - archive.writestr( - zipfile.ZipInfo(str(record_path), time.gmtime(SOURCE_EPOCH_ZIP)[:6]), RECORD - ) - archive.close() + # Currently the wheel API receives a directory and dump all its contents + # inside of a wheel. So let's use a temporary directory. + with TemporaryDirectory(suffix=archive_name) as tmp: + tmp_dist_info = Path(tmp, Path(self.dist_info_dir).name) + shutil.copytree(self.dist_info_dir, tmp_dist_info) + pth = Path(tmp, f"_editable.{editable_name}.pth") + pth.write_text(f"{_normalize_path(self.target)}\n", encoding="utf-8") + + with WheelFile(wheel_path, "w") as wf: + wf.write_files(tmp) + + return wheel_path + + +def _normalize_path(filename): + """Normalize a file/dir name for comparison purposes""" + # See pkg_resources.normalize_path + file = os.path.abspath(filename) if sys.platform == 'cygwin' else filename + return os.path.normcase(os.path.realpath(os.path.normpath(file))) -- cgit v1.2.1 From aee32452f9cf2ceb3234a1749ccd0ad3eb166e11 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Mon, 4 Apr 2022 01:06:29 +0100 Subject: Re-use dist-info dir for build_editable --- setuptools/command/editable_wheel.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index 199dbd7c..d44d216f 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -62,10 +62,11 @@ class editable_wheel(Command): self.dist_info_dir = dist_info.dist_info_dir else: assert str(self.dist_info_dir).endswith(".dist-info") - assert list(Path(self.dist_info_dir).glob("*.dist-info/METADATA")) + assert Path(self.dist_info_dir, "METADATA").exists() def _create_wheel_file(self, bdist_wheel): from wheel.wheelfile import WheelFile + dist_info = self.get_finalized_command("dist_info") tag = "-".join(bdist_wheel.get_tag()) editable_name = dist_info.name -- cgit v1.2.1 From 06fbc3688726c224c6f8ddc6ad434ce36f8c1ed7 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Tue, 5 Apr 2022 10:33:07 +0100 Subject: Start adding APIs for handling multiple editable modes --- setuptools/command/editable_wheel.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index d44d216f..7aa73c52 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -22,12 +22,13 @@ class editable_wheel(Command): ("dist-info-dir=", "I", "path to a pre-build .dist-info directory"), ] - boolean_options = [] + boolean_options = ["strict"] def initialize_options(self): self.dist_dir = None self.dist_info_dir = None self.project_dir = None + self.strict = False def finalize_options(self): dist = self.distribution @@ -81,15 +82,35 @@ class editable_wheel(Command): with TemporaryDirectory(suffix=archive_name) as tmp: tmp_dist_info = Path(tmp, Path(self.dist_info_dir).name) shutil.copytree(self.dist_info_dir, tmp_dist_info) - pth = Path(tmp, f"_editable.{editable_name}.pth") - pth.write_text(f"{_normalize_path(self.target)}\n", encoding="utf-8") - + self._populate_wheel(editable_name, tmp) with WheelFile(wheel_path, "w") as wf: wf.write_files(tmp) return wheel_path + def _best_strategy(self): + if self.strict: + return self._link_tree + + dist = self.distribution + if set(dist.packages) == {""}: + # src-layout(ish) package detected. These kind of packages are relatively + # safe so we can simply add the src directory to the pth file. + return self._top_level_pth + + if self._can_symlink(): + return self._top_level_symlinks + + # >>> def _targets(self): + # >>> build_py.find_modules() + # >>> self.dist.packages + + def _populate_wheel(self, dist_id, unpacked_wheel_dir): + pth = Path(unpacked_wheel_dir, f"_editable.{dist_id}.pth") + pth.write_text(f"{_normalize_path(self.target)}\n", encoding="utf-8") + + def _normalize_path(filename): """Normalize a file/dir name for comparison purposes""" # See pkg_resources.normalize_path -- cgit v1.2.1 From 7b402eb104681ca0cfbe36ac8e10bafa2030a9d8 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Wed, 6 Apr 2022 17:09:12 +0100 Subject: Temporarily support deprecated namespace packages --- setuptools/command/editable_wheel.py | 38 +++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index 7aa73c52..e03eb671 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -7,10 +7,12 @@ Create a wheel that, when installed, will make the source package 'editable' import os import shutil import sys -from distutils.core import Command from pathlib import Path from tempfile import TemporaryDirectory +from setuptools import Command +from setuptools import namespaces + class editable_wheel(Command): """Build 'editable' wheel for development""" @@ -39,7 +41,7 @@ class editable_wheel(Command): @property def target(self): package_dir = self.distribution.package_dir or {} - return package_dir.get("") or self.project_dir + return _normalize_path(package_dir.get("") or self.project_dir) def run(self): self._ensure_dist_info() @@ -65,6 +67,15 @@ class editable_wheel(Command): assert str(self.dist_info_dir).endswith(".dist-info") assert Path(self.dist_info_dir, "METADATA").exists() + def _install_namespaces(self, installation_dir, pth_prefix): + # XXX: Only required to support the deprecated namespace practice + dist = self.distribution + if not dist.namespace_packages: + return + + installer = _NamespaceInstaller(dist, installation_dir, pth_prefix, self.target) + installer.install_namespaces() + def _create_wheel_file(self, bdist_wheel): from wheel.wheelfile import WheelFile @@ -82,13 +93,13 @@ class editable_wheel(Command): with TemporaryDirectory(suffix=archive_name) as tmp: tmp_dist_info = Path(tmp, Path(self.dist_info_dir).name) shutil.copytree(self.dist_info_dir, tmp_dist_info) + self._install_namespaces(tmp, editable_name) self._populate_wheel(editable_name, tmp) with WheelFile(wheel_path, "w") as wf: wf.write_files(tmp) return wheel_path - def _best_strategy(self): if self.strict: return self._link_tree @@ -107,8 +118,25 @@ class editable_wheel(Command): # >>> self.dist.packages def _populate_wheel(self, dist_id, unpacked_wheel_dir): - pth = Path(unpacked_wheel_dir, f"_editable.{dist_id}.pth") - pth.write_text(f"{_normalize_path(self.target)}\n", encoding="utf-8") + pth = Path(unpacked_wheel_dir, f"__editable__.{dist_id}.pth") + pth.write_text(f"{self.target}\n", encoding="utf-8") + + +class _NamespaceInstaller(namespaces.Installer): + def __init__(self, distribution, installation_dir, editable_name, src_root): + self.distribution = distribution + self.src_root = src_root + self.installation_dir = installation_dir + self.editable_name = editable_name + self.outputs = [] + + def _get_target(self): + """Installation target.""" + return os.path.join(self.installation_dir, self.editable_name) + + def _get_root(self): + """Where the modules/packages should be loaded from.""" + return repr(str(self.src_root)) def _normalize_path(filename): -- cgit v1.2.1 From a31df2ee1147038d4af63b3b93679de882ada4eb Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Wed, 13 Apr 2022 09:44:03 +0100 Subject: Remove unused function --- setuptools/command/editable_wheel.py | 17 ----------------- 1 file changed, 17 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index e03eb671..573c4b87 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -100,23 +100,6 @@ class editable_wheel(Command): return wheel_path - def _best_strategy(self): - if self.strict: - return self._link_tree - - dist = self.distribution - if set(dist.packages) == {""}: - # src-layout(ish) package detected. These kind of packages are relatively - # safe so we can simply add the src directory to the pth file. - return self._top_level_pth - - if self._can_symlink(): - return self._top_level_symlinks - - # >>> def _targets(self): - # >>> build_py.find_modules() - # >>> self.dist.packages - def _populate_wheel(self, dist_id, unpacked_wheel_dir): pth = Path(unpacked_wheel_dir, f"__editable__.{dist_id}.pth") pth.write_text(f"{self.target}\n", encoding="utf-8") -- cgit v1.2.1 From 7ae5a6903c83eb43a96ad02c874990d02ee92672 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sat, 9 Apr 2022 10:54:37 +0100 Subject: Reorganise editable_wheel to allow different strategies --- setuptools/command/editable_wheel.py | 136 +++++++++++++++++++++++++++++------ 1 file changed, 116 insertions(+), 20 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index 573c4b87..a57f74e3 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -1,7 +1,13 @@ """ Create a wheel that, when installed, will make the source package 'editable' (add it to the interpreter's path, including metadata) per PEP 660. Replaces -'setup.py develop'. Based on the setuptools develop command. +'setup.py develop'. + +.. note:: + One of the mechanisms briefly mentioned in PEP 660 to implement editable installs is + to create a separated directory inside ``build`` and use a .pth file to point to that + directory. In the context of this file such directory is referred as + *auxiliary build directory* or ``auxiliary_build_dir``. """ import os @@ -9,9 +15,13 @@ import shutil import sys from pathlib import Path from tempfile import TemporaryDirectory +from typing import Dict, Iterable, Iterator, List, Union + +from setuptools import Command, namespaces +from setuptools.discovery import find_package_path +from setuptools.dist import Distribution -from setuptools import Command -from setuptools import namespaces +_Path = Union[str, Path] class editable_wheel(Command): @@ -35,14 +45,10 @@ class editable_wheel(Command): def finalize_options(self): dist = self.distribution self.project_dir = dist.src_root or os.curdir + self.package_dir = dist.package_dir or {} self.dist_dir = Path(self.dist_dir or os.path.join(self.project_dir, "dist")) self.dist_dir.mkdir(exist_ok=True) - @property - def target(self): - package_dir = self.distribution.package_dir or {} - return _normalize_path(package_dir.get("") or self.project_dir) - def run(self): self._ensure_dist_info() @@ -73,7 +79,8 @@ class editable_wheel(Command): if not dist.namespace_packages: return - installer = _NamespaceInstaller(dist, installation_dir, pth_prefix, self.target) + target = Path(self.project_dir, self.pakcage_dir.get("", ".")).resolve() + installer = _NamespaceInstaller(dist, installation_dir, pth_prefix, target) installer.install_namespaces() def _create_wheel_file(self, bdist_wheel): @@ -94,15 +101,111 @@ class editable_wheel(Command): tmp_dist_info = Path(tmp, Path(self.dist_info_dir).name) shutil.copytree(self.dist_info_dir, tmp_dist_info) self._install_namespaces(tmp, editable_name) - self._populate_wheel(editable_name, tmp) + populate = self._populate_strategy(editable_name, tag) + populate(tmp) with WheelFile(wheel_path, "w") as wf: wf.write_files(tmp) return wheel_path - def _populate_wheel(self, dist_id, unpacked_wheel_dir): - pth = Path(unpacked_wheel_dir, f"__editable__.{dist_id}.pth") - pth.write_text(f"{self.target}\n", encoding="utf-8") + def _populate_strategy(self, name, tag): + """Decides which strategy to use to implement an editable installation.""" + dist = self.distribution + build_name = f"__editable__.{name}-{tag}" + project_dir = Path(self.project_dir) + auxiliar_build_dir = Path(self.project_dir, "build", build_name) + + if self.strict: + # The LinkTree strategy will only link files, so it can be implemented in + # any OS, even if that means using hardlinks instead of symlinks + auxiliar_build_dir = _empty_dir(auxiliar_build_dir) + # TODO: return _LinkTree(dist, name, auxiliar_build_dir) + raise NotImplementedError + + packages = _find_packages(dist) + has_simple_layout = _simple_layout(packages, self.package_dir, project_dir) + if set(self.package_dir) == {""} and has_simple_layout: + # src-layout(ish) package detected. These kind of packages are relatively + # safe so we can simply add the src directory to the pth file. + return _StaticPth(dist, name, [Path(project_dir, self.package_dir[""])]) + + # >>> msg = "TODO: Explain limitations with meta path finder" + # >>> warnings.warn(msg) + paths = [Path(project_dir, p) for p in (".", self.package_dir.get("")) if p] + # TODO: return _TopLevelFinder(dist, name, auxiliar_build_dir) + return _StaticPth(dist, name, paths) + + +class _StaticPth: + def __init__(self, dist: Distribution, name: str, path_entries: List[Path]): + self.dist = dist + self.name = name + self.path_entries = path_entries + + def __call__(self, unpacked_wheel_dir: Path): + pth = Path(unpacked_wheel_dir, f"__editable__.{self.name}.pth") + entries = "\n".join((str(p.resolve()) for p in self.path_entries)) + pth.write_text(f"{entries}\n", encoding="utf-8") + + +def _simple_layout( + packages: Iterable[str], package_dir: Dict[str, str], project_dir: Path +) -> bool: + """Make sure all packages are contained by the same parent directory. + + >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"": "src"}, "/tmp/myproj") + True + >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"a": "a", "b": "b"}, ".") + True + >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"a": "_a", "b": "_b"}, ".") + False + >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"a": "_a"}, "/tmp/myproj") + False + >>> _simple_layout( + ... ['a', 'a.a1', 'a.a1.a2', 'b'], + ... {"a": "_a", "a.a1.a2": "_a2", "b": "_b"}, + ... ".", + ... ) + False + """ + layout = { + pkg: find_package_path(pkg, package_dir, project_dir) + for pkg in packages + } + parent = os.path.commonpath(list(layout.values())) + return all( + _normalize_path(Path(parent, *key.split('.'))) == _normalize_path(value) + for key, value in layout.items() + ) + + +def _find_packages(dist: Distribution) -> Iterator[str]: + yield from iter(dist.packages or []) + + py_modules = dist.py_modules or [] + nested_modules = [mod for mod in py_modules if "." in mod] + if dist.ext_package: + yield dist.ext_package + else: + ext_modules = dist.ext_modules or [] + nested_modules += [x.name for x in ext_modules if "." in x.name] + + for module in nested_modules: + package, _, _ = module.rpartition(".") + yield package + + +def _normalize_path(filename: _Path) -> str: + """Normalize a file/dir name for comparison purposes""" + # See pkg_resources.normalize_path + file = os.path.abspath(filename) if sys.platform == 'cygwin' else filename + return os.path.normcase(os.path.realpath(os.path.normpath(file))) + + +def _empty_dir(dir_: Path) -> Path: + shutil.rmtree(dir_, ignore_errors=True) + dir_.mkdir() + return dir_ class _NamespaceInstaller(namespaces.Installer): @@ -120,10 +223,3 @@ class _NamespaceInstaller(namespaces.Installer): def _get_root(self): """Where the modules/packages should be loaded from.""" return repr(str(self.src_root)) - - -def _normalize_path(filename): - """Normalize a file/dir name for comparison purposes""" - # See pkg_resources.normalize_path - file = os.path.abspath(filename) if sys.platform == 'cygwin' else filename - return os.path.normcase(os.path.realpath(os.path.normpath(file))) -- cgit v1.2.1 From 14eb8555a817cc69d68378be179459f73383c36e Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sat, 9 Apr 2022 11:01:06 +0100 Subject: Rename variable for clarity --- setuptools/command/editable_wheel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index a57f74e3..bbf6999f 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -79,8 +79,8 @@ class editable_wheel(Command): if not dist.namespace_packages: return - target = Path(self.project_dir, self.pakcage_dir.get("", ".")).resolve() - installer = _NamespaceInstaller(dist, installation_dir, pth_prefix, target) + src_root = Path(self.project_dir, self.pakcage_dir.get("", ".")).resolve() + installer = _NamespaceInstaller(dist, installation_dir, pth_prefix, src_root) installer.install_namespaces() def _create_wheel_file(self, bdist_wheel): -- cgit v1.2.1 From 1a531db35955b16ecd703deca9789c5de48e74be Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sat, 9 Apr 2022 15:02:02 +0100 Subject: Add template for MetaPathFinder that can be used in editable mode --- setuptools/command/editable_wheel.py | 55 ++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index bbf6999f..06a4a5d4 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -223,3 +223,58 @@ class _NamespaceInstaller(namespaces.Installer): def _get_root(self): """Where the modules/packages should be loaded from.""" return repr(str(self.src_root)) + + +_FINDER_TEMPLATE = """ +class __EditableFinder: + MAPPING = {mapping!r} + NAMESPACES = {namespaces!r} + + @classmethod + def install(cls): + import sys + + if not any(finder == cls for finder in sys.meta_path): + sys.meta_path.append(cls) + + @classmethod + def find_spec(cls, fullname, path, target=None): + if fullname in cls.NAMESPACES: + return cls._namespace_spec(fullname) + + for pkg, pkg_path in reversed(cls.MAPPING.items()): + if fullname.startswith(pkg): + return cls._find_spec(fullname, pkg, pkg_path) + + return None + + @classmethod + def _namespace_spec(cls, name): + # Since `cls` is appended to the path, this will only trigger + # when no other package is installed in the same namespace + from importlib.machinery import ModuleSpec + + # PEP 451 mentions setting loader to None for namespaces: + return ModuleSpec(name, None, is_package=True) + + @classmethod + def _find_spec(cls, fullname, parent, parent_path): + from importlib.machinery import all_suffixes as module_suffixes + from importlib.util import spec_from_file_location + from itertools import chain + + rest = fullname.replace(parent, "").strip(".").split(".") + candidate_path = Path(parent_path, *rest) + + init = candidate_path / "__init__.py" + candidates = (candidate_path.with_suffix(x) for x in module_suffixes()) + for candidate in chain([init], candidates): + if candidate.exists(): + spec = spec_from_file_location(fullname, candidate) + return spec + + return None + + +__EditableFinder.install() +""" -- cgit v1.2.1 From 994ca214cb0d9f01f72694758ddfe93cba0e26c5 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sat, 9 Apr 2022 22:55:12 +0100 Subject: Add editable strategy with MetaPathFinder for top-level packages --- setuptools/command/editable_wheel.py | 168 ++++++++++++++++++++++++++++++----- 1 file changed, 146 insertions(+), 22 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index 06a4a5d4..6d210a64 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -11,11 +11,12 @@ Create a wheel that, when installed, will make the source package 'editable' """ import os +import re import shutil import sys from pathlib import Path from tempfile import TemporaryDirectory -from typing import Dict, Iterable, Iterator, List, Union +from typing import Dict, Iterable, Iterator, List, Mapping, Set, Union from setuptools import Command, namespaces from setuptools.discovery import find_package_path @@ -131,9 +132,7 @@ class editable_wheel(Command): # >>> msg = "TODO: Explain limitations with meta path finder" # >>> warnings.warn(msg) - paths = [Path(project_dir, p) for p in (".", self.package_dir.get("")) if p] - # TODO: return _TopLevelFinder(dist, name, auxiliar_build_dir) - return _StaticPth(dist, name, paths) + return _TopLevelFinder(dist, name) class _StaticPth: @@ -148,11 +147,38 @@ class _StaticPth: pth.write_text(f"{entries}\n", encoding="utf-8") +class _TopLevelFinder: + def __init__(self, dist: Distribution, name: str): + self.dist = dist + self.name = name + + def __call__(self, unpacked_wheel_dir: Path): + src_root = self.dist.src_root or os.curdir + package_dir = self.dist.package_dir or {} + packages = _find_packages(self.dist) + pkg_roots = _find_pkg_roots(packages, package_dir, src_root) + namespaces_ = set(_find_mapped_namespaces(pkg_roots)) + + finder = _make_identifier(f"__editable__.{self.name}.finder") + content = _finder_template(pkg_roots, namespaces_) + Path(unpacked_wheel_dir, f"{finder}.py").write_text(content, encoding="utf-8") + + pth = f"__editable__.{self.name}.pth" + content = f"import {finder}; {finder}.install()" + Path(unpacked_wheel_dir, pth).write_text(content, encoding="utf-8") + + def _simple_layout( packages: Iterable[str], package_dir: Dict[str, str], project_dir: Path ) -> bool: """Make sure all packages are contained by the same parent directory. + >>> _simple_layout(['a'], {"": "src"}, "/tmp/myproj") + True + >>> _simple_layout(['a', 'a.b'], {"": "src"}, "/tmp/myproj") + True + >>> _simple_layout(['a', 'a.b'], {}, "/tmp/myproj") + True >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"": "src"}, "/tmp/myproj") True >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"a": "a", "b": "b"}, ".") @@ -172,13 +198,25 @@ def _simple_layout( pkg: find_package_path(pkg, package_dir, project_dir) for pkg in packages } - parent = os.path.commonpath(list(layout.values())) + parent = os.path.commonpath([_parent_path(k, v) for k, v in layout.items()]) return all( _normalize_path(Path(parent, *key.split('.'))) == _normalize_path(value) for key, value in layout.items() ) +def _parent_path(pkg, pkg_path): + """Infer the parent path for a package if possible. When the pkg is directly mapped + into a directory with a different name, return its own path. + >>> _parent_path("a", "src/a") + 'src' + >>> _parent_path("b", "src/c") + 'src/c' + """ + parent = pkg_path[:-len(pkg)] if pkg_path.endswith(pkg) else pkg_path + return parent.rstrip("/" + os.sep) + + def _find_packages(dist: Distribution) -> Iterator[str]: yield from iter(dist.packages or []) @@ -195,6 +233,76 @@ def _find_packages(dist: Distribution) -> Iterator[str]: yield package +def _find_pkg_roots( + packages: Iterable[str], + package_dir: Mapping[str, str], + src_root: _Path, +) -> Dict[str, str]: + pkg_roots: Dict[str, str] = { + pkg: _absolute_root(find_package_path(pkg, package_dir, src_root)) + for pkg in sorted(packages) + } + + return _remove_nested(pkg_roots) + + +def _absolute_root(path: _Path) -> str: + """Works for packages and top-level modules""" + path_ = Path(path) + parent = path_.parent + + if path_.exists(): + return str(path_.resolve()) + else: + return str(parent.resolve() / path_.name) + + +def _find_mapped_namespaces(pkg_roots: Dict[str, str]) -> Iterator[str]: + """By carefully designing ``package_dir``, it is possible to implement + PEP 420 compatible namespaces without creating extra folders. + This function will try to find this kind of namespaces. + """ + for pkg in pkg_roots: + if "." not in pkg: + continue + parts = pkg.split(".") + for i in range(len(parts) - 1, 0, -1): + partial_name = ".".join(parts[:i]) + path = find_package_path(partial_name, pkg_roots, "") + if not Path(path, "__init__.py").exists(): + yield partial_name + + +def _remove_nested(pkg_roots: Dict[str, str]) -> Dict[str, str]: + output = dict(pkg_roots.copy()) + + for pkg, path in reversed(pkg_roots.items()): + if any( + pkg != other and _is_nested(pkg, path, other, other_path) + for other, other_path in pkg_roots.items() + ): + output.pop(pkg) + + return output + + +def _is_nested(pkg: str, pkg_path: str, parent: str, parent_path: str) -> bool: + """ + >>> _is_nested("a.b", "path/a/b", "a", "path/a") + True + >>> _is_nested("a.b", "path/a/b", "a", "otherpath/a") + False + >>> _is_nested("a.b", "path/a/b", "c", "path/c") + False + """ + norm_pkg_path = _normalize_path(pkg_path) + rest = pkg.replace(parent, "").strip(".").split(".") + return ( + pkg.startswith(parent) + and norm_pkg_path == _normalize_path(Path(parent_path, *rest)) + ) + + def _normalize_path(filename: _Path) -> str: """Normalize a file/dir name for comparison purposes""" # See pkg_resources.normalize_path @@ -208,6 +316,18 @@ def _empty_dir(dir_: Path) -> Path: return dir_ +def _make_identifier(name: str) -> str: + """Make a string safe to be used as Python identifier. + >>> _make_identifier("12abc") + '_12abc' + >>> _make_identifier("__editable__.myns.pkg-78.9.3_local") + '__editable___myns_pkg_78_9_3_local' + """ + safe = re.sub(r'\W|^(?=\d)', '_', name) + assert safe.isidentifier() + return safe + + class _NamespaceInstaller(namespaces.Installer): def __init__(self, distribution, installation_dir, editable_name, src_root): self.distribution = distribution @@ -225,18 +345,18 @@ class _NamespaceInstaller(namespaces.Installer): return repr(str(self.src_root)) -_FINDER_TEMPLATE = """ +_FINDER_TEMPLATE = """\ +import sys +from importlib.machinery import all_suffixes as module_suffixes +from importlib.machinery import ModuleSpec +from importlib.util import spec_from_file_location +from itertools import chain +from pathlib import Path + class __EditableFinder: MAPPING = {mapping!r} NAMESPACES = {namespaces!r} - @classmethod - def install(cls): - import sys - - if not any(finder == cls for finder in sys.meta_path): - sys.meta_path.append(cls) - @classmethod def find_spec(cls, fullname, path, target=None): if fullname in cls.NAMESPACES: @@ -251,18 +371,12 @@ class __EditableFinder: @classmethod def _namespace_spec(cls, name): # Since `cls` is appended to the path, this will only trigger - # when no other package is installed in the same namespace - from importlib.machinery import ModuleSpec - - # PEP 451 mentions setting loader to None for namespaces: + # when no other package is installed in the same namespace. return ModuleSpec(name, None, is_package=True) + # ^-- PEP 451 mentions setting loader to None for namespaces. @classmethod def _find_spec(cls, fullname, parent, parent_path): - from importlib.machinery import all_suffixes as module_suffixes - from importlib.util import spec_from_file_location - from itertools import chain - rest = fullname.replace(parent, "").strip(".").split(".") candidate_path = Path(parent_path, *rest) @@ -273,8 +387,18 @@ class __EditableFinder: spec = spec_from_file_location(fullname, candidate) return spec + if candidate_path.exists(): + return cls._namespace_spec(fullname) + return None -__EditableFinder.install() +def install(): + if not any(finder == __EditableFinder for finder in sys.meta_path): + sys.meta_path.append(__EditableFinder) """ + + +def _finder_template(mapping: Mapping[str, str], namespaces: Set[str]): + mapping = dict(sorted(mapping.items(), key=lambda p: p[0])) + return _FINDER_TEMPLATE.format(mapping=mapping, namespaces=namespaces) -- cgit v1.2.1 From fba8717bf503e819b9ad5a52c605ce57bbab205c Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sat, 9 Apr 2022 23:27:26 +0100 Subject: Make sure top-level modules can be imported in editable install --- setuptools/command/editable_wheel.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index 6d210a64..77e77336 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -14,6 +14,7 @@ import os import re import shutil import sys +from itertools import chain from pathlib import Path from tempfile import TemporaryDirectory from typing import Dict, Iterable, Iterator, List, Mapping, Set, Union @@ -154,8 +155,8 @@ class _TopLevelFinder: def __call__(self, unpacked_wheel_dir: Path): src_root = self.dist.src_root or os.curdir + packages = chain(_find_packages(self.dist), _find_top_level_modules(self.dist)) package_dir = self.dist.package_dir or {} - packages = _find_packages(self.dist) pkg_roots = _find_pkg_roots(packages, package_dir, src_root) namespaces_ = set(_find_mapped_namespaces(pkg_roots)) @@ -198,6 +199,8 @@ def _simple_layout( pkg: find_package_path(pkg, package_dir, project_dir) for pkg in packages } + if not layout: + return False parent = os.path.commonpath([_parent_path(k, v) for k, v in layout.items()]) return all( _normalize_path(Path(parent, *key.split('.'))) == _normalize_path(value) @@ -233,6 +236,15 @@ def _find_packages(dist: Distribution) -> Iterator[str]: yield package +def _find_top_level_modules(dist: Distribution) -> Iterator[str]: + py_modules = dist.py_modules or [] + yield from (mod for mod in py_modules if "." not in mod) + + if not dist.ext_package: + ext_modules = dist.ext_modules or [] + yield from (x.name for x in ext_modules if "." not in x.name) + + def _find_pkg_roots( packages: Iterable[str], package_dir: Mapping[str, str], -- cgit v1.2.1 From 3ba7ec4bcb9055c01502741c72830458b6e99e61 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sat, 9 Apr 2022 23:38:48 +0100 Subject: Add logging messages to editable install --- setuptools/command/editable_wheel.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index 77e77336..536b1cb4 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -14,7 +14,9 @@ import os import re import shutil import sys +import logging from itertools import chain +from inspect import cleandoc from pathlib import Path from tempfile import TemporaryDirectory from typing import Dict, Iterable, Iterator, List, Mapping, Set, Union @@ -24,6 +26,7 @@ from setuptools.discovery import find_package_path from setuptools.dist import Distribution _Path = Union[str, Path] +_logger = logging.getLogger(__name__) class editable_wheel(Command): @@ -122,6 +125,11 @@ class editable_wheel(Command): # any OS, even if that means using hardlinks instead of symlinks auxiliar_build_dir = _empty_dir(auxiliar_build_dir) # TODO: return _LinkTree(dist, name, auxiliar_build_dir) + msg = """ + Strict editable install will be performed using a link tree. + New files will not be automatically picked up without a new installation. + """ + _logger.info(cleandoc(msg)) raise NotImplementedError packages = _find_packages(dist) @@ -129,10 +137,17 @@ class editable_wheel(Command): if set(self.package_dir) == {""} and has_simple_layout: # src-layout(ish) package detected. These kind of packages are relatively # safe so we can simply add the src directory to the pth file. - return _StaticPth(dist, name, [Path(project_dir, self.package_dir[""])]) - - # >>> msg = "TODO: Explain limitations with meta path finder" - # >>> warnings.warn(msg) + src_dir = self.package_dir[""] + msg = f"Editable install will be performed using .pth file to {src_dir}." + _logger.info(msg) + return _StaticPth(dist, name, [Path(project_dir, src_dir)]) + + msg = """ + Editable install will be performed using a meta path finder. + If you add any top-level packages or modules, they might not be automatically + picked up without a new installation. + """ + _logger.info(cleandoc(msg)) return _TopLevelFinder(dist, name) -- cgit v1.2.1 From 01ceef68343a42a3bb6592111305fac694a16acf Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sun, 10 Apr 2022 03:12:18 +0100 Subject: Fix error with Python 3.7 --- setuptools/command/editable_wheel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index 536b1cb4..1ee90f57 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -303,7 +303,7 @@ def _find_mapped_namespaces(pkg_roots: Dict[str, str]) -> Iterator[str]: def _remove_nested(pkg_roots: Dict[str, str]) -> Dict[str, str]: output = dict(pkg_roots.copy()) - for pkg, path in reversed(pkg_roots.items()): + for pkg, path in reversed(list(pkg_roots.items())): if any( pkg != other and _is_nested(pkg, path, other, other_path) for other, other_path in pkg_roots.items() @@ -389,7 +389,7 @@ class __EditableFinder: if fullname in cls.NAMESPACES: return cls._namespace_spec(fullname) - for pkg, pkg_path in reversed(cls.MAPPING.items()): + for pkg, pkg_path in reversed(list(cls.MAPPING.items())): if fullname.startswith(pkg): return cls._find_spec(fullname, pkg, pkg_path) -- cgit v1.2.1 From 50afbf41f470e75f67afe957023b3e03e3d41edc Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sun, 10 Apr 2022 03:07:33 +0100 Subject: Add LinkTree strategy for build executable --- setuptools/command/build_ext.py | 6 +- setuptools/command/build_py.py | 11 +++ setuptools/command/editable_wheel.py | 165 +++++++++++++++++++++++++++-------- 3 files changed, 139 insertions(+), 43 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/build_ext.py b/setuptools/command/build_ext.py index c59eff8b..1719d17a 100644 --- a/setuptools/command/build_ext.py +++ b/setuptools/command/build_ext.py @@ -3,7 +3,6 @@ import sys import itertools from importlib.machinery import EXTENSION_SUFFIXES from distutils.command.build_ext import build_ext as _du_build_ext -from distutils.file_util import copy_file from distutils.ccompiler import new_compiler from distutils.sysconfig import customize_compiler, get_config_var from distutils.errors import DistutilsError @@ -96,10 +95,7 @@ class build_ext(_build_ext): # Always copy, even if source is older than destination, to ensure # that the right extensions for the current Python/platform are # used. - copy_file( - src_filename, dest_filename, verbose=self.verbose, - dry_run=self.dry_run - ) + build_py.copy_file(src_filename, dest_filename) if ext._needs_stub: self.write_stub(package_dir or os.curdir, ext, True) diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index 2fced3d6..9575cdf8 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -36,6 +36,17 @@ class build_py(orig.build_py): if 'data_files' in self.__dict__: del self.__dict__['data_files'] self.__updated_files = [] + self.use_links = None + + def copy_file(self, infile, outfile, preserve_mode=1, preserve_times=1, + link=None, level=1): + # Overwrite base class to allow using links + link = getattr(self, "use_links", None) if link is None else link + if link: + infile = str(Path(infile).resolve()) + outfile = str(Path(outfile).resolve()) + return super().copy_file(infile, outfile, preserve_mode, + preserve_times, link, level) def run(self): """Build modules, packages, and copy data files to build directory""" diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index 1ee90f57..cf263a25 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -61,10 +61,6 @@ class editable_wheel(Command): bdist_wheel = self.reinitialize_command("bdist_wheel") bdist_wheel.write_wheelfile(self.dist_info_dir) - # Build extensions in-place - self.reinitialize_command("build_ext", inplace=1) - self.run_command("build_ext") - self._create_wheel_file(bdist_wheel) def _ensure_dist_info(self): @@ -92,63 +88,91 @@ class editable_wheel(Command): from wheel.wheelfile import WheelFile dist_info = self.get_finalized_command("dist_info") + dist_name = dist_info.name tag = "-".join(bdist_wheel.get_tag()) - editable_name = dist_info.name build_tag = "0.editable" # According to PEP 427 needs to start with digit - archive_name = f"{editable_name}-{build_tag}-{tag}.whl" + archive_name = f"{dist_name}-{build_tag}-{tag}.whl" wheel_path = Path(self.dist_dir, archive_name) if wheel_path.exists(): wheel_path.unlink() # Currently the wheel API receives a directory and dump all its contents # inside of a wheel. So let's use a temporary directory. - with TemporaryDirectory(suffix=archive_name) as tmp: - tmp_dist_info = Path(tmp, Path(self.dist_info_dir).name) - shutil.copytree(self.dist_info_dir, tmp_dist_info) - self._install_namespaces(tmp, editable_name) - populate = self._populate_strategy(editable_name, tag) - populate(tmp) + unpacked_tmp = TemporaryDirectory(suffix=archive_name) + build_tmp = TemporaryDirectory(suffix=".build-temp") + + with unpacked_tmp as unpacked, build_tmp as tmp: + unpacked_dist_info = Path(unpacked, Path(self.dist_info_dir).name) + shutil.copytree(self.dist_info_dir, unpacked_dist_info) + self._install_namespaces(unpacked, dist_info.name) + + # Add non-editable files to the wheel + _configure_build(dist_name, self.distribution, Path(unpacked), tmp) + self._run_install("headers") + self._run_install("scripts") + self._run_install("data") + + self._populate_wheel(dist_info.name, tag, unpacked, tmp) with WheelFile(wheel_path, "w") as wf: - wf.write_files(tmp) + wf.write_files(unpacked) return wheel_path - def _populate_strategy(self, name, tag): + def _run_install(self, category: str): + has_category = getattr(self.distribution, f"has_{category}", None) + if has_category and has_category(): + _logger.info(f"Installing {category} as non editable") + self.run_command(f"install_{category}") + + def _populate_wheel(self, name: str, tag: str, unpacked_dir: Path, tmp: Path): """Decides which strategy to use to implement an editable installation.""" - dist = self.distribution build_name = f"__editable__.{name}-{tag}" project_dir = Path(self.project_dir) - auxiliar_build_dir = Path(self.project_dir, "build", build_name) - - if self.strict: - # The LinkTree strategy will only link files, so it can be implemented in - # any OS, even if that means using hardlinks instead of symlinks - auxiliar_build_dir = _empty_dir(auxiliar_build_dir) - # TODO: return _LinkTree(dist, name, auxiliar_build_dir) - msg = """ - Strict editable install will be performed using a link tree. - New files will not be automatically picked up without a new installation. - """ - _logger.info(cleandoc(msg)) - raise NotImplementedError - - packages = _find_packages(dist) + + if self.strict or os.getenv("SETUPTOOLS_EDITABLE", None) == "strict": + return self._populate_link_tree(name, build_name, unpacked_dir, tmp) + + # Build extensions in-place + self.reinitialize_command("build_ext", inplace=1) + self.run_command("build_ext") + + packages = _find_packages(self.distribution) has_simple_layout = _simple_layout(packages, self.package_dir, project_dir) if set(self.package_dir) == {""} and has_simple_layout: - # src-layout(ish) package detected. These kind of packages are relatively - # safe so we can simply add the src directory to the pth file. - src_dir = self.package_dir[""] - msg = f"Editable install will be performed using .pth file to {src_dir}." - _logger.info(msg) - return _StaticPth(dist, name, [Path(project_dir, src_dir)]) + # src-layout(ish) is relatively safe for a simple pth file + return self._populate_static_pth(name, project_dir, unpacked_dir) + + # Use a MetaPathFinder to avoid adding accidental top-level packages/modules + self._populate_finder(name, unpacked_dir) + def _populate_link_tree( + self, name: str, build_name: str, unpacked_dir: Path, tmp: str + ): + auxiliary_build_dir = _empty_dir(Path(self.project_dir, "build", build_name)) + msg = """ + Strict editable install will be performed using a link tree. + New files will not be automatically picked up without a new installation. + """ + _logger.info(cleandoc(msg)) + populate = _LinkTree(self.distribution, name, auxiliary_build_dir, tmp) + populate(unpacked_dir) + + def _populate_static_pth(self, name: str, project_dir: Path, unpacked_dir: Path): + src_dir = self.package_dir[""] + msg = f"Editable install will be performed using .pth file to {src_dir}." + _logger.info(msg) + populate = _StaticPth(self.distribution, name, [Path(project_dir, src_dir)]) + populate(unpacked_dir) + + def _populate_finder(self, name: str, unpacked_dir: Path): msg = """ Editable install will be performed using a meta path finder. If you add any top-level packages or modules, they might not be automatically picked up without a new installation. """ _logger.info(cleandoc(msg)) - return _TopLevelFinder(dist, name) + populate = _TopLevelFinder(self.distribution, name) + populate(unpacked_dir) class _StaticPth: @@ -163,6 +187,36 @@ class _StaticPth: pth.write_text(f"{entries}\n", encoding="utf-8") +class _LinkTree(_StaticPth): + # The LinkTree strategy will only link files (not dirs), so it can be implemented in + # any OS, even if that means using hardlinks instead of symlinks + def __init__( + self, dist: Distribution, name: str, auxiliary_build_dir: Path, tmp: str + ): + super().__init__(dist, name, [auxiliary_build_dir]) + self.auxiliary_build_dir = auxiliary_build_dir + self.tmp = tmp + + def _build_py(self): + build_py = self.dist.get_command_obj("build_py") + build_py.ensure_finalized() + # Force build_py to use links instead of copying files + build_py.use_links = "sym" if _can_symlink_files() else "hard" + build_py.run() + + def _build_ext(self): + build_ext = self.dist.get_command_obj("build_ext") + build_ext.ensure_finalized() + # Extensions are not editable, so we just have to build them in the right dir + build_ext.run() + + def __call__(self, unpacked_wheel_dir: Path): + _configure_build(self.name, self.dist, self.auxiliary_build_dir, self.tmp) + self._build_py() + self._build_ext() + super().__call__(unpacked_wheel_dir) + + class _TopLevelFinder: def __init__(self, dist: Distribution, name: str): self.dist = dist @@ -184,6 +238,41 @@ class _TopLevelFinder: Path(unpacked_wheel_dir, pth).write_text(content, encoding="utf-8") +def _configure_build(name: str, dist: Distribution, target_dir: Path, tmp: str): + target = str(target_dir) + data = str(target_dir / f"{name}.data/data") + headers = str(target_dir / f"{name}.data/include") + scripts = str(target_dir / f"{name}.data/scripts") + + build = dist.reinitialize_command("build", reinit_subcommands=True) + install = dist.reinitialize_command("install", reinit_subcommands=True) + + build.build_platlib = build.build_purelib = build.build_lib = target + install.install_purelib = install.install_platlib = install.install_lib = target + install.install_scripts = build.build_scripts = scripts + install.install_headers = headers + install.install_data = data + + build.build_temp = tmp + + build_py = dist.get_command_obj("build_py") + build_py.compile = False + + build.ensure_finalized() + install.ensure_finalized() + + +def _can_symlink_files(): + try: + with TemporaryDirectory() as tmp: + path1, path2 = Path(tmp, "file1.txt"), Path(tmp, "file2.txt") + path1.write_text("file1", encoding="utf-8") + os.symlink(path1, path2) + return path2.is_symlink() and path2.read_text(encoding="utf-8") == "file1" + except (AttributeError, NotImplementedError, OSError): + return False + + def _simple_layout( packages: Iterable[str], package_dir: Dict[str, str], project_dir: Path ) -> bool: @@ -339,7 +428,7 @@ def _normalize_path(filename: _Path) -> str: def _empty_dir(dir_: Path) -> Path: shutil.rmtree(dir_, ignore_errors=True) - dir_.mkdir() + dir_.mkdir(parents=True) return dir_ -- cgit v1.2.1 From 40f2f0f37d129d58cfd2ef04897b941fb6bfd58f Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Mon, 11 Apr 2022 10:52:38 +0100 Subject: Improve clarity of 'editable_wheel' code --- setuptools/command/editable_wheel.py | 99 +++++++++++++++++++++--------------- 1 file changed, 58 insertions(+), 41 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index cf263a25..d5a7d530 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -16,19 +16,29 @@ import shutil import sys import logging from itertools import chain -from inspect import cleandoc from pathlib import Path from tempfile import TemporaryDirectory -from typing import Dict, Iterable, Iterator, List, Mapping, Set, Union +from typing import Dict, Iterable, Iterator, List, Mapping, Set, Union, TypeVar from setuptools import Command, namespaces from setuptools.discovery import find_package_path from setuptools.dist import Distribution _Path = Union[str, Path] +_P = TypeVar("_P", bound=_Path) _logger = logging.getLogger(__name__) +_STRICT_WARNING = """ +New or renamed files may not be automatically picked up without a new installation. +""" + +_LAX_WARNING = """ +Options like `package-data`, `include/exclude-package-data` or +`packages.find.exclude/include` may have no effect. +""" + + class editable_wheel(Command): """Build 'editable' wheel for development""" @@ -107,7 +117,7 @@ class editable_wheel(Command): self._install_namespaces(unpacked, dist_info.name) # Add non-editable files to the wheel - _configure_build(dist_name, self.distribution, Path(unpacked), tmp) + _configure_build(dist_name, self.distribution, unpacked, tmp) self._run_install("headers") self._run_install("scripts") self._run_install("data") @@ -124,7 +134,7 @@ class editable_wheel(Command): _logger.info(f"Installing {category} as non editable") self.run_command(f"install_{category}") - def _populate_wheel(self, name: str, tag: str, unpacked_dir: Path, tmp: Path): + def _populate_wheel(self, name: str, tag: str, unpacked_dir: Path, tmp: _Path): """Decides which strategy to use to implement an editable installation.""" build_name = f"__editable__.{name}-{tag}" project_dir = Path(self.project_dir) @@ -146,31 +156,27 @@ class editable_wheel(Command): self._populate_finder(name, unpacked_dir) def _populate_link_tree( - self, name: str, build_name: str, unpacked_dir: Path, tmp: str + self, name: str, build_name: str, unpacked_dir: Path, tmp: _Path ): + """Populate wheel using the "strict" ``link tree`` strategy.""" + msg = "Strict editable install will be performed using a link tree.\n" + _logger.warning(msg + _STRICT_WARNING) auxiliary_build_dir = _empty_dir(Path(self.project_dir, "build", build_name)) - msg = """ - Strict editable install will be performed using a link tree. - New files will not be automatically picked up without a new installation. - """ - _logger.info(cleandoc(msg)) populate = _LinkTree(self.distribution, name, auxiliary_build_dir, tmp) populate(unpacked_dir) def _populate_static_pth(self, name: str, project_dir: Path, unpacked_dir: Path): + """Populate wheel using the "lax" ``.pth`` file strategy, for ``src-layout``.""" src_dir = self.package_dir[""] - msg = f"Editable install will be performed using .pth file to {src_dir}." - _logger.info(msg) + msg = f"Editable install will be performed using .pth file to {src_dir}.\n" + _logger.warning(msg + _LAX_WARNING) populate = _StaticPth(self.distribution, name, [Path(project_dir, src_dir)]) populate(unpacked_dir) def _populate_finder(self, name: str, unpacked_dir: Path): - msg = """ - Editable install will be performed using a meta path finder. - If you add any top-level packages or modules, they might not be automatically - picked up without a new installation. - """ - _logger.info(cleandoc(msg)) + """Populate wheel using the "lax" MetaPathFinder strategy.""" + msg = "Editable install will be performed using a meta path finder.\n" + _logger.warning(msg + _LAX_WARNING) populate = _TopLevelFinder(self.distribution, name) populate(unpacked_dir) @@ -188,10 +194,17 @@ class _StaticPth: class _LinkTree(_StaticPth): - # The LinkTree strategy will only link files (not dirs), so it can be implemented in - # any OS, even if that means using hardlinks instead of symlinks + """ + Creates a ``.pth`` file that points to a link tree in the ``auxiliary_build_dir``. + + This strategy will only link files (not dirs), so it can be implemented in + any OS, even if that means using hardlinks instead of symlinks. + + By collocating ``auxiliary_build_dir`` and the original source code, limitations + with hardlinks should be avoided. + """ def __init__( - self, dist: Distribution, name: str, auxiliary_build_dir: Path, tmp: str + self, dist: Distribution, name: str, auxiliary_build_dir: Path, tmp: _Path ): super().__init__(dist, name, [auxiliary_build_dir]) self.auxiliary_build_dir = auxiliary_build_dir @@ -224,13 +237,13 @@ class _TopLevelFinder: def __call__(self, unpacked_wheel_dir: Path): src_root = self.dist.src_root or os.curdir - packages = chain(_find_packages(self.dist), _find_top_level_modules(self.dist)) + top_level = chain(_find_packages(self.dist), _find_top_level_modules(self.dist)) package_dir = self.dist.package_dir or {} - pkg_roots = _find_pkg_roots(packages, package_dir, src_root) - namespaces_ = set(_find_mapped_namespaces(pkg_roots)) + roots = _find_package_roots(top_level, package_dir, src_root) + namespaces_ = set(_find_mapped_namespaces(roots)) finder = _make_identifier(f"__editable__.{self.name}.finder") - content = _finder_template(pkg_roots, namespaces_) + content = _finder_template(roots, namespaces_) Path(unpacked_wheel_dir, f"{finder}.py").write_text(content, encoding="utf-8") pth = f"__editable__.{self.name}.pth" @@ -238,11 +251,11 @@ class _TopLevelFinder: Path(unpacked_wheel_dir, pth).write_text(content, encoding="utf-8") -def _configure_build(name: str, dist: Distribution, target_dir: Path, tmp: str): +def _configure_build(name: str, dist: Distribution, target_dir: _Path, tmp_dir: _Path): target = str(target_dir) - data = str(target_dir / f"{name}.data/data") - headers = str(target_dir / f"{name}.data/include") - scripts = str(target_dir / f"{name}.data/scripts") + data = str(Path(target_dir, f"{name}.data", "data")) + headers = str(Path(target_dir, f"{name}.data", "include")) + scripts = str(Path(target_dir, f"{name}.data", "scripts")) build = dist.reinitialize_command("build", reinit_subcommands=True) install = dist.reinitialize_command("install", reinit_subcommands=True) @@ -253,7 +266,7 @@ def _configure_build(name: str, dist: Distribution, target_dir: Path, tmp: str): install.install_headers = headers install.install_data = data - build.build_temp = tmp + build.build_temp = str(tmp_dir) build_py = dist.get_command_obj("build_py") build_py.compile = False @@ -276,7 +289,9 @@ def _can_symlink_files(): def _simple_layout( packages: Iterable[str], package_dir: Dict[str, str], project_dir: Path ) -> bool: - """Make sure all packages are contained by the same parent directory. + """Return ``True`` if: + - all packages are contained by the same parent directory, **and** + - all packages become importable if the parent directory is added to ``sys.path``. >>> _simple_layout(['a'], {"": "src"}, "/tmp/myproj") True @@ -292,11 +307,7 @@ def _simple_layout( False >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"a": "_a"}, "/tmp/myproj") False - >>> _simple_layout( - ... ['a', 'a.a1', 'a.a1.a2', 'b'], - ... {"a": "_a", "a.a1.a2": "_a2", "b": "_b"}, - ... ".", - ... ) + >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"a.a1.a2": "_a2"}, ".") False """ layout = { @@ -313,8 +324,10 @@ def _simple_layout( def _parent_path(pkg, pkg_path): - """Infer the parent path for a package if possible. When the pkg is directly mapped - into a directory with a different name, return its own path. + """Infer the parent path containing a package, that if added to ``sys.path`` would + allow importing that package. + When ``pkg`` is directly mapped into a directory with a different name, return its + own path. >>> _parent_path("a", "src/a") 'src' >>> _parent_path("b", "src/c") @@ -349,7 +362,7 @@ def _find_top_level_modules(dist: Distribution) -> Iterator[str]: yield from (x.name for x in ext_modules if "." not in x.name) -def _find_pkg_roots( +def _find_package_roots( packages: Iterable[str], package_dir: Mapping[str, str], src_root: _Path, @@ -404,6 +417,8 @@ def _remove_nested(pkg_roots: Dict[str, str]) -> Dict[str, str]: def _is_nested(pkg: str, pkg_path: str, parent: str, parent_path: str) -> bool: """ + Return ``True`` if ``pkg`` is nested inside ``parent`` both logically and in the + file system. >>> _is_nested("a.b", "path/a/b", "a", "path/a") True >>> _is_nested("a.b", "path/a/b", "a", "otherpath/a") @@ -426,9 +441,10 @@ def _normalize_path(filename: _Path) -> str: return os.path.normcase(os.path.realpath(os.path.normpath(file))) -def _empty_dir(dir_: Path) -> Path: +def _empty_dir(dir_: _P) -> _P: + """Create a directory ensured to be empty. Existing files may be removed.""" shutil.rmtree(dir_, ignore_errors=True) - dir_.mkdir(parents=True) + os.makedirs(dir_) return dir_ @@ -516,5 +532,6 @@ def install(): def _finder_template(mapping: Mapping[str, str], namespaces: Set[str]): + """Create a string containing the code for a ``MetaPathFinder``.""" mapping = dict(sorted(mapping.items(), key=lambda p: p[0])) return _FINDER_TEMPLATE.format(mapping=mapping, namespaces=namespaces) -- cgit v1.2.1 From f3786f3453e4c4d2dde05beb53a5090b4b449aa2 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Mon, 11 Apr 2022 19:24:35 +0100 Subject: Allow egg-info directory to be ignored in manifest --- setuptools/command/editable_wheel.py | 12 ++++++++++++ setuptools/command/egg_info.py | 13 ++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index d5a7d530..354f6b9f 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -47,6 +47,7 @@ class editable_wheel(Command): user_options = [ ("dist-dir=", "d", "directory to put final built distributions in"), ("dist-info-dir=", "I", "path to a pre-build .dist-info directory"), + ("strict", None, "perform an strict installation"), ] boolean_options = ["strict"] @@ -211,6 +212,9 @@ class _LinkTree(_StaticPth): self.tmp = tmp def _build_py(self): + if not self.dist.has_pure_modules(): + return + build_py = self.dist.get_command_obj("build_py") build_py.ensure_finalized() # Force build_py to use links instead of copying files @@ -218,6 +222,9 @@ class _LinkTree(_StaticPth): build_py.run() def _build_ext(self): + if not self.dist.has_ext_modules(): + return + build_ext = self.dist.get_command_obj("build_ext") build_ext.ensure_finalized() # Extensions are not editable, so we just have to build them in the right dir @@ -257,6 +264,11 @@ def _configure_build(name: str, dist: Distribution, target_dir: _Path, tmp_dir: headers = str(Path(target_dir, f"{name}.data", "include")) scripts = str(Path(target_dir, f"{name}.data", "scripts")) + # egg-info will be generated again to create a manifest (used for package data) + egg_info = dist.reinitialize_command("egg_info", reinit_subcommands=True) + egg_info.egg_base = str(tmp_dir) + egg_info.ignore_egg_info_in_manifest = True + build = dist.reinitialize_command("build", reinit_subcommands=True) install = dist.reinitialize_command("install", reinit_subcommands=True) diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py index c37ab81f..0c9d45ae 100644 --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py @@ -182,6 +182,7 @@ class egg_info(InfoCommon, Command): self.egg_info = None self.egg_version = None self.broken_egg_info = False + self.ignore_egg_info_in_manifest = False #################################### # allow the 'tag_svn_revision' to be detected and @@ -311,6 +312,7 @@ class egg_info(InfoCommon, Command): """Generate SOURCES.txt manifest file""" manifest_filename = os.path.join(self.egg_info, "SOURCES.txt") mm = manifest_maker(self.distribution) + mm.ignore_egg_info_dir = self.ignore_egg_info_in_manifest mm.manifest = manifest_filename mm.run() self.filelist = mm.filelist @@ -334,6 +336,10 @@ class egg_info(InfoCommon, Command): class FileList(_FileList): # Implementations of the various MANIFEST.in commands + def __init__(self, warn=None, debug_print=None, ignore_egg_info_dir=False): + super().__init__(warn, debug_print) + self.ignore_egg_info_dir = ignore_egg_info_dir + def process_template_line(self, line): # Parse the line: split it up, make sure the right number of words # is there, and return the relevant words. 'action' is always @@ -523,6 +529,10 @@ class FileList(_FileList): return False try: + # ignore egg-info paths + is_egg_info = ".egg-info" in u_path or b".egg-info" in utf8_path + if self.ignore_egg_info_dir and is_egg_info: + return False # accept is either way checks out if os.path.exists(u_path) or os.path.exists(utf8_path): return True @@ -539,12 +549,13 @@ class manifest_maker(sdist): self.prune = 1 self.manifest_only = 1 self.force_manifest = 1 + self.ignore_egg_info_dir = False def finalize_options(self): pass def run(self): - self.filelist = FileList() + self.filelist = FileList(ignore_egg_info_dir=self.ignore_egg_info_dir) if not os.path.exists(self.manifest): self.write_manifest() # it must exist so it'll get in the list self.add_defaults() -- cgit v1.2.1 From 3c71c872d9e23997d383e2adafe04d4cb1723109 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sun, 17 Apr 2022 00:45:07 +0100 Subject: Correctly handle namespace spec via PathEntryFinder --- setuptools/command/editable_wheel.py | 104 ++++++++++++++++++++++------------- 1 file changed, 65 insertions(+), 39 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index 354f6b9f..0a56fa0c 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -18,7 +18,7 @@ import logging from itertools import chain from pathlib import Path from tempfile import TemporaryDirectory -from typing import Dict, Iterable, Iterator, List, Mapping, Set, Union, TypeVar +from typing import Dict, Iterable, Iterator, List, Mapping, Union, Tuple, TypeVar from setuptools import Command, namespaces from setuptools.discovery import find_package_path @@ -247,10 +247,15 @@ class _TopLevelFinder: top_level = chain(_find_packages(self.dist), _find_top_level_modules(self.dist)) package_dir = self.dist.package_dir or {} roots = _find_package_roots(top_level, package_dir, src_root) - namespaces_ = set(_find_mapped_namespaces(roots)) - finder = _make_identifier(f"__editable__.{self.name}.finder") - content = _finder_template(roots, namespaces_) + namespaces_: Dict[str, List[str]] = dict(chain( + _find_namespaces(self.dist.packages, roots), + ((ns, []) for ns in _find_virtual_namespaces(roots)), + )) + + name = f"__editable__.{self.name}.finder" + finder = _make_identifier(name) + content = _finder_template(name, roots, namespaces_) Path(unpacked_wheel_dir, f"{finder}.py").write_text(content, encoding="utf-8") pth = f"__editable__.{self.name}.pth" @@ -398,9 +403,9 @@ def _absolute_root(path: _Path) -> str: return str(parent.resolve() / path_.name) -def _find_mapped_namespaces(pkg_roots: Dict[str, str]) -> Iterator[str]: - """By carefully designing ``package_dir``, it is possible to implement - PEP 420 compatible namespaces without creating extra folders. +def _find_virtual_namespaces(pkg_roots: Dict[str, str]) -> Iterator[str]: + """By carefully designing ``package_dir``, it is possible to implement the logical + structure of PEP 420 in a package without the corresponding directories. This function will try to find this kind of namespaces. """ for pkg in pkg_roots: @@ -409,11 +414,20 @@ def _find_mapped_namespaces(pkg_roots: Dict[str, str]) -> Iterator[str]: parts = pkg.split(".") for i in range(len(parts) - 1, 0, -1): partial_name = ".".join(parts[:i]) - path = find_package_path(partial_name, pkg_roots, "") - if not Path(path, "__init__.py").exists(): + path = Path(find_package_path(partial_name, pkg_roots, "")) + if not path.exists(): yield partial_name +def _find_namespaces( + packages: List[str], pkg_roots: Dict[str, str] +) -> Iterator[Tuple[str, List[str]]]: + for pkg in packages: + path = find_package_path(pkg, pkg_roots, "") + if Path(path).exists() and not Path(path, "__init__.py").exists(): + yield (pkg, [path]) + + def _remove_nested(pkg_roots: Dict[str, str]) -> Dict[str, str]: output = dict(pkg_roots.copy()) @@ -491,59 +505,71 @@ class _NamespaceInstaller(namespaces.Installer): _FINDER_TEMPLATE = """\ import sys -from importlib.machinery import all_suffixes as module_suffixes from importlib.machinery import ModuleSpec +from importlib.machinery import all_suffixes as module_suffixes from importlib.util import spec_from_file_location from itertools import chain from pathlib import Path -class __EditableFinder: - MAPPING = {mapping!r} - NAMESPACES = {namespaces!r} +MAPPING = {mapping!r} +NAMESPACES = {namespaces!r} +PATH_PLACEHOLDER = {name!r} + ".__path_hook__" - @classmethod - def find_spec(cls, fullname, path, target=None): - if fullname in cls.NAMESPACES: - return cls._namespace_spec(fullname) - for pkg, pkg_path in reversed(list(cls.MAPPING.items())): +class _EditableFinder: # MetaPathFinder + @classmethod + def find_spec(cls, fullname, path=None, target=None): + for pkg, pkg_path in reversed(list(MAPPING.items())): if fullname.startswith(pkg): - return cls._find_spec(fullname, pkg, pkg_path) + rest = fullname.replace(pkg, "").strip(".").split(".") + return cls._find_spec(fullname, Path(pkg_path, *rest)) return None @classmethod - def _namespace_spec(cls, name): - # Since `cls` is appended to the path, this will only trigger - # when no other package is installed in the same namespace. - return ModuleSpec(name, None, is_package=True) - # ^-- PEP 451 mentions setting loader to None for namespaces. - - @classmethod - def _find_spec(cls, fullname, parent, parent_path): - rest = fullname.replace(parent, "").strip(".").split(".") - candidate_path = Path(parent_path, *rest) - + def _find_spec(cls, fullname, candidate_path): init = candidate_path / "__init__.py" candidates = (candidate_path.with_suffix(x) for x in module_suffixes()) for candidate in chain([init], candidates): if candidate.exists(): - spec = spec_from_file_location(fullname, candidate) - return spec + return spec_from_file_location(fullname, candidate) - if candidate_path.exists(): - return cls._namespace_spec(fullname) +class _EditableNamespaceFinder: # PathEntryFinder + @classmethod + def _path_hook(cls, path): + if path == PATH_PLACEHOLDER: + return cls + raise ImportError + + @classmethod + def find_spec(cls, fullname, target=None): + if fullname in NAMESPACES: + spec = ModuleSpec(fullname, None, is_package=True) + paths = NAMESPACES[fullname] or MAPPING.get(fullname) or [PATH_PLACEHOLDER] + # ^ We have to ensure submodule_search_locations to not be empty for the + # spec to be considered a namespace + spec.submodule_search_locations = paths + return spec return None def install(): - if not any(finder == __EditableFinder for finder in sys.meta_path): - sys.meta_path.append(__EditableFinder) + if not any(finder == _EditableFinder for finder in sys.meta_path): + sys.meta_path.append(_EditableFinder) + if not any(hook == _EditableNamespaceFinder._path_hook for hook in sys.path_hooks): + # PathEntryFinder is needed to create NamespaceSpec without private APIS + sys.path_hooks.append(_EditableNamespaceFinder._path_hook) + if PATH_PLACEHOLDER not in sys.path: + sys.path.append(PATH_PLACEHOLDER) # Used just to trigger the path hook """ -def _finder_template(mapping: Mapping[str, str], namespaces: Set[str]): - """Create a string containing the code for a ``MetaPathFinder``.""" +def _finder_template( + name: str, mapping: Mapping[str, str], namespaces: Dict[str, List[str]] +) -> str: + """Create a string containing the code for the``MetaPathFinder`` and + ``PathEntryFinder``. + """ mapping = dict(sorted(mapping.items(), key=lambda p: p[0])) - return _FINDER_TEMPLATE.format(mapping=mapping, namespaces=namespaces) + return _FINDER_TEMPLATE.format(name=name, mapping=mapping, namespaces=namespaces) -- cgit v1.2.1 From 501aec9d41667e9c18811b3b8c77b4f0820d50b8 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sun, 17 Apr 2022 01:36:24 +0100 Subject: Add missing methods to PathEntryFinder --- setuptools/command/editable_wheel.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index 0a56fa0c..e3936de5 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -542,17 +542,23 @@ class _EditableNamespaceFinder: # PathEntryFinder return cls raise ImportError + @classmethod + def _paths(cls, fullname): + # Ensure __path__ is not empty for the spec to be considered a namespace. + return NAMESPACES[fullname] or MAPPING.get(fullname) or [PATH_PLACEHOLDER] + @classmethod def find_spec(cls, fullname, target=None): if fullname in NAMESPACES: spec = ModuleSpec(fullname, None, is_package=True) - paths = NAMESPACES[fullname] or MAPPING.get(fullname) or [PATH_PLACEHOLDER] - # ^ We have to ensure submodule_search_locations to not be empty for the - # spec to be considered a namespace - spec.submodule_search_locations = paths + spec.submodule_search_locations = cls._paths(fullname) return spec return None + @classmethod + def find_module(cls, fullname): + return None + def install(): if not any(finder == _EditableFinder for finder in sys.meta_path): -- cgit v1.2.1 From 4687243374368ed83d19ec70b8cf10bb22dee958 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sun, 17 Apr 2022 10:16:56 +0100 Subject: Avoid adding PathEntryFinder if not necessary --- setuptools/command/editable_wheel.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index e3936de5..48202990 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -563,6 +563,10 @@ class _EditableNamespaceFinder: # PathEntryFinder def install(): if not any(finder == _EditableFinder for finder in sys.meta_path): sys.meta_path.append(_EditableFinder) + + if not NAMESPACES: + return + if not any(hook == _EditableNamespaceFinder._path_hook for hook in sys.path_hooks): # PathEntryFinder is needed to create NamespaceSpec without private APIS sys.path_hooks.append(_EditableNamespaceFinder._path_hook) -- cgit v1.2.1 From ebf8369b13b43ff2d5f6a58875246218fe922c9c Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Thu, 16 Jun 2022 00:59:52 +0100 Subject: Add translator for config_settings to build_meta --- setuptools/command/dist_info.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/dist_info.py b/setuptools/command/dist_info.py index aa7af48c..323dbefc 100644 --- a/setuptools/command/dist_info.py +++ b/setuptools/command/dist_info.py @@ -25,13 +25,21 @@ class dist_info(Command): " DEPRECATED: use --output-dir."), ('output-dir=', 'o', "directory inside of which the .dist-info will be" "created (default: top of the source tree)"), + ('tag-date', 'd', "Add date stamp (e.g. 20050528) to version number"), + ('tag-build=', 'b', "Specify explicit tag to add to version number"), + ('no-date', 'D', "Don't include date stamp [default]"), ] + boolean_options = ['tag-date'] + negative_opt = {'no-date': 'tag-date'} + def initialize_options(self): self.egg_base = None self.output_dir = None self.name = None self.dist_info_dir = None + self.tag_date = None + self.tag_build = None def finalize_options(self): if self.egg_base: @@ -43,8 +51,14 @@ class dist_info(Command): project_dir = dist.src_root or os.curdir self.output_dir = Path(self.output_dir or project_dir) - egg_info = self.reinitialize_command('egg_info') + self.set_undefined_options( + "egg_info", ("tag_date", "tag_date"), ("tag_build", "tag_build") + ) + + egg_info = self.reinitialize_command("egg_info") egg_info.egg_base = str(self.output_dir) + egg_info.tag_date = self.tag_date + egg_info.tag_build = self.tag_build egg_info.finalize_options() self.egg_info = egg_info -- cgit v1.2.1 From 6f680c986759cb03922df9e2b275efbb5a17f796 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Thu, 16 Jun 2022 01:27:47 +0100 Subject: Ensure new options for dist-info work --- setuptools/command/dist_info.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/dist_info.py b/setuptools/command/dist_info.py index 323dbefc..39a74e1e 100644 --- a/setuptools/command/dist_info.py +++ b/setuptools/command/dist_info.py @@ -51,14 +51,19 @@ class dist_info(Command): project_dir = dist.src_root or os.curdir self.output_dir = Path(self.output_dir or project_dir) - self.set_undefined_options( - "egg_info", ("tag_date", "tag_date"), ("tag_build", "tag_build") - ) - egg_info = self.reinitialize_command("egg_info") egg_info.egg_base = str(self.output_dir) - egg_info.tag_date = self.tag_date - egg_info.tag_build = self.tag_build + + if self.tag_date: + egg_info.tag_date = self.tag_date + else: + self.tag_date = egg_info.tag_date + + if self.tag_build: + egg_info.tag_build = self.tag_build + else: + self.tag_build = egg_info.tag_build + egg_info.finalize_options() self.egg_info = egg_info -- cgit v1.2.1 From 01d961a8b990d50e326ea4759dd62dd215ad2b4d Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Thu, 16 Jun 2022 18:16:55 +0100 Subject: Add warning with information for the user about link tree --- setuptools/command/editable_wheel.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index 48202990..2776577f 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -15,6 +15,7 @@ import re import shutil import sys import logging +import warnings from itertools import chain from pathlib import Path from tempfile import TemporaryDirectory @@ -166,6 +167,15 @@ class editable_wheel(Command): populate = _LinkTree(self.distribution, name, auxiliary_build_dir, tmp) populate(unpacked_dir) + msg = f"""\n + Strict editable installation performed using the auxiliary directory: + {auxiliary_build_dir} + + Please be careful to not remove this directory, otherwise you might not be able + to import/use your package. + """ + warnings.warn(msg, InformationOnly) + def _populate_static_pth(self, name: str, project_dir: Path, unpacked_dir: Path): """Populate wheel using the "lax" ``.pth`` file strategy, for ``src-layout``.""" src_dir = self.package_dir[""] @@ -583,3 +593,11 @@ def _finder_template( """ mapping = dict(sorted(mapping.items(), key=lambda p: p[0])) return _FINDER_TEMPLATE.format(name=name, mapping=mapping, namespaces=namespaces) + + +class InformationOnly(UserWarning): + """Currently there is no clear way of displaying messages to the users + that use the setuptools backend directly via ``pip``. + The only thing that might work is a warning, although it is not the + most appropriate tool for the job... + """ -- cgit v1.2.1 From 84d5133b63dd6c2f64cc38ced7ec8f93f1725cac Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sat, 18 Jun 2022 09:39:27 +0100 Subject: build_py: Allow get_outputs() to work without re-running egg-info --- setuptools/command/build_py.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index 9575cdf8..dab81327 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -29,6 +29,8 @@ class build_py(orig.build_py): 'py_modules' and 'packages' in the same setup operation. """ + existing_egg_info_dir = None #: Private API, setuptools internal use only. + def finalize_options(self): orig.build_py.finalize_options(self) self.package_data = self.distribution.package_data @@ -143,10 +145,19 @@ class build_py(orig.build_py): # Locate package source directory src_dirs[assert_relative(self.get_package_dir(package))] = package - self.run_command('egg_info') + if ( + getattr(self, 'existing_egg_info_dir', None) + and Path(self.existing_egg_info_dir, "SOURCES.txt").exists() + ): + manifest = Path(self.existing_egg_info_dir, "SOURCES.txt") + files = manifest.read_text(encoding="utf-8").splitlines() + else: + self.run_command('egg_info') + ei_cmd = self.get_finalized_command('egg_info') + files = ei_cmd.filelist.files + check = _IncludePackageDataAbuse() - ei_cmd = self.get_finalized_command('egg_info') - for path in ei_cmd.filelist.files: + for path in files: d, f = os.path.split(assert_relative(path)) prev = None oldf = f -- cgit v1.2.1 From 207da8caa9e2673ab8470f78dc10901e0d60a96f Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sun, 19 Jun 2022 11:58:56 +0100 Subject: Make it possible for build_py to export the output mapping --- setuptools/command/build_py.py | 49 +++++++++++++++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 8 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index dab81327..3163752f 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -11,6 +11,8 @@ import itertools import stat import warnings from pathlib import Path +from typing import Dict, Iterator, List, Optional, Tuple + from setuptools._deprecation_warning import SetuptoolsDeprecationWarning from setuptools.extern.more_itertools import unique_everseen @@ -28,8 +30,13 @@ class build_py(orig.build_py): Also, this version of the 'build_py' command allows you to specify both 'py_modules' and 'packages' in the same setup operation. """ + editable_mode: bool = False + existing_egg_info_dir: Optional[str] = None #: Private API, internal use only. - existing_egg_info_dir = None #: Private API, setuptools internal use only. + def initialize_options(self): + super().initialize_options() + self.editable_mode = False + self.existing_egg_info_dir = None def finalize_options(self): orig.build_py.finalize_options(self) @@ -52,7 +59,8 @@ class build_py(orig.build_py): def run(self): """Build modules, packages, and copy data files to build directory""" - if not self.py_modules and not self.packages: + # if self.editable_mode or not (self.py_modules and self.packages): + if not (self.py_modules or self.packages) or self.editable_mode: return if self.py_modules: @@ -125,16 +133,41 @@ class build_py(orig.build_py): ) return self.exclude_data_files(package, src_dir, files) - def build_package_data(self): - """Copy data files into build directory""" + def get_outputs(self, include_bytecode=1) -> List[str]: + """See :class:`setuptools.commands.build.SubCommand`""" + if self.editable_mode: + return list(self.get_output_mapping().keys()) + return super().get_outputs(include_bytecode) + + def get_output_mapping(self) -> Dict[str, str]: + """See :class:`setuptools.commands.build.SubCommand`""" + mapping = itertools.chain( + self._get_package_data_output_mapping(), + self._get_module_mapping(), + ) + return dict(sorted(mapping, key=lambda x: x[0])) + + def _get_module_mapping(self) -> Iterator[Tuple[str, str]]: + """Iterate over all modules producing (dest, src) pairs.""" + for (package, module, module_file) in self.find_all_modules(): + package = package.split('.') + filename = self.get_module_outfile(self.build_lib, package, module) + yield (filename, module_file) + + def _get_package_data_output_mapping(self) -> Iterator[Tuple[str, str]]: + """Iterate over package data producing (dest, src) pairs.""" for package, src_dir, build_dir, filenames in self.data_files: for filename in filenames: target = os.path.join(build_dir, filename) - self.mkpath(os.path.dirname(target)) srcfile = os.path.join(src_dir, filename) - outf, copied = self.copy_file(srcfile, target) - make_writable(target) - srcfile = os.path.abspath(srcfile) + yield (target, srcfile) + + def build_package_data(self): + """Copy data files into build directory""" + for target, srcfile in self._get_package_data_output_mapping(): + self.mkpath(os.path.dirname(target)) + _outf, _copied = self.copy_file(srcfile, target) + make_writable(target) def analyze_manifest(self): self.manifest_files = mf = {} -- cgit v1.2.1 From 075a3c801910a6b65dccf8b9eb26a0637315627a Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sun, 19 Jun 2022 18:12:37 +0100 Subject: Make it possible for build_ext to export the output mapping --- setuptools/command/build_ext.py | 118 ++++++++++++++++++++++++++++++---------- 1 file changed, 90 insertions(+), 28 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/build_ext.py b/setuptools/command/build_ext.py index 1719d17a..42e67dcd 100644 --- a/setuptools/command/build_ext.py +++ b/setuptools/command/build_ext.py @@ -2,13 +2,17 @@ import os import sys import itertools from importlib.machinery import EXTENSION_SUFFIXES +from importlib.util import cache_from_source as _compiled_file_name +from pathlib import Path +from typing import Dict, Iterator, List, Tuple, Union + from distutils.command.build_ext import build_ext as _du_build_ext from distutils.ccompiler import new_compiler from distutils.sysconfig import customize_compiler, get_config_var -from distutils.errors import DistutilsError from distutils import log -from setuptools.extension import Library +from setuptools.errors import BaseError +from setuptools.extension import Extension, Library try: # Attempt to use Cython for building extensions, if available @@ -72,6 +76,9 @@ def get_abi3_suffix(): class build_ext(_build_ext): + editable_mode: bool = False + inplace: bool = False + def run(self): """Build extensions in build directory, then copy if --inplace""" old_inplace, self.inplace = self.inplace, 0 @@ -80,24 +87,61 @@ class build_ext(_build_ext): if old_inplace: self.copy_extensions_to_source() + def _get_inplace_equivalent(self, build_py, ext: Extension) -> Tuple[str, str]: + fullname = self.get_ext_fullname(ext.name) + filename = self.get_ext_filename(fullname) + modpath = fullname.split('.') + package = '.'.join(modpath[:-1]) + package_dir = build_py.get_package_dir(package) + inplace_file = os.path.join(package_dir, os.path.basename(filename)) + regular_file = os.path.join(self.build_lib, filename) + return (inplace_file, regular_file) + def copy_extensions_to_source(self): build_py = self.get_finalized_command('build_py') for ext in self.extensions: - fullname = self.get_ext_fullname(ext.name) - filename = self.get_ext_filename(fullname) - modpath = fullname.split('.') - package = '.'.join(modpath[:-1]) - package_dir = build_py.get_package_dir(package) - dest_filename = os.path.join(package_dir, - os.path.basename(filename)) - src_filename = os.path.join(self.build_lib, filename) + inplace_file, regular_file = self._get_inplace_equivalent(build_py, ext) # Always copy, even if source is older than destination, to ensure # that the right extensions for the current Python/platform are # used. - build_py.copy_file(src_filename, dest_filename) + build_py.copy_file(regular_file, inplace_file) + if ext._needs_stub: - self.write_stub(package_dir or os.curdir, ext, True) + inplace_stub = self._get_equivalent_stub(ext, inplace_file) + self._write_stub_file(inplace_stub, ext, compile=True) + # Always compile stub and remove the original (leave the cache behind) + # (this behaviour was observed in previous iterations of the code) + + def _get_equivalent_stub(self, ext: Extension, output_file: str) -> str: + dir_ = os.path.dirname(output_file) + _, _, name = ext.name.rpartition(".") + return f"{os.path.join(dir_, name)}.py" + + def _get_output_mapping(self) -> Iterator[Tuple[str, str]]: + if not self.inplace: + return + + build_py = self.get_finalized_command('build_py') + opt = self.get_finalized_command('install_lib').optimize or "" + + for ext in self.extensions: + inplace_file, regular_file = self._get_inplace_equivalent(build_py, ext) + yield (regular_file, inplace_file) + + if ext._needs_stub: + # This version of `build_ext` always builds artifacts in another dir, + # when "inplace=True" is given it just copies them back. + # This is done in the `copy_extensions_to_source` function, which + # always compile stub files via `_compile_and_remove_stub`. + # At the end of the process, a `.pyc` stub file is created without the + # corresponding `.py`. + + inplace_stub = self._get_equivalent_stub(ext, inplace_file) + regular_stub = self._get_equivalent_stub(ext, regular_file) + inplace_cache = _compiled_file_name(inplace_stub, optimization=opt) + output_cache = _compiled_file_name(regular_stub, optimization=opt) + yield (output_cache, inplace_cache) def get_ext_filename(self, fullname): so_ext = os.getenv('SETUPTOOLS_EXT_SUFFIX') @@ -127,6 +171,7 @@ class build_ext(_build_ext): self.shlib_compiler = None self.shlibs = [] self.ext_map = {} + self.editable_mode = False def finalize_options(self): _build_ext.finalize_options(self) @@ -157,6 +202,9 @@ class build_ext(_build_ext): if ltd and use_stubs and os.curdir not in ext.runtime_library_dirs: ext.runtime_library_dirs.append(os.curdir) + if self.editable_mode: + self.inplace = True + def setup_shlib_compiler(self): compiler = self.shlib_compiler = new_compiler( compiler=self.compiler, dry_run=self.dry_run, force=self.force @@ -197,8 +245,8 @@ class build_ext(_build_ext): self.compiler = self.shlib_compiler _build_ext.build_extension(self, ext) if ext._needs_stub: - cmd = self.get_finalized_command('build_py').build_lib - self.write_stub(cmd, ext) + build_lib = self.get_finalized_command('build_py').build_lib + self.write_stub(build_lib, ext) finally: self.compiler = _compiler @@ -212,7 +260,14 @@ class build_ext(_build_ext): return any(pkg + libname in libnames for libname in ext.libraries) def get_outputs(self): - return _build_ext.get_outputs(self) + self.__get_stubs_outputs() + if self.inplace: + return list(self.get_output_mapping().keys()) + return sorted(_build_ext.get_outputs(self) + self.__get_stubs_outputs()) + + def get_output_mapping(self) -> Dict[str, str]: + """See :class:`setuptools.commands.build.SubCommand`""" + mapping = self._get_output_mapping() + return dict(sorted(mapping, key=lambda x: x[0])) def __get_stubs_outputs(self): # assemble the base name for each extension that needs a stub @@ -232,12 +287,13 @@ class build_ext(_build_ext): yield '.pyo' def write_stub(self, output_dir, ext, compile=False): - log.info("writing stub loader for %s to %s", ext._full_name, - output_dir) - stub_file = (os.path.join(output_dir, *ext._full_name.split('.')) + - '.py') + stub_file = os.path.join(output_dir, *ext._full_name.split('.')) + '.py' + self._write_stub_file(stub_file, ext, compile) + + def _write_stub_file(self, stub_file: str, ext: Extension, compile=False): + log.info("writing stub loader for %s to %s", ext._full_name, stub_file) if compile and os.path.exists(stub_file): - raise DistutilsError(stub_file + " already exists! Please delete.") + raise BaseError(stub_file + " already exists! Please delete.") if not self.dry_run: f = open(stub_file, 'w') f.write( @@ -270,17 +326,23 @@ class build_ext(_build_ext): ) f.close() if compile: - from distutils.util import byte_compile + self._compile_and_remove_stub(stub_file) - byte_compile([stub_file], optimize=0, + def _compile_and_remove_stub(self, stub_file: str): + from distutils.util import byte_compile + + byte_compile([stub_file], optimize=0, + force=True, dry_run=self.dry_run) + optimize = self.get_finalized_command('install_lib').optimize + if optimize > 0: + byte_compile([stub_file], optimize=optimize, force=True, dry_run=self.dry_run) - optimize = self.get_finalized_command('install_lib').optimize - if optimize > 0: - byte_compile([stub_file], optimize=optimize, - force=True, dry_run=self.dry_run) - if os.path.exists(stub_file) and not self.dry_run: - os.unlink(stub_file) + if os.path.exists(stub_file) and not self.dry_run: + os.unlink(stub_file) + +def _file_with_suffix(directory: str, name: str, suffix: str) -> str: + return f"{os.path.join(directory, name)}.{suffix}" if use_stubs or os.name == 'nt': # Build shared libraries -- cgit v1.2.1 From d019f498ecd5ad45e58e95df64afd9730090df6a Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Mon, 20 Jun 2022 07:24:24 +0100 Subject: Fix linting errors --- setuptools/command/build_ext.py | 8 ++------ setuptools/command/build_py.py | 7 ++----- 2 files changed, 4 insertions(+), 11 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/build_ext.py b/setuptools/command/build_ext.py index 42e67dcd..31ef47bf 100644 --- a/setuptools/command/build_ext.py +++ b/setuptools/command/build_ext.py @@ -3,8 +3,7 @@ import sys import itertools from importlib.machinery import EXTENSION_SUFFIXES from importlib.util import cache_from_source as _compiled_file_name -from pathlib import Path -from typing import Dict, Iterator, List, Tuple, Union +from typing import Dict, Iterator, List, Tuple from distutils.command.build_ext import build_ext as _du_build_ext from distutils.ccompiler import new_compiler @@ -259,7 +258,7 @@ class build_ext(_build_ext): pkg = '.'.join(ext._full_name.split('.')[:-1] + ['']) return any(pkg + libname in libnames for libname in ext.libraries) - def get_outputs(self): + def get_outputs(self) -> List[str]: if self.inplace: return list(self.get_output_mapping().keys()) return sorted(_build_ext.get_outputs(self) + self.__get_stubs_outputs()) @@ -341,9 +340,6 @@ class build_ext(_build_ext): os.unlink(stub_file) -def _file_with_suffix(directory: str, name: str, suffix: str) -> str: - return f"{os.path.join(directory, name)}.{suffix}" - if use_stubs or os.name == 'nt': # Build shared libraries # diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index 3163752f..a2a6fe2c 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -33,11 +33,6 @@ class build_py(orig.build_py): editable_mode: bool = False existing_egg_info_dir: Optional[str] = None #: Private API, internal use only. - def initialize_options(self): - super().initialize_options() - self.editable_mode = False - self.existing_egg_info_dir = None - def finalize_options(self): orig.build_py.finalize_options(self) self.package_data = self.distribution.package_data @@ -244,6 +239,8 @@ class build_py(orig.build_py): def initialize_options(self): self.packages_checked = {} orig.build_py.initialize_options(self) + self.editable_mode = False + self.existing_egg_info_dir = None def get_package_dir(self, package): res = orig.build_py.get_package_dir(self, package) -- cgit v1.2.1 From fd891afb3020eceab6d8528dfbc0d844865361d4 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Tue, 21 Jun 2022 09:57:20 +0100 Subject: Allow dist-info to keep original egg-info directory During a PEP 517 build, this directory can later be passed to ``build_py`` via the ``existing_egg_info_dir attribute`` (which in turn avoids re-running ``egg_info`` to create a manifest). --- setuptools/command/dist_info.py | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/dist_info.py b/setuptools/command/dist_info.py index 39a74e1e..7f35a476 100644 --- a/setuptools/command/dist_info.py +++ b/setuptools/command/dist_info.py @@ -5,7 +5,10 @@ As defined in the wheel specification import os import re +import shutil +import sys import warnings +from contextlib import contextmanager from inspect import cleandoc from pathlib import Path @@ -28,9 +31,10 @@ class dist_info(Command): ('tag-date', 'd', "Add date stamp (e.g. 20050528) to version number"), ('tag-build=', 'b', "Specify explicit tag to add to version number"), ('no-date', 'D', "Don't include date stamp [default]"), + ('keep-egg-info', None, "*TRANSITIONAL* will be removed in the future"), ] - boolean_options = ['tag-date'] + boolean_options = ['tag-date', 'keep-egg-info'] negative_opt = {'no-date': 'tag-date'} def initialize_options(self): @@ -40,6 +44,7 @@ class dist_info(Command): self.dist_info_dir = None self.tag_date = None self.tag_build = None + self.keep_egg_info = False def finalize_options(self): if self.egg_base: @@ -72,14 +77,32 @@ class dist_info(Command): self.name = f"{name}-{version}" self.dist_info_dir = os.path.join(self.output_dir, f"{self.name}.dist-info") + @contextmanager + def _maybe_bkp_dir(self, dir_path: str, requires_bkp: bool): + if requires_bkp: + bkp_name = f"__bkp__.{dir_path}.__bkp__" + _rm(bkp_name, ignore_errors=True) + _copy(dir_path, bkp_name, dirs_exist_ok=True, symlinks=True) + try: + yield + finally: + _rm(dir_path, ignore_errors=True) + shutil.move(bkp_name, dir_path) + else: + yield + def run(self): self.output_dir.mkdir(parents=True, exist_ok=True) self.egg_info.run() egg_info_dir = self.egg_info.egg_info + assert os.path.isdir(egg_info_dir), ".egg-info dir should have been created" + log.info("creating '{}'".format(os.path.abspath(self.dist_info_dir))) bdist_wheel = self.get_finalized_command('bdist_wheel') - bdist_wheel.egg2dist(egg_info_dir, self.dist_info_dir) - assert os.path.exists(egg_info_dir) is False + + # TODO: if bdist_wheel if merged into setuptools, just add "keep_egg_info" there + with self._maybe_bkp_dir(egg_info_dir, self.keep_egg_info): + bdist_wheel.egg2dist(egg_info_dir, self.dist_info_dir) def _safe(component: str) -> str: @@ -106,3 +129,14 @@ def _version(version: str) -> str: """ warnings.warn(cleandoc(msg)) return _safe(v).strip("_") + + +def _rm(dir_name, **opts): + if os.path.isdir(dir_name): + shutil.rmtree(dir_name, **opts) + + +def _copy(src, dst, **opts): + if sys.version_info < (3, 8): + opts.pop("dirs_exist_ok", None) + shutil.copytree(src, dst, **opts) -- cgit v1.2.1 From 90c31496690312f271a1619a79e9dc98d0b4ae3c Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Tue, 21 Jun 2022 13:43:19 +0100 Subject: Fix problems with backup directory --- setuptools/command/dist_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/dist_info.py b/setuptools/command/dist_info.py index 7f35a476..0685c945 100644 --- a/setuptools/command/dist_info.py +++ b/setuptools/command/dist_info.py @@ -80,7 +80,7 @@ class dist_info(Command): @contextmanager def _maybe_bkp_dir(self, dir_path: str, requires_bkp: bool): if requires_bkp: - bkp_name = f"__bkp__.{dir_path}.__bkp__" + bkp_name = f"{dir_path}.__bkp__" _rm(bkp_name, ignore_errors=True) _copy(dir_path, bkp_name, dirs_exist_ok=True, symlinks=True) try: -- cgit v1.2.1 From f2da32f6e9c7cc7d9185adb071bee01daa5cfcb1 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 24 Jun 2022 12:08:45 +0100 Subject: Rely on get_outputs() and get_output_mapping() in editable_wheel --- setuptools/command/editable_wheel.py | 374 +++++++++++++++++++++++++---------- 1 file changed, 273 insertions(+), 101 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index 2776577f..b02486d0 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -7,24 +7,39 @@ Create a wheel that, when installed, will make the source package 'editable' One of the mechanisms briefly mentioned in PEP 660 to implement editable installs is to create a separated directory inside ``build`` and use a .pth file to point to that directory. In the context of this file such directory is referred as - *auxiliary build directory* or ``auxiliary_build_dir``. + *auxiliary build directory* or ``auxiliary_dir``. """ +import logging import os import re import shutil import sys -import logging import warnings +from contextlib import suppress from itertools import chain from pathlib import Path from tempfile import TemporaryDirectory -from typing import Dict, Iterable, Iterator, List, Mapping, Union, Tuple, TypeVar - -from setuptools import Command, namespaces +from typing import ( + TYPE_CHECKING, + Dict, + Iterable, + Iterator, + List, + Mapping, + Optional, + Tuple, + TypeVar, + Union +) + +from setuptools import Command, errors, namespaces from setuptools.discovery import find_package_path from setuptools.dist import Distribution +if TYPE_CHECKING: + from wheel.wheelfile import WheelFile # noqa + _Path = Union[str, Path] _P = TypeVar("_P", bound=_Path) _logger = logging.getLogger(__name__) @@ -64,9 +79,9 @@ class editable_wheel(Command): self.project_dir = dist.src_root or os.curdir self.package_dir = dist.package_dir or {} self.dist_dir = Path(self.dist_dir or os.path.join(self.project_dir, "dist")) - self.dist_dir.mkdir(exist_ok=True) def run(self): + self.dist_dir.mkdir(exist_ok=True) self._ensure_dist_info() # Add missing dist_info files @@ -96,6 +111,140 @@ class editable_wheel(Command): installer = _NamespaceInstaller(dist, installation_dir, pth_prefix, src_root) installer.install_namespaces() + def _find_egg_info_dir(self) -> Optional[str]: + parent_dir = Path(self.dist_info_dir).parent if self.dist_info_dir else Path() + candidates = map(str, parent_dir.glob("*.egg-info")) + return next(candidates, None) + + def _configure_build( + self, name: str, unpacked_wheel: _Path, build_lib: _Path, tmp_dir: _Path + ): + """Configure commands to behave in the following ways: + + - Build commands can write to ``build_lib`` if they really want to... + (but this folder is expected to be ignored and modules are expected to live + in the project directory...) + - Binary extensions should be built in-place (editable_mode = True) + - Data/header/script files are not part of the "editable" specification + so they are written directly to the unpacked_wheel directory. + """ + # Non-editable files (data, headers, scripts) are written directly to the + # unpacked_wheel + + dist = self.distribution + wheel = str(unpacked_wheel) + build_lib = str(build_lib) + data = str(Path(unpacked_wheel, f"{name}.data", "data")) + headers = str(Path(unpacked_wheel, f"{name}.data", "include")) + scripts = str(Path(unpacked_wheel, f"{name}.data", "scripts")) + + # egg-info may be generated again to create a manifest (used for package data) + egg_info = dist.reinitialize_command("egg_info", reinit_subcommands=True) + egg_info.egg_base = str(tmp_dir) + egg_info.ignore_egg_info_in_manifest = True + + build = dist.reinitialize_command("build", reinit_subcommands=True) + install = dist.reinitialize_command("install", reinit_subcommands=True) + + build.build_platlib = build.build_purelib = build.build_lib = build_lib + install.install_purelib = install.install_platlib = install.install_lib = wheel + install.install_scripts = build.build_scripts = scripts + install.install_headers = headers + install.install_data = data + + install_scripts = dist.get_command_obj("install_scripts") + install_scripts.no_ep = True + + build.build_temp = str(tmp_dir) + + build_py = dist.get_command_obj("build_py") + build_py.compile = False + build_py.existing_egg_info_dir = self._find_egg_info_dir() + + self._set_editable_mode() + + build.ensure_finalized() + install.ensure_finalized() + + def _set_editable_mode(self): + """Set the ``editable_mode`` flag in the build sub-commands""" + dist = self.distribution + build = dist.get_command_obj("build") + for cmd_name in build.get_sub_commands(): + cmd = dist.get_command_obj(cmd_name) + if hasattr(cmd, "editable_mode"): + cmd.editable_mode = True + + def _find_existing_source(self, file: Path) -> Optional[Path]: + """Given a file path relative to ``build_lib`` try to guess + what would be its original source file. + """ + dist = self.distribution + package = str(file.parent).replace(os.sep, ".") + package_path = find_package_path(package, dist.package_dir, self.project_dir) + candidate = Path(package_path, file.name) + return candidate if candidate.exists() else None + + def _collect_reminiscent_outputs( + self, build_lib: _Path + ) -> Tuple[List[str], Dict[str, str]]: + """People have been overwriting setuptools for a long time, and not everyone + might be respecting the new ``get_output_mapping`` APIs, so we have to do our + best to handle this scenario. + """ + files: List[str] = [] + mapping: Dict[str, str] = {} + + for dirpath, _dirnames, filenames in os.walk(build_lib): + for name in filenames: + file = Path(dirpath, name) + source = self._find_existing_source(file.relative_to(build_lib)) + if source: + mapping[str(file)] = str(source) + else: + files.append(str(file)) + + return files, mapping + + def _combine_outputs(self, *outputs: List[str]) -> List[str]: + return sorted({os.path.normpath(f) for f in chain.from_iterable(outputs)}) + + def _combine_output_mapping(self, *mappings: Dict[str, str]) -> Dict[str, str]: + mapping = ( + (os.path.normpath(k), os.path.normpath(v)) + for k, v in chain.from_iterable(m.items() for m in mappings) + ) + return dict(sorted(mapping)) + + def _collect_build_outputs(self) -> Tuple[List[str], Dict[str, str]]: + files: List[str] = [] + mapping: Dict[str, str] = {} + build = self.get_finalized_command("build") + + for cmd_name in build.get_sub_commands(): + cmd = self.get_finalized_command(cmd_name) + if hasattr(cmd, "get_outputs"): + files.extend(cmd.get_outputs() or []) + if hasattr(cmd, "get_output_mapping"): + mapping.update(cmd.get_output_mapping() or {}) + rfiles, rmapping = self._collect_reminiscent_outputs(build.build_lib) + + return ( + self._combine_outputs(files, rfiles), + self._combine_output_mapping(mapping, rmapping), + ) + + def _run_build_commands( + self, dist_name: str, unpacked_wheel: _Path, build_lib: _Path, tmp_dir: _Path + ) -> Tuple[List[str], Dict[str, str]]: + self._configure_build(dist_name, unpacked_wheel, build_lib, tmp_dir) + self.run_command("build") + files, mapping = self._collect_build_outputs() + self._run_install("headers") + self._run_install("scripts") + self._run_install("data") + return files, mapping + def _create_wheel_file(self, bdist_wheel): from wheel.wheelfile import WheelFile @@ -110,22 +259,19 @@ class editable_wheel(Command): # Currently the wheel API receives a directory and dump all its contents # inside of a wheel. So let's use a temporary directory. - unpacked_tmp = TemporaryDirectory(suffix=archive_name) + unpacked_wheel = TemporaryDirectory(suffix=archive_name) + build_lib = TemporaryDirectory(suffix=".build-lib") build_tmp = TemporaryDirectory(suffix=".build-temp") - with unpacked_tmp as unpacked, build_tmp as tmp: + with unpacked_wheel as unpacked, build_lib as lib, build_tmp as tmp: unpacked_dist_info = Path(unpacked, Path(self.dist_info_dir).name) shutil.copytree(self.dist_info_dir, unpacked_dist_info) self._install_namespaces(unpacked, dist_info.name) - - # Add non-editable files to the wheel - _configure_build(dist_name, self.distribution, unpacked, tmp) - self._run_install("headers") - self._run_install("scripts") - self._run_install("data") - - self._populate_wheel(dist_info.name, tag, unpacked, tmp) + files, mapping = self._run_build_commands(dist_name, unpacked, lib, tmp) with WheelFile(wheel_path, "w") as wf: + self._populate_wheel( + wf, dist_info.name, tag, unpacked, lib, tmp, files, mapping + ) wf.write_files(unpacked) return wheel_path @@ -136,13 +282,25 @@ class editable_wheel(Command): _logger.info(f"Installing {category} as non editable") self.run_command(f"install_{category}") - def _populate_wheel(self, name: str, tag: str, unpacked_dir: Path, tmp: _Path): + def _populate_wheel( + self, + wheel: "WheelFile", + name: str, + tag: str, + unpacked_dir: Path, + build_lib: _Path, + tmp: _Path, + outputs: List[str], + output_mapping: Dict[str, str], + ): """Decides which strategy to use to implement an editable installation.""" build_name = f"__editable__.{name}-{tag}" project_dir = Path(self.project_dir) if self.strict or os.getenv("SETUPTOOLS_EDITABLE", None) == "strict": - return self._populate_link_tree(name, build_name, unpacked_dir, tmp) + return self._populate_link_tree( + name, build_name, wheel, build_lib, outputs, output_mapping + ) # Build extensions in-place self.reinitialize_command("build_ext", inplace=1) @@ -152,44 +310,57 @@ class editable_wheel(Command): has_simple_layout = _simple_layout(packages, self.package_dir, project_dir) if set(self.package_dir) == {""} and has_simple_layout: # src-layout(ish) is relatively safe for a simple pth file - return self._populate_static_pth(name, project_dir, unpacked_dir) + return self._populate_static_pth(name, project_dir, wheel) # Use a MetaPathFinder to avoid adding accidental top-level packages/modules - self._populate_finder(name, unpacked_dir) + self._populate_finder(name, wheel) def _populate_link_tree( - self, name: str, build_name: str, unpacked_dir: Path, tmp: _Path + self, + name: str, + build_name: str, + wheel: "WheelFile", + build_lib: _Path, + outputs: List[str], + output_mapping: Dict[str, str], ): """Populate wheel using the "strict" ``link tree`` strategy.""" msg = "Strict editable install will be performed using a link tree.\n" _logger.warning(msg + _STRICT_WARNING) - auxiliary_build_dir = _empty_dir(Path(self.project_dir, "build", build_name)) - populate = _LinkTree(self.distribution, name, auxiliary_build_dir, tmp) - populate(unpacked_dir) + auxiliary_dir = _empty_dir(Path(self.project_dir, "build", build_name)) + populate = _LinkTree( + self.distribution, + name, + auxiliary_dir, + build_lib, + outputs, + output_mapping, + ) + populate(wheel) msg = f"""\n Strict editable installation performed using the auxiliary directory: - {auxiliary_build_dir} + {auxiliary_dir} Please be careful to not remove this directory, otherwise you might not be able to import/use your package. """ warnings.warn(msg, InformationOnly) - def _populate_static_pth(self, name: str, project_dir: Path, unpacked_dir: Path): + def _populate_static_pth(self, name: str, project_dir: Path, wheel: "WheelFile"): """Populate wheel using the "lax" ``.pth`` file strategy, for ``src-layout``.""" src_dir = self.package_dir[""] msg = f"Editable install will be performed using .pth file to {src_dir}.\n" _logger.warning(msg + _LAX_WARNING) populate = _StaticPth(self.distribution, name, [Path(project_dir, src_dir)]) - populate(unpacked_dir) + populate(wheel) - def _populate_finder(self, name: str, unpacked_dir: Path): + def _populate_finder(self, name: str, wheel: "WheelFile"): """Populate wheel using the "lax" MetaPathFinder strategy.""" msg = "Editable install will be performed using a meta path finder.\n" _logger.warning(msg + _LAX_WARNING) populate = _TopLevelFinder(self.distribution, name) - populate(unpacked_dir) + populate(wheel) class _StaticPth: @@ -198,53 +369,69 @@ class _StaticPth: self.name = name self.path_entries = path_entries - def __call__(self, unpacked_wheel_dir: Path): - pth = Path(unpacked_wheel_dir, f"__editable__.{self.name}.pth") + def __call__(self, wheel: "WheelFile"): entries = "\n".join((str(p.resolve()) for p in self.path_entries)) - pth.write_text(f"{entries}\n", encoding="utf-8") + contents = bytes(f"{entries}\n", "utf-8") + wheel.writestr(f"__editable__.{self.name}.pth", contents) class _LinkTree(_StaticPth): """ - Creates a ``.pth`` file that points to a link tree in the ``auxiliary_build_dir``. + Creates a ``.pth`` file that points to a link tree in the ``auxiliary_dir``. This strategy will only link files (not dirs), so it can be implemented in any OS, even if that means using hardlinks instead of symlinks. - By collocating ``auxiliary_build_dir`` and the original source code, limitations + By collocating ``auxiliary_dir`` and the original source code, limitations with hardlinks should be avoided. """ def __init__( - self, dist: Distribution, name: str, auxiliary_build_dir: Path, tmp: _Path + self, dist: Distribution, + name: str, + auxiliary_dir: _Path, + build_lib: _Path, + outputs: List[str], + output_mapping: Dict[str, str], ): - super().__init__(dist, name, [auxiliary_build_dir]) - self.auxiliary_build_dir = auxiliary_build_dir - self.tmp = tmp + self.auxiliary_dir = Path(auxiliary_dir) + self.build_lib = Path(build_lib).resolve() + self.outputs = outputs + self.output_mapping = output_mapping + self._file = dist.get_command_obj("build_py").copy_file + super().__init__(dist, name, [self.auxiliary_dir]) + + def __call__(self, wheel: "WheelFile"): + self._create_links() + super().__call__(wheel) + + def _normalize_output(self, file: str) -> Optional[str]: + # Files relative to build_lib will be normalized to None + with suppress(ValueError): + path = Path(file).resolve().relative_to(self.build_lib) + return str(path).replace(os.sep, '/') + return None - def _build_py(self): - if not self.dist.has_pure_modules(): - return + def _create_file(self, relative_output: str, src_file: str, link=None): + dest = self.auxiliary_dir / relative_output + if not dest.parent.is_dir(): + dest.parent.mkdir(parents=True) + self._file(src_file, dest, link=link) - build_py = self.dist.get_command_obj("build_py") - build_py.ensure_finalized() - # Force build_py to use links instead of copying files - build_py.use_links = "sym" if _can_symlink_files() else "hard" - build_py.run() + def _create_links(self): + link_type = "sym" if _can_symlink_files() else "hard" + mappings = { + self._normalize_output(k): v + for k, v in self.output_mapping.items() + } + mappings.pop(None, None) # remove files that are not relative to build_lib - def _build_ext(self): - if not self.dist.has_ext_modules(): - return + for output in self.outputs: + relative = self._normalize_output(output) + if relative and relative not in mappings: + self._create_file(relative, output) - build_ext = self.dist.get_command_obj("build_ext") - build_ext.ensure_finalized() - # Extensions are not editable, so we just have to build them in the right dir - build_ext.run() - - def __call__(self, unpacked_wheel_dir: Path): - _configure_build(self.name, self.dist, self.auxiliary_build_dir, self.tmp) - self._build_py() - self._build_ext() - super().__call__(unpacked_wheel_dir) + for relative, src in mappings.items(): + self._create_file(relative, src, link=link_type) class _TopLevelFinder: @@ -252,7 +439,7 @@ class _TopLevelFinder: self.dist = dist self.name = name - def __call__(self, unpacked_wheel_dir: Path): + def __call__(self, wheel: "WheelFile"): src_root = self.dist.src_root or os.curdir top_level = chain(_find_packages(self.dist), _find_top_level_modules(self.dist)) package_dir = self.dist.package_dir or {} @@ -265,51 +452,30 @@ class _TopLevelFinder: name = f"__editable__.{self.name}.finder" finder = _make_identifier(name) - content = _finder_template(name, roots, namespaces_) - Path(unpacked_wheel_dir, f"{finder}.py").write_text(content, encoding="utf-8") - - pth = f"__editable__.{self.name}.pth" - content = f"import {finder}; {finder}.install()" - Path(unpacked_wheel_dir, pth).write_text(content, encoding="utf-8") + content = bytes(_finder_template(name, roots, namespaces_), "utf-8") + wheel.writestr(f"{finder}.py", content) + content = bytes(f"import {finder}; {finder}.install()", "utf-8") + wheel.writestr(f"__editable__.{self.name}.pth", content) -def _configure_build(name: str, dist: Distribution, target_dir: _Path, tmp_dir: _Path): - target = str(target_dir) - data = str(Path(target_dir, f"{name}.data", "data")) - headers = str(Path(target_dir, f"{name}.data", "include")) - scripts = str(Path(target_dir, f"{name}.data", "scripts")) - # egg-info will be generated again to create a manifest (used for package data) - egg_info = dist.reinitialize_command("egg_info", reinit_subcommands=True) - egg_info.egg_base = str(tmp_dir) - egg_info.ignore_egg_info_in_manifest = True - - build = dist.reinitialize_command("build", reinit_subcommands=True) - install = dist.reinitialize_command("install", reinit_subcommands=True) - - build.build_platlib = build.build_purelib = build.build_lib = target - install.install_purelib = install.install_platlib = install.install_lib = target - install.install_scripts = build.build_scripts = scripts - install.install_headers = headers - install.install_data = data - - build.build_temp = str(tmp_dir) - - build_py = dist.get_command_obj("build_py") - build_py.compile = False - - build.ensure_finalized() - install.ensure_finalized() - - -def _can_symlink_files(): - try: - with TemporaryDirectory() as tmp: - path1, path2 = Path(tmp, "file1.txt"), Path(tmp, "file2.txt") - path1.write_text("file1", encoding="utf-8") +def _can_symlink_files() -> bool: + with TemporaryDirectory() as tmp: + path1, path2 = Path(tmp, "file1.txt"), Path(tmp, "file2.txt") + path1.write_text("file1", encoding="utf-8") + with suppress(AttributeError, NotImplementedError, OSError): os.symlink(path1, path2) - return path2.is_symlink() and path2.read_text(encoding="utf-8") == "file1" - except (AttributeError, NotImplementedError, OSError): + if path2.is_symlink() and path2.read_text(encoding="utf-8") == "file1": + return True + + try: + os.link(path1, path2) # Ensure hard links can be created + except Exception as ex: + msg = ( + "File system does not seem to support either symlinks or hard links. " + "Strict editable installs require one of them to be supported." + ) + raise LinksNotSupported(msg) from ex return False @@ -336,6 +502,8 @@ def _simple_layout( False >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"a.a1.a2": "_a2"}, ".") False + >>> _simple_layout(['a', 'a.b'], {"": "src", "a.b": "_ab"}, "/tmp/myproj") + False """ layout = { pkg: find_package_path(pkg, package_dir, project_dir) @@ -601,3 +769,7 @@ class InformationOnly(UserWarning): The only thing that might work is a warning, although it is not the most appropriate tool for the job... """ + + +class LinksNotSupported(errors.FileError): + """File system does not seem to support either symlinks or hard links.""" -- cgit v1.2.1 From 7a9bc7601925b2e5743c8d821d2d1ef82277c22a Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 24 Jun 2022 12:12:45 +0100 Subject: Remove unnecessary complexity --- setuptools/command/editable_wheel.py | 47 +----------------------------------- 1 file changed, 1 insertion(+), 46 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index b02486d0..c1726efb 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -175,47 +175,6 @@ class editable_wheel(Command): if hasattr(cmd, "editable_mode"): cmd.editable_mode = True - def _find_existing_source(self, file: Path) -> Optional[Path]: - """Given a file path relative to ``build_lib`` try to guess - what would be its original source file. - """ - dist = self.distribution - package = str(file.parent).replace(os.sep, ".") - package_path = find_package_path(package, dist.package_dir, self.project_dir) - candidate = Path(package_path, file.name) - return candidate if candidate.exists() else None - - def _collect_reminiscent_outputs( - self, build_lib: _Path - ) -> Tuple[List[str], Dict[str, str]]: - """People have been overwriting setuptools for a long time, and not everyone - might be respecting the new ``get_output_mapping`` APIs, so we have to do our - best to handle this scenario. - """ - files: List[str] = [] - mapping: Dict[str, str] = {} - - for dirpath, _dirnames, filenames in os.walk(build_lib): - for name in filenames: - file = Path(dirpath, name) - source = self._find_existing_source(file.relative_to(build_lib)) - if source: - mapping[str(file)] = str(source) - else: - files.append(str(file)) - - return files, mapping - - def _combine_outputs(self, *outputs: List[str]) -> List[str]: - return sorted({os.path.normpath(f) for f in chain.from_iterable(outputs)}) - - def _combine_output_mapping(self, *mappings: Dict[str, str]) -> Dict[str, str]: - mapping = ( - (os.path.normpath(k), os.path.normpath(v)) - for k, v in chain.from_iterable(m.items() for m in mappings) - ) - return dict(sorted(mapping)) - def _collect_build_outputs(self) -> Tuple[List[str], Dict[str, str]]: files: List[str] = [] mapping: Dict[str, str] = {} @@ -227,12 +186,8 @@ class editable_wheel(Command): files.extend(cmd.get_outputs() or []) if hasattr(cmd, "get_output_mapping"): mapping.update(cmd.get_output_mapping() or {}) - rfiles, rmapping = self._collect_reminiscent_outputs(build.build_lib) - return ( - self._combine_outputs(files, rfiles), - self._combine_output_mapping(mapping, rmapping), - ) + return files, mapping def _run_build_commands( self, dist_name: str, unpacked_wheel: _Path, build_lib: _Path, tmp_dir: _Path -- cgit v1.2.1 From 5b7498ba18599b0ecfea2a6541ec4a64ae8c2bae Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 24 Jun 2022 12:30:30 +0100 Subject: editable_wheel: Move warnings/logging inside the strategy classes --- setuptools/command/editable_wheel.py | 58 +++++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 18 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index c1726efb..6cfdefdd 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -280,8 +280,6 @@ class editable_wheel(Command): output_mapping: Dict[str, str], ): """Populate wheel using the "strict" ``link tree`` strategy.""" - msg = "Strict editable install will be performed using a link tree.\n" - _logger.warning(msg + _STRICT_WARNING) auxiliary_dir = _empty_dir(Path(self.project_dir, "build", build_name)) populate = _LinkTree( self.distribution, @@ -291,31 +289,21 @@ class editable_wheel(Command): outputs, output_mapping, ) - populate(wheel) - - msg = f"""\n - Strict editable installation performed using the auxiliary directory: - {auxiliary_dir} - - Please be careful to not remove this directory, otherwise you might not be able - to import/use your package. - """ - warnings.warn(msg, InformationOnly) + with populate: + populate(wheel) def _populate_static_pth(self, name: str, project_dir: Path, wheel: "WheelFile"): """Populate wheel using the "lax" ``.pth`` file strategy, for ``src-layout``.""" src_dir = self.package_dir[""] - msg = f"Editable install will be performed using .pth file to {src_dir}.\n" - _logger.warning(msg + _LAX_WARNING) populate = _StaticPth(self.distribution, name, [Path(project_dir, src_dir)]) - populate(wheel) + with populate: + populate(wheel) def _populate_finder(self, name: str, wheel: "WheelFile"): """Populate wheel using the "lax" MetaPathFinder strategy.""" - msg = "Editable install will be performed using a meta path finder.\n" - _logger.warning(msg + _LAX_WARNING) populate = _TopLevelFinder(self.distribution, name) - populate(wheel) + with populate: + populate(wheel) class _StaticPth: @@ -329,6 +317,17 @@ class _StaticPth: contents = bytes(f"{entries}\n", "utf-8") wheel.writestr(f"__editable__.{self.name}.pth", contents) + def __enter__(self): + msg = f""" + Editable install will be performed using .pth file to extend `sys.path` with: + {self.path_entries!r} + """ + _logger.warning(msg + _LAX_WARNING) + return self + + def __exit__(self, _exc_type, _exc_value, _traceback): + ... + class _LinkTree(_StaticPth): """ @@ -388,6 +387,21 @@ class _LinkTree(_StaticPth): for relative, src in mappings.items(): self._create_file(relative, src, link=link_type) + def __enter__(self): + msg = "Strict editable install will be performed using a link tree.\n" + _logger.warning(msg + _STRICT_WARNING) + return self + + def __exit__(self, _exc_type, _exc_value, _traceback): + msg = f"""\n + Strict editable installation performed using the auxiliary directory: + {self.auxiliary_dir} + + Please be careful to not remove this directory, otherwise you might not be able + to import/use your package. + """ + warnings.warn(msg, InformationOnly) + class _TopLevelFinder: def __init__(self, dist: Distribution, name: str): @@ -413,6 +427,14 @@ class _TopLevelFinder: content = bytes(f"import {finder}; {finder}.install()", "utf-8") wheel.writestr(f"__editable__.{self.name}.pth", content) + def __enter__(self): + msg = "Editable install will be performed using a meta path finder.\n" + _logger.warning(msg + _LAX_WARNING) + return self + + def __exit__(self, _exc_type, _exc_value, _traceback): + ... + def _can_symlink_files() -> bool: with TemporaryDirectory() as tmp: -- cgit v1.2.1 From 5f231b9fc86fa9ae9a0ed81f441b9113f416c5b1 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 24 Jun 2022 12:39:30 +0100 Subject: editable_wheel: simplify strategy instantiation --- setuptools/command/editable_wheel.py | 68 ++++++++---------------------------- 1 file changed, 15 insertions(+), 53 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index 6cfdefdd..c6c08688 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -212,8 +212,6 @@ class editable_wheel(Command): if wheel_path.exists(): wheel_path.unlink() - # Currently the wheel API receives a directory and dump all its contents - # inside of a wheel. So let's use a temporary directory. unpacked_wheel = TemporaryDirectory(suffix=archive_name) build_lib = TemporaryDirectory(suffix=".build-lib") build_tmp = TemporaryDirectory(suffix=".build-temp") @@ -223,10 +221,9 @@ class editable_wheel(Command): shutil.copytree(self.dist_info_dir, unpacked_dist_info) self._install_namespaces(unpacked, dist_info.name) files, mapping = self._run_build_commands(dist_name, unpacked, lib, tmp) - with WheelFile(wheel_path, "w") as wf: - self._populate_wheel( - wf, dist_info.name, tag, unpacked, lib, tmp, files, mapping - ) + strategy = self._select_strategy(dist_name, tag, lib, files, mapping) + with strategy, WheelFile(wheel_path, "w") as wf: + strategy(wf) wf.write_files(unpacked) return wheel_path @@ -237,14 +234,11 @@ class editable_wheel(Command): _logger.info(f"Installing {category} as non editable") self.run_command(f"install_{category}") - def _populate_wheel( + def _select_strategy( self, - wheel: "WheelFile", name: str, tag: str, - unpacked_dir: Path, build_lib: _Path, - tmp: _Path, outputs: List[str], output_mapping: Dict[str, str], ): @@ -253,57 +247,25 @@ class editable_wheel(Command): project_dir = Path(self.project_dir) if self.strict or os.getenv("SETUPTOOLS_EDITABLE", None) == "strict": - return self._populate_link_tree( - name, build_name, wheel, build_lib, outputs, output_mapping + auxiliary_dir = _empty_dir(Path(self.project_dir, "build", build_name)) + return _LinkTree( + self.distribution, + name, + auxiliary_dir, + build_lib, + outputs, + output_mapping, ) - # Build extensions in-place - self.reinitialize_command("build_ext", inplace=1) - self.run_command("build_ext") - packages = _find_packages(self.distribution) has_simple_layout = _simple_layout(packages, self.package_dir, project_dir) if set(self.package_dir) == {""} and has_simple_layout: # src-layout(ish) is relatively safe for a simple pth file - return self._populate_static_pth(name, project_dir, wheel) + src_dir = self.package_dir[""] + return _StaticPth(self.distribution, name, [Path(project_dir, src_dir)]) # Use a MetaPathFinder to avoid adding accidental top-level packages/modules - self._populate_finder(name, wheel) - - def _populate_link_tree( - self, - name: str, - build_name: str, - wheel: "WheelFile", - build_lib: _Path, - outputs: List[str], - output_mapping: Dict[str, str], - ): - """Populate wheel using the "strict" ``link tree`` strategy.""" - auxiliary_dir = _empty_dir(Path(self.project_dir, "build", build_name)) - populate = _LinkTree( - self.distribution, - name, - auxiliary_dir, - build_lib, - outputs, - output_mapping, - ) - with populate: - populate(wheel) - - def _populate_static_pth(self, name: str, project_dir: Path, wheel: "WheelFile"): - """Populate wheel using the "lax" ``.pth`` file strategy, for ``src-layout``.""" - src_dir = self.package_dir[""] - populate = _StaticPth(self.distribution, name, [Path(project_dir, src_dir)]) - with populate: - populate(wheel) - - def _populate_finder(self, name: str, wheel: "WheelFile"): - """Populate wheel using the "lax" MetaPathFinder strategy.""" - populate = _TopLevelFinder(self.distribution, name) - with populate: - populate(wheel) + return _TopLevelFinder(self.distribution, name) class _StaticPth: -- cgit v1.2.1 From a376bf930cabee548f5f1125a8679cf6fecfde4f Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 24 Jun 2022 12:49:20 +0100 Subject: editable_wheel: Improve strategy interface --- setuptools/command/editable_wheel.py | 59 +++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 27 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index c6c08688..bd580acb 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -40,6 +40,13 @@ from setuptools.dist import Distribution if TYPE_CHECKING: from wheel.wheelfile import WheelFile # noqa +if sys.version_info >= (3, 8): + from typing import Protocol +elif TYPE_CHECKING: + from typing_extensions import Protocol +else: + from abc import ABC as Protocol + _Path = Union[str, Path] _P = TypeVar("_P", bound=_Path) _logger = logging.getLogger(__name__) @@ -221,10 +228,10 @@ class editable_wheel(Command): shutil.copytree(self.dist_info_dir, unpacked_dist_info) self._install_namespaces(unpacked, dist_info.name) files, mapping = self._run_build_commands(dist_name, unpacked, lib, tmp) - strategy = self._select_strategy(dist_name, tag, lib, files, mapping) - with strategy, WheelFile(wheel_path, "w") as wf: - strategy(wf) - wf.write_files(unpacked) + strategy = self._select_strategy(dist_name, tag, lib) + with strategy, WheelFile(wheel_path, "w") as wheel_obj: + strategy(wheel_obj, files, mapping) + wheel_obj.write_files(unpacked) return wheel_path @@ -239,23 +246,14 @@ class editable_wheel(Command): name: str, tag: str, build_lib: _Path, - outputs: List[str], - output_mapping: Dict[str, str], - ): + ) -> "EditableStrategy": """Decides which strategy to use to implement an editable installation.""" build_name = f"__editable__.{name}-{tag}" project_dir = Path(self.project_dir) if self.strict or os.getenv("SETUPTOOLS_EDITABLE", None) == "strict": auxiliary_dir = _empty_dir(Path(self.project_dir, "build", build_name)) - return _LinkTree( - self.distribution, - name, - auxiliary_dir, - build_lib, - outputs, - output_mapping, - ) + return _LinkTree(self.distribution, name, auxiliary_dir, build_lib) packages = _find_packages(self.distribution) has_simple_layout = _simple_layout(packages, self.package_dir, project_dir) @@ -268,13 +266,24 @@ class editable_wheel(Command): return _TopLevelFinder(self.distribution, name) +class EditableStrategy(Protocol): + def __call__(self, wheel: "WheelFile", files: List[str], mapping: Dict[str, str]): + ... + + def __enter__(self): + ... + + def __exit__(self, _exc_type, _exc_value, _traceback): + ... + + class _StaticPth: def __init__(self, dist: Distribution, name: str, path_entries: List[Path]): self.dist = dist self.name = name self.path_entries = path_entries - def __call__(self, wheel: "WheelFile"): + def __call__(self, wheel: "WheelFile", files: List[str], mapping: Dict[str, str]): entries = "\n".join((str(p.resolve()) for p in self.path_entries)) contents = bytes(f"{entries}\n", "utf-8") wheel.writestr(f"__editable__.{self.name}.pth", contents) @@ -306,19 +315,15 @@ class _LinkTree(_StaticPth): name: str, auxiliary_dir: _Path, build_lib: _Path, - outputs: List[str], - output_mapping: Dict[str, str], ): self.auxiliary_dir = Path(auxiliary_dir) self.build_lib = Path(build_lib).resolve() - self.outputs = outputs - self.output_mapping = output_mapping self._file = dist.get_command_obj("build_py").copy_file super().__init__(dist, name, [self.auxiliary_dir]) - def __call__(self, wheel: "WheelFile"): - self._create_links() - super().__call__(wheel) + def __call__(self, wheel: "WheelFile", files: List[str], mapping: Dict[str, str]): + self._create_links(files, mapping) + super().__call__(wheel, files, mapping) def _normalize_output(self, file: str) -> Optional[str]: # Files relative to build_lib will be normalized to None @@ -333,15 +338,15 @@ class _LinkTree(_StaticPth): dest.parent.mkdir(parents=True) self._file(src_file, dest, link=link) - def _create_links(self): + def _create_links(self, outputs, output_mapping): link_type = "sym" if _can_symlink_files() else "hard" mappings = { self._normalize_output(k): v - for k, v in self.output_mapping.items() + for k, v in output_mapping.items() } mappings.pop(None, None) # remove files that are not relative to build_lib - for output in self.outputs: + for output in outputs: relative = self._normalize_output(output) if relative and relative not in mappings: self._create_file(relative, output) @@ -370,7 +375,7 @@ class _TopLevelFinder: self.dist = dist self.name = name - def __call__(self, wheel: "WheelFile"): + def __call__(self, wheel: "WheelFile", files: List[str], mapping: Dict[str, str]): src_root = self.dist.src_root or os.curdir top_level = chain(_find_packages(self.dist), _find_top_level_modules(self.dist)) package_dir = self.dist.package_dir or {} -- cgit v1.2.1 From fdd9ab363a88665b3678c661721c073f19982737 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 24 Jun 2022 15:40:55 +0100 Subject: Add docstrings for command classes --- setuptools/command/build.py | 108 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 107 insertions(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/build.py b/setuptools/command/build.py index 12a43622..bf4f71a7 100644 --- a/setuptools/command/build.py +++ b/setuptools/command/build.py @@ -1,8 +1,17 @@ -from distutils.command.build import build as _build +import sys import warnings +from typing import TYPE_CHECKING, List, Dict +from distutils.command.build import build as _build from setuptools import SetuptoolsDeprecationWarning +if sys.version_info >= (3, 8): + from typing import Protocol +elif TYPE_CHECKING: + from typing_extensions import Protocol +else: + from abc import ABC as Protocol + _ORIGINAL_SUBCOMMANDS = {"build_py", "build_clib", "build_ext", "build_scripts"} @@ -22,3 +31,100 @@ class build(_build): warnings.warn(msg, SetuptoolsDeprecationWarning) self.sub_commands = _build.sub_commands super().run() + + +class SubCommand(Protocol): + """In order to support editable installations (see :pep:`660`) all + build subcommands **SHOULD** implement this protocol. They also **MUST** inherit + from ``setuptools.Command``. + + When creating an :pep:`editable wheel <660>`, ``setuptools`` will try to evaluate + custom ``build`` subcommands using the following procedure: + + 1. ``setuptools`` will set the ``editable_mode`` flag will be set to ``True`` + 2. ``setuptools`` will execute the ``run()`` command. + + .. important:: + Subcommands **SHOULD** take advantage of ``editable_mode=True`` to adequate + its behaviour or perform optimisations. + + For example, if a subcommand don't need to generate any extra file and + everything it does is to copy a source file into the build directory, + ``run()`` **SHOULD** simply "early return". + + Similarly, if the subcommand creates files that would be placed alongside + Python files in the final distribution, during an editable install + the command **SHOULD** generate these files "in place" (i.e. write them to + the original source directory, instead of using the build directory). + Note that ``get_output_mapping()`` should reflect that and include mappings + for "in place" builds accordingly. + + 3. ``setuptools`` use any knowledge it can derive from the return values of + ``get_outputs()`` and ``get_output_mapping()`` to create an editable wheel. + When relevant ``setuptools`` **MAY** attempt to use file links based on the value + of ``get_output_mapping()``. Alternatively, ``setuptools`` **MAY** attempt to use + :doc:`import hooks ` to redirect any attempt to import + to the directory with the original source code and other files built in place. + """ + + editable_mode: bool = False + """Boolean flag that will be set to ``True`` when setuptools is used for an + editable installation (see :pep:`660`). + Implementations **SHOULD** explicitly set the default value of this attribute to + ``False``. + When subcommands run, they can use this flag to perform optimizations or change + their behaviour accordingly. + """ + + build_lib: str + """String representing the directory where the build artifacts should be stored, + e.g. ``build/lib``. + For example, if a distribution wants to provide a Python module named ``pkg.mod``, + then a corresponding file should be written to ``{build_lib}/package/module.py``. + A way of thinking about this is that the files saved under ``build_lib`` + would be eventually copied to one of the directories in :obj:`site.PREFIXES` + upon installation. + + A command that produces platform-independent files (e.g. compiling text templates + into Python functions), **CAN** initialize ``build_lib`` by copying its value from + the ``build_py`` command. On the other hand, a command that produces + platform-specific files **CAN** initialize ``build_lib`` by copying its value from + the ``build_ext`` command. In general this is done inside the ``finalize_options`` + method with the help of the ``set_undefined_options`` command:: + + def finalize_options(self): + self.set_undefined_options("build_py", ("build_lib", "build_lib")) + ... + """ + + def initialize_options(self): + """(Required by the original :class:`setuptools.Command` interface)""" + + def finalize_options(self): + """(Required by the original :class:`setuptools.Command` interface)""" + + def run(self): + """(Required by the original :class:`setuptools.Command` interface)""" + + def get_outputs(self) -> List[str]: + """ + Return a list of files intended for distribution as they would have been + produced by the build. + These files should be strings in the form of + ``"{build_lib}/destination/file/path"``. + + .. note:: + The return value of ``get_output()`` should include all files used as keys + in ``get_output_mapping()`` plus files that are generated during the build + and don't correspond to any source file already present in the project. + """ + + def get_output_mapping(self) -> Dict[str, str]: + """ + Return a mapping between destination files as they would be produced by the + build (dict keys) into the respective existing (source) files (dict values). + Existing (source) files should be represented as strings relative to the project + root directory. + Destination files should be strings in the form of + ``"{build_lib}/destination/file/path"``. + """ -- cgit v1.2.1 From 965458d1d271553d25f431ffebdb68bd12938f9a Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 24 Jun 2022 15:57:06 +0100 Subject: Revert addition of use_links to build_lib --- setuptools/command/build_ext.py | 2 +- setuptools/command/build_py.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/build_ext.py b/setuptools/command/build_ext.py index 31ef47bf..7ad5a87a 100644 --- a/setuptools/command/build_ext.py +++ b/setuptools/command/build_ext.py @@ -104,7 +104,7 @@ class build_ext(_build_ext): # Always copy, even if source is older than destination, to ensure # that the right extensions for the current Python/platform are # used. - build_py.copy_file(regular_file, inplace_file) + self.copy_file(regular_file, inplace_file, level=self.verbose) if ext._needs_stub: inplace_stub = self._get_equivalent_stub(ext, inplace_file) diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index a2a6fe2c..923a3232 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -40,17 +40,15 @@ class build_py(orig.build_py): if 'data_files' in self.__dict__: del self.__dict__['data_files'] self.__updated_files = [] - self.use_links = None def copy_file(self, infile, outfile, preserve_mode=1, preserve_times=1, link=None, level=1): # Overwrite base class to allow using links - link = getattr(self, "use_links", None) if link is None else link if link: infile = str(Path(infile).resolve()) outfile = str(Path(outfile).resolve()) - return super().copy_file(infile, outfile, preserve_mode, - preserve_times, link, level) + return super().copy_file(infile, outfile, preserve_mode, preserve_times, + link, level) def run(self): """Build modules, packages, and copy data files to build directory""" -- cgit v1.2.1 From c675d781f89f2057c8e5e0e53896adf468cfbac1 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 24 Jun 2022 19:53:25 +0100 Subject: sdist: Add files from build subcommands (get_source_files) --- setuptools/command/build.py | 16 ++++++++++++++++ setuptools/command/sdist.py | 14 ++++++++++++++ 2 files changed, 30 insertions(+) (limited to 'setuptools/command') diff --git a/setuptools/command/build.py b/setuptools/command/build.py index bf4f71a7..c35dc3fc 100644 --- a/setuptools/command/build.py +++ b/setuptools/command/build.py @@ -65,6 +65,11 @@ class SubCommand(Protocol): of ``get_output_mapping()``. Alternatively, ``setuptools`` **MAY** attempt to use :doc:`import hooks ` to redirect any attempt to import to the directory with the original source code and other files built in place. + + Please note that custom sub-commands **SHOULD NOT** rely on ``run()`` being + executed (or not) to provide correct return values for ``get_outputs()``, + ``get_output_mapping()`` or ``get_source_files()``. The ``get_*`` methods should + work independently of ``run()``. """ editable_mode: bool = False @@ -106,6 +111,17 @@ class SubCommand(Protocol): def run(self): """(Required by the original :class:`setuptools.Command` interface)""" + def get_source_files(self) -> List[str]: + """ + Return a list of all files that are used by the command to create the expected + outputs. + For example, if your build command transpiles Java files into Python, you should + list here all the Java files. + The primary purpose of this function is to help populating the ``sdist`` + with all the files necessary to build the distribution. + All files should be strings relative to the project root directory. + """ + def get_outputs(self) -> List[str]: """ Return a list of files intended for distribution as they would have been diff --git a/setuptools/command/sdist.py b/setuptools/command/sdist.py index 0ffeacf3..4a8cde7e 100644 --- a/setuptools/command/sdist.py +++ b/setuptools/command/sdist.py @@ -4,10 +4,12 @@ import os import sys import io import contextlib +from itertools import chain from .py36compat import sdist_add_defaults from .._importlib import metadata +from .build import _ORIGINAL_SUBCOMMANDS _default_revctrl = list @@ -100,6 +102,10 @@ class sdist(sdist_add_defaults, orig.sdist): if orig_val is not NoValue: setattr(os, 'link', orig_val) + def add_defaults(self): + super().add_defaults() + self._add_defaults_build_sub_commands() + def _add_defaults_optional(self): super()._add_defaults_optional() if os.path.isfile('pyproject.toml'): @@ -112,6 +118,14 @@ class sdist(sdist_add_defaults, orig.sdist): self.filelist.extend(build_py.get_source_files()) self._add_data_files(self._safe_data_files(build_py)) + def _add_defaults_build_sub_commands(self): + build = self.get_finalized_command("build") + missing_cmds = set(build.get_sub_commands()) - _ORIGINAL_SUBCOMMANDS + # ^-- the original built-in sub-commands are already handled by default. + cmds = (self.get_finalized_command(c) for c in missing_cmds) + files = (c.get_source_files() for c in cmds if hasattr(c, "get_source_files")) + self.filelist.extend(chain.from_iterable(files)) + def _safe_data_files(self, build_py): """ Since the ``sdist`` class is also used to compute the MANIFEST -- cgit v1.2.1 From 0a82048ee585efbd85656047843e29792cf84a3a Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 24 Jun 2022 23:23:59 +0100 Subject: Test symlinks in the target directory Improve chances that symlinks are tested directly on the relevant file system (some machines might have multiple disks with different link support). --- setuptools/command/editable_wheel.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index bd580acb..8a0fb8bc 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -339,7 +339,8 @@ class _LinkTree(_StaticPth): self._file(src_file, dest, link=link) def _create_links(self, outputs, output_mapping): - link_type = "sym" if _can_symlink_files() else "hard" + self.auxiliary_dir.mkdir(parents=True, exist_ok=True) + link_type = "sym" if _can_symlink_files(self.auxiliary_dir) else "hard" mappings = { self._normalize_output(k): v for k, v in output_mapping.items() @@ -403,8 +404,8 @@ class _TopLevelFinder: ... -def _can_symlink_files() -> bool: - with TemporaryDirectory() as tmp: +def _can_symlink_files(base_dir: Path) -> bool: + with TemporaryDirectory(dir=str(base_dir.resolve())) as tmp: path1, path2 = Path(tmp, "file1.txt"), Path(tmp, "file2.txt") path1.write_text("file1", encoding="utf-8") with suppress(AttributeError, NotImplementedError, OSError): -- cgit v1.2.1 From a3e8d4da41e651884848f5bb43148c12faa12f2f Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sat, 25 Jun 2022 20:05:38 +0100 Subject: Add help message to editable install exception --- setuptools/command/editable_wheel.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index 8a0fb8bc..7a396bbe 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -15,8 +15,10 @@ import os import re import shutil import sys +import traceback import warnings from contextlib import suppress +from inspect import cleandoc from itertools import chain from pathlib import Path from tempfile import TemporaryDirectory @@ -88,14 +90,29 @@ class editable_wheel(Command): self.dist_dir = Path(self.dist_dir or os.path.join(self.project_dir, "dist")) def run(self): - self.dist_dir.mkdir(exist_ok=True) - self._ensure_dist_info() + try: + self.dist_dir.mkdir(exist_ok=True) + self._ensure_dist_info() + + # Add missing dist_info files + bdist_wheel = self.reinitialize_command("bdist_wheel") + bdist_wheel.write_wheelfile(self.dist_info_dir) + + self._create_wheel_file(bdist_wheel) + except Exception as ex: + traceback.print_exc() + msg = """ + Support for editable installs via PEP 660 was recently introduced + in `setuptools`. If you are seeing this error, please report to: + + https://github.com/pypa/setuptools/issues - # Add missing dist_info files - bdist_wheel = self.reinitialize_command("bdist_wheel") - bdist_wheel.write_wheelfile(self.dist_info_dir) + Meanwhile you can try the legacy behavior by setting an + environment variable and trying to install again: - self._create_wheel_file(bdist_wheel) + SETUPTOOLS_ENABLE_FEATURES="legacy-editable" + """ + raise errors.InternalError(cleandoc(msg)) from ex def _ensure_dist_info(self): if self.dist_info_dir is None: -- cgit v1.2.1 From 17311b1f8eb8f14afd23d84bb54e7a62c666e7fe Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sat, 25 Jun 2022 20:41:23 +0100 Subject: Add interfaces to docs --- setuptools/command/build.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/build.py b/setuptools/command/build.py index c35dc3fc..1396afd5 100644 --- a/setuptools/command/build.py +++ b/setuptools/command/build.py @@ -44,20 +44,20 @@ class SubCommand(Protocol): 1. ``setuptools`` will set the ``editable_mode`` flag will be set to ``True`` 2. ``setuptools`` will execute the ``run()`` command. - .. important:: - Subcommands **SHOULD** take advantage of ``editable_mode=True`` to adequate - its behaviour or perform optimisations. - - For example, if a subcommand don't need to generate any extra file and - everything it does is to copy a source file into the build directory, - ``run()`` **SHOULD** simply "early return". - - Similarly, if the subcommand creates files that would be placed alongside - Python files in the final distribution, during an editable install - the command **SHOULD** generate these files "in place" (i.e. write them to - the original source directory, instead of using the build directory). - Note that ``get_output_mapping()`` should reflect that and include mappings - for "in place" builds accordingly. + .. important:: + Subcommands **SHOULD** take advantage of ``editable_mode=True`` to adequate + its behaviour or perform optimisations. + + For example, if a subcommand don't need to generate any extra file and + everything it does is to copy a source file into the build directory, + ``run()`` **SHOULD** simply "early return". + + Similarly, if the subcommand creates files that would be placed alongside + Python files in the final distribution, during an editable install + the command **SHOULD** generate these files "in place" (i.e. write them to + the original source directory, instead of using the build directory). + Note that ``get_output_mapping()`` should reflect that and include mappings + for "in place" builds accordingly. 3. ``setuptools`` use any knowledge it can derive from the return values of ``get_outputs()`` and ``get_output_mapping()`` to create an editable wheel. -- cgit v1.2.1 From 8bf3658177611de085373ace052605be321b5b0c Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Mon, 27 Jun 2022 22:49:35 +0100 Subject: Fix header location --- setuptools/command/editable_wheel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index 7a396bbe..5e205a4d 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -159,7 +159,7 @@ class editable_wheel(Command): wheel = str(unpacked_wheel) build_lib = str(build_lib) data = str(Path(unpacked_wheel, f"{name}.data", "data")) - headers = str(Path(unpacked_wheel, f"{name}.data", "include")) + headers = str(Path(unpacked_wheel, f"{name}.data", "headers")) scripts = str(Path(unpacked_wheel, f"{name}.data", "scripts")) # egg-info may be generated again to create a manifest (used for package data) -- cgit v1.2.1 From 36981f1419b4bcf88b074a8296f235f602e8e8c0 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Tue, 28 Jun 2022 11:16:58 +0200 Subject: setuptools: drop support for installing an entrypoint dependencies It does not work (broken since `v60.8.0`: the code in `_install_dependencies` forgets to add the newly installed egg to `sys.path`), and is unnecessary as it's the job of the code handling `setup_requires` to ensure all necessary build requirements are installed. --- setuptools/command/egg_info.py | 1 - 1 file changed, 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py index c37ab81f..42a0178f 100644 --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py @@ -296,7 +296,6 @@ class egg_info(InfoCommon, Command): self.mkpath(self.egg_info) os.utime(self.egg_info, None) for ep in metadata.entry_points(group='egg_info.writers'): - self.distribution._install_dependencies(ep) writer = ep.load() writer(self, ep.name, os.path.join(self.egg_info, ep.name)) -- cgit v1.2.1 From b422ddc51bfb6a79cad5c8bfa4ab2619400839ed Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sun, 3 Jul 2022 17:13:04 +0100 Subject: Fix grammar problems with paragraph in docs --- setuptools/command/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/build.py b/setuptools/command/build.py index 1396afd5..283999da 100644 --- a/setuptools/command/build.py +++ b/setuptools/command/build.py @@ -41,7 +41,7 @@ class SubCommand(Protocol): When creating an :pep:`editable wheel <660>`, ``setuptools`` will try to evaluate custom ``build`` subcommands using the following procedure: - 1. ``setuptools`` will set the ``editable_mode`` flag will be set to ``True`` + 1. ``setuptools`` will set the ``editable_mode`` attribute to ``True`` 2. ``setuptools`` will execute the ``run()`` command. .. important:: -- cgit v1.2.1 From 24286270cfd87e386d190be7a5af81ae67a69d37 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 27 Jul 2022 05:11:23 -0400 Subject: Revert "Limit the scope of deprecation of the upload_docs command." This reverts commit 995d309317c6895a123c03df28bc8f51f6ead5f5. Ref #2971. --- setuptools/command/upload_docs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/upload_docs.py b/setuptools/command/upload_docs.py index a5480005..bcbaf233 100644 --- a/setuptools/command/upload_docs.py +++ b/setuptools/command/upload_docs.py @@ -59,6 +59,7 @@ class upload_docs(upload): self.target_dir = None def finalize_options(self): + log.warn("Upload_docs command is deprecated. Use RTD instead.") upload.finalize_options(self) if self.upload_dir is None: if self.has_sphinx(): @@ -70,8 +71,6 @@ class upload_docs(upload): else: self.ensure_dirname('upload_dir') self.target_dir = self.upload_dir - if 'pypi.python.org' in self.repository: - log.warn("Upload_docs command is deprecated for PyPi. Use RTD instead.") self.announce('Using upload directory %s' % self.target_dir) def create_zipfile(self, filename): -- cgit v1.2.1 From b8d50cf4f7431ed617957e7d6e432a1729656524 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 31 Jul 2022 14:59:16 -0400 Subject: Update 'bdist' format addition to assume a single 'format_commands' as a dictionary, but fall back to the dual dict/list model for compatibility with stdlib. --- setuptools/command/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/__init__.py b/setuptools/command/__init__.py index b966dcea..5acd7687 100644 --- a/setuptools/command/__init__.py +++ b/setuptools/command/__init__.py @@ -2,7 +2,11 @@ from distutils.command.bdist import bdist import sys if 'egg' not in bdist.format_commands: - bdist.format_command['egg'] = ('bdist_egg', "Python .egg file") - bdist.format_commands.append('egg') + try: + bdist.format_commands['egg'] = ('bdist_egg', "Python .egg file") + except TypeError: + # For backward compatibility with older distutils (stdlib) + bdist.format_command['egg'] = ('bdist_egg', "Python .egg file") + bdist.format_commands.append('egg') del bdist, sys -- cgit v1.2.1 From d8f4d5de6263020e250b6def2fea481d3ed9c7d0 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 2 Aug 2022 20:58:36 -0400 Subject: spell out read the docs Co-authored-by: Pradyun Gedam --- setuptools/command/upload_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/upload_docs.py b/setuptools/command/upload_docs.py index bcbaf233..7ee0221b 100644 --- a/setuptools/command/upload_docs.py +++ b/setuptools/command/upload_docs.py @@ -59,7 +59,7 @@ class upload_docs(upload): self.target_dir = None def finalize_options(self): - log.warn("Upload_docs command is deprecated. Use RTD instead.") + log.warn("Upload_docs command is deprecated. Use Read the Docs (https://readthedocs.org) instead.") upload.finalize_options(self) if self.upload_dir is None: if self.has_sphinx(): -- cgit v1.2.1 From 871e04e592f7408787ceae5787a6e196b2fc1783 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 2 Aug 2022 21:20:27 -0400 Subject: =?UTF-8?q?=F0=9F=91=B9=20Feed=20the=20hobgoblins=20(delint).?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- setuptools/command/upload_docs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/upload_docs.py b/setuptools/command/upload_docs.py index 7ee0221b..3263f07f 100644 --- a/setuptools/command/upload_docs.py +++ b/setuptools/command/upload_docs.py @@ -59,7 +59,9 @@ class upload_docs(upload): self.target_dir = None def finalize_options(self): - log.warn("Upload_docs command is deprecated. Use Read the Docs (https://readthedocs.org) instead.") + log.warn( + "Upload_docs command is deprecated. Use Read the Docs " + "(https://readthedocs.org) instead.") upload.finalize_options(self) if self.upload_dir is None: if self.has_sphinx(): -- cgit v1.2.1 From da5af04f0a8c2ab79454e0670039e19b54f5eddc Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Tue, 2 Aug 2022 18:11:45 +0100 Subject: Add compat mode to editable install This is only a transitional measure that can be temporarily used by users to help them to adapt to the PEP 660 implementation. In this commit the argument for the `editable_wheel` command is changed from the flag form `--strict` to `--mode=strict`. --- setuptools/command/editable_wheel.py | 48 +++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 11 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index 5e205a4d..2fb4b07d 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -18,6 +18,7 @@ import sys import traceback import warnings from contextlib import suppress +from enum import Enum from inspect import cleandoc from itertools import chain from pathlib import Path @@ -54,34 +55,57 @@ _P = TypeVar("_P", bound=_Path) _logger = logging.getLogger(__name__) +class _EditableMode(Enum): + """ + Possible editable installation modes: + a) `lenient` (new files automatically added to the package - DEFAULT) + b) `strict` (requires a new installation when files are added/removed) + c) `compat` (attempts to replicate `python setup.py develop` - DEPRECATED) + """ + STRICT = "strict" + LENIENT = "lenient" + COMPAT = "compat" # TODO: Remove `compat` after Dec/2022. + + @classmethod + def convert(cls, mode: Optional[str]) -> "_EditableMode": + if not mode: + return _EditableMode.LENIENT # default + + _mode = mode.upper() + if _mode not in _EditableMode.__members__: + raise errors.OptionError(f"Invalid editable mode: {mode!r}. Try: 'strict'.") + + return _EditableMode[_mode] + + _STRICT_WARNING = """ New or renamed files may not be automatically picked up without a new installation. """ -_LAX_WARNING = """ +_LENIENT_WARNING = """ Options like `package-data`, `include/exclude-package-data` or `packages.find.exclude/include` may have no effect. """ class editable_wheel(Command): - """Build 'editable' wheel for development""" + """Build 'editable' wheel for development. + (This command is reserved for internal use of setuptools). + """ description = "create a PEP 660 'editable' wheel" user_options = [ ("dist-dir=", "d", "directory to put final built distributions in"), ("dist-info-dir=", "I", "path to a pre-build .dist-info directory"), - ("strict", None, "perform an strict installation"), + ("mode=", None, cleandoc(_EditableMode.__doc__ or "")), ] - boolean_options = ["strict"] - def initialize_options(self): self.dist_dir = None self.dist_info_dir = None self.project_dir = None - self.strict = False + self.mode = None def finalize_options(self): dist = self.distribution @@ -267,16 +291,18 @@ class editable_wheel(Command): """Decides which strategy to use to implement an editable installation.""" build_name = f"__editable__.{name}-{tag}" project_dir = Path(self.project_dir) + mode = _EditableMode.convert(self.mode) - if self.strict or os.getenv("SETUPTOOLS_EDITABLE", None) == "strict": + if mode is _EditableMode.STRICT: auxiliary_dir = _empty_dir(Path(self.project_dir, "build", build_name)) return _LinkTree(self.distribution, name, auxiliary_dir, build_lib) packages = _find_packages(self.distribution) has_simple_layout = _simple_layout(packages, self.package_dir, project_dir) - if set(self.package_dir) == {""} and has_simple_layout: + is_compat_mode = mode is _EditableMode.COMPAT + if set(self.package_dir) == {""} and has_simple_layout or is_compat_mode: # src-layout(ish) is relatively safe for a simple pth file - src_dir = self.package_dir[""] + src_dir = self.package_dir.get("", ".") return _StaticPth(self.distribution, name, [Path(project_dir, src_dir)]) # Use a MetaPathFinder to avoid adding accidental top-level packages/modules @@ -310,7 +336,7 @@ class _StaticPth: Editable install will be performed using .pth file to extend `sys.path` with: {self.path_entries!r} """ - _logger.warning(msg + _LAX_WARNING) + _logger.warning(msg + _LENIENT_WARNING) return self def __exit__(self, _exc_type, _exc_value, _traceback): @@ -414,7 +440,7 @@ class _TopLevelFinder: def __enter__(self): msg = "Editable install will be performed using a meta path finder.\n" - _logger.warning(msg + _LAX_WARNING) + _logger.warning(msg + _LENIENT_WARNING) return self def __exit__(self, _exc_type, _exc_value, _traceback): -- cgit v1.2.1 From d57e68b7208cb6e21b5ddd877e83477488a52121 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Tue, 2 Aug 2022 18:28:42 +0100 Subject: Add deprecation warning for compat editable mode --- setuptools/command/editable_wheel.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index 2fb4b07d..4110897d 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -33,10 +33,10 @@ from typing import ( Optional, Tuple, TypeVar, - Union + Union, ) -from setuptools import Command, errors, namespaces +from setuptools import Command, SetuptoolsDeprecationWarning, errors, namespaces from setuptools.discovery import find_package_path from setuptools.dist import Distribution @@ -62,6 +62,7 @@ class _EditableMode(Enum): b) `strict` (requires a new installation when files are added/removed) c) `compat` (attempts to replicate `python setup.py develop` - DEPRECATED) """ + STRICT = "strict" LENIENT = "lenient" COMPAT = "compat" # TODO: Remove `compat` after Dec/2022. @@ -75,6 +76,18 @@ class _EditableMode(Enum): if _mode not in _EditableMode.__members__: raise errors.OptionError(f"Invalid editable mode: {mode!r}. Try: 'strict'.") + if _mode == "COMPAT": + msg = """ + The 'compat' editable mode is transitional and will be removed + in future versions of `setuptools`. + Please adapt your code accordingly to use either the 'strict' or the + 'lenient' modes. + + For more information, please check: + https://setuptools.pypa.io/en/latest/userguide/development_mode.html + """ + warnings.warn(msg, SetuptoolsDeprecationWarning) + return _EditableMode[_mode] -- cgit v1.2.1 From eb363c321b1283802b75c641d1f6dcf9fd863032 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Tue, 2 Aug 2022 19:24:14 +0100 Subject: Use better wording in description of 'compat' editable mode --- setuptools/command/editable_wheel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index 4110897d..f6749ed1 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -60,7 +60,7 @@ class _EditableMode(Enum): Possible editable installation modes: a) `lenient` (new files automatically added to the package - DEFAULT) b) `strict` (requires a new installation when files are added/removed) - c) `compat` (attempts to replicate `python setup.py develop` - DEPRECATED) + c) `compat` (attempts to emulate `python setup.py develop` - DEPRECATED) """ STRICT = "strict" -- cgit v1.2.1 From 4f4c6ac09b11039d4af85ce5c08d1f3e3c0f29b0 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Tue, 2 Aug 2022 19:26:49 +0100 Subject: Improve editable_wheel command help --- setuptools/command/editable_wheel.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index f6749ed1..a44d24be 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -58,9 +58,9 @@ _logger = logging.getLogger(__name__) class _EditableMode(Enum): """ Possible editable installation modes: - a) `lenient` (new files automatically added to the package - DEFAULT) - b) `strict` (requires a new installation when files are added/removed) - c) `compat` (attempts to emulate `python setup.py develop` - DEPRECATED) + `lenient` (new files automatically added to the package - DEFAULT); + `strict` (requires a new installation when files are added/removed); or + `compat` (attempts to emulate `python setup.py develop` - DEPRECATED). """ STRICT = "strict" -- cgit v1.2.1 From 35dbc918752391dbd213ab082b19a9c6c967c96a Mon Sep 17 00:00:00 2001 From: leonixyz Date: Thu, 11 Aug 2022 15:58:56 +0100 Subject: Fix typo in editable_wheel.py --- setuptools/command/editable_wheel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index a44d24be..eb86f2b9 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -168,7 +168,7 @@ class editable_wheel(Command): if not dist.namespace_packages: return - src_root = Path(self.project_dir, self.pakcage_dir.get("", ".")).resolve() + src_root = Path(self.project_dir, self.package_dir.get("", ".")).resolve() installer = _NamespaceInstaller(dist, installation_dir, pth_prefix, src_root) installer.install_namespaces() -- cgit v1.2.1 From ae7662f1019b5bfdfba43a70a774eca37597731f Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Thu, 11 Aug 2022 18:09:52 +0100 Subject: editable_install: Fix missing dry-run for legacy namespaces --- setuptools/command/editable_wheel.py | 1 + 1 file changed, 1 insertion(+) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index eb86f2b9..eb79608b 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -672,6 +672,7 @@ class _NamespaceInstaller(namespaces.Installer): self.installation_dir = installation_dir self.editable_name = editable_name self.outputs = [] + self.dry_run = False def _get_target(self): """Installation target.""" -- cgit v1.2.1 From 216bf5e6cd7a5ce19ccf04753d03dc6348003163 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Thu, 11 Aug 2022 18:52:28 +0100 Subject: Fix problem with editable install and single module packages --- setuptools/command/editable_wheel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index a44d24be..bd8ac82c 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -439,7 +439,7 @@ class _TopLevelFinder: roots = _find_package_roots(top_level, package_dir, src_root) namespaces_: Dict[str, List[str]] = dict(chain( - _find_namespaces(self.dist.packages, roots), + _find_namespaces(self.dist.packages or [], roots), ((ns, []) for ns in _find_virtual_namespaces(roots)), )) -- cgit v1.2.1 From c056758b664fd3a9fb042a254d1c1462282ce127 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Thu, 11 Aug 2022 20:37:37 +0100 Subject: Filter external egg_info files --- setuptools/command/build_py.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index 923a3232..796d7bdd 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -11,7 +11,7 @@ import itertools import stat import warnings from pathlib import Path -from typing import Dict, Iterator, List, Optional, Tuple +from typing import Dict, Iterable, Iterator, List, Optional, Tuple from setuptools._deprecation_warning import SetuptoolsDeprecationWarning from setuptools.extern.more_itertools import unique_everseen @@ -175,15 +175,17 @@ class build_py(orig.build_py): getattr(self, 'existing_egg_info_dir', None) and Path(self.existing_egg_info_dir, "SOURCES.txt").exists() ): - manifest = Path(self.existing_egg_info_dir, "SOURCES.txt") + egg_info_dir = self.existing_egg_info_dir + manifest = Path(egg_info_dir, "SOURCES.txt") files = manifest.read_text(encoding="utf-8").splitlines() else: self.run_command('egg_info') ei_cmd = self.get_finalized_command('egg_info') + egg_info_dir = ei_cmd.egg_info files = ei_cmd.filelist.files check = _IncludePackageDataAbuse() - for path in files: + for path in _filter_absolute_egg_info(files, egg_info_dir): d, f = os.path.split(assert_relative(path)) prev = None oldf = f @@ -346,3 +348,15 @@ class _IncludePackageDataAbuse: msg = textwrap.dedent(self.MESSAGE).format(importable=importable) warnings.warn(msg, SetuptoolsDeprecationWarning, stacklevel=2) self._already_warned.add(importable) + + +def _filter_absolute_egg_info(files: Iterable[str], egg_info: str) -> Iterator[str]: + """ + ``build_meta`` may try to create egg_info outside of the project directory, + and this can be problematic for certain plugins (reported in issue #3500). + This function should filter this case of invalid files out. + """ + egg_info_name = Path(egg_info).name + for file in files: + if not (egg_info_name in file and os.path.isabs(file)): + yield file -- cgit v1.2.1 From e2fb005beed79dba58ec0ecfa9bcdadf03d6666a Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Thu, 11 Aug 2022 21:52:18 +0100 Subject: Avoid mentioning project names in CHANGELOG ... since they may change implementantion --- setuptools/command/build_py.py | 1 - 1 file changed, 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index 796d7bdd..8b1a3320 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -52,7 +52,6 @@ class build_py(orig.build_py): def run(self): """Build modules, packages, and copy data files to build directory""" - # if self.editable_mode or not (self.py_modules and self.packages): if not (self.py_modules or self.packages) or self.editable_mode: return -- cgit v1.2.1 From 048633afd7d6e4f852766d06c975ad602a035193 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 12 Aug 2022 03:59:23 +0100 Subject: Safeguard editable installs against build_py errors --- setuptools/command/build.py | 4 ++-- setuptools/command/editable_wheel.py | 45 +++++++++++++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 3 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/build.py b/setuptools/command/build.py index 283999da..c0676d8e 100644 --- a/setuptools/command/build.py +++ b/setuptools/command/build.py @@ -20,7 +20,7 @@ class build(_build): # copy to avoid sharing the object with parent class sub_commands = _build.sub_commands[:] - def run(self): + def get_sub_commands(self): subcommands = {cmd[0] for cmd in _build.sub_commands} if subcommands - _ORIGINAL_SUBCOMMANDS: msg = """ @@ -30,7 +30,7 @@ class build(_build): """ warnings.warn(msg, SetuptoolsDeprecationWarning) self.sub_commands = _build.sub_commands - super().run() + return super().get_sub_commands() class SubCommand(Protocol): diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index 1bb7ddfb..2c98983f 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -37,6 +37,7 @@ from typing import ( ) from setuptools import Command, SetuptoolsDeprecationWarning, errors, namespaces +from setuptools.command.build_py import build_py as build_py_cls from setuptools.discovery import find_package_path from setuptools.dist import Distribution @@ -254,13 +255,55 @@ class editable_wheel(Command): self, dist_name: str, unpacked_wheel: _Path, build_lib: _Path, tmp_dir: _Path ) -> Tuple[List[str], Dict[str, str]]: self._configure_build(dist_name, unpacked_wheel, build_lib, tmp_dir) - self.run_command("build") + self._run_build_subcommands() files, mapping = self._collect_build_outputs() self._run_install("headers") self._run_install("scripts") self._run_install("data") return files, mapping + def _run_build_subcommands(self): + """ + Issue #3501 indicates that some plugins/customizations might rely on: + + 1. ``build_py`` not running + 2. ``build_py`` always copying files to ``build_lib`` + + However both these assumptions may be false in editable_wheel. + This method implements a temporary workaround to support the ecosystem + while the implementations catch up. + """ + # TODO: Once plugins/customisations had the chance to catch up, replace + # `self._run_build_subcommands()` with `self.run_command("build")`. + # Also remove _safely_run, TestCustomBuildPy. Suggested date: Aug/2023. + build: Command = self.get_finalized_command("build") + for name in build.get_sub_commands(): + cmd = self.distribution.get_command_obj(name) + if name == "build_py" and type(cmd) != build_py_cls: + self._safely_run(name) + else: + self.run_command(name) + + def _safely_run(self, cmd_name: str): + try: + return self.run_command(cmd_name) + except Exception: + msg = f"""{traceback.format_exc()}\n + If you are seeing this warning it is very likely that a setuptools + plugin or customization overrides the `build_py` command, without + tacking into consideration how editable installs run build steps + starting from v64.0.0. + + Plugin authors and developers relying on custom build steps are encouraged + to update their `build_py` implementation considering the information in + https://setuptools.pypa.io/en/latest/userguide/extension.html + about editable installs. + + For the time being `setuptools` will silence this error and ignore + the faulty command, but this behaviour will change in future versions.\n + """ + warnings.warn(msg, SetuptoolsDeprecationWarning, stacklevel=2) + def _create_wheel_file(self, bdist_wheel): from wheel.wheelfile import WheelFile -- cgit v1.2.1 From 66994535aceb1d4865957ab7ec70762d15716c25 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 12 Aug 2022 04:22:31 +0100 Subject: Parametrise command name --- setuptools/command/editable_wheel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index 2c98983f..8a53de65 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -290,12 +290,12 @@ class editable_wheel(Command): except Exception: msg = f"""{traceback.format_exc()}\n If you are seeing this warning it is very likely that a setuptools - plugin or customization overrides the `build_py` command, without + plugin or customization overrides the `{cmd_name}` command, without tacking into consideration how editable installs run build steps starting from v64.0.0. Plugin authors and developers relying on custom build steps are encouraged - to update their `build_py` implementation considering the information in + to update their `{cmd_name}` implementation considering the information in https://setuptools.pypa.io/en/latest/userguide/extension.html about editable installs. -- cgit v1.2.1 From 9e2295af2ef4de51d1112d8215e387ae691dc46c Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 12 Aug 2022 12:32:49 +0100 Subject: Emulate accidental namespaces from regular installation --- setuptools/command/editable_wheel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index 1bb7ddfb..c4b7ff9e 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -594,7 +594,7 @@ def _find_virtual_namespaces(pkg_roots: Dict[str, str]) -> Iterator[str]: for i in range(len(parts) - 1, 0, -1): partial_name = ".".join(parts[:i]) path = Path(find_package_path(partial_name, pkg_roots, "")) - if not path.exists(): + if not path.exists() or partial_name not in pkg_roots: yield partial_name -- cgit v1.2.1 From 53353125e74448389704f0b3fb96367f8bd17ccd Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 12 Aug 2022 12:59:23 +0100 Subject: Add comments about accidental virtual namespaces --- setuptools/command/editable_wheel.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index c4b7ff9e..ccd0e60e 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -585,7 +585,14 @@ def _absolute_root(path: _Path) -> str: def _find_virtual_namespaces(pkg_roots: Dict[str, str]) -> Iterator[str]: """By carefully designing ``package_dir``, it is possible to implement the logical structure of PEP 420 in a package without the corresponding directories. - This function will try to find this kind of namespaces. + + Moreover a parent package can be purposefully/accidentally skipped in the discovery + phase (e.g. ``find_packages(include=["mypkg.*"])``, when ``mypkg.foo`` is included + by ``mypkg`` itself is not). + We consider this case to also be a virtual namespace (ignoring the original + directory) to emulate a non-editable installation. + + This function will try to find these kinds of namespaces. """ for pkg in pkg_roots: if "." not in pkg: @@ -595,6 +602,7 @@ def _find_virtual_namespaces(pkg_roots: Dict[str, str]) -> Iterator[str]: partial_name = ".".join(parts[:i]) path = Path(find_package_path(partial_name, pkg_roots, "")) if not path.exists() or partial_name not in pkg_roots: + # partial_name not in pkg_roots ==> purposefully/accidentally skipped yield partial_name -- cgit v1.2.1 From 15bebc7a4ca6925c26e8685ddbf0c2d8ca788b93 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 12 Aug 2022 14:55:01 +0100 Subject: Prevent optional extensions from failing in build_ext --- setuptools/command/build_ext.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/build_ext.py b/setuptools/command/build_ext.py index 7ad5a87a..cbfe3ec1 100644 --- a/setuptools/command/build_ext.py +++ b/setuptools/command/build_ext.py @@ -104,7 +104,8 @@ class build_ext(_build_ext): # Always copy, even if source is older than destination, to ensure # that the right extensions for the current Python/platform are # used. - self.copy_file(regular_file, inplace_file, level=self.verbose) + if os.path.exists(regular_file) or not ext.optional: + self.copy_file(regular_file, inplace_file, level=self.verbose) if ext._needs_stub: inplace_stub = self._get_equivalent_stub(ext, inplace_file) -- cgit v1.2.1 From 4ba12764a540f390b08ff5dd7e298ede44dee01d Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 12 Aug 2022 18:01:48 +0100 Subject: Ensure commands are finalized before using them --- setuptools/command/editable_wheel.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index 2631a082..560efebd 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -133,7 +133,8 @@ class editable_wheel(Command): self._ensure_dist_info() # Add missing dist_info files - bdist_wheel = self.reinitialize_command("bdist_wheel") + self.reinitialize_command("bdist_wheel") + bdist_wheel = self.get_finalized_command("bdist_wheel") bdist_wheel.write_wheelfile(self.dist_info_dir) self._create_wheel_file(bdist_wheel) @@ -156,7 +157,7 @@ class editable_wheel(Command): if self.dist_info_dir is None: dist_info = self.reinitialize_command("dist_info") dist_info.output_dir = self.dist_dir - dist_info.finalize_options() + dist_info.ensure_finalized() dist_info.run() self.dist_info_dir = dist_info.dist_info_dir else: @@ -278,7 +279,7 @@ class editable_wheel(Command): # Also remove _safely_run, TestCustomBuildPy. Suggested date: Aug/2023. build: Command = self.get_finalized_command("build") for name in build.get_sub_commands(): - cmd = self.distribution.get_command_obj(name) + cmd = self.get_finalized_command(name) if name == "build_py" and type(cmd) != build_py_cls: self._safely_run(name) else: -- cgit v1.2.1 From d4aed72d5d821820d29584a1e839a159d0893e59 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 12 Aug 2022 18:50:04 +0100 Subject: Filter out temporary source files from being added to sdist --- setuptools/command/build_py.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index 8b1a3320..ec062742 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -184,7 +184,7 @@ class build_py(orig.build_py): files = ei_cmd.filelist.files check = _IncludePackageDataAbuse() - for path in _filter_absolute_egg_info(files, egg_info_dir): + for path in self._filter_build_files(files, egg_info_dir): d, f = os.path.split(assert_relative(path)) prev = None oldf = f @@ -202,6 +202,25 @@ class build_py(orig.build_py): check.warn(importable) mf.setdefault(src_dirs[d], []).append(path) + def _filter_build_files(self, files: Iterable[str], egg_info: str) -> Iterator[str]: + """ + ``build_meta`` may try to create egg_info outside of the project directory, + and this can be problematic for certain plugins (reported in issue #3500). + + Extensions might also include between their sources files created on the + ``build_lib`` and ``build_temp`` directories. + + This function should filter this case of invalid files out. + """ + build = self.get_finalized_command("build") + build_dirs = (egg_info, self.build_lib, build.build_temp, build.build_base) + norm_dirs = [os.path.normpath(p) for p in build_dirs if p] + + for file in files: + norm_path = os.path.normpath(file) + if not os.path.isabs(file) or all(d not in norm_path for d in norm_dirs): + yield file + def get_data_files(self): pass # Lazily compute data files in _get_data_files() function. @@ -347,15 +366,3 @@ class _IncludePackageDataAbuse: msg = textwrap.dedent(self.MESSAGE).format(importable=importable) warnings.warn(msg, SetuptoolsDeprecationWarning, stacklevel=2) self._already_warned.add(importable) - - -def _filter_absolute_egg_info(files: Iterable[str], egg_info: str) -> Iterator[str]: - """ - ``build_meta`` may try to create egg_info outside of the project directory, - and this can be problematic for certain plugins (reported in issue #3500). - This function should filter this case of invalid files out. - """ - egg_info_name = Path(egg_info).name - for file in files: - if not (egg_info_name in file and os.path.isabs(file)): - yield file -- cgit v1.2.1 From 4371f3b79c3a65d453459e25de38cf03b98ff91d Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Mon, 15 Aug 2022 17:39:35 +0100 Subject: Fix backward compatibility with distutils --- setuptools/command/editable_wheel.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index 560efebd..ea214841 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -237,6 +237,8 @@ class editable_wheel(Command): cmd = dist.get_command_obj(cmd_name) if hasattr(cmd, "editable_mode"): cmd.editable_mode = True + elif hasattr(cmd, "inplace"): + cmd.inplace = True # backward compatibility with distutils def _collect_build_outputs(self) -> Tuple[List[str], Dict[str, str]]: files: List[str] = [] -- cgit v1.2.1 From 2372749be2e92df5d2aab11b1407948a8e25bf2f Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sun, 21 Aug 2022 06:34:41 +0100 Subject: Fix recursion problem in finder --- setuptools/command/editable_wheel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index ea214841..cd535e48 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -755,7 +755,7 @@ class _EditableFinder: # MetaPathFinder def find_spec(cls, fullname, path=None, target=None): for pkg, pkg_path in reversed(list(MAPPING.items())): if fullname.startswith(pkg): - rest = fullname.replace(pkg, "").strip(".").split(".") + rest = fullname.replace(pkg, "", 1).strip(".").split(".") return cls._find_spec(fullname, Path(pkg_path, *rest)) return None -- cgit v1.2.1 From d6095e99c846aef17f4d5a6d8a78432a54aacc52 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sun, 21 Aug 2022 06:43:34 +0100 Subject: Limit number of string replacements --- setuptools/command/editable_wheel.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index cd535e48..b908298f 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -684,9 +684,13 @@ def _is_nested(pkg: str, pkg_path: str, parent: str, parent_path: str) -> bool: False >>> _is_nested("a.b", "path/a/b", "c", "path/c") False + >>> _is_nested("a.a", "path/a/a", "a", "path/a") + True + >>> _is_nested("b.a", "path/b/a", "a", "path/a") + False """ norm_pkg_path = _normalize_path(pkg_path) - rest = pkg.replace(parent, "").strip(".").split(".") + rest = pkg.replace(parent, "", 1).strip(".").split(".") return ( pkg.startswith(parent) and norm_pkg_path == _normalize_path(Path(parent_path, *rest)) -- cgit v1.2.1 From 1387232226db86d22135907302de1cff0a6b0989 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Wed, 24 Aug 2022 09:44:52 +0100 Subject: Prevent accidental partial name matching in editable hooks --- setuptools/command/editable_wheel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index b908298f..d05c3a75 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -758,7 +758,7 @@ class _EditableFinder: # MetaPathFinder @classmethod def find_spec(cls, fullname, path=None, target=None): for pkg, pkg_path in reversed(list(MAPPING.items())): - if fullname.startswith(pkg): + if fullname == pkg or fullname.startswith(f"{{pkg}}."): rest = fullname.replace(pkg, "", 1).strip(".").split(".") return cls._find_spec(fullname, Path(pkg_path, *rest)) -- cgit v1.2.1 From ac7a0ddd49fc2a520dc498526e4feaebfbfeaadd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Blondon?= Date: Thu, 25 Aug 2022 20:47:43 +0200 Subject: remove python2 cases --- setuptools/command/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/test.py b/setuptools/command/test.py index 652f3e4a..8dde513c 100644 --- a/setuptools/command/test.py +++ b/setuptools/command/test.py @@ -118,7 +118,7 @@ class test(Command): return list(self._test_args()) def _test_args(self): - if not self.test_suite and sys.version_info >= (2, 7): + if not self.test_suite: yield 'discover' if self.verbose: yield '--verbose' -- cgit v1.2.1 From 62f9a0bde39aec0b4a13be441e5f12f568fc58b4 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 26 Aug 2022 14:44:56 +0100 Subject: Improve warnings in editable install --- setuptools/command/editable_wheel.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index d05c3a75..80c6d5d9 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -393,7 +393,7 @@ class _StaticPth: def __enter__(self): msg = f""" Editable install will be performed using .pth file to extend `sys.path` with: - {self.path_entries!r} + {list(map(os.fspath, self.path_entries))!r} """ _logger.warning(msg + _LENIENT_WARNING) return self @@ -503,7 +503,11 @@ class _TopLevelFinder: return self def __exit__(self, _exc_type, _exc_value, _traceback): - ... + msg = """\n + Please be careful with folders in your working directory with the same + name as your package as they may take precedence during imports. + """ + warnings.warn(msg, InformationOnly) def _can_symlink_files(base_dir: Path) -> bool: -- cgit v1.2.1 From 5434cb0a190f93571b86f54627890593905570cf Mon Sep 17 00:00:00 2001 From: alexzorin Date: Wed, 12 Oct 2022 05:47:28 +1100 Subject: fix small error message typo in editable_wheel.py --- setuptools/command/editable_wheel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index d05c3a75..ece1b590 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -294,7 +294,7 @@ class editable_wheel(Command): msg = f"""{traceback.format_exc()}\n If you are seeing this warning it is very likely that a setuptools plugin or customization overrides the `{cmd_name}` command, without - tacking into consideration how editable installs run build steps + taking into consideration how editable installs run build steps starting from v64.0.0. Plugin authors and developers relying on custom build steps are encouraged -- cgit v1.2.1 From 367a82b839e42ed2910193fdaae54d4e6353541a Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Wed, 12 Oct 2022 17:00:40 +0100 Subject: Handle no-packages projects in editable_wheel --- setuptools/command/editable_wheel.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index d05c3a75..a219ec3c 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -551,13 +551,18 @@ def _simple_layout( False >>> _simple_layout(['a', 'a.b'], {"": "src", "a.b": "_ab"}, "/tmp/myproj") False + >>> # Special cases, no packages yet: + >>> _simple_layout([], {"": "src"}, "/tmp/myproj") + True + >>> _simple_layout([], {"a": "_a", "": "src"}, "/tmp/myproj") + False """ layout = { pkg: find_package_path(pkg, package_dir, project_dir) for pkg in packages } if not layout: - return False + return set(package_dir) in ({}, {""}) parent = os.path.commonpath([_parent_path(k, v) for k, v in layout.items()]) return all( _normalize_path(Path(parent, *key.split('.'))) == _normalize_path(value) -- cgit v1.2.1 From c068081a7234a0c5c322a9312654e7d0f4aaa8d1 Mon Sep 17 00:00:00 2001 From: "Brett M. Morris" Date: Mon, 31 Oct 2022 09:39:00 -0400 Subject: Typo corrections --- setuptools/command/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/command') diff --git a/setuptools/command/build.py b/setuptools/command/build.py index c0676d8e..fa3c99ef 100644 --- a/setuptools/command/build.py +++ b/setuptools/command/build.py @@ -48,8 +48,8 @@ class SubCommand(Protocol): Subcommands **SHOULD** take advantage of ``editable_mode=True`` to adequate its behaviour or perform optimisations. - For example, if a subcommand don't need to generate any extra file and - everything it does is to copy a source file into the build directory, + For example, if a subcommand doesn't need to generate an extra file and + all it does is to copy a source file into the build directory, ``run()`` **SHOULD** simply "early return". Similarly, if the subcommand creates files that would be placed alongside -- cgit v1.2.1 From 95234bb9002dfb259d37125ae5382a404d0f5ab1 Mon Sep 17 00:00:00 2001 From: Daniel Garcia Moreno Date: Mon, 14 Nov 2022 12:56:32 +0100 Subject: Make clib build reproducible by sorting sources Fix https://github.com/pypa/setuptools/issues/3678 --- setuptools/command/build_clib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/build_clib.py b/setuptools/command/build_clib.py index 67ce2444..09483e69 100644 --- a/setuptools/command/build_clib.py +++ b/setuptools/command/build_clib.py @@ -28,7 +28,7 @@ class build_clib(orig.build_clib): "in 'libraries' option (library '%s'), " "'sources' must be present and must be " "a list of source filenames" % lib_name) - sources = list(sources) + sources = sorted(list(sources)) log.info("building '%s' library", lib_name) -- cgit v1.2.1 From 069dca9c534bf380659da7fc7a1a83476419a2da Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Sun, 20 Nov 2022 19:40:56 +0100 Subject: The default encoding of Python sources is UTF-8 Python 3 assumes the encoding is UTF-8 by default, and so do or should do tools such as text editors when opening Python files. No need to explicitly set `*- coding: utf-8 -*-`. --- setuptools/command/upload_docs.py | 1 - 1 file changed, 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/upload_docs.py b/setuptools/command/upload_docs.py index 3263f07f..63eb28c7 100644 --- a/setuptools/command/upload_docs.py +++ b/setuptools/command/upload_docs.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """upload_docs Implements a Distutils 'upload_docs' subcommand (upload documentation to -- cgit v1.2.1 From fb3313fe7fb650c822e9ec89660915b37b7d891e Mon Sep 17 00:00:00 2001 From: Andrzej Kaczmarczyk Date: Sat, 26 Nov 2022 13:57:38 +0100 Subject: Catching the exception of utime in order to provide a path to the file that was subject to the utime call; originally, the exception from utime does not point to this filepath. Ref #3667. --- setuptools/command/egg_info.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'setuptools/command') diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py index 25888ed8..95c81845 100644 --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py @@ -295,7 +295,11 @@ class egg_info(InfoCommon, Command): def run(self): self.mkpath(self.egg_info) - os.utime(self.egg_info, None) + try: + os.utime(self.egg_info, None) + except OSError as e: + msg = f"Cannot update time stamp of directory '{self.egg_info}'" + raise distutils.errors.DistutilsFileError(msg) from e for ep in metadata.entry_points(group='egg_info.writers'): writer = ep.load() writer(self, ep.name, os.path.join(self.egg_info, ep.name)) -- cgit v1.2.1