Automatically add files referenced by configuration to sdist (#3779)

9 files changed, 103 insertions, 22 deletions

diff --git a/changelog.d/3779.change.rst b/changelog.d/3779.change.rst
new file mode 100644
index 00000000..55c0fbf1
--- /dev/null
+++ b/changelog.d/3779.change.rst
@@ -0,0 +1,3 @@
+Files referenced by ``file:`` in ``setup.cfg`` and by ``project.readme.file``,
+``project.license.file`` or ``tool.setuptools.dynamic.*.file`` in
+``pyproject.toml`` are now automatically included in the generated sdists.
diff --git a/docs/userguide/declarative_config.rst b/docs/userguide/declarative_config.rst
index 6303e66f..d5735166 100644
--- a/docs/userguide/declarative_config.rst
+++ b/docs/userguide/declarative_config.rst
@@ -169,11 +169,14 @@ Special directives:
       The ``file:`` directive is sandboxed and won't reach anything outside the
       project directory (i.e. the directory containing ``setup.cfg``/``pyproject.toml``).
 
-  .. attention::
-      When using the ``file:`` directive, please make sure that all necessary
-      files are included in the ``sdist``. You can do that via ``MANIFEST.in``
-      or using plugins such as ``setuptools-scm``.
-      Please have a look on :doc:`/userguide/miscellaneous` for more information.
+  .. note::
+      If you are using an old version of ``setuptools``, you might need to ensure
+      that all files referenced by the ``file:`` directive are included in the ``sdist``
+      (you can do that via ``MANIFEST.in`` or using plugins such as ``setuptools-scm``,
+      please have a look on :doc:`/userguide/miscellaneous` for more information).
+
+      .. versionchanged:: 66.1.0
+         Newer versions of ``setuptools`` will automatically add these files to the ``sdist``.
 
 
 Metadata
diff --git a/docs/userguide/pyproject_config.rst b/docs/userguide/pyproject_config.rst
index 633f4de7..c97984ba 100644
--- a/docs/userguide/pyproject_config.rst
+++ b/docs/userguide/pyproject_config.rst
@@ -220,11 +220,14 @@ however please keep in mind that all non-comment lines must conform with :pep:`5
 (``pip``-specify syntaxes, e.g. ``-c/-r/-e`` flags, are not supported).
 
 
-.. attention::
-   When using the ``file`` directive, please make sure that all necessary
-   files are included in the ``sdist``. You can do that via ``MANIFEST.in``
-   or using plugins such as ``setuptools-scm``.
-   Please have a look on :doc:`/userguide/miscellaneous` for more information.
+.. note::
+   If you are using an old version of ``setuptools``, you might need to ensure
+   that all files referenced by the ``file`` directive are included in the ``sdist``
+   (you can do that via ``MANIFEST.in`` or using plugins such as ``setuptools-scm``,
+   please have a look on :doc:`/userguide/miscellaneous` for more information).
+
+   .. versionchanged:: 66.1.0
+      Newer versions of ``setuptools`` will automatically add these files to the ``sdist``.
 
 ----
 
diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py
index 1885efb0..86e99dd2 100644
--- a/setuptools/command/egg_info.py
+++ b/setuptools/command/egg_info.py
@@ -565,6 +565,7 @@ class manifest_maker(sdist):
         if os.path.exists(self.template):
             self.read_template()
         self.add_license_files()
+        self._add_referenced_files()
         self.prune_file_list()
         self.filelist.sort()
         self.filelist.remove_duplicates()
@@ -619,9 +620,16 @@ class manifest_maker(sdist):
         license_files = self.distribution.metadata.license_files or []
         for lf in license_files:
             log.info("adding license file '%s'", lf)
-            pass
         self.filelist.extend(license_files)
 
+    def _add_referenced_files(self):
+        """Add files referenced by the config (e.g. `file:` directive) to filelist"""
+        referenced = getattr(self.distribution, '_referenced_files', [])
+        # ^-- fallback if dist comes from distutils or is a custom class
+        for rf in referenced:
+            log.debug("adding file referenced by config '%s'", rf)
+        self.filelist.extend(referenced)
+
     def prune_file_list(self):
         build = self.get_finalized_command('build')
         base_dir = self.distribution.get_fullname()
diff --git a/setuptools/config/_apply_pyprojecttoml.py b/setuptools/config/_apply_pyprojecttoml.py
index 8af55616..c805e639 100644
--- a/setuptools/config/_apply_pyprojecttoml.py
+++ b/setuptools/config/_apply_pyprojecttoml.py
@@ -16,7 +16,7 @@ from functools import partial, reduce
 from itertools import chain
 from types import MappingProxyType
 from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, Tuple,
-                    Type, Union)
+                    Type, Union, cast)
 
 from setuptools._deprecation_warning import SetuptoolsDeprecationWarning
 
@@ -142,22 +142,29 @@ def _long_description(dist: "Distribution", val: _DictOrStr, root_dir: _Path):
     from setuptools.config import expand
 
     if isinstance(val, str):
-        text = expand.read_files(val, root_dir)
+        file: Union[str, list] = val
+        text = expand.read_files(file, root_dir)
         ctype = _guess_content_type(val)
     else:
-        text = val.get("text") or expand.read_files(val.get("file", []), root_dir)
+        file = val.get("file") or []
+        text = val.get("text") or expand.read_files(file, root_dir)
         ctype = val["content-type"]
 
     _set_config(dist, "long_description", text)
+
     if ctype:
         _set_config(dist, "long_description_content_type", ctype)
 
+    if file:
+        dist._referenced_files.add(cast(str, file))
+
 
 def _license(dist: "Distribution", val: dict, root_dir: _Path):
     from setuptools.config import expand
 
     if "file" in val:
         _set_config(dist, "license", expand.read_files([val["file"]], root_dir))
+        dist._referenced_files.add(val["file"])
     else:
         _set_config(dist, "license", val["text"])
 
diff --git a/setuptools/config/pyprojecttoml.py b/setuptools/config/pyprojecttoml.py
index fee6fac6..cedf5675 100644
--- a/setuptools/config/pyprojecttoml.py
+++ b/setuptools/config/pyprojecttoml.py
@@ -8,7 +8,7 @@ import os
 import warnings
 from contextlib import contextmanager
 from functools import partial
-from typing import TYPE_CHECKING, Callable, Dict, Optional, Mapping, Union
+from typing import TYPE_CHECKING, Callable, Dict, Optional, Mapping, Set, Union
 
 from setuptools.errors import FileError, OptionError
 
@@ -84,8 +84,8 @@ def read_configuration(
 
     :param Distribution|None: Distribution object to which the configuration refers.
         If not given a dummy object will be created and discarded after the
-        configuration is read. This is used for auto-discovery of packages in the case
-        a dynamic configuration (e.g. ``attr`` or ``cmdclass``) is expanded.
+        configuration is read. This is used for auto-discovery of packages and in the
+        case a dynamic configuration (e.g. ``attr`` or ``cmdclass``) is expanded.
         When ``expand=False`` this object is simply ignored.
 
     :rtype: dict
@@ -211,6 +211,7 @@ class _ConfigExpander:
         self.dynamic_cfg = self.setuptools_cfg.get("dynamic", {})
         self.ignore_option_errors = ignore_option_errors
         self._dist = dist
+        self._referenced_files: Set[str] = set()
 
     def _ensure_dist(self) -> "Distribution":
         from setuptools.dist import Distribution
@@ -241,6 +242,7 @@ class _ConfigExpander:
             self._expand_cmdclass(package_dir)
             self._expand_all_dynamic(dist, package_dir)
 
+        dist._referenced_files.update(self._referenced_files)
         return self.config
 
     def _expand_packages(self):
@@ -310,6 +312,7 @@ class _ConfigExpander:
         with _ignore_errors(self.ignore_option_errors):
             root_dir = self.root_dir
             if "file" in directive:
+                self._referenced_files.update(directive["file"])
                 return _expand.read_files(directive["file"], root_dir)
             if "attr" in directive:
                 return _expand.read_attr(directive["attr"], package_dir, root_dir)
diff --git a/setuptools/config/setupcfg.py b/setuptools/config/setupcfg.py
index c2a974de..3df3b6e7 100644
--- a/setuptools/config/setupcfg.py
+++ b/setuptools/config/setupcfg.py
@@ -12,7 +12,7 @@ from collections import defaultdict
 from functools import partial
 from functools import wraps
 from typing import (TYPE_CHECKING, Callable, Any, Dict, Generic, Iterable, List,
-                    Optional, Tuple, TypeVar, Union)
+                    Optional, Set, Tuple, TypeVar, Union)
 
 from distutils.errors import DistutilsOptionError, DistutilsFileError
 from setuptools.extern.packaging.requirements import Requirement, InvalidRequirement
@@ -172,6 +172,9 @@ def parse_configuration(
             distribution.src_root,
         )
         meta.parse()
+        distribution._referenced_files.update(
+            options._referenced_files, meta._referenced_files
+        )
 
     return meta, options
 
@@ -247,6 +250,10 @@ class ConfigHandler(Generic[Target]):
         self.sections = sections
         self.set_options: List[str] = []
         self.ensure_discovered = ensure_discovered
+        self._referenced_files: Set[str] = set()
+        """After parsing configurations, this property will enumerate
+        all files referenced by the "file:" directive. Private API for setuptools only.
+        """
 
     @property
     def parsers(self):
@@ -365,8 +372,7 @@ class ConfigHandler(Generic[Target]):
 
         return parser
 
-    @classmethod
-    def _parse_file(cls, value, root_dir: _Path):
+    def _parse_file(self, value, root_dir: _Path):
         """Represents value as a string, allowing including text
         from nearest files using `file:` directive.
 
@@ -388,7 +394,8 @@ class ConfigHandler(Generic[Target]):
             return value
 
         spec = value[len(include_directive) :]
-        filepaths = (path.strip() for path in spec.split(','))
+        filepaths = [path.strip() for path in spec.split(',')]
+        self._referenced_files.update(filepaths)
         return expand.read_files(filepaths, root_dir)
 
     def _parse_attr(self, value, package_dir, root_dir: _Path):
diff --git a/setuptools/dist.py b/setuptools/dist.py
index 1c71e5ee..cd34d74a 100644
--- a/setuptools/dist.py
+++ b/setuptools/dist.py
@@ -17,7 +17,7 @@ from distutils.fancy_getopt import translate_longopt
 from glob import iglob
 import itertools
 import textwrap
-from typing import List, Optional, TYPE_CHECKING
+from typing import List, Optional, Set, TYPE_CHECKING
 from pathlib import Path
 
 from collections import defaultdict
@@ -481,6 +481,11 @@ class Distribution(_Distribution):
             },
         )
 
+        # Private API (setuptools-use only, not restricted to Distribution)
+        # Stores files that are referenced by the configuration and need to be in the
+        # sdist (e.g. `version = file: VERSION.txt`)
+        self._referenced_files: Set[str] = set()
+
         # Save the original dependencies before they are processed into the egg format
         self._orig_extras_require = {}
         self._orig_install_requires = []
diff --git a/setuptools/tests/test_sdist.py b/setuptools/tests/test_sdist.py
index 30631c24..11de75c7 100644
--- a/setuptools/tests/test_sdist.py
+++ b/setuptools/tests/test_sdist.py
@@ -498,6 +498,48 @@ class TestSdistTest:
             filename = filename.decode('latin-1')
             filename not in cmd.filelist.files
 
+    _EXAMPLE_DIRECTIVES = {
+        "setup.cfg - long_description and version": """
+            [metadata]
+            name = testing
+            version = file: VERSION.txt
+            license_files = DOWHATYOUWANT
+            long_description = file: README.rst, USAGE.rst
+            """,
+        "pyproject.toml - static readme/license files and dynamic version": """
+            [project]
+            name = "testing"
+            readme = "USAGE.rst"
+            license = {file = "DOWHATYOUWANT"}
+            dynamic = ["version"]
+            [tool.setuptools.dynamic]
+            version = {file = ["VERSION.txt"]}
+            """
+    }
+
+    @pytest.mark.parametrize("config", _EXAMPLE_DIRECTIVES.keys())
+    def test_add_files_referenced_by_config_directives(self, tmp_path, config):
+        config_file, _, _ = config.partition(" - ")
+        config_text = self._EXAMPLE_DIRECTIVES[config]
+        (tmp_path / 'VERSION.txt').write_text("0.42", encoding="utf-8")
+        (tmp_path / 'README.rst').write_text("hello world!", encoding="utf-8")
+        (tmp_path / 'USAGE.rst').write_text("hello world!", encoding="utf-8")
+        (tmp_path / 'DOWHATYOUWANT').write_text("hello world!", encoding="utf-8")
+        (tmp_path / config_file).write_text(config_text, encoding="utf-8")
+
+        dist = Distribution({"packages": []})
+        dist.script_name = 'setup.py'
+        dist.parse_config_files()
+
+        cmd = sdist(dist)
+        cmd.ensure_finalized()
+        with quiet():
+            cmd.run()
+
+        assert 'VERSION.txt' in cmd.filelist.files
+        assert 'USAGE.rst' in cmd.filelist.files
+        assert 'DOWHATYOUWANT' in cmd.filelist.files
+
     def test_pyproject_toml_in_sdist(self, tmpdir):
         """
         Check if pyproject.toml is included in source distribution if present