From 4f6fc8537842c14b03c4a1ffd25b88f2f4c276c6 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 22 May 2016 14:55:31 -0400 Subject: Use itertools.chain for more lenient support of any iterable types and also more uniform indentation. --- setuptools/command/build_py.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'setuptools/command/build_py.py') diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index 8623c777..3849b6ad 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -94,8 +94,10 @@ class build_py(orig.build_py, Mixin2to3): def find_data_files(self, package, src_dir): """Return filenames for package's data files in 'src_dir'""" - globs = (self.package_data.get('', []) - + self.package_data.get(package, [])) + globs = itertools.chain( + self.package_data.get('', []), + self.package_data.get(package, []), + ) files = self.manifest_files.get(package, [])[:] for pattern in globs: # Each pattern has to be converted to a platform-specific path -- cgit v1.2.1 From 2d765a69b19bb92882582249942d607684dd14eb Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 22 May 2016 15:18:04 -0400 Subject: Refactor build_py.find_data_files to use iterables, constructing the files list directly. Ref #261. --- setuptools/command/build_py.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'setuptools/command/build_py.py') diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index 3849b6ad..34f39037 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -98,10 +98,18 @@ class build_py(orig.build_py, Mixin2to3): self.package_data.get('', []), self.package_data.get(package, []), ) - files = self.manifest_files.get(package, [])[:] - for pattern in globs: + globs_expanded = ( # Each pattern has to be converted to a platform-specific path - files.extend(glob(os.path.join(src_dir, convert_path(pattern)))) + glob(os.path.join(src_dir, convert_path(pattern))) + for pattern in globs + ) + # flatten the expanded globs into an iterable of matches + globs_matches = itertools.chain.from_iterable(globs_expanded) + glob_files = globs_matches + files = list(itertools.chain( + self.manifest_files.get(package, []), + glob_files, + )) return self.exclude_data_files(package, src_dir, files) def build_package_data(self): -- cgit v1.2.1 From 8f0ac47db9fe2934725aa9c8a7b0089451ed033d Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 22 May 2016 15:18:32 -0400 Subject: Filter non-files in find_data_files. Fixes #261. --- setuptools/command/build_py.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/command/build_py.py') diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index 34f39037..23e8b31c 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -9,7 +9,7 @@ import distutils.errors import collections import itertools -from setuptools.extern.six.moves import map +from setuptools.extern.six.moves import map, filter try: from setuptools.lib2to3_ex import Mixin2to3 @@ -105,7 +105,7 @@ class build_py(orig.build_py, Mixin2to3): ) # flatten the expanded globs into an iterable of matches globs_matches = itertools.chain.from_iterable(globs_expanded) - glob_files = globs_matches + glob_files = filter(os.path.isfile, globs_matches) files = list(itertools.chain( self.manifest_files.get(package, []), glob_files, -- cgit v1.2.1 From dde1bfbf9aca70f4d78c349059ba43b3d40eb0e5 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 22 May 2016 15:21:28 -0400 Subject: Rewrite globs as chain of iterables. --- setuptools/command/build_py.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'setuptools/command/build_py.py') diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index 23e8b31c..4ce35314 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -194,9 +194,9 @@ class build_py(orig.build_py, Mixin2to3): def exclude_data_files(self, package, src_dir, files): """Filter filenames for package's data files in 'src_dir'""" - globs = ( - self.exclude_package_data.get('', []) - + self.exclude_package_data.get(package, []) + globs = itertools.chain( + self.exclude_package_data.get('', []), + self.exclude_package_data.get(package, []), ) bad = set( item -- cgit v1.2.1 From ba0b4d2eb91961a2237064850deca9e711ee4368 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 22 May 2016 15:22:09 -0400 Subject: Allow files to be iterable in exclude_data_files --- setuptools/command/build_py.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'setuptools/command/build_py.py') diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index 4ce35314..41b2660c 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -106,10 +106,10 @@ class build_py(orig.build_py, Mixin2to3): # flatten the expanded globs into an iterable of matches globs_matches = itertools.chain.from_iterable(globs_expanded) glob_files = filter(os.path.isfile, globs_matches) - files = list(itertools.chain( + files = itertools.chain( self.manifest_files.get(package, []), glob_files, - )) + ) return self.exclude_data_files(package, src_dir, files) def build_package_data(self): @@ -198,6 +198,7 @@ class build_py(orig.build_py, Mixin2to3): self.exclude_package_data.get('', []), self.exclude_package_data.get(package, []), ) + files = list(files) bad = set( item for pattern in globs -- cgit v1.2.1 From c05fd0799bb2a83ab877f317e8630a403a3514f7 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 22 May 2016 15:40:01 -0400 Subject: Rewrite find_data_files and exclude_data_files to follow the same pattern for building platform_patterns. --- setuptools/command/build_py.py | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) (limited to 'setuptools/command/build_py.py') diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index 41b2660c..cd91a85e 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -94,15 +94,17 @@ class build_py(orig.build_py, Mixin2to3): def find_data_files(self, package, src_dir): """Return filenames for package's data files in 'src_dir'""" - globs = itertools.chain( - self.package_data.get('', []), - self.package_data.get(package, []), + spec = self.package_data + raw_patterns = itertools.chain( + spec.get('', []), + spec.get(package, []), ) - globs_expanded = ( + platform_patterns = ( # Each pattern has to be converted to a platform-specific path - glob(os.path.join(src_dir, convert_path(pattern))) - for pattern in globs + os.path.join(src_dir, convert_path(pattern)) + for pattern in raw_patterns ) + globs_expanded = map(glob, platform_patterns) # flatten the expanded globs into an iterable of matches globs_matches = itertools.chain.from_iterable(globs_expanded) glob_files = filter(os.path.isfile, globs_matches) @@ -194,19 +196,24 @@ class build_py(orig.build_py, Mixin2to3): def exclude_data_files(self, package, src_dir, files): """Filter filenames for package's data files in 'src_dir'""" - globs = itertools.chain( - self.exclude_package_data.get('', []), - self.exclude_package_data.get(package, []), - ) files = list(files) - bad = set( - item - for pattern in globs - for item in fnmatch.filter( - files, - os.path.join(src_dir, convert_path(pattern)), - ) + spec = self.exclude_package_data + raw_patterns = itertools.chain( + spec.get('', []), + spec.get(package, []), + ) + platform_patterns = ( + # Each pattern has to be converted to a platform-specific path + os.path.join(src_dir, convert_path(pattern)) + for pattern in raw_patterns + ) + match_groups = ( + fnmatch.filter(files, pattern) + for pattern in platform_patterns ) + # flatten the groups of matches into an iterable of matches + matches = itertools.chain.from_iterable(match_groups) + bad = set(matches) seen = collections.defaultdict(itertools.count) return [ fn -- cgit v1.2.1 From 1af011b1f1dac17485c0cf8fe9eb08c43b5b6f2a Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 22 May 2016 15:53:06 -0400 Subject: Extract duplicate code into a single method. --- setuptools/command/build_py.py | 48 ++++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 20 deletions(-) (limited to 'setuptools/command/build_py.py') diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index cd91a85e..1db0acb9 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -94,17 +94,12 @@ class build_py(orig.build_py, Mixin2to3): def find_data_files(self, package, src_dir): """Return filenames for package's data files in 'src_dir'""" - spec = self.package_data - raw_patterns = itertools.chain( - spec.get('', []), - spec.get(package, []), - ) - platform_patterns = ( - # Each pattern has to be converted to a platform-specific path - os.path.join(src_dir, convert_path(pattern)) - for pattern in raw_patterns + patterns = self._get_platform_patterns( + self.package_data, + package, + src_dir, ) - globs_expanded = map(glob, platform_patterns) + globs_expanded = map(glob, patterns) # flatten the expanded globs into an iterable of matches globs_matches = itertools.chain.from_iterable(globs_expanded) glob_files = filter(os.path.isfile, globs_matches) @@ -197,19 +192,14 @@ class build_py(orig.build_py, Mixin2to3): def exclude_data_files(self, package, src_dir, files): """Filter filenames for package's data files in 'src_dir'""" files = list(files) - spec = self.exclude_package_data - raw_patterns = itertools.chain( - spec.get('', []), - spec.get(package, []), - ) - platform_patterns = ( - # Each pattern has to be converted to a platform-specific path - os.path.join(src_dir, convert_path(pattern)) - for pattern in raw_patterns + patterns = self._get_platform_patterns( + self.exclude_package_data, + package, + src_dir, ) match_groups = ( fnmatch.filter(files, pattern) - for pattern in platform_patterns + for pattern in patterns ) # flatten the groups of matches into an iterable of matches matches = itertools.chain.from_iterable(match_groups) @@ -223,6 +213,24 @@ class build_py(orig.build_py, Mixin2to3): and not next(seen[fn]) ] + @staticmethod + def _get_platform_patterns(spec, package, src_dir): + """ + yield platfrom-specific path patterns (suitable for glob + or fn_match) from a glob-based spec (such as + self.package_data or self.exclude_package_data) + matching package in src_dir. + """ + raw_patterns = itertools.chain( + spec.get('', []), + spec.get(package, []), + ) + return ( + # Each pattern has to be converted to a platform-specific path + os.path.join(src_dir, convert_path(pattern)) + for pattern in raw_patterns + ) + def assert_relative(path): if not os.path.isabs(path): -- cgit v1.2.1 From e75bb0594bad10cb307620359abcd3b5779ecfd2 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 22 May 2016 15:59:45 -0400 Subject: Re-use unique_everseen from itertools recipes. --- setuptools/command/build_py.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) (limited to 'setuptools/command/build_py.py') diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index 1db0acb9..758a3fdf 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -6,10 +6,9 @@ import fnmatch import textwrap import io import distutils.errors -import collections import itertools -from setuptools.extern.six.moves import map, filter +from setuptools.extern.six.moves import map, filter, filterfalse try: from setuptools.lib2to3_ex import Mixin2to3 @@ -204,14 +203,13 @@ class build_py(orig.build_py, Mixin2to3): # flatten the groups of matches into an iterable of matches matches = itertools.chain.from_iterable(match_groups) bad = set(matches) - seen = collections.defaultdict(itertools.count) - return [ + keepers = ( fn for fn in files if fn not in bad - # ditch dupes - and not next(seen[fn]) - ] + ) + # ditch dupes + return list(_unique_everseen(keepers)) @staticmethod def _get_platform_patterns(spec, package, src_dir): @@ -232,6 +230,25 @@ class build_py(orig.build_py, Mixin2to3): ) +# from Python docs +def _unique_everseen(iterable, key=None): + "List unique elements, preserving order. Remember all elements ever seen." + # unique_everseen('AAAABBBCCDAABBB') --> A B C D + # unique_everseen('ABBCcAD', str.lower) --> A B C D + seen = set() + seen_add = seen.add + if key is None: + for element in filterfalse(seen.__contains__, iterable): + seen_add(element) + yield element + else: + for element in iterable: + k = key(element) + if k not in seen: + seen_add(k) + yield element + + def assert_relative(path): if not os.path.isabs(path): return path -- cgit v1.2.1 From 11d5bb9e47754f0f4ded5379450e83e29d3661fb Mon Sep 17 00:00:00 2001 From: Benjamin Bach Date: Thu, 2 Jun 2016 11:58:43 +0200 Subject: Fixing #190 following proposal by @jaraco --- setuptools/command/build_py.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'setuptools/command/build_py.py') diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index 758a3fdf..0bad8295 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -8,6 +8,7 @@ import io import distutils.errors import itertools +from setuptools.extern import six from setuptools.extern.six.moves import map, filter, filterfalse try: @@ -66,6 +67,9 @@ class build_py(orig.build_py, Mixin2to3): return orig.build_py.__getattr__(self, attr) def build_module(self, module, module_file, package): + if six.PY2 and isinstance(package, six.string_types): + # avoid errors on Python 2 when unicode is passed (#190) + package = package.split('.') outfile, copied = orig.build_py.build_module(self, module, module_file, package) if copied: -- cgit v1.2.1