diff options
| -rw-r--r-- | distutils2/_backport/pkgutil.py | 5 | ||||
| -rw-r--r-- | distutils2/_backport/sysconfig.py | 8 | ||||
| -rw-r--r-- | distutils2/command/install_data.py | 68 | ||||
| -rw-r--r-- | distutils2/command/install_dist.py | 4 | ||||
| -rw-r--r-- | distutils2/command/install_distinfo.py | 27 | ||||
| -rw-r--r-- | distutils2/config.py | 26 | ||||
| -rw-r--r-- | distutils2/datafiles.py | 71 | ||||
| -rw-r--r-- | distutils2/tests/__init__.py | 1 | ||||
| -rw-r--r-- | distutils2/tests/test_datafiles.py | 127 | ||||
| -rw-r--r-- | docs/design/wiki.rst | 6 | ||||
| -rw-r--r-- | docs/source/setupcfg.rst | 221 |
11 files changed, 514 insertions, 50 deletions
diff --git a/distutils2/_backport/pkgutil.py b/distutils2/_backport/pkgutil.py index 6f2d278..42a7b69 100644 --- a/distutils2/_backport/pkgutil.py +++ b/distutils2/_backport/pkgutil.py @@ -613,7 +613,7 @@ def get_data(package, resource): # PEP 376 Implementation # ########################## -DIST_FILES = ('INSTALLER', 'METADATA', 'RECORD', 'REQUESTED',) +DIST_FILES = ('INSTALLER', 'METADATA', 'RECORD', 'REQUESTED', 'DATAFILES') # Cache _cache_name = {} # maps names to Distribution instances @@ -1165,3 +1165,6 @@ def get_file_users(path): for dist in get_distributions(): if dist.uses(path): yield dist + +def open(distribution_name, relative_path): + pass diff --git a/distutils2/_backport/sysconfig.py b/distutils2/_backport/sysconfig.py index 99b0a3e..7760e89 100644 --- a/distutils2/_backport/sysconfig.py +++ b/distutils2/_backport/sysconfig.py @@ -120,6 +120,14 @@ def _expand_vars(scheme, vars): res[key] = os.path.normpath(_subst_vars(value, vars)) return res +def format_value(value, vars): + def _replacer(matchobj): + name = matchobj.group(1) + if name in vars: + return vars[name] + return matchobj.group(0) + return _VAR_REPL.sub(_replacer, value) + def _get_default_scheme(): if os.name == 'posix': diff --git a/distutils2/command/install_data.py b/distutils2/command/install_data.py index e77b11c..59cf90e 100644 --- a/distutils2/command/install_data.py +++ b/distutils2/command/install_data.py @@ -9,6 +9,7 @@ platform-independent data files.""" import os from distutils2.command.cmd import Command from distutils2.util import change_root, convert_path +from distutils2._backport.sysconfig import get_paths, format_value class install_data(Command): @@ -28,6 +29,7 @@ class install_data(Command): def initialize_options(self): self.install_dir = None self.outfiles = [] + self.data_files_out = [] self.root = None self.force = 0 self.data_files = self.distribution.data_files @@ -40,54 +42,34 @@ class install_data(Command): def run(self): self.mkpath(self.install_dir) - for f in self.data_files: - if isinstance(f, str): - # it's a simple file, so copy it - f = convert_path(f) - if self.warn_dir: - self.warn("setup script did not provide a directory for " - "'%s' -- installing right in '%s'" % - (f, self.install_dir)) - (out, _) = self.copy_file(f, self.install_dir) - self.outfiles.append(out) - else: - # it's a tuple with path to install to and a list of files - dir = convert_path(f[0]) - if not os.path.isabs(dir): - dir = os.path.join(self.install_dir, dir) - elif self.root: - dir = change_root(self.root, dir) - self.mkpath(dir) - - if f[1] == []: - # If there are no files listed, the user must be - # trying to create an empty directory, so add the - # directory to the list of output files. - self.outfiles.append(dir) - else: - # Copy files, adding them to the list of output files. - for data in f[1]: - data = convert_path(data) - (out, _) = self.copy_file(data, dir) - self.outfiles.append(out) + for file in self.data_files.items(): + destination = convert_path(self.expand_categories(file[1])) + dir_dest = os.path.abspath(os.path.dirname(destination)) + + self.mkpath(dir_dest) + (out, _) = self.copy_file(file[0], dir_dest) + + self.outfiles.append(out) + self.data_files_out.append((file[0], destination)) + + def expand_categories(self, path_with_categories): + local_vars = get_paths() + local_vars['distribution.name'] = self.distribution.metadata['Name'] + expanded_path = format_value(path_with_categories, local_vars) + expanded_path = format_value(expanded_path, local_vars) + if '{' in expanded_path and '}' in expanded_path: + self.warn("Unable to expand %s, some categories may missing." % + path_with_categories) + return expanded_path def get_source_files(self): - sources = [] - for item in self.data_files: - if isinstance(item, str): # plain file - item = convert_path(item) - if os.path.isfile(item): - sources.append(item) - else: # a (dirname, filenames) tuple - dirname, filenames = item - for f in filenames: - f = convert_path(f) - if os.path.isfile(f): - sources.append(f) - return sources + return self.data_files.keys() def get_inputs(self): return self.data_files or [] def get_outputs(self): return self.outfiles + + def get_datafiles_out(self): + return self.data_files_out
\ No newline at end of file diff --git a/distutils2/command/install_dist.py b/distutils2/command/install_dist.py index 146c905..23f3c5f 100644 --- a/distutils2/command/install_dist.py +++ b/distutils2/command/install_dist.py @@ -87,6 +87,8 @@ class install_dist(Command): ('record=', None, "filename in which to record a list of installed files " "(not PEP 376-compliant)"), + ('datafiles=', None, + "data files mapping"), # .dist-info related arguments, read by install_dist_info ('no-distinfo', None, @@ -184,12 +186,14 @@ class install_dist(Command): #self.install_info = None self.record = None + self.datafiles = None # .dist-info related options self.no_distinfo = None self.installer = None self.requested = None self.no_record = None + self.no_datafiles = None # -- Option finalizing methods ------------------------------------- # (This is rather more involved than for most commands, diff --git a/distutils2/command/install_distinfo.py b/distutils2/command/install_distinfo.py index 6e76546..4357dd9 100644 --- a/distutils2/command/install_distinfo.py +++ b/distutils2/command/install_distinfo.py @@ -39,9 +39,11 @@ class install_distinfo(Command): "do not generate a REQUESTED file"), ('no-record', None, "do not generate a RECORD file"), + ('no-datafiles', None, + "do not generate a DATAFILES list installed file") ] - boolean_options = ['requested', 'no-record'] + boolean_options = ['requested', 'no-record', 'no-datafiles'] negative_opt = {'no-requested': 'requested'} @@ -50,6 +52,7 @@ class install_distinfo(Command): self.installer = None self.requested = None self.no_record = None + self.no_datafiles = None def finalize_options(self): self.set_undefined_options('install_dist', @@ -66,6 +69,9 @@ class install_distinfo(Command): self.requested = True if self.no_record is None: self.no_record = False + if self.no_datafiles is None: + self.no_datafiles = False + metadata = self.distribution.metadata @@ -113,6 +119,24 @@ class install_distinfo(Command): f.close() self.outputs.append(requested_path) + + if not self.no_datafiles: + install_data = self.get_finalized_command('install_data') + if install_data.get_datafiles_out() != []: + datafiles_path = os.path.join(self.distinfo_dir, 'DATAFILES') + logger.info('creating %s', datafiles_path) + f = open(datafiles_path, 'wb') + try: + writer = csv.writer(f, delimiter=',', + lineterminator=os.linesep, + quotechar='"') + for tuple in install_data.get_datafiles_out(): + writer.writerow(tuple) + + self.outputs.append(datafiles_path) + finally: + f.close() + if not self.no_record: record_path = os.path.join(self.distinfo_dir, 'RECORD') logger.info('creating %s', record_path) @@ -142,6 +166,7 @@ class install_distinfo(Command): finally: f.close() + def get_outputs(self): return self.outputs diff --git a/distutils2/config.py b/distutils2/config.py index a62c9d8..9910edc 100644 --- a/distutils2/config.py +++ b/distutils2/config.py @@ -2,8 +2,10 @@ Know how to read all config files Distutils2 uses. """ +import os.path import os import sys +import re from ConfigParser import RawConfigParser from distutils2 import logger @@ -11,6 +13,7 @@ from distutils2.errors import DistutilsOptionError from distutils2.util import check_environ, resolve_name, strtobool from distutils2.compiler import set_compiler from distutils2.command import set_command +from distutils2.datafiles import resources_dests class Config(object): """Reads configuration files and work with the Distribution instance @@ -82,7 +85,7 @@ class Config(object): if v != ''] return value - def _read_setup_cfg(self, parser): + def _read_setup_cfg(self, parser, filename): content = {} for section in parser.sections(): content[section] = dict(parser.items(section)) @@ -182,6 +185,25 @@ class Config(object): # manifest template self.dist.extra_files = files.get('extra_files', []) + if 'resources' in content: + resources = [] + for glob, destination in content['resources'].iteritems(): + splitted_glob = glob.split(' ', 1) + if len(splitted_glob) == 1: + prefix = '' + suffix = splitted_glob[0] + else: + prefix = splitted_glob[0] + suffix = splitted_glob[1] + if destination == '<exclude>': + destination = None + resources.append((prefix, suffix, destination)) + + dir = os.path.dirname(os.path.join(os.getcwd(), filename)) + data_files = resources_dests(dir, resources) + self.dist.data_files = data_files + + def parse_config_files(self, filenames=None): if filenames is None: filenames = self.find_config_files() @@ -195,7 +217,7 @@ class Config(object): parser.read(filename) if os.path.split(filename)[-1] == 'setup.cfg': - self._read_setup_cfg(parser) + self._read_setup_cfg(parser, filename) for section in parser.sections(): if section == 'global': diff --git a/distutils2/datafiles.py b/distutils2/datafiles.py new file mode 100644 index 0000000..df3f1e7 --- /dev/null +++ b/distutils2/datafiles.py @@ -0,0 +1,71 @@ +import os +import re +from glob import iglob as simple_iglob + +__all__ = ['iglob', 'resources_dests'] + +class SmartGlob(object): + + def __init__(self, base, suffix): + self.base = base + self.suffix = suffix + + + def expand(self, basepath, destination): + if self.base: + base = os.path.join(basepath, self.base) + else: + base = basepath + if '*' in base or '{' in base or '}' in base: + raise NotImplementedError('glob are not supported into base part of datafiles definition. %r is an invalide basepath' % base) + absglob = os.path.join(base, self.suffix) + for file in iglob(absglob): + path_suffix = file[len(base):].lstrip('/') + relpath = file[len(basepath):].lstrip('/') + dest = os.path.join(destination, path_suffix) + yield relpath, dest + +RICH_GLOB = re.compile(r'\{([^}]*)\}') + +# r'\\\{' match "\{" + +def iglob(path_glob): + rich_path_glob = RICH_GLOB.split(path_glob, 1) + if len(rich_path_glob) > 1: + assert len(rich_path_glob) == 3, rich_path_glob + prefix, set, suffix = rich_path_glob + for item in set.split(','): + for path in iglob( ''.join((prefix, item, suffix))): + yield path + else: + if '**' not in path_glob: + for item in simple_iglob(path_glob): + yield item + else: + prefix, radical = path_glob.split('**', 1) + if prefix == '': + prefix = '.' + if radical == '': + radical = '*' + else: + radical = radical.lstrip('/') + for (path, dir, files) in os.walk(prefix): + for file in iglob(os.path.join(prefix, path, radical)): + yield os.path.join(prefix, file) + +def resources_dests(resources_dir, rules): + destinations = {} + for (base, suffix, glob_dest) in rules: + sglob = SmartGlob(base, suffix) + if glob_dest is None: + delete = True + dest = '' + else: + delete = False + dest = glob_dest + for file, file_dest in sglob.expand(resources_dir, dest): + if delete and file in destinations: + del destinations[file] + else: + destinations[file] = file_dest + return destinations diff --git a/distutils2/tests/__init__.py b/distutils2/tests/__init__.py index 1e8b09f..4d9208a 100644 --- a/distutils2/tests/__init__.py +++ b/distutils2/tests/__init__.py @@ -28,6 +28,7 @@ else: # external release of same package for older versions import unittest2 as unittest except ImportError: + raise sys.exit('Error: You have to install unittest2') diff --git a/distutils2/tests/test_datafiles.py b/distutils2/tests/test_datafiles.py new file mode 100644 index 0000000..bb1cd04 --- /dev/null +++ b/distutils2/tests/test_datafiles.py @@ -0,0 +1,127 @@ +# -*- encoding: utf-8 -*- +"""Tests for distutils.data.""" +import os +import sys +from StringIO import StringIO + +from distutils2.tests import unittest, support, run_unittest +from distutils2.datafiles import resources_dests, RICH_GLOB +import re + + + + +class DataFilesTestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + def setUp(self): + super(DataFilesTestCase, self).setUp() + self.addCleanup(setattr, sys, 'stdout', sys.stdout) + self.addCleanup(setattr, sys, 'stderr', sys.stderr) + + + def build_spec(self, spec, clean=True): + tempdir = self.mkdtemp() + for filepath in spec: + filepath = os.path.join(tempdir, *filepath.split('/')) + dirname = os.path.dirname(filepath) + if dirname and not os.path.exists(dirname): + os.makedirs(dirname) + self.write_file(filepath, 'babar') + if clean: + for key, value in list(spec.items()): + if value is None: + del spec[key] + return tempdir + + def assertFindGlob(self, rules, spec): + tempdir = self.build_spec(spec) + result = resources_dests(tempdir, rules) + self.assertEquals(spec, result) + + def test_regex_rich_glob(self): + matches = RICH_GLOB.findall(r"babar aime les {fraises} est les {huitres}") + self.assertEquals(["fraises","huitres"], matches) + + def test_simple_glob(self): + rules = [('', '*.tpl', '{data}')] + spec = {'coucou.tpl': '{data}/coucou.tpl', + 'Donotwant': None} + self.assertFindGlob(rules, spec) + + def test_multiple_match(self): + rules = [('scripts', '*.bin', '{appdata}'), + ('scripts', '*', '{appscript}')] + spec = {'scripts/script.bin': '{appscript}/script.bin', + 'Babarlikestrawberry': None} + self.assertFindGlob(rules, spec) + + def test_set_match(self): + rules = [('scripts', '*.{bin,sh}', '{appscript}')] + spec = {'scripts/script.bin': '{appscript}/script.bin', + 'scripts/babar.sh': '{appscript}/babar.sh', + 'Babarlikestrawberry': None} + self.assertFindGlob(rules, spec) + + def test_set_match_multiple(self): + rules = [('scripts', 'script{s,}.{bin,sh}', '{appscript}')] + spec = {'scripts/scripts.bin': '{appscript}/scripts.bin', + 'scripts/script.sh': '{appscript}/script.sh', + 'Babarlikestrawberry': None} + self.assertFindGlob(rules, spec) + + def test_set_match_exclude(self): + rules = [('scripts', '*', '{appscript}'), + ('', '**/*.sh', None)] + spec = {'scripts/scripts.bin': '{appscript}/scripts.bin', + 'scripts/script.sh': None, + 'Babarlikestrawberry': None} + self.assertFindGlob(rules, spec) + + def test_glob_in_base(self): + rules = [('scrip*', '*.bin', '{appscript}')] + spec = {'scripts/scripts.bin': '{appscript}/scripts.bin', + 'Babarlikestrawberry': None} + tempdir = self.build_spec(spec) + self.assertRaises(NotImplementedError, resources_dests, tempdir, rules) + + def test_recursive_glob(self): + rules = [('', '**/*.bin', '{binary}')] + spec = {'binary0.bin': '{binary}/binary0.bin', + 'scripts/binary1.bin': '{binary}/scripts/binary1.bin', + 'scripts/bin/binary2.bin': '{binary}/scripts/bin/binary2.bin', + 'you/kill/pandabear.guy': None} + self.assertFindGlob(rules, spec) + + def test_final_exemple_glob(self): + rules = [ + ('mailman/database/schemas/','*', '{appdata}/schemas'), + ('', '**/*.tpl', '{appdata}/templates'), + ('', 'developer-docs/**/*.txt', '{doc}'), + ('', 'README', '{doc}'), + ('mailman/etc/', '*', '{config}'), + ('mailman/foo/', '**/bar/*.cfg', '{config}/baz'), + ('mailman/foo/', '**/*.cfg', '{config}/hmm'), + ('', 'some-new-semantic.sns', '{funky-crazy-category}') + ] + spec = { + 'README': '{doc}/README', + 'some.tpl': '{appdata}/templates/some.tpl', + 'some-new-semantic.sns': '{funky-crazy-category}/some-new-semantic.sns', + 'mailman/database/mailman.db': None, + 'mailman/database/schemas/blah.schema': '{appdata}/schemas/blah.schema', + 'mailman/etc/my.cnf': '{config}/my.cnf', + 'mailman/foo/some/path/bar/my.cfg': '{config}/hmm/some/path/bar/my.cfg', + 'mailman/foo/some/path/other.cfg': '{config}/hmm/some/path/other.cfg', + 'developer-docs/index.txt': '{doc}/developer-docs/index.txt', + 'developer-docs/api/toc.txt': '{doc}/developer-docs/api/toc.txt', + } + self.maxDiff = None + self.assertFindGlob(rules, spec) + +def test_suite(): + return unittest.makeSuite(DataFilesTestCase) + +if __name__ == '__main__': + run_unittest(test_suite()) diff --git a/docs/design/wiki.rst b/docs/design/wiki.rst index 11b3058..e8ed1bf 100644 --- a/docs/design/wiki.rst +++ b/docs/design/wiki.rst @@ -250,8 +250,8 @@ Here's where we want the files to end up in a typical Linux distribution: == ==================================== =================================================================================================== 1 mailman/database/schemas/blah.schema /var/mailman/schemas/blah.schema 2 some.tpl /var/mailman/templates/some.tpl -3 path/to/some.tpl /var/mailman/templates/path/to/some.tpl -4 mailman/database/mailman.db /var/mailman/database/mailman.db +3 path/to/some.tpl /var/mailman/templates/path/to/some.tpl ! +4 mailman/database/mailman.db /var/mailman/database/mailman.db ! 5 developer-docs/index.txt /usr/share/doc/mailman/developer-docs/index.txt 6 developer-docs/api/toc.txt /usr/share/doc/mailman/developer-docs/api/toc.txt 7 README /usr/share/doc/mailman/README @@ -259,7 +259,7 @@ Here's where we want the files to end up in a typical Linux distribution: 9 mailman/foo/some/path/bar/my.cfg /etc/mailman/baz/some/path/bar/my.cfg AND /etc/mailman/hmm/some/path/bar/my.cfg + emit a warning -10 mailman/foo/some/path/other.cfg /etc/mailman/some/path/other.cfg +10 mailman/foo/some/path/other.cfg /etc/mailman/some/path/other.cfg ! 11 some-new-semantic.sns /var/funky/mailman/some-new-semantic.sns == ==================================== =================================================================================================== diff --git a/docs/source/setupcfg.rst b/docs/source/setupcfg.rst index d8d316b..68a37eb 100644 --- a/docs/source/setupcfg.rst +++ b/docs/source/setupcfg.rst @@ -149,6 +149,227 @@ Example:: extra_files = setup.py +data-files +========== + + +TODO : + + ### + source -> destination + + final-path = destination + source + + There is an {alias} for each categories of datafiles + ----- + source may be a glob (*, ?, **, {}) + + order + + exclude + -- + base-prefix + + #### + overwrite system config for {alias} + + #### + extra-categories + +This section describes the files used by the project which must not be installed in the same place that python modules or libraries. + +The format for specifing data files is : + + **source** = **destination** + +Example:: + + scripts/script1.bin = {scripts} + +It means that the file scripts/script1.bin will be placed + +It means that every file which match the glob_syntax will be placed in the destination. A part of the path of the file will be stripped when it will be expanded and another part will be append to the destination. For more informations about which part of the path will be stripped or not, take a look at next sub-section globsyntax_. + +The destination path will be expanded at the installation time using categories's default-path in the sysconfig.cfg file in the system. For more information about categories's default-paths, take a look at next next sub-section destination_. + + +.. _globsyntax: + +glob_syntax +----------- + +The glob syntax is traditionnal glob syntax (with unix separator **/**) with one more information : what part of the path will be stripped when path will be expanded ? + +The special character which indicate the end of the part that will be stripped and the beginning of the part that will be added is whitespace, which can follow or replace a path separator. + +Example:: + + scripts/ *.bin + +is equivalent to:: + + scripts *.bin + +Theses examples means that all files with extensions bin in the directory scripts will be placed directly on **destination** directory. + +This glob example:: + + scripts/*.bin + +means that all files with extensions bin in the directory scripts will be placed directly on **destination/scripts** directory. + +.. _destination: + +destination +----------- + +The destination is a traditionnal path (with unix separator **/**) where some parts will be expanded at installation time. These parts look like **{category}**, they will be expanded by reading system-wide default-path stored in sysconfig.cfg. Defaults categories are : + +* config +* appdata +* appdata.arch +* appdata.persistent +* appdata.disposable +* help +* icon +* scripts +* doc +* info +* man + +A special category exists, named {distribution.name} which will be expanded into your distribution name. You should not use it in your destination path, as they are may be used in defaults categories:: + + [globals] + # These are the useful categories that are sometimes referenced at runtime, + # using pkgutil.open(): + # Configuration files + config = {confdir}/{distribution.name} + # Non-writable data that is independent of architecture (images, many xml/text files) + appdata = {datadir}/{distribution.name} + # Non-writable data that is architecture-dependent (some binary data formats) + appdata.arch = {libdir}/{distribution.name} + # Data, written by the package, that must be preserved (databases) + appdata.persistent = {statedir}/lib/{distribution.name} + # Data, written by the package, that can be safely discarded (cache) + appdata.disposable = {statedir}/cache/{distribution.name} + # Help or documentation files referenced at runtime + help = {datadir}/{distribution.name} + icon = {datadir}/pixmaps + scripts = {base}/bin + + # Non-runtime files. These are valid categories for marking files for + # install, but they should not be referenced by the app at runtime: + # Help or documentation files not referenced by the package at runtime + doc = {datadir}/doc/{distribution.name} + # GNU info documentation files + info = {datadir}/info + # man pages + man = {datadir}/man + +So, if you have this destination path : **{help}/api**, it will be expanded into **{datadir}/{distribution.name}/api**. {datadir} will be expanded depending on your system value (ex : confdir = datadir = /usr/share/). + + +Simple-example +-------------- + +Source tree:: + + babar-1.0/ + README + babar.sh + launch.sh + babar.py + +Setup.cfg:: + + [RESOURCES] + README = {doc} + *.sh = {scripts} + +So babar.sh and launch.sh will be placed in {scripts} directory. + +Now let's create to move all the scripts into a scripts/directory. + +Second-example +-------------- + +Source tree:: + + babar-1.1/ + README + scripts/ + babar.sh + launch.sh + LAUNCH + babar.py + +Setup.cfg:: + + [RESOURCES] + README = {doc} + scripts/ LAUNCH = {scripts} + scripts/ *.sh = {scripts} + +It's important to use the separator after scripts/ to install all the bash scripts into {scripts} instead of {scripts}/scripts. + +Now let's add some docs. + +Third-example +------------- + +Source tree:: + + babar-1.2/ + README + scripts/ + babar.sh + launch.sh + LAUNCH + docs/ + api + man + babar.py + +Setup.cfg:: + + [RESOURCES] + README = {doc} + scripts/ LAUNCH = {doc} + scripts/ *.sh = {scripts} + doc/ * = {doc} + doc/ man = {man} + +You want to place all the file in the docs script into {doc} category, instead of man, which must be placed into {man} category, we will use the order of declaration of globs to choose the destination, the last glob that match the file is used. + +Now let's add some scripts for windows users. + +Final example +------------- + +Source tree:: + + babar-1.3/ + README + doc/ + api + man + scripts/ + babar.sh + launch.sh + babar.bat + launch.bat + LAUNCH + +Setup.cfg:: + + [RESOURCES] + README = {doc} + scripts/ LAUNCH = {doc} + scripts/ *.{sh,bat} = {scripts} + doc/ * = {doc} + doc/ man = {man} + +We use brace expansion syntax to place all the bash and batch scripts into {scripts} category. command sections ================ |