diff options
Diffstat (limited to 'command/build_src.py')
| -rw-r--r-- | command/build_src.py | 662 |
1 files changed, 662 insertions, 0 deletions
diff --git a/command/build_src.py b/command/build_src.py new file mode 100644 index 000000000..5e12f59f3 --- /dev/null +++ b/command/build_src.py @@ -0,0 +1,662 @@ +""" Build swig, f2py, weave, sources. +""" + +import os +import re +import sys + +from distutils.command import build_ext +from distutils.dep_util import newer_group, newer +from distutils.util import get_platform + +from numpy.distutils import log +from numpy.distutils.misc_util import fortran_ext_match, \ + appendpath, is_string, is_sequence +from numpy.distutils.from_template import process_file as process_f_file +from numpy.distutils.conv_template import process_file as process_c_file + +class build_src(build_ext.build_ext): + + description = "build sources from SWIG, F2PY files or a function" + + user_options = [ + ('build-src=', 'd', "directory to \"build\" sources to"), + ('f2pyflags=', None, "additonal flags to f2py"), + ('swigflags=', None, "additional flags to swig"), + ('force', 'f', "forcibly build everything (ignore file timestamps)"), + ('inplace', 'i', + "ignore build-lib and put compiled extensions into the source " + + "directory alongside your pure Python modules"), + ] + + boolean_options = ['force','inplace'] + + help_options = [] + + def initialize_options(self): + self.extensions = None + self.package = None + self.py_modules = None + self.py_modules_dict = None + self.build_src = None + self.build_lib = None + self.build_base = None + self.force = None + self.inplace = None + self.package_dir = None + self.f2pyflags = None + self.swigflags = None + return + + def finalize_options(self): + self.set_undefined_options('build', + ('build_base', 'build_base'), + ('build_lib', 'build_lib'), + ('force', 'force')) + if self.package is None: + self.package = self.distribution.ext_package + self.extensions = self.distribution.ext_modules + self.libraries = self.distribution.libraries or [] + self.py_modules = self.distribution.py_modules or [] + self.data_files = self.distribution.data_files or [] + + if self.build_src is None: + plat_specifier = ".%s-%s" % (get_platform(), sys.version[0:3]) + self.build_src = os.path.join(self.build_base, 'src'+plat_specifier) + if self.inplace is None: + build_ext = self.get_finalized_command('build_ext') + self.inplace = build_ext.inplace + + # py_modules_dict is used in build_py.find_package_modules + self.py_modules_dict = {} + + if self.f2pyflags is None: + self.f2pyflags = [] + else: + self.f2pyflags = self.f2pyflags.split() # XXX spaces?? + + if self.swigflags is None: + self.swigflags = [] + else: + self.swigflags = self.swigflags.split() # XXX spaces?? + return + + def run(self): + if not (self.extensions or self.libraries): + return + self.build_sources() + + return + + def build_sources(self): + + if self.inplace: + self.get_package_dir = self.get_finalized_command('build_py')\ + .get_package_dir + + self.build_py_modules_sources() + + for libname_info in self.libraries: + self.build_library_sources(*libname_info) + + if self.extensions: + self.check_extensions_list(self.extensions) + + for ext in self.extensions: + self.build_extension_sources(ext) + + self.build_data_files_sources() + + return + + def build_data_files_sources(self): + if not self.data_files: + return + log.info('building data_files sources') + from numpy.distutils.misc_util import get_data_files + new_data_files = [] + for data in self.data_files: + if isinstance(data,str): + new_data_files.append(data) + elif isinstance(data,tuple): + d,files = data + if self.inplace: + build_dir = self.get_package_dir('.'.join(d.split(os.sep))) + else: + build_dir = os.path.join(self.build_src,d) + funcs = filter(callable,files) + files = filter(lambda f:not callable(f), files) + for f in funcs: + if f.func_code.co_argcount==1: + s = f(build_dir) + else: + s = f() + if s is not None: + if isinstance(s,list): + files.extend(s) + elif isinstance(s,str): + files.append(s) + else: + raise TypeError(repr(s)) + filenames = get_data_files((d,files)) + new_data_files.append((d, filenames)) + else: + raise + self.data_files[:] = new_data_files + return + + def build_py_modules_sources(self): + if not self.py_modules: + return + log.info('building py_modules sources') + new_py_modules = [] + for source in self.py_modules: + if is_sequence(source) and len(source)==3: + package, module_base, source = source + if self.inplace: + build_dir = self.get_package_dir(package) + else: + build_dir = os.path.join(self.build_src, + os.path.join(*package.split('.'))) + if callable(source): + target = os.path.join(build_dir, module_base + '.py') + source = source(target) + if source is None: + continue + modules = [(package, module_base, source)] + if not self.py_modules_dict.has_key(package): + self.py_modules_dict[package] = [] + self.py_modules_dict[package] += modules + else: + new_py_modules.append(source) + self.py_modules[:] = new_py_modules + return + + def build_library_sources(self, lib_name, build_info): + sources = list(build_info.get('sources',[])) + + if not sources: + return + + log.info('building library "%s" sources' % (lib_name)) + + sources = self.generate_sources(sources, (lib_name, build_info)) + + sources = self.template_sources(sources, (lib_name, build_info)) + + sources, h_files = self.filter_h_files(sources) + + if h_files: + print self.package,'- nothing done with h_files=',h_files + + #for f in h_files: + # self.distribution.headers.append((lib_name,f)) + + build_info['sources'] = sources + return + + def build_extension_sources(self, ext): + + sources = list(ext.sources) + + log.info('building extension "%s" sources' % (ext.name)) + + fullname = self.get_ext_fullname(ext.name) + + modpath = fullname.split('.') + package = '.'.join(modpath[0:-1]) + + if self.inplace: + self.ext_target_dir = self.get_package_dir(package) + + sources = self.generate_sources(sources, ext) + + sources = self.template_sources(sources, ext) + + sources = self.swig_sources(sources, ext) + + sources = self.f2py_sources(sources, ext) + + sources = self.pyrex_sources(sources, ext) + + sources, py_files = self.filter_py_files(sources) + + if not self.py_modules_dict.has_key(package): + self.py_modules_dict[package] = [] + modules = [] + for f in py_files: + module = os.path.splitext(os.path.basename(f))[0] + modules.append((package, module, f)) + self.py_modules_dict[package] += modules + + sources, h_files = self.filter_h_files(sources) + + if h_files: + print package,'- nothing done with h_files=',h_files + #for f in h_files: + # self.distribution.headers.append((package,f)) + + ext.sources = sources + + return + + def generate_sources(self, sources, extension): + new_sources = [] + func_sources = [] + for source in sources: + if is_string(source): + new_sources.append(source) + else: + func_sources.append(source) + if not func_sources: + return new_sources + if self.inplace and not is_sequence(extension): + build_dir = self.ext_target_dir + else: + if is_sequence(extension): + name = extension[0] + # if not extension[1].has_key('include_dirs'): + # extension[1]['include_dirs'] = [] + # incl_dirs = extension[1]['include_dirs'] + else: + name = extension.name + # incl_dirs = extension.include_dirs + #if self.build_src not in incl_dirs: + # incl_dirs.append(self.build_src) + build_dir = os.path.join(*([self.build_src]\ + +name.split('.')[:-1])) + self.mkpath(build_dir) + for func in func_sources: + source = func(extension, build_dir) + if not source: + continue + if is_sequence(source): + [log.info(" adding '%s' to sources." % (s,)) for s in source] + new_sources.extend(source) + else: + log.info(" adding '%s' to sources." % (source,)) + new_sources.append(source) + + return new_sources + + def filter_py_files(self, sources): + return self.filter_files(sources,['.py']) + + def filter_h_files(self, sources): + return self.filter_files(sources,['.h','.hpp','.inc']) + + def filter_files(self, sources, exts = []): + new_sources = [] + files = [] + for source in sources: + (base, ext) = os.path.splitext(source) + if ext in exts: + files.append(source) + else: + new_sources.append(source) + return new_sources, files + + def template_sources(self, sources, extension): + new_sources = [] + if is_sequence(extension): + depends = extension[1].get('depends') + include_dirs = extension[1].get('include_dirs') + else: + depends = extension.depends + include_dirs = extension.include_dirs + for source in sources: + (base, ext) = os.path.splitext(source) + if ext == '.src': # Template file + if self.inplace: + target_dir = os.path.dirname(base) + else: + target_dir = appendpath(self.build_src, os.path.dirname(base)) + self.mkpath(target_dir) + target_file = os.path.join(target_dir,os.path.basename(base)) + if (self.force or newer_group([source] + depends, target_file)): + if _f_pyf_ext_match(base): + log.info("from_template:> %s" % (target_file)) + outstr = process_f_file(source) + else: + log.info("conv_template:> %s" % (target_file)) + outstr = process_c_file(source) + fid = open(target_file,'w') + fid.write(outstr) + fid.close() + if _header_ext_match(target_file): + d = os.path.dirname(target_file) + if d not in include_dirs: + log.info(" adding '%s' to include_dirs." % (d)) + include_dirs.append(d) + new_sources.append(target_file) + else: + new_sources.append(source) + return new_sources + + def pyrex_sources(self, sources, extension): + have_pyrex = False + try: + import Pyrex + have_pyrex = True + except ImportError: + pass + new_sources = [] + ext_name = extension.name.split('.')[-1] + for source in sources: + (base, ext) = os.path.splitext(source) + if ext == '.pyx': + if self.inplace or not have_pyrex: + target_dir = os.path.dirname(base) + else: + target_dir = appendpath(self.build_src, os.path.dirname(base)) + target_file = os.path.join(target_dir, ext_name + '.c') + depends = [source] + extension.depends + if (self.force or newer_group(depends, target_file, 'newer')): + if have_pyrex: + log.info("pyrexc:> %s" % (target_file)) + self.mkpath(target_dir) + from Pyrex.Compiler import Main + options = Main.CompilationOptions( + defaults=Main.default_options, + output_file=target_file) + pyrex_result = Main.compile(source, options=options) + if pyrex_result.num_errors != 0: + raise RuntimeError("%d errors in Pyrex compile" % + pyrex_result.num_errors) + else: + log.warn("Pyrex needed to compile %s but not available."\ + " Using old target %s"\ + % (source, target_file)) + new_sources.append(target_file) + else: + new_sources.append(source) + return new_sources + + def f2py_sources(self, sources, extension): + new_sources = [] + f2py_sources = [] + f_sources = [] + f2py_targets = {} + target_dirs = [] + ext_name = extension.name.split('.')[-1] + skip_f2py = 0 + + for source in sources: + (base, ext) = os.path.splitext(source) + if ext == '.pyf': # F2PY interface file + if self.inplace: + target_dir = os.path.dirname(base) + else: + target_dir = appendpath(self.build_src, os.path.dirname(base)) + if os.path.isfile(source): + name = get_f2py_modulename(source) + if name != ext_name: + raise ValueError('mismatch of extension names: %s ' + 'provides %r but expected %r' % ( + source, name, ext_name)) + target_file = os.path.join(target_dir,name+'module.c') + else: + log.debug(' source %s does not exist: skipping f2py\'ing.' \ + % (source)) + name = ext_name + skip_f2py = 1 + target_file = os.path.join(target_dir,name+'module.c') + if not os.path.isfile(target_file): + log.debug(' target %s does not exist:\n '\ + 'Assuming %smodule.c was generated with '\ + '"build_src --inplace" command.' \ + % (target_file, name)) + target_dir = os.path.dirname(base) + target_file = os.path.join(target_dir,name+'module.c') + if not os.path.isfile(target_file): + raise ValueError("%r missing" % (target_file,)) + log.debug(' Yes! Using %s as up-to-date target.' \ + % (target_file)) + target_dirs.append(target_dir) + f2py_sources.append(source) + f2py_targets[source] = target_file + new_sources.append(target_file) + elif fortran_ext_match(ext): + f_sources.append(source) + else: + new_sources.append(source) + + if not (f2py_sources or f_sources): + return new_sources + + map(self.mkpath, target_dirs) + + f2py_options = extension.f2py_options + self.f2pyflags + + if self.distribution.libraries: + for name,build_info in self.distribution.libraries: + if name in extension.libraries: + f2py_options.extend(build_info.get('f2py_options',[])) + + log.info("f2py options: %s" % (f2py_options)) + + if f2py_sources: + if len(f2py_sources) != 1: + raise ValueError( + 'only one .pyf file is allowed per extension module but got'\ + ' more: %r' % (f2py_sources,)) + source = f2py_sources[0] + target_file = f2py_targets[source] + target_dir = os.path.dirname(target_file) or '.' + depends = [source] + extension.depends + if (self.force or newer_group(depends, target_file,'newer')) \ + and not skip_f2py: + log.info("f2py: %s" % (source)) + import numpy.f2py as f2py2e + f2py2e.run_main(f2py_options + ['--build-dir',target_dir,source]) + else: + log.debug(" skipping '%s' f2py interface (up-to-date)" % (source)) + else: + #XXX TODO: --inplace support for sdist command + if is_sequence(extension): + name = extension[0] + else: name = extension.name + target_dir = os.path.join(*([self.build_src]\ + +name.split('.')[:-1])) + target_file = os.path.join(target_dir,ext_name + 'module.c') + new_sources.append(target_file) + depends = f_sources + extension.depends + if (self.force or newer_group(depends, target_file, 'newer')) \ + and not skip_f2py: + import numpy.f2py as f2py2e + log.info("f2py:> %s" % (target_file)) + self.mkpath(target_dir) + f2py2e.run_main(f2py_options + ['--lower', + '--build-dir',target_dir]+\ + ['-m',ext_name]+f_sources) + else: + log.debug(" skipping f2py fortran files for '%s' (up-to-date)"\ + % (target_file)) + + if not os.path.isfile(target_file): + raise ValueError("%r missing" % (target_file,)) + + target_c = os.path.join(self.build_src,'fortranobject.c') + target_h = os.path.join(self.build_src,'fortranobject.h') + log.info(" adding '%s' to sources." % (target_c)) + new_sources.append(target_c) + if self.build_src not in extension.include_dirs: + log.info(" adding '%s' to include_dirs." \ + % (self.build_src)) + extension.include_dirs.append(self.build_src) + + if not skip_f2py: + import numpy.f2py as f2py2e + d = os.path.dirname(f2py2e.__file__) + source_c = os.path.join(d,'src','fortranobject.c') + source_h = os.path.join(d,'src','fortranobject.h') + if newer(source_c,target_c) or newer(source_h,target_h): + self.mkpath(os.path.dirname(target_c)) + self.copy_file(source_c,target_c) + self.copy_file(source_h,target_h) + else: + if not os.path.isfile(target_c): + raise ValueError("%r missing" % (target_c,)) + if not os.path.isfile(target_h): + raise ValueError("%r missing" % (target_h,)) + + for name_ext in ['-f2pywrappers.f','-f2pywrappers2.f90']: + filename = os.path.join(target_dir,ext_name + name_ext) + if os.path.isfile(filename): + log.info(" adding '%s' to sources." % (filename)) + f_sources.append(filename) + + return new_sources + f_sources + + def swig_sources(self, sources, extension): + # Assuming SWIG 1.3.14 or later. See compatibility note in + # http://www.swig.org/Doc1.3/Python.html#Python_nn6 + + new_sources = [] + swig_sources = [] + swig_targets = {} + target_dirs = [] + py_files = [] # swig generated .py files + target_ext = '.c' + typ = None + is_cpp = 0 + skip_swig = 0 + ext_name = extension.name.split('.')[-1] + + for source in sources: + (base, ext) = os.path.splitext(source) + if ext == '.i': # SWIG interface file + if self.inplace: + target_dir = os.path.dirname(base) + py_target_dir = self.ext_target_dir + else: + target_dir = appendpath(self.build_src, os.path.dirname(base)) + py_target_dir = target_dir + if os.path.isfile(source): + name = get_swig_modulename(source) + if name != ext_name[1:]: + raise ValueError( + 'mismatch of extension names: %s provides %r' + ' but expected %r' % (source, name, ext_name[1:])) + if typ is None: + typ = get_swig_target(source) + is_cpp = typ=='c++' + if is_cpp: + target_ext = '.cpp' + else: + assert typ == get_swig_target(source), repr(typ) + target_file = os.path.join(target_dir,'%s_wrap%s' \ + % (name, target_ext)) + else: + log.debug(' source %s does not exist: skipping swig\'ing.' \ + % (source)) + name = ext_name[1:] + skip_swig = 1 + target_file = _find_swig_target(target_dir, name) + if not os.path.isfile(target_file): + log.debug(' target %s does not exist:\n '\ + 'Assuming %s_wrap.{c,cpp} was generated with '\ + '"build_src --inplace" command.' \ + % (target_file, name)) + target_dir = os.path.dirname(base) + target_file = _find_swig_target(target_dir, name) + if not os.path.isfile(target_file): + raise ValueError("%r missing" % (target_file,)) + log.debug(' Yes! Using %s as up-to-date target.' \ + % (target_file)) + target_dirs.append(target_dir) + new_sources.append(target_file) + py_files.append(os.path.join(py_target_dir, name+'.py')) + swig_sources.append(source) + swig_targets[source] = new_sources[-1] + else: + new_sources.append(source) + + if not swig_sources: + return new_sources + + if skip_swig: + return new_sources + py_files + + map(self.mkpath, target_dirs) + swig = self.find_swig() + swig_cmd = [swig, "-python"] + if is_cpp: + swig_cmd.append('-c++') + for d in extension.include_dirs: + swig_cmd.append('-I'+d) + for source in swig_sources: + target = swig_targets[source] + depends = [source] + extension.depends + if self.force or newer_group(depends, target, 'newer'): + log.info("%s: %s" % (os.path.basename(swig) \ + + (is_cpp and '++' or ''), source)) + self.spawn(swig_cmd + self.swigflags \ + + ["-o", target, '-outdir', py_target_dir, source]) + else: + log.debug(" skipping '%s' swig interface (up-to-date)" \ + % (source)) + + return new_sources + py_files + +_f_pyf_ext_match = re.compile(r'.*[.](f90|f95|f77|for|ftn|f|pyf)\Z',re.I).match +_header_ext_match = re.compile(r'.*[.](inc|h|hpp)\Z',re.I).match + +#### SWIG related auxiliary functions #### +_swig_module_name_match = re.compile(r'\s*%module\s*(.*\(\s*package\s*=\s*"(?P<package>[\w_]+)".*\)|)\s*(?P<name>[\w_]+)', + re.I).match +_has_c_header = re.compile(r'-[*]-\s*c\s*-[*]-',re.I).search +_has_cpp_header = re.compile(r'-[*]-\s*c[+][+]\s*-[*]-',re.I).search + +def get_swig_target(source): + f = open(source,'r') + result = 'c' + line = f.readline() + if _has_cpp_header(line): + result = 'c++' + if _has_c_header(line): + result = 'c' + f.close() + return result + +def get_swig_modulename(source): + f = open(source,'r') + f_readlines = getattr(f,'xreadlines',f.readlines) + name = None + for line in f_readlines(): + m = _swig_module_name_match(line) + if m: + name = m.group('name') + break + f.close() + return name + +def _find_swig_target(target_dir,name): + for ext in ['.cpp','.c']: + target = os.path.join(target_dir,'%s_wrap%s' % (name, ext)) + if os.path.isfile(target): + break + return target + +#### F2PY related auxiliary functions #### + +_f2py_module_name_match = re.compile(r'\s*python\s*module\s*(?P<name>[\w_]+)', + re.I).match +_f2py_user_module_name_match = re.compile(r'\s*python\s*module\s*(?P<name>[\w_]*?'\ + '__user__[\w_]*)',re.I).match + +def get_f2py_modulename(source): + name = None + f = open(source) + f_readlines = getattr(f,'xreadlines',f.readlines) + for line in f_readlines(): + m = _f2py_module_name_match(line) + if m: + if _f2py_user_module_name_match(line): # skip *__user__* names + continue + name = m.group('name') + break + f.close() + return name + +########################################## |