Merge pull request #5597 from zerothi/ENH-distutils

BLD, ENH: Reading of extra flags from site.cfg to extend flexibility

7 files changed, 314 insertions, 2 deletions

diff --git a/doc/release/1.10.0-notes.rst b/doc/release/1.10.0-notes.rst
index a7c0e2852..6a6bbd4c6 100644
--- a/doc/release/1.10.0-notes.rst
+++ b/doc/release/1.10.0-notes.rst
@@ -8,6 +8,9 @@ Highlights
 ==========
 * numpy.distutils now supports parallel compilation via the --jobs/-j argument
   passed to setup.py build
+* numpy.distutils now supports additional customization via site.cfg to
+  control compilation parameters, i.e. runtime libraries, extra
+  linking/compilation flags.
 * Addition of *np.linalg.multi_dot*: compute the dot product of two or more
   arrays in a single function call, while automatically selecting the fastest
   evaluation order.
@@ -86,6 +89,20 @@ output for ufuncs with multiple outputs, is deprecated, and will result in a
 New Features
 ============
 
+Reading extra flags from site.cfg
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Previously customization of compilation of dependency libraries and numpy 
+itself was only accomblishable via code changes in the distutils package.
+Now numpy.distutils reads in the following extra flags from each group of the
+*site.cfg*:
+
+* ``runtime_library_dirs/rpath``, sets runtime library directories to override
+    ``LD_LIBRARY_PATH``
+* ``extra_compile_args``, add extra flags to the compilation of sources
+* ``extra_link_args``, add extra flags when linking libraries
+
+This should, at least partially, complete user customization.
+
 *np.cbrt* to compute cube root for real floats
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 *np.cbrt* wraps the C99 cube root function *cbrt*.
diff --git a/numpy/distutils/fcompiler/intel.py b/numpy/distutils/fcompiler/intel.py
index f76174c7a..63436e4ed 100644
--- a/numpy/distutils/fcompiler/intel.py
+++ b/numpy/distutils/fcompiler/intel.py
@@ -14,12 +14,17 @@ def intel_version_match(type):
     # Match against the important stuff in the version string
     return simple_version_match(start=r'Intel.*?Fortran.*?(?:%s).*?Version' % (type,))
 
+
 class BaseIntelFCompiler(FCompiler):
     def update_executables(self):
         f = dummy_fortran_file()
         self.executables['version_cmd'] = ['<F77>', '-FI', '-V', '-c',
                                            f + '.f', '-o', f + '.o']
 
+    def runtime_library_dir_option(self, dir):
+        return '-Wl,-rpath="%s"' % dir
+
+
 class IntelFCompiler(BaseIntelFCompiler):
 
     compiler_type = 'intel'
@@ -71,6 +76,7 @@ class IntelFCompiler(BaseIntelFCompiler):
             opt[idx:idx] = ['-dynamiclib', '-Wl,-undefined,dynamic_lookup']
         return opt
 
+
 class IntelItaniumFCompiler(IntelFCompiler):
     compiler_type = 'intele'
     compiler_aliases = ()
@@ -90,6 +96,7 @@ class IntelItaniumFCompiler(IntelFCompiler):
         'ranlib'       : ["ranlib"]
         }
 
+
 class IntelEM64TFCompiler(IntelFCompiler):
     compiler_type = 'intelem'
     compiler_aliases = ()
@@ -122,6 +129,7 @@ class IntelEM64TFCompiler(IntelFCompiler):
 # Is there no difference in the version string between the above compilers
 # and the Visual compilers?
 
+
 class IntelVisualFCompiler(BaseIntelFCompiler):
     compiler_type = 'intelv'
     description = 'Intel Visual Fortran Compiler for 32-bit apps'
@@ -167,6 +175,10 @@ class IntelVisualFCompiler(BaseIntelFCompiler):
     def get_flags_arch(self):
         return ["/arch:IA-32", "/QaxSSE3"]
 
+    def runtime_library_dir_option(self, dir):
+        raise NotImplementedError
+
+
 class IntelItaniumVisualFCompiler(IntelVisualFCompiler):
     compiler_type = 'intelev'
     description = 'Intel Visual Fortran Compiler for Itanium apps'
@@ -186,6 +198,7 @@ class IntelItaniumVisualFCompiler(IntelVisualFCompiler):
         'ranlib'       : None
         }
 
+
 class IntelEM64VisualFCompiler(IntelVisualFCompiler):
     compiler_type = 'intelvem'
     description = 'Intel Visual Fortran Compiler for 64-bit apps'
diff --git a/numpy/distutils/fcompiler/pg.py b/numpy/distutils/fcompiler/pg.py
index f3f5ea22b..ee357c6d0 100644
--- a/numpy/distutils/fcompiler/pg.py
+++ b/numpy/distutils/fcompiler/pg.py
@@ -51,6 +51,9 @@ class PGroupFCompiler(FCompiler):
         def get_flags_linker_so(self):
             return ["-dynamic", '-undefined', 'dynamic_lookup']
 
+    def runtime_library_dir_option(self, dir):
+        return '-R"%s"' % dir
+
 if __name__ == '__main__':
     from distutils import log
     log.set_verbosity(2)
diff --git a/numpy/distutils/fcompiler/sun.py b/numpy/distutils/fcompiler/sun.py
index 0955f14a1..76ce1cabc 100644
--- a/numpy/distutils/fcompiler/sun.py
+++ b/numpy/distutils/fcompiler/sun.py
@@ -43,6 +43,9 @@ class SunFCompiler(FCompiler):
         opt.extend(['fsu', 'sunmath', 'mvec'])
         return opt
 
+    def runtime_library_dir_option(self, dir):
+        return '-R"%s"' % dir
+
 if __name__ == '__main__':
     from distutils import log
     log.set_verbosity(2)
diff --git a/numpy/distutils/system_info.py b/numpy/distutils/system_info.py
index 25e8a63df..3459f67f2 100644
--- a/numpy/distutils/system_info.py
+++ b/numpy/distutils/system_info.py
@@ -198,6 +198,7 @@ if sys.platform == 'win32':
     default_lib_dirs = ['C:\\',
                         os.path.join(distutils.sysconfig.EXEC_PREFIX,
                                      'libs')]
+    default_runtime_dirs = []
     default_include_dirs = []
     default_src_dirs = ['.']
     default_x11_lib_dirs = []
@@ -205,6 +206,7 @@ if sys.platform == 'win32':
 else:
     default_lib_dirs = libpaths(['/usr/local/lib', '/opt/lib', '/usr/lib',
                                  '/opt/local/lib', '/sw/lib'], platform_bits)
+    default_runtime_dirs = []
     default_include_dirs = ['/usr/local/include',
                             '/opt/include', '/usr/include',
                             # path of umfpack under macports
@@ -254,6 +256,7 @@ if os.path.join(sys.prefix, 'lib') not in default_lib_dirs:
     default_src_dirs.append(os.path.join(sys.prefix, 'src'))
 
 default_lib_dirs = [_m for _m in default_lib_dirs if os.path.isdir(_m)]
+default_runtime_dirs = [_m for _m in default_runtime_dirs if os.path.isdir(_m)]
 default_include_dirs = [_m for _m in default_include_dirs if os.path.isdir(_m)]
 default_src_dirs = [_m for _m in default_src_dirs if os.path.isdir(_m)]
 
@@ -470,8 +473,12 @@ class system_info(object):
         defaults = {}
         defaults['library_dirs'] = os.pathsep.join(default_lib_dirs)
         defaults['include_dirs'] = os.pathsep.join(default_include_dirs)
+        defaults['runtime_library_dirs'] = os.pathsep.join(default_runtime_dirs)
+        defaults['rpath'] = ''
         defaults['src_dirs'] = os.pathsep.join(default_src_dirs)
         defaults['search_static_first'] = str(self.search_static_first)
+        defaults['extra_compile_args'] = ''
+        defaults['extra_link_args'] = ''
         self.cp = ConfigParser(defaults)
         self.files = []
         self.files.extend(get_standard_file('.numpy-site.cfg'))
@@ -491,6 +498,11 @@ class system_info(object):
     def calc_libraries_info(self):
         libs = self.get_libraries()
         dirs = self.get_lib_dirs()
+        # The extensions use runtime_library_dirs
+        r_dirs = self.get_runtime_lib_dirs() 
+        # Intrinsic distutils use rpath, we simply append both entries
+        # as though they were one entry
+        r_dirs.extend(self.get_runtime_lib_dirs(key='rpath'))
         info = {}
         for lib in libs:
             i = self.check_libs(dirs, [lib])
@@ -498,17 +510,46 @@ class system_info(object):
                 dict_append(info, **i)
             else:
                 log.info('Library %s was not found. Ignoring' % (lib))
+            i = self.check_libs(r_dirs, [lib])
+            if i is not None:
+                # Swap library keywords found to runtime_library_dirs
+                # the libraries are insisting on the user having defined
+                # them using the library_dirs, and not necessarily by
+                # runtime_library_dirs
+                del i['libraries']
+                i['runtime_library_dirs'] = i.pop('library_dirs')
+                dict_append(info, **i)
+            else:
+                log.info('Runtime library %s was not found. Ignoring' % (lib))
         return info
 
     def set_info(self, **info):
         if info:
             lib_info = self.calc_libraries_info()
             dict_append(info, **lib_info)
+            # Update extra information
+            extra_info = self.calc_extra_info()
+            dict_append(info, **extra_info)
         self.saved_results[self.__class__.__name__] = info
 
     def has_info(self):
         return self.__class__.__name__ in self.saved_results
 
+    def calc_extra_info(self):
+        """ Updates the information in the current information with
+        respect to these flags:
+          extra_compile_args
+          extra_link_args
+        """
+        info = {}
+        for key in ['extra_compile_args', 'extra_link_args']:
+            # Get values
+            opt = self.cp.get(self.section, key)
+            if opt:
+                tmp = {key : [opt]}
+                dict_append(info, **tmp)
+        return info
+
     def get_info(self, notfound_action=0):
         """ Return a dictonary with items that are compatible
             with numpy.distutils.setup keyword arguments.
@@ -603,6 +644,9 @@ class system_info(object):
     def get_lib_dirs(self, key='library_dirs'):
         return self.get_paths(self.section, key)
 
+    def get_runtime_lib_dirs(self, key='runtime_library_dirs'):
+        return self.get_paths(self.section, key)
+
     def get_include_dirs(self, key='include_dirs'):
         return self.get_paths(self.section, key)
 
@@ -2307,7 +2351,9 @@ def dict_append(d, **kws):
             languages.append(v)
             continue
         if k in d:
-            if k in ['library_dirs', 'include_dirs', 'define_macros']:
+            if k in ['library_dirs', 'include_dirs', 
+                     'extra_compile_args', 'extra_link_args',
+                     'runtime_library_dirs', 'define_macros']:
                 [d[k].append(vv) for vv in v if vv not in d[k]]
             else:
                 d[k].extend(v)
diff --git a/numpy/distutils/tests/test_system_info.py b/numpy/distutils/tests/test_system_info.py
new file mode 100644
index 000000000..d459acde7
--- /dev/null
+++ b/numpy/distutils/tests/test_system_info.py
@@ -0,0 +1,203 @@
+from __future__ import division, print_function
+
+import os
+import shutil
+from tempfile import mkstemp, mkdtemp
+
+from numpy.distutils import ccompiler
+from numpy.testing import TestCase, run_module_suite, assert_, assert_equal
+from numpy.distutils.system_info import system_info, ConfigParser
+from numpy.distutils.system_info import default_lib_dirs, default_include_dirs
+
+def get_class(name, notfound_action=1):
+    """
+    notfound_action:
+      0 - do nothing
+      1 - display warning message
+      2 - raise error
+    """
+    cl = {'temp1': TestTemp1,
+          'temp2': TestTemp2
+          }.get(name.lower(), test_system_info)
+    return cl()
+
+simple_site = """
+[ALL]
+library_dirs = {dir1:s}:{dir2:s}
+libraries = {lib1:s},{lib2:s}
+extra_compile_args = -I/fake/directory
+runtime_library_dirs = {dir1:s}
+
+[temp1]
+library_dirs = {dir1:s}
+libraries = {lib1:s}
+runtime_library_dirs = {dir1:s}
+
+[temp2]
+library_dirs = {dir2:s}
+libraries = {lib2:s}
+extra_link_args = -Wl,-rpath={lib2:s}
+rpath = {dir2:s}
+"""
+site_cfg = simple_site
+
+fakelib_c_text = """
+/* This file is generated from numpy/distutils/testing/test_system_info.py */
+#include<stdio.h>
+void foo(void) {
+   printf("Hello foo");
+}
+void bar(void) {
+   printf("Hello bar");
+}
+"""
+
+
+class test_system_info(system_info):
+    def __init__(self,
+                 default_lib_dirs=default_lib_dirs,
+                 default_include_dirs=default_include_dirs,
+                 verbosity=1,
+                 ):
+        self.__class__.info = {}
+        self.local_prefixes = []
+        defaults = {}
+        defaults['library_dirs'] = ''
+        defaults['include_dirs'] = ''
+        defaults['runtime_library_dirs'] = ''
+        defaults['rpath'] = ''
+        defaults['src_dirs'] = ''
+        defaults['search_static_first'] = "0"
+        defaults['extra_compile_args'] = ''
+        defaults['extra_link_args'] = ''
+        self.cp = ConfigParser(defaults)
+        # We have to parse the config files afterwards
+        # to have a consistent temporary filepath
+        
+    def _check_libs(self, lib_dirs, libs, opt_libs, exts):
+        """Override _check_libs to return with all dirs """
+        info = {'libraries' : libs , 'library_dirs' : lib_dirs}
+        return info
+
+
+class TestTemp1(test_system_info):
+    section = 'temp1'
+
+
+class TestTemp2(test_system_info):
+    section = 'temp2'
+
+
+class TestSystemInfoReading(TestCase):
+
+    def setUp(self):
+        """ Create the libraries """
+        # Create 2 sources and 2 libraries
+        self._dir1 = mkdtemp()
+        self._src1 = os.path.join(self._dir1, 'foo.c')
+        self._lib1 = os.path.join(self._dir1, 'libfoo.so')
+        self._dir2 = mkdtemp()
+        self._src2 = os.path.join(self._dir2, 'bar.c')
+        self._lib2 = os.path.join(self._dir2, 'libbar.so')
+        # Update local site.cfg
+        global simple_site, site_cfg
+        site_cfg = simple_site.format(**{
+                'dir1' : self._dir1,
+                'lib1' : self._lib1,
+                'dir2' : self._dir2,
+                'lib2' : self._lib2 
+                })
+        # Write site.cfg
+        fd, self._sitecfg = mkstemp()
+        os.close(fd)
+        with open(self._sitecfg, 'w') as fd:
+            fd.write(site_cfg)
+        # Write the sources
+        with open(self._src1, 'w') as fd:
+            fd.write(fakelib_c_text)
+        with open(self._src2, 'w') as fd:
+            fd.write(fakelib_c_text)
+        # We create all class-instances 
+        def site_and_parse(c, site_cfg):
+            c.files = [site_cfg]
+            c.parse_config_files()
+            return c
+        self.c_default = site_and_parse(get_class('default'), self._sitecfg)
+        self.c_temp1 = site_and_parse(get_class('temp1'), self._sitecfg)
+        self.c_temp2 = site_and_parse(get_class('temp2'), self._sitecfg)
+
+    def tearDown(self):
+        # Do each removal separately
+        try: 
+            shutil.rmtree(self._dir1)
+        except:
+            pass
+        try: 
+            shutil.rmtree(self._dir2)
+        except:
+            pass
+        try: 
+            os.remove(self._sitecfg)
+        except:
+            pass
+
+    def test_all(self):
+        # Read in all information in the ALL block
+        tsi = self.c_default
+        assert_equal(tsi.get_lib_dirs(), [self._dir1, self._dir2])
+        assert_equal(tsi.get_libraries(), [self._lib1, self._lib2])
+        assert_equal(tsi.get_runtime_lib_dirs(), [self._dir1])
+        extra = tsi.calc_extra_info()
+        assert_equal(extra['extra_compile_args'], ['-I/fake/directory'])
+
+    def test_temp1(self):
+        # Read in all information in the temp1 block
+        tsi = self.c_temp1
+        assert_equal(tsi.get_lib_dirs(), [self._dir1])
+        assert_equal(tsi.get_libraries(), [self._lib1])
+        assert_equal(tsi.get_runtime_lib_dirs(), [self._dir1])
+
+    def test_temp2(self):
+        # Read in all information in the temp2 block
+        tsi = self.c_temp2
+        assert_equal(tsi.get_lib_dirs(), [self._dir2])
+        assert_equal(tsi.get_libraries(), [self._lib2])
+        # Now from rpath and not runtime_library_dirs
+        assert_equal(tsi.get_runtime_lib_dirs(key='rpath'), [self._dir2])
+        extra = tsi.calc_extra_info()
+        assert_equal(extra['extra_link_args'], ['-Wl,-rpath='+self._lib2])
+
+    def test_compile1(self):
+        # Compile source and link the first source
+        tsi = self.c_temp1
+        c = ccompiler.new_compiler()
+        try:
+            # Change directory to not screw up directories
+            previousDir = os.getcwd()
+            os.chdir(self._dir1)
+            c.compile([os.path.basename(self._src1)], output_dir=self._dir1)
+            # Ensure that the object exists
+            assert_(os.path.isfile(self._src1.replace('.c', '.o')))
+            os.chdir(previousDir)
+        except OSError:
+            pass
+
+    def test_compile2(self):
+        # Compile source and link the second source
+        tsi = self.c_temp2
+        c = ccompiler.new_compiler()
+        extra_link_args = tsi.calc_extra_info()['extra_link_args']
+        try:
+            # Change directory to not screw up directories
+            previousDir = os.getcwd()
+            os.chdir(self._dir2)
+            c.compile([os.path.basename(self._src2)], output_dir=self._dir2,
+                      extra_postargs=extra_link_args)
+            # Ensure that the object exists
+            assert_(os.path.isfile(self._src2.replace('.c', '.o')))
+            os.chdir(previousDir)
+        except OSError:
+            pass
+        
+if __name__ == '__main__':
+    run_module_suite()
diff --git a/site.cfg.example b/site.cfg.example
index cc92edb59..04105e866 100644
--- a/site.cfg.example
+++ b/site.cfg.example
@@ -52,6 +52,28 @@
 #       True) to tell numpy.distutils to prefer static libraries (.a) over
 #       shared libraries (.so). It is turned off by default.
 #           search_static_first = false
+#
+#   runtime_library_dirs/rpath
+#       List of directories that contains the libraries that should be 
+#       used at runtime, thereby disregarding the LD_LIBRARY_PATH variable.
+#       See 'library_dirs' for formatting on different platforms.
+#           runtime_library_dirs = /opt/blas/lib:/opt/lapack/lib
+#       or equivalently
+#           rpath = /opt/blas/lib:/opt/lapack/lib
+#
+#   extra_compile_args
+#       Add additional arguments to the compilation of sources.
+#       Simple variable with no parsing done. 
+#       Provide a single line with all complete flags.
+#           extra_compile_args = -g -ftree-vectorize
+#
+#   extra_link_args
+#       Add additional arguments to when libraries/executables
+#       are linked.
+#       Simple variable with no parsing done. 
+#       Provide a single line with all complete flags.
+#           extra_link_args = -lgfortran
+#
 
 # Defaults
 # ========
@@ -59,9 +81,10 @@
 # This is a good place to add general library and include directories like
 # /usr/local/{lib,include}
 #
-#[DEFAULT]
+#[ALL]
 #library_dirs = /usr/local/lib
 #include_dirs = /usr/local/include
+#
 
 # Atlas
 # -----
@@ -82,6 +105,9 @@
 # instead of Atlas, use this section instead of the above, adjusting as needed
 # for your configuration (in the following example we installed OpenBLAS with
 # ``make install PREFIX=/opt/OpenBLAS``.
+# OpenBLAS is generically installed as a shared library, to force the OpenBLAS
+# library linked to also be used at runtime you can utilize the 
+# runtime_library_dirs variable.
 #
 # **Warning**: OpenBLAS, by default, is built in multithreaded mode. Due to the
 # way Python's multiprocessing is implemented, a multithreaded OpenBLAS can
@@ -102,6 +128,7 @@
 # libraries = openblas
 # library_dirs = /opt/OpenBLAS/lib
 # include_dirs = /opt/OpenBLAS/include
+# runtime_library_dirs = /opt/OpenBLAS/lib
 
 # MKL
 #----