distutils: handle unlinkable object files in build_clib/build_ext, not gnu

Add concept of unlinkable Fortran object files on the level of
build_clib/build_ext.

Make build_clib generate fake static libs when unlinkable object files are
present, postponing the actual linkage to build_ext.

This enables MSVC+gfortran DLL chaining to only involve those DLLs that
are actually necessary for each .pyd file, rather than linking everything
in to every file. Linking everything to everywhere has issues due to
potential symbol clashes and the fact that library build order is
unspecified.

Record shared_libs on disk instead of in system_info. This is necessary
for partial builds -- it is not guaranteed the compiler is actually called
for all of the DLL files.

Remove magic from openblas msvc detection. That this worked previously
relied on the side effect that the generated openblas DLL would be added
to shared_libs, and then being linked to all generated outputs.

6 files changed, 264 insertions, 175 deletions

diff --git a/numpy/distutils/command/build_clib.py b/numpy/distutils/command/build_clib.py
index 594c3f67f..910493a77 100644
--- a/numpy/distutils/command/build_clib.py
+++ b/numpy/distutils/command/build_clib.py
@@ -12,8 +12,8 @@ from distutils.errors import DistutilsSetupError, DistutilsError, \
 from numpy.distutils import log
 from distutils.dep_util import newer_group
 from numpy.distutils.misc_util import filter_sources, has_f_sources,\
-    has_cxx_sources, all_strings, get_lib_source_files, is_sequence, \
-    get_numpy_include_dirs, get_library_names
+     has_cxx_sources, all_strings, get_lib_source_files, is_sequence, \
+     get_numpy_include_dirs
 
 # Fix Python distutils bug sf #1718574:
 _l = old_build_clib.user_options
@@ -131,11 +131,6 @@ class build_clib(old_build_clib):
         return filenames
 
     def build_libraries(self, libraries):
-        library_names = get_library_names()
-        library_order = {v: k for k, v in enumerate(library_names)}
-        libraries = sorted(
-            libraries, key=lambda library: library_order[library[0]])
-
         for (lib_name, build_info) in libraries:
             self.build_a_library(build_info, lib_name, libraries)
 
@@ -292,13 +287,32 @@ class build_clib(old_build_clib):
         else:
             f_objects = []
 
-        objects.extend(f_objects)
-
-        # assume that default linker is suitable for
-        # linking Fortran object files
-        compiler.create_static_lib(objects, lib_name,
-                                   output_dir=self.build_clib,
-                                   debug=self.debug)
+        if f_objects and not fcompiler.can_ccompiler_link(compiler):
+            # Default linker cannot link Fortran object files, and results
+            # need to be wrapped later. Instead of creating a real static
+            # library, just keep track of the object files.
+            listfn = os.path.join(self.build_clib,
+                                  lib_name + '.fobjects')
+            with open(listfn, 'w') as f:
+                f.write("\n".join(os.path.abspath(obj) for obj in f_objects))
+
+            listfn = os.path.join(self.build_clib,
+                                  lib_name + '.cobjects')
+            with open(listfn, 'w') as f:
+                f.write("\n".join(os.path.abspath(obj) for obj in objects))
+
+            # create empty "library" file for dependency tracking
+            lib_fname = os.path.join(self.build_clib,
+                                     lib_name + compiler.static_lib_extension)
+            with open(lib_fname, 'wb') as f:
+                pass
+        else:
+            # assume that default linker is suitable for
+            # linking Fortran object files
+            objects.extend(f_objects)
+            compiler.create_static_lib(objects, lib_name,
+                                       output_dir=self.build_clib,
+                                       debug=self.debug)
 
         # fix library dependencies
         clib_libraries = build_info.get('libraries', [])
diff --git a/numpy/distutils/command/build_ext.py b/numpy/distutils/command/build_ext.py
index cf0747d2f..d935a3303 100644
--- a/numpy/distutils/command/build_ext.py
+++ b/numpy/distutils/command/build_ext.py
@@ -119,6 +119,11 @@ class build_ext (old_build_ext):
         self.compiler.customize_cmd(self)
         self.compiler.show_customization()
 
+        # Setup directory for storing generated extra DLL files on Windows
+        self.extra_dll_dir = os.path.join(self.build_temp, 'extra-dll')
+        if not os.path.isdir(self.extra_dll_dir):
+            os.makedirs(self.extra_dll_dir)
+
         # Create mapping of libraries built by build_clib:
         clibs = {}
         if build_clib is not None:
@@ -256,18 +261,16 @@ class build_ext (old_build_ext):
         # Build extensions
         self.build_extensions()
 
-        shared_libs = system_info.shared_libs
-        if shared_libs:
-            runtime_lib_dir = os.path.join(
-                self.build_lib, self.distribution.get_name(), '_lib')
-            try:
+        # Copy over any extra DLL files
+        runtime_lib_dir = os.path.join(
+            self.build_lib, self.distribution.get_name(), 'extra-dll')
+        for fn in os.listdir(self.extra_dll_dir):
+            if not fn.lower().endswith('.dll'):
+                continue
+            if not os.path.isdir(runtime_lib_dir):
                 os.makedirs(runtime_lib_dir)
-            except OSError:
-                pass
-
-            for runtime_lib in shared_libs:
-                if runtime_lib:
-                    copy_file(runtime_lib, runtime_lib_dir)
+            runtime_lib = os.path.join(self.extra_dll_dir, fn)
+            copy_file(runtime_lib, runtime_lib_dir)
 
     def swig_sources(self, sources):
         # Do nothing. Swig sources have beed handled in build_src command.
@@ -422,7 +425,12 @@ class build_ext (old_build_ext):
                                            extra_postargs=extra_postargs,
                                            depends=ext.depends)
 
-        objects = c_objects + f_objects
+        if f_objects and not fcompiler.can_ccompiler_link(self.compiler):
+            unlinkable_fobjects = f_objects
+            objects = c_objects
+        else:
+            unlinkable_fobjects = []
+            objects = c_objects + f_objects
 
         if ext.extra_objects:
             objects.extend(ext.extra_objects)
@@ -443,6 +451,12 @@ class build_ext (old_build_ext):
         if ext.language == 'c++' and cxx_compiler is not None:
             linker = cxx_compiler.link_shared_object
 
+        if fcompiler is not None:
+            objects, libraries = self._process_unlinkable_fobjects(
+                    objects, libraries,
+                    fcompiler, library_dirs,
+                    unlinkable_fobjects)
+
         linker(objects, ext_filename,
                libraries=libraries,
                library_dirs=library_dirs,
@@ -462,6 +476,38 @@ class build_ext (old_build_ext):
         self.compiler.create_static_lib(
             objects, "_gfortran_workaround", output_dir=build_clib, debug=self.debug)
 
+    def _process_unlinkable_fobjects(self, objects, libraries,
+                                     fcompiler, library_dirs,
+                                     unlinkable_fobjects):
+        libraries = list(libraries)
+        objects = list(objects)
+        unlinkable_fobjects = list(unlinkable_fobjects)
+
+        # Expand possible fake static libraries to objects
+        for lib in list(libraries):
+            for libdir in library_dirs:
+                fake_lib = os.path.join(libdir, lib + '.fobjects')
+                if os.path.isfile(fake_lib):
+                    # Replace fake static library
+                    libraries.remove(lib)
+                    with open(fake_lib, 'r') as f:
+                        unlinkable_fobjects.extend(f.read().splitlines())
+
+                    # Expand C objects
+                    c_lib = os.path.join(libdir, lib + '.cobjects')
+                    with open(c_lib, 'r') as f:
+                        objects.extend(f.read().splitlines())
+
+        # Wrap unlinkable objects to a linkable one
+        if unlinkable_fobjects:
+            fobjects = [os.path.relpath(obj) for obj in unlinkable_fobjects]
+            wrapped = fcompiler.wrap_unlinkable_objects(
+                    fobjects, output_dir=self.build_temp,
+                    extra_dll_dir=self.extra_dll_dir)
+            objects.extend(wrapped)
+
+        return objects, libraries
+
     def _libs_with_msvc_and_fortran(self, fcompiler, c_libraries,
                                     c_library_dirs):
         if fcompiler is None:
diff --git a/numpy/distutils/fcompiler/__init__.py b/numpy/distutils/fcompiler/__init__.py
index 9d465e9d8..1d558319d 100644
--- a/numpy/distutils/fcompiler/__init__.py
+++ b/numpy/distutils/fcompiler/__init__.py
@@ -698,6 +698,38 @@ class FCompiler(CCompiler):
         else:
             return hook_name()
 
+    def can_ccompiler_link(self, ccompiler):
+        """
+        Check if the given C compiler can link objects produced by
+        this compiler.
+        """
+        return True
+
+    def wrap_unlinkable_objects(self, objects, output_dir, extra_dll_dir):
+        """
+        Convert a set of object files that are not compatible with the default
+        linker, to a file that is compatible.
+
+        Parameters
+        ----------
+        objects : list
+            List of object files to include.
+        output_dir : str
+            Output directory to place generated object files.
+        extra_dll_dir : str
+            Output directory to place extra DLL files that need to be
+            included on Windows.
+
+        Returns
+        -------
+        converted_objects : list of str
+             List of converted object files.
+             Note that the number of output files is not necessarily
+             the same as inputs.
+
+        """
+        raise NotImplementedError()
+
     ## class FCompiler
 
 _default_compilers = (
diff --git a/numpy/distutils/fcompiler/gnu.py b/numpy/distutils/fcompiler/gnu.py
index 0b2c1c6de..17bd320c3 100644
--- a/numpy/distutils/fcompiler/gnu.py
+++ b/numpy/distutils/fcompiler/gnu.py
@@ -6,8 +6,8 @@ import sys
 import warnings
 import platform
 import tempfile
-import random
-import string
+import hashlib
+import base64
 from subprocess import Popen, PIPE, STDOUT
 from copy import copy
 from numpy.distutils.fcompiler import FCompiler
@@ -305,8 +305,6 @@ class Gnu95FCompiler(GnuFCompiler):
         return arch_flags
 
     def get_flags(self):
-        if self.c_compiler.compiler_type == "msvc" and not is_win64():
-            return ['-O0']
         flags = GnuFCompiler.get_flags(self)
         arch_flags = self._universal_flags(self.compiler_f90)
         if arch_flags:
@@ -361,16 +359,22 @@ class Gnu95FCompiler(GnuFCompiler):
                 return m.group(1)
         return ""
 
-    @staticmethod
-    def _generate_id(size=6, chars=string.ascii_uppercase + string.digits):
-        return ''.join(random.choice(chars) for _ in range(size))
-
-    def link_wrapper_lib(self,
-                         objects=[],
-                         libraries=[],
-                         library_dirs=[],
-                         output_dir=None,
-                         debug=False):
+    def _hash_files(self, filenames):
+        h = hashlib.sha1()
+        for fn in filenames:
+            with open(fn, 'rb') as f:
+                while True:
+                    block = f.read(131072)
+                    if not block:
+                        break
+                    h.update(block)
+        text = base64.b32encode(h.digest())
+        if sys.version_info[0] >= 3:
+            text = text.decode('ascii')
+        return text.rstrip('=')
+
+    def _link_wrapper_lib(self, objects, output_dir, extra_dll_dir,
+                          chained_dlls, is_archive):
         """Create a wrapper shared library for the given objects
 
         Return an MSVC-compatible lib
@@ -380,32 +384,42 @@ class Gnu95FCompiler(GnuFCompiler):
         if c_compiler.compiler_type != "msvc":
             raise ValueError("This method only supports MSVC")
 
+        object_hash = self._hash_files(objects)
+
         if is_win64():
             tag = 'win_amd64'
         else:
             tag = 'win32'
 
-        root_name = self._generate_id() + '.gfortran-' + tag
+        basename = 'lib' + os.path.splitext(
+            os.path.basename(objects[0]))[0][:8]
+        root_name = basename + '.' + object_hash + '.gfortran-' + tag
         dll_name = root_name + '.dll'
-        dll_path = os.path.join(output_dir, dll_name)
-        def_path = root_name + '.def'
-        lib_path = os.path.join(output_dir, root_name + '.lib')
-
+        def_name = root_name + '.def'
+        lib_name = root_name + '.lib'
+        dll_path = os.path.join(extra_dll_dir, dll_name)
+        def_path = os.path.join(output_dir, def_name)
+        lib_path = os.path.join(output_dir, lib_name)
+
+        if os.path.isfile(lib_path):
+            # Nothing to do
+            return lib_path, dll_path
+
+        if is_archive:
+            objects = (["-Wl,--whole-archive"] + list(objects) +
+                       ["-Wl,--no-whole-archive"])
         self.link_shared_object(
             objects,
             dll_name,
-            output_dir=output_dir,
-            extra_postargs=list(system_info.shared_libs) + [
+            output_dir=extra_dll_dir,
+            extra_postargs=list(chained_dlls) + [
+                '-Wl,--allow-multiple-definition',
                 '-Wl,--output-def,' + def_path,
                 '-Wl,--export-all-symbols',
                 '-Wl,--enable-auto-import',
                 '-static',
                 '-mlong-double-64',
-            ] + ['-l' + library for library in libraries] +
-            ['-L' + lib_dir for lib_dir in library_dirs],
-            debug=debug)
-
-        system_info.shared_libs.add(dll_path)
+            ])
 
         # No PowerPC!
         if is_win64():
@@ -419,49 +433,56 @@ class Gnu95FCompiler(GnuFCompiler):
             c_compiler.initialize()
         c_compiler.spawn([c_compiler.lib] + lib_args)
 
-        return lib_path
+        return lib_path, dll_path
 
-    def compile(self,
-                sources,
-                output_dir,
-                macros=[],
-                include_dirs=[],
-                debug=False,
-                extra_postargs=[],
-                depends=[],
-                **kwargs):
-        c_compiler = self.c_compiler
-        if c_compiler and c_compiler.compiler_type == "msvc":
-            # MSVC cannot link objects compiled by GNU fortran
-            # so we need to contain the damage here. Immediately
-            # compile a DLL and return the lib for the DLL as
+    def can_ccompiler_link(self, compiler):
+        # MSVC cannot link objects compiled by GNU fortran
+        return compiler.compiler_type not in ("msvc", )
+
+    def wrap_unlinkable_objects(self, objects, output_dir, extra_dll_dir):
+        """
+        Convert a set of object files that are not compatible with the default
+        linker, to a file that is compatible.
+        """
+        if self.c_compiler.compiler_type == "msvc":
+            # Compile a DLL and return the lib for the DLL as
             # the object. Also keep track of previous DLLs that
             # we have compiled so that we can link against them.
-            objects = GnuFCompiler.compile(
-                self,
-                sources,
-                output_dir=output_dir,
-                macros=macros,
-                include_dirs=include_dirs,
-                debug=debug,
-                extra_postargs=extra_postargs,
-                depends=depends)
-
-            lib_path = self.link_wrapper_lib(objects, output_dir=output_dir)
-
-            # Return the lib that we created as the "object"
-            return [lib_path]
-        else:
-            return GnuFCompiler.compile(
-                self,
-                sources,
+
+            # If there are .a archives, assume they are self-contained
+            # static libraries, and build separate DLLs for each
+            archives = []
+            plain_objects = []
+            for obj in objects:
+                if obj.lower().endswith('.a'):
+                    archives.append(obj)
+                else:
+                    plain_objects.append(obj)
+
+            chained_libs = []
+            chained_dlls = []
+            for archive in archives[::-1]:
+                lib, dll = self._link_wrapper_lib(
+                    [archive],
+                    output_dir,
+                    extra_dll_dir,
+                    chained_dlls=chained_dlls,
+                    is_archive=True)
+                chained_libs.insert(0, lib)
+                chained_dlls.insert(0, dll)
+
+            if not plain_objects:
+                return chained_libs
+
+            lib, dll = self._link_wrapper_lib(
+                plain_objects,
                 output_dir,
-                macros=macros,
-                include_dirs=include_dirs,
-                debug=debug,
-                extra_postargs=extra_postargs,
-                depends=depends,
-                **kwargs)
+                extra_dll_dir,
+                chained_dlls=chained_dlls,
+                is_archive=False)
+            return [lib] + chained_libs
+        else:
+            raise ValueError("Unsupported C compiler")
 
 
 def _can_target(cmd, arch):
diff --git a/numpy/distutils/misc_util.py b/numpy/distutils/misc_util.py
index 3221e4da3..01376a7ff 100644
--- a/numpy/distutils/misc_util.py
+++ b/numpy/distutils/misc_util.py
@@ -33,14 +33,6 @@ def clean_up_temporary_directory():
 
 atexit.register(clean_up_temporary_directory)
 
-# stores the order in which the libraries were added
-_ldata = []
-def get_library_names():
-    """Get the library names in order"""
-    global _ldata
-
-    return _ldata
-
 from numpy.distutils.compat import get_exception
 from numpy.compat import basestring
 from numpy.compat import npy_load_module
@@ -1565,10 +1557,6 @@ class Configuration(object):
         name = name #+ '__OF__' + self.name
         build_info['sources'] = sources
 
-        global _ldata
-        # Track the library order
-        _ldata += [name]
-
         # Sometimes, depends is not set up to an empty list by default, and if
         # depends is not given to add_library, distutils barfs (#1134)
         if not 'depends' in build_info:
@@ -2077,7 +2065,6 @@ class Configuration(object):
         """
         self.py_modules.append((self.name, name, generate_config_py))
 
-
     def get_info(self,*names):
         """Get resources information.
 
@@ -2295,13 +2282,12 @@ def generate_config_py(target):
     f.write('# It contains system_info results at the time of building this package.\n')
     f.write('__all__ = ["get_info","show"]\n\n')
 
-    if system_info.shared_libs:
-        f.write("""
-
+    # For gfortran+msvc combination, extra shared libraries may exist
+    f.write("""
 import os
-
-os.environ["PATH"] += os.pathsep + os.path.join(os.path.dirname(__file__), '_lib')
-
+extra_dll_dir = os.path.join(os.path.dirname(__file__), 'extra-dll')
+if os.path.isdir(extra_dll_dir):
+    os.environ["PATH"] += os.pathsep + extra_dll_dir
 """)
 
     for k, i in system_info.saved_results.items():
diff --git a/numpy/distutils/system_info.py b/numpy/distutils/system_info.py
index 05eedfd3f..65d2fdb4e 100644
--- a/numpy/distutils/system_info.py
+++ b/numpy/distutils/system_info.py
@@ -470,8 +470,6 @@ class system_info(object):
     verbosity = 1
     saved_results = {}
 
-    shared_libs = set()
-
     notfounderror = NotFoundError
 
     def __init__(self,
@@ -687,9 +685,14 @@ class system_info(object):
             return self.get_libs(key, '')
 
     def library_extensions(self):
-        static_exts = ['.a']
+        c = distutils.ccompiler.new_compiler()
+        c.customize('')
+        static_exts = []
+        if c.compiler_type != 'msvc':
+            # MSVC doesn't understand binutils
+            static_exts.append('.a')
         if sys.platform == 'win32':
-            static_exts.append('.lib')  # .lib is used by MSVC
+            static_exts.append('.lib')  # .lib is used by MSVC and others
         if self.search_static_first:
             exts = static_exts + [so_ext]
         else:
@@ -1742,12 +1745,29 @@ class openblas_info(blas_info):
         return True
 
     def calc_info(self):
+        c = distutils.ccompiler.new_compiler()
+        c.customize('')
+
         lib_dirs = self.get_lib_dirs()
 
         openblas_libs = self.get_libs('libraries', self._lib_names)
         if openblas_libs == self._lib_names: # backward compat with 1.8.0
             openblas_libs = self.get_libs('openblas_libs', self._lib_names)
+
         info = self.check_libs(lib_dirs, openblas_libs, [])
+
+        if c.compiler_type == "msvc" and info is None:
+            from numpy.distutils.fcompiler import new_fcompiler
+            f = new_fcompiler(c_compiler=c)
+            if f.compiler_type == 'gnu95':
+                # Try gfortran-compatible library files
+                info = self.check_msvc_gfortran_libs(lib_dirs, openblas_libs)
+                # Skip lapack check, we'd need build_ext to do it
+                assume_lapack = True
+        else:
+            assume_lapack = False
+            info['language'] = 'c'
+
         if info is None:
             return
 
@@ -1755,13 +1775,42 @@ class openblas_info(blas_info):
         extra_info = self.calc_extra_info()
         dict_append(info, **extra_info)
 
-        if not self.check_embedded_lapack(info):
+        if not (assume_lapack or self.check_embedded_lapack(info)):
             return
 
-        info['language'] = 'c'
         info['define_macros'] = [('HAVE_CBLAS', None)]
         self.set_info(**info)
 
+    def check_msvc_gfortran_libs(self, library_dirs, libraries):
+        # First, find the full path to each library directory
+        library_paths = []
+        for library in libraries:
+            for library_dir in library_dirs:
+                # MinGW static ext will be .a
+                fullpath = os.path.join(library_dir, library + '.a')
+                if os.path.isfile(fullpath):
+                    library_paths.append(fullpath)
+                    break
+            else:
+                return None
+
+        # Generate numpy.distutils virtual static library file
+        tmpdir = os.path.join(os.getcwd(), 'build', 'openblas')
+        if not os.path.isdir(tmpdir):
+            os.makedirs(tmpdir)
+
+        info = {'library_dirs': [tmpdir],
+                'libraries': ['openblas'],
+                'language': 'f77'}
+
+        fake_lib_file = os.path.join(tmpdir, 'openblas.fobjects')
+        fake_clib_file = os.path.join(tmpdir, 'openblas.cobjects')
+        with open(fake_lib_file, 'w') as f:
+            f.write("\n".join(library_paths))
+        with open(fake_clib_file, 'w') as f:
+            pass
+
+        return info
 
 class openblas_lapack_info(openblas_info):
     section = 'openblas'
@@ -1769,70 +1818,11 @@ class openblas_lapack_info(openblas_info):
     _lib_names = ['openblas']
     notfounderror = BlasNotFoundError
 
-    def create_msvc_openblas_lib(self, info):
-        from numpy.distutils.fcompiler import new_fcompiler
-        
-        try:
-            c = distutils.ccompiler.new_compiler()
-            if c.compiler_type != "msvc":
-                return False
-
-            f = new_fcompiler(compiler="gnu95", c_compiler=c)
-            f.customize('')
-
-            libraries=info['libraries']
-            library_dirs=info['library_dirs']
-
-            # For each gfortran-compatible static library, 
-            # we need to generate a dynamic library with 
-            # no dependencies.
-
-            # First, find the full path to each library directory
-            library_paths = []
-            for library in libraries:
-                for library_dir in library_dirs:
-                    # MinGW static ext will be .a
-                    fullpath = os.path.join(library_dir, library + '.a')
-                    if os.path.isfile(fullpath):
-                        library_paths.append(fullpath)
-                        break   
-                else:
-                    return False
-
-
-            tmpdir = tempfile.mkdtemp()
-            # Next, convert each library to MSVC format
-            for library, library_path in zip(libraries, library_paths):
-                mingw_lib = os.path.join(tmpdir, 'lib'+library+'.a')
-                shutil.copy(library_path, mingw_lib)
-                msvc_lib = f.link_wrapper_lib(['-Wl,--whole-archive',
-                                                mingw_lib,
-                                                '-Wl,--no-whole-archive',
-                                                ],
-                                            output_dir=tmpdir)
-
-                msvc_lib_path = library + '.lib'
-
-                # Now copy
-                print('copying ' + msvc_lib + ' -> ' + msvc_lib_path)
-                shutil.copy(msvc_lib, msvc_lib_path)
-
-                atexit.register(os.remove, msvc_lib_path)
-
-            system_info.shared_libs.add('')
-
-            return True
-        except:
-            return False
-
     def check_embedded_lapack(self, info):
         res = False
         c = distutils.ccompiler.new_compiler()
         c.customize('')
 
-        if c.compiler_type == "msvc" and not self.create_msvc_openblas_lib(info):
-            return False
-
         tmpdir = tempfile.mkdtemp()
         s = """void zungqr();
         int main(int argc, const char *argv[])