distutils: handle unlinkable object files in build_clib/build_ext, not gnu

Add concept of unlinkable Fortran object files on the level of
build_clib/build_ext.

Make build_clib generate fake static libs when unlinkable object files are
present, postponing the actual linkage to build_ext.

This enables MSVC+gfortran DLL chaining to only involve those DLLs that
are actually necessary for each .pyd file, rather than linking everything
in to every file. Linking everything to everywhere has issues due to
potential symbol clashes and the fact that library build order is
unspecified.

Record shared_libs on disk instead of in system_info. This is necessary
for partial builds -- it is not guaranteed the compiler is actually called
for all of the DLL files.

Remove magic from openblas msvc detection. That this worked previously
relied on the side effect that the generated openblas DLL would be added
to shared_libs, and then being linked to all generated outputs.

2 files changed, 86 insertions, 26 deletions

diff --git a/numpy/distutils/command/build_clib.py b/numpy/distutils/command/build_clib.py
index 594c3f67f..910493a77 100644
--- a/numpy/distutils/command/build_clib.py
+++ b/numpy/distutils/command/build_clib.py
@@ -12,8 +12,8 @@ from distutils.errors import DistutilsSetupError, DistutilsError, \
 from numpy.distutils import log
 from distutils.dep_util import newer_group
 from numpy.distutils.misc_util import filter_sources, has_f_sources,\
-    has_cxx_sources, all_strings, get_lib_source_files, is_sequence, \
-    get_numpy_include_dirs, get_library_names
+     has_cxx_sources, all_strings, get_lib_source_files, is_sequence, \
+     get_numpy_include_dirs
 
 # Fix Python distutils bug sf #1718574:
 _l = old_build_clib.user_options
@@ -131,11 +131,6 @@ class build_clib(old_build_clib):
         return filenames
 
     def build_libraries(self, libraries):
-        library_names = get_library_names()
-        library_order = {v: k for k, v in enumerate(library_names)}
-        libraries = sorted(
-            libraries, key=lambda library: library_order[library[0]])
-
         for (lib_name, build_info) in libraries:
             self.build_a_library(build_info, lib_name, libraries)
 
@@ -292,13 +287,32 @@ class build_clib(old_build_clib):
         else:
             f_objects = []
 
-        objects.extend(f_objects)
-
-        # assume that default linker is suitable for
-        # linking Fortran object files
-        compiler.create_static_lib(objects, lib_name,
-                                   output_dir=self.build_clib,
-                                   debug=self.debug)
+        if f_objects and not fcompiler.can_ccompiler_link(compiler):
+            # Default linker cannot link Fortran object files, and results
+            # need to be wrapped later. Instead of creating a real static
+            # library, just keep track of the object files.
+            listfn = os.path.join(self.build_clib,
+                                  lib_name + '.fobjects')
+            with open(listfn, 'w') as f:
+                f.write("\n".join(os.path.abspath(obj) for obj in f_objects))
+
+            listfn = os.path.join(self.build_clib,
+                                  lib_name + '.cobjects')
+            with open(listfn, 'w') as f:
+                f.write("\n".join(os.path.abspath(obj) for obj in objects))
+
+            # create empty "library" file for dependency tracking
+            lib_fname = os.path.join(self.build_clib,
+                                     lib_name + compiler.static_lib_extension)
+            with open(lib_fname, 'wb') as f:
+                pass
+        else:
+            # assume that default linker is suitable for
+            # linking Fortran object files
+            objects.extend(f_objects)
+            compiler.create_static_lib(objects, lib_name,
+                                       output_dir=self.build_clib,
+                                       debug=self.debug)
 
         # fix library dependencies
         clib_libraries = build_info.get('libraries', [])
diff --git a/numpy/distutils/command/build_ext.py b/numpy/distutils/command/build_ext.py
index cf0747d2f..d935a3303 100644
--- a/numpy/distutils/command/build_ext.py
+++ b/numpy/distutils/command/build_ext.py
@@ -119,6 +119,11 @@ class build_ext (old_build_ext):
         self.compiler.customize_cmd(self)
         self.compiler.show_customization()
 
+        # Setup directory for storing generated extra DLL files on Windows
+        self.extra_dll_dir = os.path.join(self.build_temp, 'extra-dll')
+        if not os.path.isdir(self.extra_dll_dir):
+            os.makedirs(self.extra_dll_dir)
+
         # Create mapping of libraries built by build_clib:
         clibs = {}
         if build_clib is not None:
@@ -256,18 +261,16 @@ class build_ext (old_build_ext):
         # Build extensions
         self.build_extensions()
 
-        shared_libs = system_info.shared_libs
-        if shared_libs:
-            runtime_lib_dir = os.path.join(
-                self.build_lib, self.distribution.get_name(), '_lib')
-            try:
+        # Copy over any extra DLL files
+        runtime_lib_dir = os.path.join(
+            self.build_lib, self.distribution.get_name(), 'extra-dll')
+        for fn in os.listdir(self.extra_dll_dir):
+            if not fn.lower().endswith('.dll'):
+                continue
+            if not os.path.isdir(runtime_lib_dir):
                 os.makedirs(runtime_lib_dir)
-            except OSError:
-                pass
-
-            for runtime_lib in shared_libs:
-                if runtime_lib:
-                    copy_file(runtime_lib, runtime_lib_dir)
+            runtime_lib = os.path.join(self.extra_dll_dir, fn)
+            copy_file(runtime_lib, runtime_lib_dir)
 
     def swig_sources(self, sources):
         # Do nothing. Swig sources have beed handled in build_src command.
@@ -422,7 +425,12 @@ class build_ext (old_build_ext):
                                            extra_postargs=extra_postargs,
                                            depends=ext.depends)
 
-        objects = c_objects + f_objects
+        if f_objects and not fcompiler.can_ccompiler_link(self.compiler):
+            unlinkable_fobjects = f_objects
+            objects = c_objects
+        else:
+            unlinkable_fobjects = []
+            objects = c_objects + f_objects
 
         if ext.extra_objects:
             objects.extend(ext.extra_objects)
@@ -443,6 +451,12 @@ class build_ext (old_build_ext):
         if ext.language == 'c++' and cxx_compiler is not None:
             linker = cxx_compiler.link_shared_object
 
+        if fcompiler is not None:
+            objects, libraries = self._process_unlinkable_fobjects(
+                    objects, libraries,
+                    fcompiler, library_dirs,
+                    unlinkable_fobjects)
+
         linker(objects, ext_filename,
                libraries=libraries,
                library_dirs=library_dirs,
@@ -462,6 +476,38 @@ class build_ext (old_build_ext):
         self.compiler.create_static_lib(
             objects, "_gfortran_workaround", output_dir=build_clib, debug=self.debug)
 
+    def _process_unlinkable_fobjects(self, objects, libraries,
+                                     fcompiler, library_dirs,
+                                     unlinkable_fobjects):
+        libraries = list(libraries)
+        objects = list(objects)
+        unlinkable_fobjects = list(unlinkable_fobjects)
+
+        # Expand possible fake static libraries to objects
+        for lib in list(libraries):
+            for libdir in library_dirs:
+                fake_lib = os.path.join(libdir, lib + '.fobjects')
+                if os.path.isfile(fake_lib):
+                    # Replace fake static library
+                    libraries.remove(lib)
+                    with open(fake_lib, 'r') as f:
+                        unlinkable_fobjects.extend(f.read().splitlines())
+
+                    # Expand C objects
+                    c_lib = os.path.join(libdir, lib + '.cobjects')
+                    with open(c_lib, 'r') as f:
+                        objects.extend(f.read().splitlines())
+
+        # Wrap unlinkable objects to a linkable one
+        if unlinkable_fobjects:
+            fobjects = [os.path.relpath(obj) for obj in unlinkable_fobjects]
+            wrapped = fcompiler.wrap_unlinkable_objects(
+                    fobjects, output_dir=self.build_temp,
+                    extra_dll_dir=self.extra_dll_dir)
+            objects.extend(wrapped)
+
+        return objects, libraries
+
     def _libs_with_msvc_and_fortran(self, fcompiler, c_libraries,
                                     c_library_dirs):
         if fcompiler is None: