summaryrefslogtreecommitdiff
path: root/numpy/distutils
diff options
context:
space:
mode:
Diffstat (limited to 'numpy/distutils')
-rw-r--r--numpy/distutils/command/build_clib.py42
-rw-r--r--numpy/distutils/command/build_ext.py70
-rw-r--r--numpy/distutils/fcompiler/__init__.py32
-rw-r--r--numpy/distutils/fcompiler/gnu.py151
-rw-r--r--numpy/distutils/misc_util.py24
-rw-r--r--numpy/distutils/system_info.py120
6 files changed, 264 insertions, 175 deletions
diff --git a/numpy/distutils/command/build_clib.py b/numpy/distutils/command/build_clib.py
index 594c3f67f..910493a77 100644
--- a/numpy/distutils/command/build_clib.py
+++ b/numpy/distutils/command/build_clib.py
@@ -12,8 +12,8 @@ from distutils.errors import DistutilsSetupError, DistutilsError, \
from numpy.distutils import log
from distutils.dep_util import newer_group
from numpy.distutils.misc_util import filter_sources, has_f_sources,\
- has_cxx_sources, all_strings, get_lib_source_files, is_sequence, \
- get_numpy_include_dirs, get_library_names
+ has_cxx_sources, all_strings, get_lib_source_files, is_sequence, \
+ get_numpy_include_dirs
# Fix Python distutils bug sf #1718574:
_l = old_build_clib.user_options
@@ -131,11 +131,6 @@ class build_clib(old_build_clib):
return filenames
def build_libraries(self, libraries):
- library_names = get_library_names()
- library_order = {v: k for k, v in enumerate(library_names)}
- libraries = sorted(
- libraries, key=lambda library: library_order[library[0]])
-
for (lib_name, build_info) in libraries:
self.build_a_library(build_info, lib_name, libraries)
@@ -292,13 +287,32 @@ class build_clib(old_build_clib):
else:
f_objects = []
- objects.extend(f_objects)
-
- # assume that default linker is suitable for
- # linking Fortran object files
- compiler.create_static_lib(objects, lib_name,
- output_dir=self.build_clib,
- debug=self.debug)
+ if f_objects and not fcompiler.can_ccompiler_link(compiler):
+ # Default linker cannot link Fortran object files, and results
+ # need to be wrapped later. Instead of creating a real static
+ # library, just keep track of the object files.
+ listfn = os.path.join(self.build_clib,
+ lib_name + '.fobjects')
+ with open(listfn, 'w') as f:
+ f.write("\n".join(os.path.abspath(obj) for obj in f_objects))
+
+ listfn = os.path.join(self.build_clib,
+ lib_name + '.cobjects')
+ with open(listfn, 'w') as f:
+ f.write("\n".join(os.path.abspath(obj) for obj in objects))
+
+ # create empty "library" file for dependency tracking
+ lib_fname = os.path.join(self.build_clib,
+ lib_name + compiler.static_lib_extension)
+ with open(lib_fname, 'wb') as f:
+ pass
+ else:
+ # assume that default linker is suitable for
+ # linking Fortran object files
+ objects.extend(f_objects)
+ compiler.create_static_lib(objects, lib_name,
+ output_dir=self.build_clib,
+ debug=self.debug)
# fix library dependencies
clib_libraries = build_info.get('libraries', [])
diff --git a/numpy/distutils/command/build_ext.py b/numpy/distutils/command/build_ext.py
index cf0747d2f..d935a3303 100644
--- a/numpy/distutils/command/build_ext.py
+++ b/numpy/distutils/command/build_ext.py
@@ -119,6 +119,11 @@ class build_ext (old_build_ext):
self.compiler.customize_cmd(self)
self.compiler.show_customization()
+ # Setup directory for storing generated extra DLL files on Windows
+ self.extra_dll_dir = os.path.join(self.build_temp, 'extra-dll')
+ if not os.path.isdir(self.extra_dll_dir):
+ os.makedirs(self.extra_dll_dir)
+
# Create mapping of libraries built by build_clib:
clibs = {}
if build_clib is not None:
@@ -256,18 +261,16 @@ class build_ext (old_build_ext):
# Build extensions
self.build_extensions()
- shared_libs = system_info.shared_libs
- if shared_libs:
- runtime_lib_dir = os.path.join(
- self.build_lib, self.distribution.get_name(), '_lib')
- try:
+ # Copy over any extra DLL files
+ runtime_lib_dir = os.path.join(
+ self.build_lib, self.distribution.get_name(), 'extra-dll')
+ for fn in os.listdir(self.extra_dll_dir):
+ if not fn.lower().endswith('.dll'):
+ continue
+ if not os.path.isdir(runtime_lib_dir):
os.makedirs(runtime_lib_dir)
- except OSError:
- pass
-
- for runtime_lib in shared_libs:
- if runtime_lib:
- copy_file(runtime_lib, runtime_lib_dir)
+ runtime_lib = os.path.join(self.extra_dll_dir, fn)
+ copy_file(runtime_lib, runtime_lib_dir)
def swig_sources(self, sources):
# Do nothing. Swig sources have beed handled in build_src command.
@@ -422,7 +425,12 @@ class build_ext (old_build_ext):
extra_postargs=extra_postargs,
depends=ext.depends)
- objects = c_objects + f_objects
+ if f_objects and not fcompiler.can_ccompiler_link(self.compiler):
+ unlinkable_fobjects = f_objects
+ objects = c_objects
+ else:
+ unlinkable_fobjects = []
+ objects = c_objects + f_objects
if ext.extra_objects:
objects.extend(ext.extra_objects)
@@ -443,6 +451,12 @@ class build_ext (old_build_ext):
if ext.language == 'c++' and cxx_compiler is not None:
linker = cxx_compiler.link_shared_object
+ if fcompiler is not None:
+ objects, libraries = self._process_unlinkable_fobjects(
+ objects, libraries,
+ fcompiler, library_dirs,
+ unlinkable_fobjects)
+
linker(objects, ext_filename,
libraries=libraries,
library_dirs=library_dirs,
@@ -462,6 +476,38 @@ class build_ext (old_build_ext):
self.compiler.create_static_lib(
objects, "_gfortran_workaround", output_dir=build_clib, debug=self.debug)
+ def _process_unlinkable_fobjects(self, objects, libraries,
+ fcompiler, library_dirs,
+ unlinkable_fobjects):
+ libraries = list(libraries)
+ objects = list(objects)
+ unlinkable_fobjects = list(unlinkable_fobjects)
+
+ # Expand possible fake static libraries to objects
+ for lib in list(libraries):
+ for libdir in library_dirs:
+ fake_lib = os.path.join(libdir, lib + '.fobjects')
+ if os.path.isfile(fake_lib):
+ # Replace fake static library
+ libraries.remove(lib)
+ with open(fake_lib, 'r') as f:
+ unlinkable_fobjects.extend(f.read().splitlines())
+
+ # Expand C objects
+ c_lib = os.path.join(libdir, lib + '.cobjects')
+ with open(c_lib, 'r') as f:
+ objects.extend(f.read().splitlines())
+
+ # Wrap unlinkable objects to a linkable one
+ if unlinkable_fobjects:
+ fobjects = [os.path.relpath(obj) for obj in unlinkable_fobjects]
+ wrapped = fcompiler.wrap_unlinkable_objects(
+ fobjects, output_dir=self.build_temp,
+ extra_dll_dir=self.extra_dll_dir)
+ objects.extend(wrapped)
+
+ return objects, libraries
+
def _libs_with_msvc_and_fortran(self, fcompiler, c_libraries,
c_library_dirs):
if fcompiler is None:
diff --git a/numpy/distutils/fcompiler/__init__.py b/numpy/distutils/fcompiler/__init__.py
index 9d465e9d8..1d558319d 100644
--- a/numpy/distutils/fcompiler/__init__.py
+++ b/numpy/distutils/fcompiler/__init__.py
@@ -698,6 +698,38 @@ class FCompiler(CCompiler):
else:
return hook_name()
+ def can_ccompiler_link(self, ccompiler):
+ """
+ Check if the given C compiler can link objects produced by
+ this compiler.
+ """
+ return True
+
+ def wrap_unlinkable_objects(self, objects, output_dir, extra_dll_dir):
+ """
+ Convert a set of object files that are not compatible with the default
+ linker, to a file that is compatible.
+
+ Parameters
+ ----------
+ objects : list
+ List of object files to include.
+ output_dir : str
+ Output directory to place generated object files.
+ extra_dll_dir : str
+ Output directory to place extra DLL files that need to be
+ included on Windows.
+
+ Returns
+ -------
+ converted_objects : list of str
+ List of converted object files.
+ Note that the number of output files is not necessarily
+ the same as inputs.
+
+ """
+ raise NotImplementedError()
+
## class FCompiler
_default_compilers = (
diff --git a/numpy/distutils/fcompiler/gnu.py b/numpy/distutils/fcompiler/gnu.py
index 0b2c1c6de..17bd320c3 100644
--- a/numpy/distutils/fcompiler/gnu.py
+++ b/numpy/distutils/fcompiler/gnu.py
@@ -6,8 +6,8 @@ import sys
import warnings
import platform
import tempfile
-import random
-import string
+import hashlib
+import base64
from subprocess import Popen, PIPE, STDOUT
from copy import copy
from numpy.distutils.fcompiler import FCompiler
@@ -305,8 +305,6 @@ class Gnu95FCompiler(GnuFCompiler):
return arch_flags
def get_flags(self):
- if self.c_compiler.compiler_type == "msvc" and not is_win64():
- return ['-O0']
flags = GnuFCompiler.get_flags(self)
arch_flags = self._universal_flags(self.compiler_f90)
if arch_flags:
@@ -361,16 +359,22 @@ class Gnu95FCompiler(GnuFCompiler):
return m.group(1)
return ""
- @staticmethod
- def _generate_id(size=6, chars=string.ascii_uppercase + string.digits):
- return ''.join(random.choice(chars) for _ in range(size))
-
- def link_wrapper_lib(self,
- objects=[],
- libraries=[],
- library_dirs=[],
- output_dir=None,
- debug=False):
+ def _hash_files(self, filenames):
+ h = hashlib.sha1()
+ for fn in filenames:
+ with open(fn, 'rb') as f:
+ while True:
+ block = f.read(131072)
+ if not block:
+ break
+ h.update(block)
+ text = base64.b32encode(h.digest())
+ if sys.version_info[0] >= 3:
+ text = text.decode('ascii')
+ return text.rstrip('=')
+
+ def _link_wrapper_lib(self, objects, output_dir, extra_dll_dir,
+ chained_dlls, is_archive):
"""Create a wrapper shared library for the given objects
Return an MSVC-compatible lib
@@ -380,32 +384,42 @@ class Gnu95FCompiler(GnuFCompiler):
if c_compiler.compiler_type != "msvc":
raise ValueError("This method only supports MSVC")
+ object_hash = self._hash_files(objects)
+
if is_win64():
tag = 'win_amd64'
else:
tag = 'win32'
- root_name = self._generate_id() + '.gfortran-' + tag
+ basename = 'lib' + os.path.splitext(
+ os.path.basename(objects[0]))[0][:8]
+ root_name = basename + '.' + object_hash + '.gfortran-' + tag
dll_name = root_name + '.dll'
- dll_path = os.path.join(output_dir, dll_name)
- def_path = root_name + '.def'
- lib_path = os.path.join(output_dir, root_name + '.lib')
-
+ def_name = root_name + '.def'
+ lib_name = root_name + '.lib'
+ dll_path = os.path.join(extra_dll_dir, dll_name)
+ def_path = os.path.join(output_dir, def_name)
+ lib_path = os.path.join(output_dir, lib_name)
+
+ if os.path.isfile(lib_path):
+ # Nothing to do
+ return lib_path, dll_path
+
+ if is_archive:
+ objects = (["-Wl,--whole-archive"] + list(objects) +
+ ["-Wl,--no-whole-archive"])
self.link_shared_object(
objects,
dll_name,
- output_dir=output_dir,
- extra_postargs=list(system_info.shared_libs) + [
+ output_dir=extra_dll_dir,
+ extra_postargs=list(chained_dlls) + [
+ '-Wl,--allow-multiple-definition',
'-Wl,--output-def,' + def_path,
'-Wl,--export-all-symbols',
'-Wl,--enable-auto-import',
'-static',
'-mlong-double-64',
- ] + ['-l' + library for library in libraries] +
- ['-L' + lib_dir for lib_dir in library_dirs],
- debug=debug)
-
- system_info.shared_libs.add(dll_path)
+ ])
# No PowerPC!
if is_win64():
@@ -419,49 +433,56 @@ class Gnu95FCompiler(GnuFCompiler):
c_compiler.initialize()
c_compiler.spawn([c_compiler.lib] + lib_args)
- return lib_path
+ return lib_path, dll_path
- def compile(self,
- sources,
- output_dir,
- macros=[],
- include_dirs=[],
- debug=False,
- extra_postargs=[],
- depends=[],
- **kwargs):
- c_compiler = self.c_compiler
- if c_compiler and c_compiler.compiler_type == "msvc":
- # MSVC cannot link objects compiled by GNU fortran
- # so we need to contain the damage here. Immediately
- # compile a DLL and return the lib for the DLL as
+ def can_ccompiler_link(self, compiler):
+ # MSVC cannot link objects compiled by GNU fortran
+ return compiler.compiler_type not in ("msvc", )
+
+ def wrap_unlinkable_objects(self, objects, output_dir, extra_dll_dir):
+ """
+ Convert a set of object files that are not compatible with the default
+ linker, to a file that is compatible.
+ """
+ if self.c_compiler.compiler_type == "msvc":
+ # Compile a DLL and return the lib for the DLL as
# the object. Also keep track of previous DLLs that
# we have compiled so that we can link against them.
- objects = GnuFCompiler.compile(
- self,
- sources,
- output_dir=output_dir,
- macros=macros,
- include_dirs=include_dirs,
- debug=debug,
- extra_postargs=extra_postargs,
- depends=depends)
-
- lib_path = self.link_wrapper_lib(objects, output_dir=output_dir)
-
- # Return the lib that we created as the "object"
- return [lib_path]
- else:
- return GnuFCompiler.compile(
- self,
- sources,
+
+ # If there are .a archives, assume they are self-contained
+ # static libraries, and build separate DLLs for each
+ archives = []
+ plain_objects = []
+ for obj in objects:
+ if obj.lower().endswith('.a'):
+ archives.append(obj)
+ else:
+ plain_objects.append(obj)
+
+ chained_libs = []
+ chained_dlls = []
+ for archive in archives[::-1]:
+ lib, dll = self._link_wrapper_lib(
+ [archive],
+ output_dir,
+ extra_dll_dir,
+ chained_dlls=chained_dlls,
+ is_archive=True)
+ chained_libs.insert(0, lib)
+ chained_dlls.insert(0, dll)
+
+ if not plain_objects:
+ return chained_libs
+
+ lib, dll = self._link_wrapper_lib(
+ plain_objects,
output_dir,
- macros=macros,
- include_dirs=include_dirs,
- debug=debug,
- extra_postargs=extra_postargs,
- depends=depends,
- **kwargs)
+ extra_dll_dir,
+ chained_dlls=chained_dlls,
+ is_archive=False)
+ return [lib] + chained_libs
+ else:
+ raise ValueError("Unsupported C compiler")
def _can_target(cmd, arch):
diff --git a/numpy/distutils/misc_util.py b/numpy/distutils/misc_util.py
index 3221e4da3..01376a7ff 100644
--- a/numpy/distutils/misc_util.py
+++ b/numpy/distutils/misc_util.py
@@ -33,14 +33,6 @@ def clean_up_temporary_directory():
atexit.register(clean_up_temporary_directory)
-# stores the order in which the libraries were added
-_ldata = []
-def get_library_names():
- """Get the library names in order"""
- global _ldata
-
- return _ldata
-
from numpy.distutils.compat import get_exception
from numpy.compat import basestring
from numpy.compat import npy_load_module
@@ -1565,10 +1557,6 @@ class Configuration(object):
name = name #+ '__OF__' + self.name
build_info['sources'] = sources
- global _ldata
- # Track the library order
- _ldata += [name]
-
# Sometimes, depends is not set up to an empty list by default, and if
# depends is not given to add_library, distutils barfs (#1134)
if not 'depends' in build_info:
@@ -2077,7 +2065,6 @@ class Configuration(object):
"""
self.py_modules.append((self.name, name, generate_config_py))
-
def get_info(self,*names):
"""Get resources information.
@@ -2295,13 +2282,12 @@ def generate_config_py(target):
f.write('# It contains system_info results at the time of building this package.\n')
f.write('__all__ = ["get_info","show"]\n\n')
- if system_info.shared_libs:
- f.write("""
-
+ # For gfortran+msvc combination, extra shared libraries may exist
+ f.write("""
import os
-
-os.environ["PATH"] += os.pathsep + os.path.join(os.path.dirname(__file__), '_lib')
-
+extra_dll_dir = os.path.join(os.path.dirname(__file__), 'extra-dll')
+if os.path.isdir(extra_dll_dir):
+ os.environ["PATH"] += os.pathsep + extra_dll_dir
""")
for k, i in system_info.saved_results.items():
diff --git a/numpy/distutils/system_info.py b/numpy/distutils/system_info.py
index 05eedfd3f..65d2fdb4e 100644
--- a/numpy/distutils/system_info.py
+++ b/numpy/distutils/system_info.py
@@ -470,8 +470,6 @@ class system_info(object):
verbosity = 1
saved_results = {}
- shared_libs = set()
-
notfounderror = NotFoundError
def __init__(self,
@@ -687,9 +685,14 @@ class system_info(object):
return self.get_libs(key, '')
def library_extensions(self):
- static_exts = ['.a']
+ c = distutils.ccompiler.new_compiler()
+ c.customize('')
+ static_exts = []
+ if c.compiler_type != 'msvc':
+ # MSVC doesn't understand binutils
+ static_exts.append('.a')
if sys.platform == 'win32':
- static_exts.append('.lib') # .lib is used by MSVC
+ static_exts.append('.lib') # .lib is used by MSVC and others
if self.search_static_first:
exts = static_exts + [so_ext]
else:
@@ -1742,12 +1745,29 @@ class openblas_info(blas_info):
return True
def calc_info(self):
+ c = distutils.ccompiler.new_compiler()
+ c.customize('')
+
lib_dirs = self.get_lib_dirs()
openblas_libs = self.get_libs('libraries', self._lib_names)
if openblas_libs == self._lib_names: # backward compat with 1.8.0
openblas_libs = self.get_libs('openblas_libs', self._lib_names)
+
info = self.check_libs(lib_dirs, openblas_libs, [])
+
+ if c.compiler_type == "msvc" and info is None:
+ from numpy.distutils.fcompiler import new_fcompiler
+ f = new_fcompiler(c_compiler=c)
+ if f.compiler_type == 'gnu95':
+ # Try gfortran-compatible library files
+ info = self.check_msvc_gfortran_libs(lib_dirs, openblas_libs)
+ # Skip lapack check, we'd need build_ext to do it
+ assume_lapack = True
+ else:
+ assume_lapack = False
+ info['language'] = 'c'
+
if info is None:
return
@@ -1755,13 +1775,42 @@ class openblas_info(blas_info):
extra_info = self.calc_extra_info()
dict_append(info, **extra_info)
- if not self.check_embedded_lapack(info):
+ if not (assume_lapack or self.check_embedded_lapack(info)):
return
- info['language'] = 'c'
info['define_macros'] = [('HAVE_CBLAS', None)]
self.set_info(**info)
+ def check_msvc_gfortran_libs(self, library_dirs, libraries):
+ # First, find the full path to each library directory
+ library_paths = []
+ for library in libraries:
+ for library_dir in library_dirs:
+ # MinGW static ext will be .a
+ fullpath = os.path.join(library_dir, library + '.a')
+ if os.path.isfile(fullpath):
+ library_paths.append(fullpath)
+ break
+ else:
+ return None
+
+ # Generate numpy.distutils virtual static library file
+ tmpdir = os.path.join(os.getcwd(), 'build', 'openblas')
+ if not os.path.isdir(tmpdir):
+ os.makedirs(tmpdir)
+
+ info = {'library_dirs': [tmpdir],
+ 'libraries': ['openblas'],
+ 'language': 'f77'}
+
+ fake_lib_file = os.path.join(tmpdir, 'openblas.fobjects')
+ fake_clib_file = os.path.join(tmpdir, 'openblas.cobjects')
+ with open(fake_lib_file, 'w') as f:
+ f.write("\n".join(library_paths))
+ with open(fake_clib_file, 'w') as f:
+ pass
+
+ return info
class openblas_lapack_info(openblas_info):
section = 'openblas'
@@ -1769,70 +1818,11 @@ class openblas_lapack_info(openblas_info):
_lib_names = ['openblas']
notfounderror = BlasNotFoundError
- def create_msvc_openblas_lib(self, info):
- from numpy.distutils.fcompiler import new_fcompiler
-
- try:
- c = distutils.ccompiler.new_compiler()
- if c.compiler_type != "msvc":
- return False
-
- f = new_fcompiler(compiler="gnu95", c_compiler=c)
- f.customize('')
-
- libraries=info['libraries']
- library_dirs=info['library_dirs']
-
- # For each gfortran-compatible static library,
- # we need to generate a dynamic library with
- # no dependencies.
-
- # First, find the full path to each library directory
- library_paths = []
- for library in libraries:
- for library_dir in library_dirs:
- # MinGW static ext will be .a
- fullpath = os.path.join(library_dir, library + '.a')
- if os.path.isfile(fullpath):
- library_paths.append(fullpath)
- break
- else:
- return False
-
-
- tmpdir = tempfile.mkdtemp()
- # Next, convert each library to MSVC format
- for library, library_path in zip(libraries, library_paths):
- mingw_lib = os.path.join(tmpdir, 'lib'+library+'.a')
- shutil.copy(library_path, mingw_lib)
- msvc_lib = f.link_wrapper_lib(['-Wl,--whole-archive',
- mingw_lib,
- '-Wl,--no-whole-archive',
- ],
- output_dir=tmpdir)
-
- msvc_lib_path = library + '.lib'
-
- # Now copy
- print('copying ' + msvc_lib + ' -> ' + msvc_lib_path)
- shutil.copy(msvc_lib, msvc_lib_path)
-
- atexit.register(os.remove, msvc_lib_path)
-
- system_info.shared_libs.add('')
-
- return True
- except:
- return False
-
def check_embedded_lapack(self, info):
res = False
c = distutils.ccompiler.new_compiler()
c.customize('')
- if c.compiler_type == "msvc" and not self.create_msvc_openblas_lib(info):
- return False
-
tmpdir = tempfile.mkdtemp()
s = """void zungqr();
int main(int argc, const char *argv[])
View Lorry Controller Status