diff options
Diffstat (limited to 'numpy/distutils/system_info.py')
| -rw-r--r-- | numpy/distutils/system_info.py | 457 |
1 files changed, 325 insertions, 132 deletions
diff --git a/numpy/distutils/system_info.py b/numpy/distutils/system_info.py index 5fd1003ab..3a6a7b29d 100644 --- a/numpy/distutils/system_info.py +++ b/numpy/distutils/system_info.py @@ -1,51 +1,8 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ This file defines a set of system_info classes for getting information about various resources (libraries, library directories, -include directories, etc.) in the system. Currently, the following -classes are available: - - atlas_info - atlas_threads_info - atlas_blas_info - atlas_blas_threads_info - lapack_atlas_info - lapack_atlas_threads_info - atlas_3_10_info - atlas_3_10_threads_info - atlas_3_10_blas_info, - atlas_3_10_blas_threads_info, - lapack_atlas_3_10_info - lapack_atlas_3_10_threads_info - flame_info - blas_info - lapack_info - openblas_info - blis_info - blas_opt_info # usage recommended - lapack_opt_info # usage recommended - fftw_info,dfftw_info,sfftw_info - fftw_threads_info,dfftw_threads_info,sfftw_threads_info - djbfft_info - x11_info - lapack_src_info - blas_src_info - numpy_info - numarray_info - numpy_info - boost_python_info - agg2_info - wx_info - gdk_pixbuf_xlib_2_info - gdk_pixbuf_2_info - gdk_x11_2_info - gtkp_x11_2_info - gtkp_2_info - xft_info - freetype2_info - umfpack_info - -Usage: +include directories, etc.) in the system. Usage: info_dict = get_info(<name>) where <name> is a string 'atlas','x11','fftw','lapack','blas', 'lapack_src', 'blas_src', etc. For a complete list of allowed names, @@ -73,19 +30,94 @@ The file 'site.cfg' is looked for in The first one found is used to get system configuration options The format is that used by ConfigParser (i.e., Windows .INI style). The -section ALL has options that are the default for each section. The -available sections are fftw, atlas, and x11. Appropriate defaults are -used if nothing is specified. +section ALL is not intended for general use. + +Appropriate defaults are used if nothing is specified. The order of finding the locations of resources is the following: 1. environment variable 2. section in site.cfg - 3. ALL section in site.cfg + 3. DEFAULT section in site.cfg + 4. System default search paths (see ``default_*`` variables below). Only the first complete match is returned. +Currently, the following classes are available, along with their section names: + + Numeric_info:Numeric + _numpy_info:Numeric + _pkg_config_info:None + accelerate_info:accelerate + agg2_info:agg2 + amd_info:amd + atlas_3_10_blas_info:atlas + atlas_3_10_blas_threads_info:atlas + atlas_3_10_info:atlas + atlas_3_10_threads_info:atlas + atlas_blas_info:atlas + atlas_blas_threads_info:atlas + atlas_info:atlas + atlas_threads_info:atlas + blas64__opt_info:ALL # usage recommended (general ILP64 BLAS, 64_ symbol suffix) + blas_ilp64_opt_info:ALL # usage recommended (general ILP64 BLAS) + blas_ilp64_plain_opt_info:ALL # usage recommended (general ILP64 BLAS, no symbol suffix) + blas_info:blas + blas_mkl_info:mkl + blas_opt_info:ALL # usage recommended + blas_src_info:blas_src + blis_info:blis + boost_python_info:boost_python + dfftw_info:fftw + dfftw_threads_info:fftw + djbfft_info:djbfft + f2py_info:ALL + fft_opt_info:ALL + fftw2_info:fftw + fftw3_info:fftw3 + fftw_info:fftw + fftw_threads_info:fftw + flame_info:flame + freetype2_info:freetype2 + gdk_2_info:gdk_2 + gdk_info:gdk + gdk_pixbuf_2_info:gdk_pixbuf_2 + gdk_pixbuf_xlib_2_info:gdk_pixbuf_xlib_2 + gdk_x11_2_info:gdk_x11_2 + gtkp_2_info:gtkp_2 + gtkp_x11_2_info:gtkp_x11_2 + lapack64__opt_info:ALL # usage recommended (general ILP64 LAPACK, 64_ symbol suffix) + lapack_atlas_3_10_info:atlas + lapack_atlas_3_10_threads_info:atlas + lapack_atlas_info:atlas + lapack_atlas_threads_info:atlas + lapack_ilp64_opt_info:ALL # usage recommended (general ILP64 LAPACK) + lapack_ilp64_plain_opt_info:ALL # usage recommended (general ILP64 LAPACK, no symbol suffix) + lapack_info:lapack + lapack_mkl_info:mkl + lapack_opt_info:ALL # usage recommended + lapack_src_info:lapack_src + mkl_info:mkl + numarray_info:numarray + numerix_info:numerix + numpy_info:numpy + openblas64__info:openblas64_ + openblas64__lapack_info:openblas64_ + openblas_clapack_info:openblas + openblas_ilp64_info:openblas_ilp64 + openblas_ilp64_lapack_info:openblas_ilp64 + openblas_info:openblas + openblas_lapack_info:openblas + sfftw_info:fftw + sfftw_threads_info:fftw + system_info:ALL + umfpack_info:umfpack + wx_info:wx + x11_info:x11 + xft_info:xft + Example: ---------- -[ALL] +[DEFAULT] +# default section library_dirs = /usr/lib:/usr/local/lib:/opt/lib include_dirs = /usr/include:/usr/local/include:/opt/include src_dirs = /usr/local/src:/opt/src @@ -120,8 +152,6 @@ this distribution for specifics. NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. """ -from __future__ import division, absolute_import, print_function - import sys import os import re @@ -132,12 +162,8 @@ import textwrap from glob import glob from functools import reduce -if sys.version_info[0] < 3: - from ConfigParser import NoOptionError - from ConfigParser import RawConfigParser as ConfigParser -else: - from configparser import NoOptionError - from configparser import RawConfigParser as ConfigParser +from configparser import NoOptionError +from configparser import RawConfigParser as ConfigParser # It seems that some people are importing ConfigParser from here so is # good to keep its class name. Use of RawConfigParser is needed in # order to be able to load path names with percent in them, like @@ -151,12 +177,11 @@ from distutils.util import get_platform from numpy.distutils.exec_command import ( find_executable, filepath_from_subprocess_output, - get_pythonexe) + ) from numpy.distutils.misc_util import (is_sequence, is_string, get_shared_lib_extension) from numpy.distutils.command.config import config as cmd_config -from numpy.distutils.compat import get_exception -from numpy.distutils import customized_ccompiler +from numpy.distutils import customized_ccompiler as _customized_ccompiler from numpy.distutils import _shell_utils import distutils.ccompiler import tempfile @@ -169,6 +194,15 @@ _bits = {'32bit': 32, '64bit': 64} platform_bits = _bits[platform.architecture()[0]] +global_compiler = None + +def customized_ccompiler(): + global global_compiler + if not global_compiler: + global_compiler = _customized_ccompiler() + return global_compiler + + def _c_string_literal(s): """ Convert a python string into a literal suitable for inclusion into C code @@ -250,32 +284,29 @@ if sys.platform == 'win32': default_include_dirs.extend( os.path.join(library_root, d) for d in _include_dirs) - if sys.version_info >= (3, 3): - # VCpkg is the de-facto package manager on windows for C/C++ - # libraries. If it is on the PATH, then we append its paths here. - # We also don't re-implement shutil.which for Python 2.7 because - # vcpkg doesn't support MSVC 2008. - vcpkg = shutil.which('vcpkg') - if vcpkg: - vcpkg_dir = os.path.dirname(vcpkg) - if platform.architecture() == '32bit': - specifier = 'x86' - else: - specifier = 'x64' - - vcpkg_installed = os.path.join(vcpkg_dir, 'installed') - for vcpkg_root in [ - os.path.join(vcpkg_installed, specifier + '-windows'), - os.path.join(vcpkg_installed, specifier + '-windows-static'), - ]: - add_system_root(vcpkg_root) - - # Conda is another popular package manager that provides libraries - conda = shutil.which('conda') - if conda: - conda_dir = os.path.dirname(conda) - add_system_root(os.path.join(conda_dir, '..', 'Library')) - add_system_root(os.path.join(conda_dir, 'Library')) + # VCpkg is the de-facto package manager on windows for C/C++ + # libraries. If it is on the PATH, then we append its paths here. + vcpkg = shutil.which('vcpkg') + if vcpkg: + vcpkg_dir = os.path.dirname(vcpkg) + if platform.architecture() == '32bit': + specifier = 'x86' + else: + specifier = 'x64' + + vcpkg_installed = os.path.join(vcpkg_dir, 'installed') + for vcpkg_root in [ + os.path.join(vcpkg_installed, specifier + '-windows'), + os.path.join(vcpkg_installed, specifier + '-windows-static'), + ]: + add_system_root(vcpkg_root) + + # Conda is another popular package manager that provides libraries + conda = shutil.which('conda') + if conda: + conda_dir = os.path.dirname(conda) + add_system_root(os.path.join(conda_dir, '..', 'Library')) + add_system_root(os.path.join(conda_dir, 'Library')) else: default_lib_dirs = libpaths(['/usr/local/lib', '/opt/lib', '/usr/lib', @@ -397,6 +428,10 @@ def get_info(name, notfound_action=0): 'lapack_mkl': lapack_mkl_info, # use lapack_opt instead 'blas_mkl': blas_mkl_info, # use blas_opt instead 'accelerate': accelerate_info, # use blas_opt instead + 'openblas64_': openblas64__info, + 'openblas64__lapack': openblas64__lapack_info, + 'openblas_ilp64': openblas_ilp64_info, + 'openblas_ilp64_lapack': openblas_ilp64_lapack_info, 'x11': x11_info, 'fft_opt': fft_opt_info, 'fftw': fftw_info, @@ -419,7 +454,13 @@ def get_info(name, notfound_action=0): 'numarray': numarray_info, 'numerix': numerix_info, 'lapack_opt': lapack_opt_info, + 'lapack_ilp64_opt': lapack_ilp64_opt_info, + 'lapack_ilp64_plain_opt': lapack_ilp64_plain_opt_info, + 'lapack64__opt': lapack64__opt_info, 'blas_opt': blas_opt_info, + 'blas_ilp64_opt': blas_ilp64_opt_info, + 'blas_ilp64_plain_opt': blas_ilp64_plain_opt_info, + 'blas64__opt': blas64__opt_info, 'boost_python': boost_python_info, 'agg2': agg2_info, 'wx': wx_info, @@ -485,6 +526,13 @@ class LapackSrcNotFoundError(LapackNotFoundError): the LAPACK_SRC environment variable.""" +class LapackILP64NotFoundError(NotFoundError): + """ + 64-bit Lapack libraries not found. + Known libraries in numpy/distutils/site.cfg file are: + openblas64_, openblas_ilp64 + """ + class BlasOptNotFoundError(NotFoundError): """ Optimized (vendor) Blas libraries are not found. @@ -499,6 +547,12 @@ class BlasNotFoundError(NotFoundError): numpy/distutils/site.cfg file (section [blas]) or by setting the BLAS environment variable.""" +class BlasILP64NotFoundError(NotFoundError): + """ + 64-bit Blas libraries not found. + Known libraries in numpy/distutils/site.cfg file are: + openblas64_, openblas_ilp64 + """ class BlasSrcNotFoundError(BlasNotFoundError): """ @@ -542,14 +596,18 @@ class UmfpackNotFoundError(NotFoundError): the UMFPACK environment variable.""" -class system_info(object): +class system_info: """ get_info() is the only public method. Don't use others. """ - section = 'ALL' dir_env_var = None - search_static_first = 0 # XXX: disabled by default, may disappear in - # future unless it is proved to be useful. + # XXX: search_static_first is disabled by default, may disappear in + # future unless it is proved to be useful. + search_static_first = 0 + # The base-class section name is a random word "ALL" and is not really + # intended for general use. It cannot be None nor can it be DEFAULT as + # these break the ConfigParser. See gh-15338 + section = 'ALL' saved_results = {} notfounderror = NotFoundError @@ -677,7 +735,7 @@ class system_info(object): return info def get_info(self, notfound_action=0): - """ Return a dictonary with items that are compatible + """ Return a dictionary with items that are compatible with numpy.distutils.setup keyword arguments. """ flag = 0 @@ -1580,7 +1638,7 @@ def get_atlas_version(**config): log.info('Status: %d', s) log.info('Output: %s', o) - if atlas_version == '3.2.1_pre3.3.6': + elif atlas_version == '3.2.1_pre3.3.6': dict_append(info, define_macros=[('NO_ATLAS_INFO', -2)]) else: dict_append(info, define_macros=[( @@ -1591,10 +1649,10 @@ def get_atlas_version(**config): class lapack_opt_info(system_info): - notfounderror = LapackNotFoundError - # Default order of LAPACK checks + # List of all known BLAS libraries, in the default order lapack_order = ['mkl', 'openblas', 'flame', 'atlas', 'accelerate', 'lapack'] + order_env_var_name = 'NPY_LAPACK_ORDER' def _calc_info_mkl(self): info = get_info('lapack_mkl') @@ -1686,8 +1744,11 @@ class lapack_opt_info(system_info): return True return False + def _calc_info(self, name): + return getattr(self, '_calc_info_{}'.format(name))() + def calc_info(self): - user_order = os.environ.get('NPY_LAPACK_ORDER', None) + user_order = os.environ.get(self.order_env_var_name, None) if user_order is None: lapack_order = self.lapack_order else: @@ -1707,7 +1768,7 @@ class lapack_opt_info(system_info): "values: {}".format(non_existing)) for lapack in lapack_order: - if getattr(self, '_calc_info_{}'.format(lapack))(): + if self._calc_info(lapack): return if 'lapack' not in lapack_order: @@ -1717,11 +1778,53 @@ class lapack_opt_info(system_info): warnings.warn(LapackSrcNotFoundError.__doc__ or '', stacklevel=2) -class blas_opt_info(system_info): +class _ilp64_opt_info_mixin: + symbol_suffix = None + symbol_prefix = None + + def _check_info(self, info): + macros = dict(info.get('define_macros', [])) + prefix = macros.get('BLAS_SYMBOL_PREFIX', '') + suffix = macros.get('BLAS_SYMBOL_SUFFIX', '') + + if self.symbol_prefix not in (None, prefix): + return False + + if self.symbol_suffix not in (None, suffix): + return False + return bool(info) + + +class lapack_ilp64_opt_info(lapack_opt_info, _ilp64_opt_info_mixin): + notfounderror = LapackILP64NotFoundError + lapack_order = ['openblas64_', 'openblas_ilp64'] + order_env_var_name = 'NPY_LAPACK_ILP64_ORDER' + + def _calc_info(self, name): + info = get_info(name + '_lapack') + if self._check_info(info): + self.set_info(**info) + return True + return False + + +class lapack_ilp64_plain_opt_info(lapack_ilp64_opt_info): + # Same as lapack_ilp64_opt_info, but fix symbol names + symbol_prefix = '' + symbol_suffix = '' + + +class lapack64__opt_info(lapack_ilp64_opt_info): + symbol_prefix = '' + symbol_suffix = '64_' + + +class blas_opt_info(system_info): notfounderror = BlasNotFoundError - # Default order of BLAS checks + # List of all known BLAS libraries, in the default order blas_order = ['mkl', 'blis', 'openblas', 'atlas', 'accelerate', 'blas'] + order_env_var_name = 'NPY_BLAS_ORDER' def _calc_info_mkl(self): info = get_info('blas_mkl') @@ -1786,8 +1889,11 @@ class blas_opt_info(system_info): self.set_info(**info) return True + def _calc_info(self, name): + return getattr(self, '_calc_info_{}'.format(name))() + def calc_info(self): - user_order = os.environ.get('NPY_BLAS_ORDER', None) + user_order = os.environ.get(self.order_env_var_name, None) if user_order is None: blas_order = self.blas_order else: @@ -1805,7 +1911,7 @@ class blas_opt_info(system_info): raise ValueError("blas_opt_info user defined BLAS order has unacceptable values: {}".format(non_existing)) for blas in blas_order: - if getattr(self, '_calc_info_{}'.format(blas))(): + if self._calc_info(blas): return if 'blas' not in blas_order: @@ -1815,6 +1921,29 @@ class blas_opt_info(system_info): warnings.warn(BlasSrcNotFoundError.__doc__ or '', stacklevel=2) +class blas_ilp64_opt_info(blas_opt_info, _ilp64_opt_info_mixin): + notfounderror = BlasILP64NotFoundError + blas_order = ['openblas64_', 'openblas_ilp64'] + order_env_var_name = 'NPY_BLAS_ILP64_ORDER' + + def _calc_info(self, name): + info = get_info(name) + if self._check_info(info): + self.set_info(**info) + return True + return False + + +class blas_ilp64_plain_opt_info(blas_ilp64_opt_info): + symbol_prefix = '' + symbol_suffix = '' + + +class blas64__opt_info(blas_ilp64_opt_info): + symbol_prefix = '' + symbol_suffix = '64_' + + class blas_info(system_info): section = 'blas' dir_env_var = 'BLAS' @@ -1914,12 +2043,24 @@ class openblas_info(blas_info): section = 'openblas' dir_env_var = 'OPENBLAS' _lib_names = ['openblas'] + _require_symbols = [] notfounderror = BlasNotFoundError - def check_embedded_lapack(self, info): - return True + @property + def symbol_prefix(self): + try: + return self.cp.get(self.section, 'symbol_prefix') + except NoOptionError: + return '' - def calc_info(self): + @property + def symbol_suffix(self): + try: + return self.cp.get(self.section, 'symbol_suffix') + except NoOptionError: + return '' + + def _calc_info(self): c = customized_ccompiler() lib_dirs = self.get_lib_dirs() @@ -1937,23 +2078,33 @@ class openblas_info(blas_info): # Try gfortran-compatible library files info = self.check_msvc_gfortran_libs(lib_dirs, openblas_libs) # Skip lapack check, we'd need build_ext to do it - assume_lapack = True + skip_symbol_check = True elif info: - assume_lapack = False + skip_symbol_check = False info['language'] = 'c' if info is None: - return + return None # Add extra info for OpenBLAS extra_info = self.calc_extra_info() dict_append(info, **extra_info) - if not (assume_lapack or self.check_embedded_lapack(info)): - return + if not (skip_symbol_check or self.check_symbols(info)): + return None info['define_macros'] = [('HAVE_CBLAS', None)] - self.set_info(**info) + if self.symbol_prefix: + info['define_macros'] += [('BLAS_SYMBOL_PREFIX', self.symbol_prefix)] + if self.symbol_suffix: + info['define_macros'] += [('BLAS_SYMBOL_SUFFIX', self.symbol_suffix)] + + return info + + def calc_info(self): + info = self._calc_info() + if info is not None: + self.set_info(**info) def check_msvc_gfortran_libs(self, library_dirs, libraries): # First, find the full path to each library directory @@ -1969,16 +2120,17 @@ class openblas_info(blas_info): return None # Generate numpy.distutils virtual static library file - tmpdir = os.path.join(os.getcwd(), 'build', 'openblas') + basename = self.__class__.__name__ + tmpdir = os.path.join(os.getcwd(), 'build', basename) if not os.path.isdir(tmpdir): os.makedirs(tmpdir) info = {'library_dirs': [tmpdir], - 'libraries': ['openblas'], + 'libraries': [basename], 'language': 'f77'} - fake_lib_file = os.path.join(tmpdir, 'openblas.fobjects') - fake_clib_file = os.path.join(tmpdir, 'openblas.cobjects') + fake_lib_file = os.path.join(tmpdir, basename + '.fobjects') + fake_clib_file = os.path.join(tmpdir, basename + '.cobjects') with open(fake_lib_file, 'w') as f: f.write("\n".join(library_paths)) with open(fake_clib_file, 'w') as f: @@ -1986,24 +2138,27 @@ class openblas_info(blas_info): return info -class openblas_lapack_info(openblas_info): - section = 'openblas' - dir_env_var = 'OPENBLAS' - _lib_names = ['openblas'] - notfounderror = BlasNotFoundError - - def check_embedded_lapack(self, info): + def check_symbols(self, info): res = False c = customized_ccompiler() tmpdir = tempfile.mkdtemp() + + prototypes = "\n".join("void %s%s%s();" % (self.symbol_prefix, + symbol_name, + self.symbol_suffix) + for symbol_name in self._require_symbols) + calls = "\n".join("%s%s%s();" % (self.symbol_prefix, + symbol_name, + self.symbol_suffix) + for symbol_name in self._require_symbols) s = textwrap.dedent("""\ - void zungqr_(); + %(prototypes)s int main(int argc, const char *argv[]) { - zungqr_(); + %(calls)s return 0; - }""") + }""") % dict(prototypes=prototypes, calls=calls) src = os.path.join(tmpdir, 'source.c') out = os.path.join(tmpdir, 'a.out') # Add the additional "extra" arguments @@ -2011,8 +2166,6 @@ class openblas_lapack_info(openblas_info): extra_args = info['extra_link_args'] except Exception: extra_args = [] - if sys.version_info < (3, 5) and sys.version_info > (3, 0) and c.compiler_type == "msvc": - extra_args.append("/MANIFEST") try: with open(src, 'wt') as f: f.write(s) @@ -2028,9 +2181,49 @@ class openblas_lapack_info(openblas_info): shutil.rmtree(tmpdir) return res +class openblas_lapack_info(openblas_info): + section = 'openblas' + dir_env_var = 'OPENBLAS' + _lib_names = ['openblas'] + _require_symbols = ['zungqr_'] + notfounderror = BlasNotFoundError + class openblas_clapack_info(openblas_lapack_info): _lib_names = ['openblas', 'lapack'] +class openblas_ilp64_info(openblas_info): + section = 'openblas_ilp64' + dir_env_var = 'OPENBLAS_ILP64' + _lib_names = ['openblas64'] + _require_symbols = ['dgemm_', 'cblas_dgemm'] + notfounderror = BlasILP64NotFoundError + + def _calc_info(self): + info = super()._calc_info() + if info is not None: + info['define_macros'] += [('HAVE_BLAS_ILP64', None)] + return info + +class openblas_ilp64_lapack_info(openblas_ilp64_info): + _require_symbols = ['dgemm_', 'cblas_dgemm', 'zungqr_', 'LAPACKE_zungqr'] + + def _calc_info(self): + info = super()._calc_info() + if info: + info['define_macros'] += [('HAVE_LAPACKE', None)] + return info + +class openblas64__info(openblas_ilp64_info): + # ILP64 Openblas, with default symbol suffix + section = 'openblas64_' + dir_env_var = 'OPENBLAS64_' + _lib_names = ['openblas64_'] + symbol_suffix = '64_' + symbol_prefix = '' + +class openblas64__lapack_info(openblas_ilp64_lapack_info, openblas64__info): + pass + class blis_info(blas_info): section = 'blis' dir_env_var = 'BLIS' @@ -2364,18 +2557,18 @@ class numerix_info(system_info): try: import numpy # noqa: F401 which = "numpy", "defaulted" - except ImportError: - msg1 = str(get_exception()) + except ImportError as e: + msg1 = str(e) try: import Numeric # noqa: F401 which = "numeric", "defaulted" - except ImportError: - msg2 = str(get_exception()) + except ImportError as e: + msg2 = str(e) try: import numarray # noqa: F401 which = "numarray", "defaulted" - except ImportError: - msg3 = str(get_exception()) + except ImportError as e: + msg3 = str(e) log.info(msg1) log.info(msg2) log.info(msg3) |