ENH: distutils: add support for ILP64 OpenBLAS (generic symbol suffix)

Generalize the ILP64 BLAS/LAPACK symbol name handling to deal with
arbitrary prefix/suffix.

The build-time behavior is changed so that HAVE_BLAS_ILP64 and
BLAS_SYMBOL_SUFFIX/PREFIX defines are added to compile options
as appropriate.

Mainly to make autodetection of BLAS/LAPACK easier for downstream
numpy.distutils users, add get_info aliases 'blas_ilp64_opt',
'blas_ilp64_plain_opt', and 'blas64__opt' for any/no/""&"64_"
prefix&suffix, and the same for lapack. (Due to the way system_info
works, each also gets a separate class.)

In addition to openblas64_ which has a fixed suffix, add the (by default
suffixless) openblas_ilp64, which correspond to the most likely cases to
be present.

4 files changed, 155 insertions, 50 deletions

diff --git a/numpy/core/setup.py b/numpy/core/setup.py
index 5bbb1c622..974ec4628 100644
--- a/numpy/core/setup.py
+++ b/numpy/core/setup.py
@@ -91,9 +91,6 @@ def is_npy_no_smp():
     #  block.
     return 'NPY_NOSMP' in os.environ
 
-def is_npy_use_blas64_():
-    return (os.environ.get('NPY_USE_BLAS64_', "0") != "0")
-
 def win32_checks(deflist):
     from numpy.distutils.misc_util import get_build_architecture
     a = get_build_architecture()
@@ -756,12 +753,12 @@ def configuration(parent_package='',top_path=None):
             join('src', 'common', 'numpyos.c'),
             ]
 
-    if is_npy_use_blas64_():
-        blas_info = get_info('blas64__opt', 2)
-        have_blas = blas_info and ('HAVE_CBLAS64_', None) in blas_info.get('define_macros', [])
+    if os.environ.get('NPY_USE_BLAS_ILP64', "0") != "0":
+        blas_info = get_info('blas_ilp64_opt', 2)
     else:
         blas_info = get_info('blas_opt', 0)
-        have_blas = blas_info and ('HAVE_CBLAS', None) in blas_info.get('define_macros', [])
+
+    have_blas = blas_info and ('HAVE_CBLAS', None) in blas_info.get('define_macros', [])
 
     if have_blas:
         extra_info = blas_info
diff --git a/numpy/distutils/system_info.py b/numpy/distutils/system_info.py
index fc2902b78..4786b3a0c 100644
--- a/numpy/distutils/system_info.py
+++ b/numpy/distutils/system_info.py
@@ -22,11 +22,16 @@ classes are available:
   lapack_info
   openblas_info
   openblas64__info
+  openblas_ilp64_info
   blis_info
   blas_opt_info       # usage recommended
   lapack_opt_info     # usage recommended
-  blas64__opt_info       # usage recommended
-  lapack64__opt_info     # usage recommended
+  blas_ilp64_opt_info    # usage recommended (general ILP64 BLAS)
+  lapack_ilp64_opt_info  # usage recommended (general ILP64 LAPACK)
+  blas_ilp64_plain_opt_info    # usage recommended (general ILP64 BLAS, no symbol suffix)
+  lapack_ilp64_plain_opt_info  # usage recommended (general ILP64 LAPACK, no symbol suffix)
+  blas64__opt_info    # usage recommended (general ILP64 BLAS, 64_ symbol suffix)
+  lapack64__opt_info  # usage recommended (general ILP64 LAPACK, 64_ symbol suffix)
   fftw_info,dfftw_info,sfftw_info
   fftw_threads_info,dfftw_threads_info,sfftw_threads_info
   djbfft_info
@@ -411,6 +416,8 @@ def get_info(name, notfound_action=0):
           'accelerate': accelerate_info,      # use blas_opt instead
           'openblas64_': openblas64__info,
           'openblas64__lapack': openblas64__lapack_info,
+          'openblas_ilp64': openblas_ilp64_info,
+          'openblas_ilp64_lapack': openblas_ilp64_lapack_info,
           'x11': x11_info,
           'fft_opt': fft_opt_info,
           'fftw': fftw_info,
@@ -433,8 +440,12 @@ def get_info(name, notfound_action=0):
           'numarray': numarray_info,
           'numerix': numerix_info,
           'lapack_opt': lapack_opt_info,
+          'lapack_ilp64_opt': lapack_ilp64_opt_info,
+          'lapack_ilp64_plain_opt': lapack_ilp64_plain_opt_info,
           'lapack64__opt': lapack64__opt_info,
           'blas_opt': blas_opt_info,
+          'blas_ilp64_opt': blas_ilp64_opt_info,
+          'blas_ilp64_plain_opt': blas_ilp64_plain_opt_info,
           'blas64__opt': blas64__opt_info,
           'boost_python': boost_python_info,
           'agg2': agg2_info,
@@ -501,14 +512,13 @@ class LapackSrcNotFoundError(LapackNotFoundError):
     the LAPACK_SRC environment variable."""
 
 
-class Lapack64_NotFoundError(NotFoundError):
+class LapackILP64NotFoundError(NotFoundError):
     """
-    64-bit Lapack libraries with '64_' symbol suffix not found.
+    64-bit Lapack libraries not found.
     Known libraries in numpy/distutils/site.cfg file are:
-    openblas64_
+    openblas64_, openblas_ilp64
     """
 
-
 class BlasOptNotFoundError(NotFoundError):
     """
     Optimized (vendor) Blas libraries are not found.
@@ -523,11 +533,11 @@ class BlasNotFoundError(NotFoundError):
     numpy/distutils/site.cfg file (section [blas]) or by setting
     the BLAS environment variable."""
 
-class Blas64_NotFoundError(NotFoundError):
+class BlasILP64NotFoundError(NotFoundError):
     """
-    64-bit Blas libraries with '64_' symbol suffix not found.
+    64-bit Blas libraries not found.
     Known libraries in numpy/distutils/site.cfg file are:
-    openblas64_
+    openblas64_, openblas_ilp64
     """
 
 class BlasSrcNotFoundError(BlasNotFoundError):
@@ -1716,6 +1726,9 @@ class lapack_opt_info(system_info):
             return True
         return False
 
+    def _calc_info(self, name):
+        return getattr(self, '_calc_info_{}'.format(name))()
+
     def calc_info(self):
         user_order = os.environ.get(self.order_env_var_name, None)
         if user_order is None:
@@ -1737,7 +1750,7 @@ class lapack_opt_info(system_info):
                                  "values: {}".format(non_existing))
 
         for lapack in lapack_order:
-            if getattr(self, '_calc_info_{}'.format(lapack))():
+            if self._calc_info(lapack):
                 return
 
         if 'lapack' not in lapack_order:
@@ -1746,19 +1759,49 @@ class lapack_opt_info(system_info):
             warnings.warn(LapackNotFoundError.__doc__ or '', stacklevel=2)
             warnings.warn(LapackSrcNotFoundError.__doc__ or '', stacklevel=2)
 
-class lapack64__opt_info(lapack_opt_info):
-    notfounderror = Lapack64_NotFoundError
-    lapack_order = ['openblas64_']
-    order_env_var_name = 'NPY_LAPACK64__ORDER'
 
-    def _calc_info_openblas64_(self):
-        info = get_info('openblas64__lapack')
-        if info:
+class _ilp64_opt_info_mixin:
+    symbol_suffix = None
+    symbol_prefix = None
+
+    def _check_info(self, info):
+        macros = dict(info.get('define_macros', []))
+        prefix = macros.get('BLAS_SYMBOL_PREFIX', '')
+        suffix = macros.get('BLAS_SYMBOL_SUFFIX', '')
+
+        if self.symbol_prefix not in (None, prefix):
+            return False
+
+        if self.symbol_suffix not in (None, suffix):
+            return False
+
+        return bool(info)
+
+
+class lapack_ilp64_opt_info(lapack_opt_info, _ilp64_opt_info_mixin):
+    notfounderror = LapackILP64NotFoundError
+    lapack_order = ['openblas64_', 'openblas_ilp64']
+    order_env_var_name = 'NPY_LAPACK_ILP64_ORDER'
+
+    def _calc_info(self, name):
+        info = get_info(name + '_lapack')
+        if self._check_info(info):
             self.set_info(**info)
             return True
         return False
 
 
+class lapack_ilp64_plain_opt_info(lapack_ilp64_opt_info):
+    # Same as lapack_ilp64_opt_info, but fix symbol names
+    symbol_prefix = ''
+    symbol_suffix = ''
+
+
+class lapack64__opt_info(lapack_ilp64_opt_info):
+    symbol_prefix = ''
+    symbol_suffix = '64_'
+
+
 class blas_opt_info(system_info):
     notfounderror = BlasNotFoundError
     # List of all known BLAS libraries, in the default order
@@ -1828,6 +1871,9 @@ class blas_opt_info(system_info):
         self.set_info(**info)
         return True
 
+    def _calc_info(self, name):
+        return getattr(self, '_calc_info_{}'.format(name))()
+
     def calc_info(self):
         user_order = os.environ.get(self.order_env_var_name, None)
         if user_order is None:
@@ -1847,7 +1893,7 @@ class blas_opt_info(system_info):
                 raise ValueError("blas_opt_info user defined BLAS order has unacceptable values: {}".format(non_existing))
 
         for blas in blas_order:
-            if getattr(self, '_calc_info_{}'.format(blas))():
+            if self._calc_info(blas):
                 return
 
         if 'blas' not in blas_order:
@@ -1857,19 +1903,29 @@ class blas_opt_info(system_info):
             warnings.warn(BlasSrcNotFoundError.__doc__ or '', stacklevel=2)
 
 
-class blas64__opt_info(blas_opt_info):
-    notfounderror = Blas64_NotFoundError
-    blas_order = ['openblas64_']
-    order_env_var_name = 'NPY_BLAS64__ORDER'
+class blas_ilp64_opt_info(blas_opt_info, _ilp64_opt_info_mixin):
+    notfounderror = BlasILP64NotFoundError
+    blas_order = ['openblas64_', 'openblas_ilp64']
+    order_env_var_name = 'NPY_BLAS_ILP64_ORDER'
 
-    def _calc_info_openblas64_(self):
-        info = get_info('openblas64_')
-        if info:
+    def _calc_info(self, name):
+        info = get_info(name)
+        if self._check_info(info):
             self.set_info(**info)
             return True
         return False
 
 
+class blas_ilp64_plain_opt_info(blas_ilp64_opt_info):
+    symbol_prefix = ''
+    symbol_suffix = ''
+
+
+class blas64__opt_info(blas_ilp64_opt_info):
+    symbol_prefix = ''
+    symbol_suffix = '64_'
+
+
 class blas_info(system_info):
     section = 'blas'
     dir_env_var = 'BLAS'
@@ -1972,6 +2028,20 @@ class openblas_info(blas_info):
     _require_symbols = []
     notfounderror = BlasNotFoundError
 
+    @property
+    def symbol_prefix(self):
+        try:
+            return self.cp.get(self.section, 'symbol_prefix')
+        except NoOptionError:
+            return ''
+
+    @property
+    def symbol_suffix(self):
+        try:
+            return self.cp.get(self.section, 'symbol_suffix')
+        except NoOptionError:
+            return ''
+
     def _calc_info(self):
         c = customized_ccompiler()
 
@@ -2006,6 +2076,10 @@ class openblas_info(blas_info):
             return None
 
         info['define_macros'] = [('HAVE_CBLAS', None)]
+        if self.symbol_prefix:
+            info['define_macros'] += [('BLAS_SYMBOL_PREFIX', self.symbol_prefix)]
+        if self.symbol_suffix:
+            info['define_macros'] += [('BLAS_SYMBOL_SUFFIX', self.symbol_suffix)]
 
         return info
 
@@ -2051,9 +2125,13 @@ class openblas_info(blas_info):
 
         tmpdir = tempfile.mkdtemp()
 
-        prototypes = "\n".join("void %s();" % symbol_name
+        prototypes = "\n".join("void %s%s%s();" % (self.symbol_prefix,
+                                                   symbol_name,
+                                                   self.symbol_suffix)
                                for symbol_name in self._require_symbols)
-        calls = "\n".join("%s();" % symbol_name
+        calls = "\n".join("%s%s%s();" % (self.symbol_prefix,
+                                         symbol_name,
+                                         self.symbol_suffix)
                           for symbol_name in self._require_symbols)
         s = textwrap.dedent("""\
             %(prototypes)s
@@ -2096,28 +2174,39 @@ class openblas_lapack_info(openblas_info):
 class openblas_clapack_info(openblas_lapack_info):
     _lib_names = ['openblas', 'lapack']
 
-class openblas64__info(openblas_info):
-    section = 'openblas64_'
-    dir_env_var = 'OPENBLAS64_'
-    _lib_names = ['openblas64_']
-    _require_symbols = ['dgemm_64_', 'cblas_dgemm64_']
-    notfounderror = Blas64_NotFoundError
+class openblas_ilp64_info(openblas_info):
+    section = 'openblas_ilp64'
+    dir_env_var = 'OPENBLAS_ILP64'
+    _lib_names = ['openblas64']
+    _require_symbols = ['dgemm_', 'cblas_dgemm']
+    notfounderror = BlasILP64NotFoundError
 
     def _calc_info(self):
         info = super()._calc_info()
         if info is not None:
-            info['define_macros'] = [('HAVE_CBLAS64_', None)]
+            info['define_macros'] += [('HAVE_BLAS_ILP64', None)]
         return info
 
-class openblas64__lapack_info(openblas64__info):
-    _require_symbols = ['dgemm_64_', 'cblas_dgemm64_', 'zungqr_64_', 'LAPACKE_zungqr64_']
+class openblas_ilp64_lapack_info(openblas_ilp64_info):
+    _require_symbols = ['dgemm_', 'cblas_dgemm', 'zungqr_', 'LAPACKE_zungqr']
 
     def _calc_info(self):
         info = super()._calc_info()
         if info:
-            info['define_macros'] += [('HAVE_LAPACKE64_', None)]
+            info['define_macros'] += [('HAVE_LAPACKE', None)]
         return info
 
+class openblas64__info(openblas_ilp64_info):
+    # ILP64 Openblas, with default symbol suffix
+    section = 'openblas64_'
+    dir_env_var = 'OPENBLAS64_'
+    _lib_names = ['openblas64_']
+    symbol_suffix = '64_'
+    symbol_prefix = ''
+
+class openblas64__lapack_info(openblas_ilp64_lapack_info, openblas64__info):
+    pass
+
 class blis_info(blas_info):
     section = 'blis'
     dir_env_var = 'BLIS'
diff --git a/numpy/linalg/setup.py b/numpy/linalg/setup.py
index f5cb04e89..6315a34b4 100644
--- a/numpy/linalg/setup.py
+++ b/numpy/linalg/setup.py
@@ -26,10 +26,8 @@ def configuration(parent_package='', top_path=None):
     ]
     all_sources = config.paths(lapack_lite_src)
 
-    if (os.environ.get('NPY_USE_BLAS64_', "0") != "0"):
-        lapack_info = get_info('lapack64__opt', 2)
-        lapack_info.setdefault('define_macros', [])
-        lapack_info['define_macros'] += [('NPY_UMATH_USE_BLAS64_', None)]
+    if os.environ.get('NPY_USE_BLAS_ILP64', "0") != "0":
+        lapack_info = get_info('lapack_ilp64_opt', 2)
     else:
         lapack_info = get_info('lapack_opt', 0)  # and {}
 
diff --git a/site.cfg.example b/site.cfg.example
index a58039df3..ba912866a 100644
--- a/site.cfg.example
+++ b/site.cfg.example
@@ -140,8 +140,9 @@
 # This is an emerging "standard" for 64-bit BLAS/LAPACK, avoiding symbol clashes
 # with 32-bit BLAS/LAPACK.
 #
-# To build Numpy with such 64-bit BLAS/LAPACK, set environment variable
-# NPY_USE_BLAS64_=1 at build time.
+# To build Numpy with such 64-bit BLAS/LAPACK, set environment
+# variables NPY_USE_BLAS_ILP64=1, NPY_BLAS_ILP64_ORDER=openblas64_,
+# NPY_LAPACK_ILP64_ORDER=openblas64_ at build time.
 #
 # See:
 # https://github.com/xianyi/OpenBLAS/issues/646
@@ -152,6 +153,26 @@
 # include_dirs = /opt/OpenBLAS/include
 # runtime_library_dirs = /opt/OpenBLAS/lib
 
+# OpenBLAS (64-bit ILP64)
+# -----------------------
+# It is possible to also use OpenBLAS compiled with 64-bit integer
+# size (ILP64) but no symbol name changes. To do that, set the
+# environment variables NPY_USE_BLAS_ILP64=1,
+# NPY_BLAS_ILP64_ORDER=openblas_ilp64,
+# NPY_LAPACK_ILP64_ORDER=openblas_ilp64 at build time.
+#
+# Note that mixing both 64-bit and 32-bit BLAS without symbol suffixes
+# in the same application may cause problems due to symbol name
+# clashes, especially with embedded Python interpreters.
+#
+# [openblas_ilp64]
+# libraries = openblas64
+# library_dirs = /opt/OpenBLAS/lib
+# include_dirs = /opt/OpenBLAS/include
+# runtime_library_dirs = /opt/OpenBLAS/lib
+# symbol_prefix =
+# symbol_suffix =
+
 # BLIS
 # ----
 # BLIS (https://github.com/flame/blis) also provides a BLAS interface.  It's a