summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.travis.yml13
-rw-r--r--azure-pipelines.yml22
-rw-r--r--azure-steps-windows.yml2
-rw-r--r--doc/release/upcoming_changes/15218.improvement.rst6
-rw-r--r--doc/source/dev/index.rst16
-rw-r--r--numpy/distutils/ccompiler.py2
-rw-r--r--numpy/distutils/system_info.py9
-rw-r--r--numpy/distutils/tests/test_ccompiler.py24
-rw-r--r--numpy/lib/_iotools.py51
-rw-r--r--numpy/lib/function_base.py29
-rw-r--r--numpy/lib/tests/test_function_base.py1
-rw-r--r--numpy/lib/tests/test_recfunctions.py1
-rw-r--r--numpy/lib/tests/test_shape_base.py1
-rw-r--r--numpy/lib/twodim_base.py4
-rw-r--r--numpy/linalg/lapack_lite/README.rst4
-rw-r--r--numpy/linalg/lapack_lite/f2c.h14
-rw-r--r--numpy/linalg/lapack_litemodule.c6
-rw-r--r--numpy/linalg/setup.py19
-rw-r--r--numpy/linalg/tests/test_linalg.py28
-rw-r--r--numpy/testing/_private/utils.py4
-rw-r--r--shippable.yml5
-rw-r--r--tools/openblas_support.py111
-rwxr-xr-xtools/pypy-test.sh15
-rwxr-xr-xtools/travis-before-install.sh6
24 files changed, 267 insertions, 126 deletions
diff --git a/.travis.yml b/.travis.yml
index 5ee27edb7..73106b4c1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -75,7 +75,17 @@ jobs:
- USE_ASV=1
- python: 3.7
- env: NPY_RELAXED_STRIDES_CHECKING=0
+ env:
+ - NPY_RELAXED_STRIDES_CHECKING=0
+ # use custom symbol-suffixed openblas build, not system ATLAS
+ - DOWNLOAD_OPENBLAS=1
+ - CHECK_BLAS=1
+ - NPY_USE_BLAS_ILP64=1
+ addons:
+ apt:
+ packages:
+ - gfortran
+ - eatmydata
- python: 3.7
env: USE_WHEEL=1 NPY_RELAXED_STRIDES_DEBUG=1
@@ -94,6 +104,7 @@ jobs:
arch: ppc64le
env:
# use ppc64le OpenBLAS build, not system ATLAS
+ - DOWNLOAD_OPENBLAS=1
- ATLAS=None
- python: 3.7
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 29fc6c614..1851df71a 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -40,8 +40,9 @@ stages:
locale-gen fr_FR && update-locale && \
apt-get -y install gfortran-5 wget && \
target=\$(python3 tools/openblas_support.py) && \
- cp -r \$target/usr/local/lib/* /usr/lib && \
- cp \$target/usr/local/include/* /usr/include && \
+ ls -lR \$target && \
+ cp -r \$target/lib/* /usr/lib && \
+ cp \$target/include/* /usr/include && \
python3 -m pip install --user --upgrade pip setuptools && \
python3 -m pip install --user -r test_requirements.txt && \
python3 -m pip install . && \
@@ -65,13 +66,21 @@ stages:
# azure config for mac os -- Microsoft has indicated
# they will patch this issue
vmImage: macOS-10.14
+ strategy:
+ maxParallel: 2
+ matrix:
+ Python36:
+ PYTHON_VERSION: '3.6'
+ Python36-ILP64:
+ PYTHON_VERSION: '3.6'
+ NPY_USE_BLAS_ILP64: '1'
steps:
# the @0 refers to the (major) version of the *task* on Microsoft's
# end, not the order in the build matrix nor anything to do
# with version of Python selected
- task: UsePythonVersion@0
inputs:
- versionSpec: '3.6'
+ versionSpec: $(PYTHON_VERSION)
addToPath: true
architecture: 'x64'
# NOTE: do we have a compelling reason to use older / newer
@@ -97,9 +106,10 @@ stages:
# primarily on file size / name details
- script: |
target=$(python tools/openblas_support.py)
+ ls -lR $target
# manually link to appropriate system paths
- cp $target/usr/local/lib/* /usr/local/lib/
- cp $target/usr/local/include/* /usr/local/include/
+ cp $target/lib/* /usr/local/lib/
+ cp $target/include/* /usr/local/include/
displayName: 'install pre-built openblas'
- script: python -m pip install --upgrade pip setuptools wheel
displayName: 'Install tools'
@@ -171,6 +181,8 @@ stages:
PYTHON_ARCH: 'x64'
TEST_MODE: full
BITS: 64
+ NPY_USE_BLAS_ILP64: '1'
+ OPENBLAS_SUFFIX: '64_'
steps:
- template: azure-steps-windows.yml
- job: Linux_PyPy3
diff --git a/azure-steps-windows.yml b/azure-steps-windows.yml
index 26d7a667d..f17039455 100644
--- a/azure-steps-windows.yml
+++ b/azure-steps-windows.yml
@@ -11,7 +11,7 @@ steps:
- powershell: |
$pyversion = python -c "from __future__ import print_function; import sys; print(sys.version.split()[0])"
Write-Host "Python Version: $pyversion"
- $target = "C:\\hostedtoolcache\\windows\\Python\\$pyversion\\$(PYTHON_ARCH)\\lib\\openblas.a"
+ $target = "C:\\hostedtoolcache\\windows\\Python\\$pyversion\\$(PYTHON_ARCH)\\lib\\openblas$env:OPENBLAS_SUFFIX.a"
Write-Host "target path: $target"
$openblas = python tools/openblas_support.py
cp $openblas $target
diff --git a/doc/release/upcoming_changes/15218.improvement.rst b/doc/release/upcoming_changes/15218.improvement.rst
new file mode 100644
index 000000000..ccbbbd66f
--- /dev/null
+++ b/doc/release/upcoming_changes/15218.improvement.rst
@@ -0,0 +1,6 @@
+Use 64-bit integer size on 64-bit platforms in fallback lapack_lite
+-------------------------------------------------------------------
+
+Use 64-bit integer size on 64-bit platforms in the fallback LAPACK library,
+which is used when the system has no LAPACK installed, allowing it to deal with
+linear algebra for large arrays.
diff --git a/doc/source/dev/index.rst b/doc/source/dev/index.rst
index 306c15069..1aacc2c49 100644
--- a/doc/source/dev/index.rst
+++ b/doc/source/dev/index.rst
@@ -231,6 +231,22 @@ Requirements
`Sphinx <http://www.sphinx-doc.org/en/stable/>`__ is needed to build
the documentation. Matplotlib, SciPy, and IPython are also required.
+The numpy documentation also depends on the
+`numpydoc <https://numpydoc.readthedocs.io/en/latest/>`__ sphinx extension
+as well as an external sphinx theme.
+These extensions are included as git submodules and must be initialized
+before building the docs.
+From the ``doc/`` directory:
+
+.. code:: sh
+
+ git submodule update --init
+
+The documentation includes mathematical formulae with LaTeX formatting.
+A working LaTeX document production system
+(e.g. `texlive <https://www.tug.org/texlive/>`__) is required for the
+proper rendering of the LaTeX math in the documentation.
+
Fixing Warnings
~~~~~~~~~~~~~~~
diff --git a/numpy/distutils/ccompiler.py b/numpy/distutils/ccompiler.py
index 684c7535b..c3a557f50 100644
--- a/numpy/distutils/ccompiler.py
+++ b/numpy/distutils/ccompiler.py
@@ -532,7 +532,7 @@ def CCompiler_customize(self, dist, need_cxx=0):
'g++' in self.compiler[0] or
'clang' in self.compiler[0]):
self._auto_depends = True
- if 'gcc' in self.compiler[0]:
+ if 'gcc' in self.compiler[0] and not need_cxx:
# add std=c99 flag for gcc
# TODO: does this need to be more specific?
self.compiler.append('-std=c99')
diff --git a/numpy/distutils/system_info.py b/numpy/distutils/system_info.py
index 4786b3a0c..fc7018af3 100644
--- a/numpy/distutils/system_info.py
+++ b/numpy/distutils/system_info.py
@@ -2102,16 +2102,17 @@ class openblas_info(blas_info):
return None
# Generate numpy.distutils virtual static library file
- tmpdir = os.path.join(os.getcwd(), 'build', 'openblas')
+ basename = self.__class__.__name__
+ tmpdir = os.path.join(os.getcwd(), 'build', basename)
if not os.path.isdir(tmpdir):
os.makedirs(tmpdir)
info = {'library_dirs': [tmpdir],
- 'libraries': ['openblas'],
+ 'libraries': [basename],
'language': 'f77'}
- fake_lib_file = os.path.join(tmpdir, 'openblas.fobjects')
- fake_clib_file = os.path.join(tmpdir, 'openblas.cobjects')
+ fake_lib_file = os.path.join(tmpdir, basename + '.fobjects')
+ fake_clib_file = os.path.join(tmpdir, basename + '.cobjects')
with open(fake_lib_file, 'w') as f:
f.write("\n".join(library_paths))
with open(fake_clib_file, 'w') as f:
diff --git a/numpy/distutils/tests/test_ccompiler.py b/numpy/distutils/tests/test_ccompiler.py
new file mode 100644
index 000000000..8b4a56b79
--- /dev/null
+++ b/numpy/distutils/tests/test_ccompiler.py
@@ -0,0 +1,24 @@
+from __future__ import division, absolute_import, print_function
+
+from distutils.ccompiler import new_compiler
+
+from numpy.distutils.numpy_distribution import NumpyDistribution
+
+def test_ccompiler():
+ '''
+ scikit-image/scikit-image issue 4369
+ We unconditionally add ``-std-c99`` to the gcc compiler in order
+ to support c99 with very old gcc compilers. However the same call
+ is used to get the flags for the c++ compiler, just with a kwarg.
+ Make sure in this case, where it would not be legal, the option is **not** added
+ '''
+ dist = NumpyDistribution()
+ compiler = new_compiler()
+ compiler.customize(dist)
+ if hasattr(compiler, 'compiler') and 'gcc' in compiler.compiler[0]:
+ assert 'c99' in ' '.join(compiler.compiler)
+
+ compiler = new_compiler()
+ compiler.customize(dist, need_cxx=True)
+ if hasattr(compiler, 'compiler') and 'gcc' in compiler.compiler[0]:
+ assert 'c99' not in ' '.join(compiler.compiler)
diff --git a/numpy/lib/_iotools.py b/numpy/lib/_iotools.py
index c392929fd..8bc336fdb 100644
--- a/numpy/lib/_iotools.py
+++ b/numpy/lib/_iotools.py
@@ -5,17 +5,10 @@ from __future__ import division, absolute_import, print_function
__docformat__ = "restructuredtext en"
-import sys
import numpy as np
import numpy.core.numeric as nx
from numpy.compat import asbytes, asunicode, bytes, basestring
-if sys.version_info[0] >= 3:
- from builtins import bool, int, float, complex, object, str
- unicode = str
-else:
- from __builtin__ import bool, int, float, complex, object, unicode, str
-
def _decode_line(line, encoding=None):
"""Decode bytes from binary input streams.
@@ -65,40 +58,6 @@ def _is_bytes_like(obj):
return True
-def _to_filehandle(fname, flag='r', return_opened=False):
- """
- Returns the filehandle corresponding to a string or a file.
- If the string ends in '.gz', the file is automatically unzipped.
-
- Parameters
- ----------
- fname : string, filehandle
- Name of the file whose filehandle must be returned.
- flag : string, optional
- Flag indicating the status of the file ('r' for read, 'w' for write).
- return_opened : boolean, optional
- Whether to return the opening status of the file.
- """
- if _is_string_like(fname):
- if fname.endswith('.gz'):
- import gzip
- fhd = gzip.open(fname, flag)
- elif fname.endswith('.bz2'):
- import bz2
- fhd = bz2.BZ2File(fname)
- else:
- fhd = file(fname, flag)
- opened = True
- elif hasattr(fname, 'seek'):
- fhd = fname
- opened = False
- else:
- raise ValueError('fname must be a string or file handle')
- if return_opened:
- return fhd, opened
- return fhd
-
-
def has_nested_fields(ndtype):
"""
Returns whether one or several fields of a dtype are nested.
@@ -210,7 +169,8 @@ class LineSplitter(object):
return lambda input: [_.strip() for _ in method(input)]
#
- def __init__(self, delimiter=None, comments='#', autostrip=True, encoding=None):
+ def __init__(self, delimiter=None, comments='#', autostrip=True,
+ encoding=None):
delimiter = _decode_line(delimiter)
comments = _decode_line(comments)
@@ -949,9 +909,10 @@ def easy_dtype(ndtype, names=None, defaultfmt="f%i", **validationargs):
elif ndtype.names is not None:
validate = NameValidator(**validationargs)
# Default initial names : should we change the format ?
- if ((ndtype.names == tuple("f%i" % i for i in range(len(ndtype.names)))) and
- (defaultfmt != "f%i")):
- ndtype.names = validate([''] * len(ndtype.names), defaultfmt=defaultfmt)
+ numbered_names = tuple("f%i" % i for i in range(len(ndtype.names)))
+ if ((ndtype.names == numbered_names) and (defaultfmt != "f%i")):
+ ndtype.names = validate([''] * len(ndtype.names),
+ defaultfmt=defaultfmt)
# Explicit initial names : just validate
else:
ndtype.names = validate(ndtype.names, defaultfmt=defaultfmt)
diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py
index 499120630..c2680b016 100644
--- a/numpy/lib/function_base.py
+++ b/numpy/lib/function_base.py
@@ -13,10 +13,10 @@ import warnings
import numpy as np
import numpy.core.numeric as _nx
-from numpy.core import atleast_1d, transpose
+from numpy.core import transpose
from numpy.core.numeric import (
ones, zeros, arange, concatenate, array, asarray, asanyarray, empty,
- empty_like, ndarray, around, floor, ceil, take, dot, where, intp,
+ ndarray, around, floor, ceil, take, dot, where, intp,
integer, isscalar, absolute
)
from numpy.core.umath import (
@@ -38,21 +38,16 @@ from numpy.core.multiarray import (
from numpy.core.umath import _add_newdoc_ufunc as add_newdoc_ufunc
from numpy.compat import long
-if sys.version_info[0] < 3:
- # Force range to be a generator, for np.delete's usage.
- range = xrange
- import __builtin__ as builtins
-else:
- import builtins
+import builtins
+
+# needed in this module for compatibility
+from numpy.lib.histograms import histogram, histogramdd
array_function_dispatch = functools.partial(
overrides.array_function_dispatch, module='numpy')
-# needed in this module for compatibility
-from numpy.lib.histograms import histogram, histogramdd
-
__all__ = [
'select', 'piecewise', 'trim_zeros', 'copy', 'iterable', 'percentile',
'diff', 'gradient', 'angle', 'unwrap', 'sort_complex', 'disp', 'flip',
@@ -70,7 +65,7 @@ def _rot90_dispatcher(m, k=None, axes=None):
@array_function_dispatch(_rot90_dispatcher)
-def rot90(m, k=1, axes=(0,1)):
+def rot90(m, k=1, axes=(0, 1)):
"""
Rotate an array by 90 degrees in the plane specified by axes.
@@ -150,7 +145,7 @@ def rot90(m, k=1, axes=(0,1)):
axes_list[axes[0]])
if k == 1:
- return transpose(flip(m,axes[1]), axes_list)
+ return transpose(flip(m, axes[1]), axes_list)
else:
# k == 3
return flip(transpose(m, axes_list), axes[1])
@@ -1612,6 +1607,7 @@ def trim_zeros(filt, trim='fb'):
last = last - 1
return filt[first:last]
+
def _extract_dispatcher(condition, arr):
return (condition, arr)
@@ -2947,6 +2943,7 @@ def hamming(M):
n = arange(0, M)
return 0.54 - 0.46*cos(2.0*pi*n/(M-1))
+
## Code from cephes for i0
_i0A = [
@@ -3489,6 +3486,7 @@ def median(a, axis=None, out=None, overwrite_input=False, keepdims=False):
else:
return r
+
def _median(a, axis=None, out=None, overwrite_input=False):
# can't be reasonably be implemented in terms of percentile as we have to
# call mean to not break astropy
@@ -3707,7 +3705,7 @@ def quantile(a, q, axis=None, out=None,
overwrite_input=False, interpolation='linear', keepdims=False):
"""
Compute the q-th quantile of the data along the specified axis.
-
+
.. versionadded:: 1.15.0
Parameters
@@ -3878,7 +3876,7 @@ def _quantile_ureduce_func(a, q, axis=None, out=None, overwrite_input=False,
"interpolation can only be 'linear', 'lower' 'higher', "
"'midpoint', or 'nearest'")
- n = np.array(False, dtype=bool) # check for nan's flag
+ n = np.array(False, dtype=bool) # check for nan's flag
if indices.dtype == intp: # take the points along axis
# Check if the array contains any nan's
if np.issubdtype(a.dtype, np.inexact):
@@ -3898,7 +3896,6 @@ def _quantile_ureduce_func(a, q, axis=None, out=None, overwrite_input=False,
indices = indices[0]
r = take(ap, indices, axis=axis, out=out)
-
else: # weight the points above and below the indices
indices_below = floor(indices).astype(intp)
indices_above = indices_below + 1
diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py
index 9075ff538..f95e0a251 100644
--- a/numpy/lib/tests/test_function_base.py
+++ b/numpy/lib/tests/test_function_base.py
@@ -4,7 +4,6 @@ import operator
import warnings
import sys
import decimal
-import types
from fractions import Fraction
import pytest
diff --git a/numpy/lib/tests/test_recfunctions.py b/numpy/lib/tests/test_recfunctions.py
index fa5f4dec2..53229e31a 100644
--- a/numpy/lib/tests/test_recfunctions.py
+++ b/numpy/lib/tests/test_recfunctions.py
@@ -772,7 +772,6 @@ class TestJoinBy(object):
def test_join_subdtype(self):
# tests the bug in https://stackoverflow.com/q/44769632/102441
- from numpy.lib import recfunctions as rfn
foo = np.array([(1,)],
dtype=[('key', int)])
bar = np.array([(1, np.array([1,2,3]))],
diff --git a/numpy/lib/tests/test_shape_base.py b/numpy/lib/tests/test_shape_base.py
index be1604a75..ff9019e3d 100644
--- a/numpy/lib/tests/test_shape_base.py
+++ b/numpy/lib/tests/test_shape_base.py
@@ -1,7 +1,6 @@
from __future__ import division, absolute_import, print_function
import numpy as np
-import warnings
import functools
import sys
import pytest
diff --git a/numpy/lib/twodim_base.py b/numpy/lib/twodim_base.py
index f45392188..2e6d30a1d 100644
--- a/numpy/lib/twodim_base.py
+++ b/numpy/lib/twodim_base.py
@@ -6,13 +6,13 @@ from __future__ import division, absolute_import, print_function
import functools
from numpy.core.numeric import (
- absolute, asanyarray, arange, zeros, greater_equal, multiply, ones,
+ asanyarray, arange, zeros, greater_equal, multiply, ones,
asarray, where, int8, int16, int32, int64, empty, promote_types, diagonal,
nonzero
)
from numpy.core.overrides import set_module
from numpy.core import overrides
-from numpy.core import iinfo, transpose
+from numpy.core import iinfo
__all__ = [
diff --git a/numpy/linalg/lapack_lite/README.rst b/numpy/linalg/lapack_lite/README.rst
index 1343d25f8..ba30aa4ed 100644
--- a/numpy/linalg/lapack_lite/README.rst
+++ b/numpy/linalg/lapack_lite/README.rst
@@ -18,9 +18,9 @@ and is unlikely to ever be ported to python 3.
The routines that ``lapack_litemodule.c`` wraps are listed in
``wrapped_routines``, along with a few exceptions that aren't picked up
properly. Assuming that you have an unpacked LAPACK source tree in
-``~/LAPACK``, you generate the new routines in a directory ``new-lite/`` with::
+``~/LAPACK``, you generate the new routines in this directory with::
-$ python2 ./make_lite.py wrapped_routines ~/LAPACK new-lite/
+$ python2 ./make_lite.py wrapped_routines ~/LAPACK
This will grab the right routines, with dependencies, put them into the
appropriate ``f2c_*.f`` files, run ``f2c`` over them, then do some scrubbing
diff --git a/numpy/linalg/lapack_lite/f2c.h b/numpy/linalg/lapack_lite/f2c.h
index 80f1a12b1..4462eaa74 100644
--- a/numpy/linalg/lapack_lite/f2c.h
+++ b/numpy/linalg/lapack_lite/f2c.h
@@ -8,15 +8,17 @@
#define F2C_INCLUDE
#include <math.h>
+#include "numpy/npy_common.h"
+#include "npy_cblas.h"
-typedef int integer;
+typedef CBLAS_INT integer;
typedef char *address;
typedef short int shortint;
typedef float real;
typedef double doublereal;
typedef struct { real r, i; } complex;
typedef struct { doublereal r, i; } doublecomplex;
-typedef int logical;
+typedef CBLAS_INT logical;
typedef short int shortlogical;
typedef char logical1;
typedef char integer1;
@@ -37,9 +39,9 @@ typedef short flag;
typedef short ftnlen;
typedef short ftnint;
#else
-typedef int flag;
-typedef int ftnlen;
-typedef int ftnint;
+typedef CBLAS_INT flag;
+typedef CBLAS_INT ftnlen;
+typedef CBLAS_INT ftnint;
#endif
/*external read, write*/
@@ -352,7 +354,7 @@ extern void s_copy(char *, char *, ftnlen, ftnlen);
extern int s_paus(char *, ftnlen);
extern integer s_rdfe(cilist *);
extern integer s_rdue(cilist *);
-extern integer s_rnge(char *, integer, char *, integer);
+extern int s_rnge(char *, int, char *, int);
extern integer s_rsfe(cilist *);
extern integer s_rsfi(icilist *);
extern integer s_rsle(cilist *);
diff --git a/numpy/linalg/lapack_litemodule.c b/numpy/linalg/lapack_litemodule.c
index 4c80317f5..56f38364f 100644
--- a/numpy/linalg/lapack_litemodule.c
+++ b/numpy/linalg/lapack_litemodule.c
@@ -416,5 +416,11 @@ initlapack_lite(void)
LapackError = PyErr_NewException("lapack_lite.LapackError", NULL, NULL);
PyDict_SetItemString(d, "LapackError", LapackError);
+#ifdef HAVE_BLAS_ILP64
+ PyDict_SetItemString(d, "_ilp64", Py_True);
+#else
+ PyDict_SetItemString(d, "_ilp64", Py_False);
+#endif
+
return RETVAL(m);
}
diff --git a/numpy/linalg/setup.py b/numpy/linalg/setup.py
index 6315a34b4..0aa0566d6 100644
--- a/numpy/linalg/setup.py
+++ b/numpy/linalg/setup.py
@@ -5,7 +5,7 @@ import sys
def configuration(parent_package='', top_path=None):
from numpy.distutils.misc_util import Configuration
- from numpy.distutils.system_info import get_info
+ from numpy.distutils.system_info import get_info, system_info
config = Configuration('linalg', parent_package, top_path)
config.add_data_dir('tests')
@@ -31,8 +31,23 @@ def configuration(parent_package='', top_path=None):
else:
lapack_info = get_info('lapack_opt', 0) # and {}
+ use_lapack_lite = not lapack_info
+
+ if use_lapack_lite:
+ # This makes numpy.distutils write the fact that lapack_lite
+ # is being used to numpy.__config__
+ class numpy_linalg_lapack_lite(system_info):
+ def calc_info(self):
+ info = {'language': 'c'}
+ if sys.maxsize > 2**32:
+ # Build lapack-lite in 64-bit integer mode
+ info['define_macros'] = [('HAVE_BLAS_ILP64', None)]
+ self.set_info(**info)
+
+ lapack_info = numpy_linalg_lapack_lite().get_info(2)
+
def get_lapack_lite_sources(ext, build_dir):
- if not lapack_info:
+ if use_lapack_lite:
print("### Warning: Using unoptimized lapack ###")
return all_sources
else:
diff --git a/numpy/linalg/tests/test_linalg.py b/numpy/linalg/tests/test_linalg.py
index bd3df1ca4..ef05b595e 100644
--- a/numpy/linalg/tests/test_linalg.py
+++ b/numpy/linalg/tests/test_linalg.py
@@ -2016,3 +2016,31 @@ def test_blas64_dot():
a[0,-1] = 1
c = np.dot(b, a)
assert_equal(c[0,-1], 1)
+
+
+@pytest.mark.xfail(not HAS_LAPACK64,
+ reason="Numpy not compiled with 64-bit BLAS/LAPACK")
+def test_blas64_geqrf_lwork_smoketest():
+ # Smoke test LAPACK geqrf lwork call with 64-bit integers
+ dtype = np.float64
+ lapack_routine = np.linalg.lapack_lite.dgeqrf
+
+ m = 2**32 + 1
+ n = 2**32 + 1
+ lda = m
+
+ # Dummy arrays, not referenced by the lapack routine, so don't
+ # need to be of the right size
+ a = np.zeros([1, 1], dtype=dtype)
+ work = np.zeros([1], dtype=dtype)
+ tau = np.zeros([1], dtype=dtype)
+
+ # Size query
+ results = lapack_routine(m, n, a, lda, tau, work, -1, 0)
+ assert_equal(results['info'], 0)
+ assert_equal(results['m'], m)
+ assert_equal(results['n'], m)
+
+ # Should result to an integer of a reasonable size
+ lwork = int(work.item())
+ assert_(2**32 < lwork < 2**42)
diff --git a/numpy/testing/_private/utils.py b/numpy/testing/_private/utils.py
index 23267a9e1..94bad4f63 100644
--- a/numpy/testing/_private/utils.py
+++ b/numpy/testing/_private/utils.py
@@ -21,7 +21,7 @@ import pprint
from numpy.core import(
intp, float32, empty, arange, array_repr, ndarray, isnat, array)
-import numpy.__config__
+import numpy.linalg.lapack_lite
if sys.version_info[0] >= 3:
from io import StringIO
@@ -54,7 +54,7 @@ verbose = 0
IS_PYPY = platform.python_implementation() == 'PyPy'
HAS_REFCOUNT = getattr(sys, 'getrefcount', None) is not None
-HAS_LAPACK64 = hasattr(numpy.__config__, 'lapack_ilp64_opt_info')
+HAS_LAPACK64 = numpy.linalg.lapack_lite._ilp64
def import_nose():
diff --git a/shippable.yml b/shippable.yml
index f64cb9937..4313a6de2 100644
--- a/shippable.yml
+++ b/shippable.yml
@@ -25,8 +25,9 @@ build:
- sudo apt-get update
- sudo apt-get install gcc gfortran
- target=$(python tools/openblas_support.py)
- - sudo cp -r "${target}"/64/lib/* /usr/lib
- - sudo cp "${target}"/64/include/* /usr/include
+ - ls -lR "${target}"
+ - sudo cp -r "${target}"/lib/* /usr/lib
+ - sudo cp "${target}"/include/* /usr/include
- python -m pip install --upgrade pip
# we will pay the ~13 minute cost of compiling Cython only when a new
diff --git a/tools/openblas_support.py b/tools/openblas_support.py
index 964adce6e..4a210cfe1 100644
--- a/tools/openblas_support.py
+++ b/tools/openblas_support.py
@@ -1,6 +1,8 @@
from __future__ import division, absolute_import, print_function
import os
import sys
+import glob
+import shutil
import textwrap
import platform
try:
@@ -40,7 +42,15 @@ def get_arch():
assert ret in ARCHITECTURES
return ret
-def download_openblas(target, arch):
+def get_ilp64():
+ if os.environ.get("NPY_USE_BLAS_ILP64", "0") == "0":
+ return None
+ if IS_32BIT:
+ raise RuntimeError("NPY_USE_BLAS_ILP64 set on 32-bit arch")
+ return "64_"
+
+def download_openblas(target, arch, ilp64):
+ fnsuffix = {None: "", "64_": "64_"}[ilp64]
filename = ''
if arch == 'arm':
# ARMv8 OpenBLAS built using script available here:
@@ -48,7 +58,7 @@ def download_openblas(target, arch):
# build done on GCC compile farm machine named gcc115
# tarball uploaded manually to an unshared Dropbox location
filename = ('https://www.dropbox.com/s/vdeckao4omss187/'
- 'openblas-{}-armv8.tar.gz?dl=1'.format(OPENBLAS_V))
+ 'openblas{}-{}-armv8.tar.gz?dl=1'.format(fnsuffix, OPENBLAS_V))
typ = 'tar.gz'
elif arch == 'ppc64':
# build script for POWER8 OpenBLAS available here:
@@ -56,28 +66,29 @@ def download_openblas(target, arch):
# built on GCC compile farm machine named gcc112
# manually uploaded tarball to an unshared Dropbox location
filename = ('https://www.dropbox.com/s/yt0d2j86x1j8nh1/'
- 'openblas-{}-ppc64le-power8.tar.gz?dl=1'.format(OPENBLAS_V))
+ 'openblas{}-{}-ppc64le-power8.tar.gz?dl=1'.format(fnsuffix, OPENBLAS_V))
typ = 'tar.gz'
elif arch == 'darwin':
- filename = '{0}/openblas-{1}-macosx_10_9_x86_64-gf_1becaaa.tar.gz'.format(
- RACKSPACE, OPENBLAS_LONG)
+ filename = '{0}/openblas{1}-{2}-macosx_10_9_x86_64-gf_1becaaa.tar.gz'.format(
+ RACKSPACE, fnsuffix, OPENBLAS_LONG)
typ = 'tar.gz'
elif arch == 'windows':
if IS_32BIT:
suffix = 'win32-gcc_7_1_0.zip'
else:
suffix = 'win_amd64-gcc_7_1_0.zip'
- filename = '{0}/openblas-{1}-{2}'.format(RACKSPACE, OPENBLAS_LONG, suffix)
+ filename = '{0}/openblas{1}-{2}-{3}'.format(RACKSPACE, fnsuffix, OPENBLAS_LONG, suffix)
typ = 'zip'
elif arch == 'x86':
if IS_32BIT:
suffix = 'manylinux1_i686.tar.gz'
else:
suffix = 'manylinux1_x86_64.tar.gz'
- filename = '{0}/openblas-{1}-{2}'.format(RACKSPACE, OPENBLAS_LONG, suffix)
+ filename = '{0}/openblas{1}-{2}-{3}'.format(RACKSPACE, fnsuffix, OPENBLAS_LONG, suffix)
typ = 'tar.gz'
if not filename:
return None
+ print("Downloading:", filename, file=sys.stderr)
try:
with open(target, 'wb') as fid:
fid.write(urlopen(filename).read())
@@ -86,7 +97,7 @@ def download_openblas(target, arch):
return None
return typ
-def setup_openblas(arch=get_arch()):
+def setup_openblas(arch=get_arch(), ilp64=get_ilp64()):
'''
Download and setup an openblas library for building. If successful,
the configuration script will find it automatically.
@@ -100,7 +111,7 @@ def setup_openblas(arch=get_arch()):
_, tmp = mkstemp()
if not arch:
raise ValueError('unknown architecture')
- typ = download_openblas(tmp, arch)
+ typ = download_openblas(tmp, arch, ilp64)
if not typ:
return ''
if arch == 'windows':
@@ -132,10 +143,33 @@ def unpack_targz(fname):
if not os.path.exists(target):
os.mkdir(target)
with tarfile.open(fname, 'r') as zf:
- # TODO: check that all the zf.getnames() files do not escape the
- # extract directory (no leading '../', '/')
- zf.extractall(target)
- return target
+ # Strip common prefix from paths when unpacking
+ prefix = os.path.commonpath(zf.getnames())
+ extract_tarfile_to(zf, target, prefix)
+ return target
+
+def extract_tarfile_to(tarfileobj, target_path, archive_path):
+ """Extract TarFile contents under archive_path/ to target_path/"""
+
+ target_path = os.path.abspath(target_path)
+
+ def get_members():
+ for member in tarfileobj.getmembers():
+ if archive_path:
+ norm_path = os.path.normpath(member.name)
+ if norm_path.startswith(archive_path + os.path.sep):
+ member.name = norm_path[len(archive_path)+1:]
+ else:
+ continue
+
+ dst_path = os.path.abspath(os.path.join(target_path, member.name))
+ if os.path.commonpath([target_path, dst_path]) != target_path:
+ # Path not under target_path, probably contains ../
+ continue
+
+ yield member
+
+ tarfileobj.extractall(target_path, members=get_members())
def make_init(dirname):
'''
@@ -180,19 +214,41 @@ def test_setup(arches):
'''
Make sure all the downloadable files exist and can be opened
'''
- for arch in arches:
+ def items():
+ for arch in arches:
+ yield arch, None
+ if arch in ('x86', 'darwin', 'windows'):
+ yield arch, '64_'
+
+ for arch, ilp64 in items():
if arch == '':
continue
+
+ target = None
try:
- target = setup_openblas(arch)
- except:
- print('Could not setup %s' % arch)
- raise
- if not target:
- raise RuntimeError('Could not setup %s' % arch)
- print(target)
-
-def test_version(expected_version):
+ try:
+ target = setup_openblas(arch, ilp64)
+ except:
+ print('Could not setup %s' % arch)
+ raise
+ if not target:
+ raise RuntimeError('Could not setup %s' % arch)
+ print(target)
+ if arch == 'windows':
+ if not target.endswith('.a'):
+ raise RuntimeError("Not .a extracted!")
+ else:
+ files = glob.glob(os.path.join(target, "lib", "*.a"))
+ if not files:
+ raise RuntimeError("No lib/*.a unpacked!")
+ finally:
+ if target is not None:
+ if os.path.isfile(target):
+ os.unlink(target)
+ else:
+ shutil.rmtree(target)
+
+def test_version(expected_version, ilp64=get_ilp64()):
"""
Assert that expected OpenBLAS version is
actually available via NumPy
@@ -201,12 +257,19 @@ def test_version(expected_version):
import ctypes
dll = ctypes.CDLL(numpy.core._multiarray_umath.__file__)
- get_config = dll.openblas_get_config
+ if ilp64 == "64_":
+ get_config = dll.openblas_get_config64_
+ else:
+ get_config = dll.openblas_get_config
get_config.restype=ctypes.c_char_p
res = get_config()
print('OpenBLAS get_config returned', str(res))
check_str = b'OpenBLAS %s' % expected_version[0].encode()
assert check_str in res
+ if ilp64:
+ assert b"USE64BITINT" in res
+ else:
+ assert b"USE64BITINT" not in res
if __name__ == '__main__':
import argparse
diff --git a/tools/pypy-test.sh b/tools/pypy-test.sh
index f4d56ba1a..33a97ad17 100755
--- a/tools/pypy-test.sh
+++ b/tools/pypy-test.sh
@@ -11,19 +11,20 @@ sudo apt-get -yq install libatlas-base-dev liblapack-dev gfortran-5
F77=gfortran-5 F90=gfortran-5 \
# Download the proper OpenBLAS x64 precompiled library
-target=$(python tools/openblas_support.py)
+target=$(python3 tools/openblas_support.py)
+ls -lR "$target"
echo getting OpenBLAS into $target
-export LD_LIBRARY_PATH=$target/usr/local/lib
-export LIB=$target/usr/local/lib
-export INCLUDE=$target/usr/local/include
+export LD_LIBRARY_PATH=$target/lib
+export LIB=$target/lib
+export INCLUDE=$target/include
# Use a site.cfg to build with local openblas
cat << EOF > site.cfg
[openblas]
libraries = openblas
-library_dirs = $target/usr/local/lib:$LIB
-include_dirs = $target/usr/local/lib:$LIB
-runtime_library_dirs = $target/usr/local/lib
+library_dirs = $target/lib:$LIB
+include_dirs = $target/lib:$LIB
+runtime_library_dirs = $target/lib
EOF
echo getting PyPy 3.6 nightly
diff --git a/tools/travis-before-install.sh b/tools/travis-before-install.sh
index 072ad3bf6..9f8b66a47 100755
--- a/tools/travis-before-install.sh
+++ b/tools/travis-before-install.sh
@@ -5,12 +5,12 @@ free -m
df -h
ulimit -a
-if [ -n "$PPC64_LE" ]; then
+if [ -n "$DOWNLOAD_OPENBLAS" ]; then
pwd
ls -ltrh
target=$(python tools/openblas_support.py)
- sudo cp -r $target/64/lib/* /usr/lib
- sudo cp $target/64/include/* /usr/include
+ sudo cp -r $target/lib/* /usr/lib
+ sudo cp $target/include/* /usr/include
fi
mkdir builds
View Lorry Controller Status