diff options
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/openblas_support.py | 99 | ||||
| -rwxr-xr-x | tools/pypy-test.sh | 9 | ||||
| -rw-r--r-- | tools/refguide_check.py | 2 | ||||
| -rwxr-xr-x | tools/travis-before-install.sh | 3 |
4 files changed, 78 insertions, 35 deletions
diff --git a/tools/openblas_support.py b/tools/openblas_support.py index 105aae51f..984dea35a 100644 --- a/tools/openblas_support.py +++ b/tools/openblas_support.py @@ -1,22 +1,42 @@ +import glob +import hashlib import os +import platform import sys -import glob import shutil +import tarfile import textwrap -import platform +import zipfile from tempfile import mkstemp, gettempdir -import zipfile -import tarfile +from urllib.request import urlopen, Request OPENBLAS_V = '0.3.9' # Temporary build of OpenBLAS to test a fix for dynamic detection of CPU OPENBLAS_LONG = 'v0.3.7-527-g79fd006c' # the 0.3.7 is misleading BASE_LOC = 'https://anaconda.org/multibuild-wheels-staging/openblas-libs' BASEURL = f'{BASE_LOC}/{OPENBLAS_LONG}/download' -ARCHITECTURES = ['', 'windows', 'darwin', 'aarch64', 'x86', 'ppc64le', 's390x'] +ARCHITECTURES = ['', 'windows', 'darwin', 'aarch64', 'x86_64', 'i686', 'ppc64le', 's390x'] +sha256_vals = { +"openblas-v0.3.7-527-g79fd006c-win_amd64-gcc_7_1_0.zip": "7249d68c02e6b6339e06edfeab1fecddf29ee1e67a3afaa77917c320c43de840", +"openblas64_-v0.3.7-527-g79fd006c-win_amd64-gcc_7_1_0.zip": "6488e0961a5926e47242f63b63b41cfdd661e6f1d267e8e313e397cde4775c17", +"openblas-v0.3.7-527-g79fd006c-win32-gcc_7_1_0.zip": "5fb0867ca70b1d0fdbf68dd387c0211f26903d74631420e4aabb49e94aa3930d", +"openblas-v0.3.7-527-g79fd006c-macosx_10_9_x86_64-gf_1becaaa.tar.gz": "69434bd626bbc495da9ce8c36b005d140c75e3c47f94e88c764a199e820f9259", +"openblas64_-v0.3.7-527-g79fd006c-macosx_10_9_x86_64-gf_1becaaa.tar.gz": "093f6d953e3fa76a86809be67bd1f0b27656671b5a55b233169cfaa43fd63e22", +"openblas-v0.3.7-527-g79fd006c-manylinux2014_aarch64.tar.gz": "42676c69dc48cd6e412251b39da6b955a5a0e00323ddd77f9137f7c259d35319", +"openblas64_-v0.3.7-527-g79fd006c-manylinux2014_aarch64.tar.gz": "5aec167af4052cf5e9e3e416c522d9794efabf03a2aea78b9bb3adc94f0b73d8", +"openblas-v0.3.7-527-g79fd006c-manylinux2010_x86_64.tar.gz": "fa67c6cc29d4cc5c70a147c80526243239a6f95fc3feadcf83a78176cd9c526b", +"openblas64_-v0.3.7-527-g79fd006c-manylinux2010_x86_64.tar.gz": "9ad34e89a5307dcf5823bf5c020580d0559a0c155fe85b44fc219752e61852b0", +"openblas-v0.3.7-527-g79fd006c-manylinux2010_i686.tar.gz": "0b8595d316c8b7be84ab1f1d5a0c89c1b35f7c987cdaf61d441bcba7ab4c7439", +"openblas-v0.3.7-527-g79fd006c-manylinux2014_ppc64le.tar.gz": "3e1c7d6472c34e7210e3605be4bac9ddd32f613d44297dc50cf2d067e720c4a9", +"openblas64_-v0.3.7-527-g79fd006c-manylinux2014_ppc64le.tar.gz": "a0885873298e21297a04be6cb7355a585df4fa4873e436b4c16c0a18fc9073ea", +"openblas-v0.3.7-527-g79fd006c-manylinux2014_s390x.tar.gz": "79b454320817574e20499d58f05259ed35213bea0158953992b910607b17f240", +"openblas64_-v0.3.7-527-g79fd006c-manylinux2014_s390x.tar.gz": "9fddbebf5301518fc4a5d2022a61886544a0566868c8c014359a1ee6b17f2814", +} + IS_32BIT = sys.maxsize < 2**32 + def get_arch(): if platform.system() == 'Windows': ret = 'windows' @@ -25,10 +45,10 @@ def get_arch(): else: ret = platform.uname().machine # What do 32 bit machines report? - # If they are a docker, they report x86_64 or i686 - if 'x86' in ret or ret == 'i686': - ret = 'x86' - assert ret in ARCHITECTURES + # If they are a docker, they can report x86_64 + if 'x86' in ret and IS_32BIT: + arch = 'i686' + assert ret in ARCHITECTURES, f'invalid architecture {ret}' return ret def get_ilp64(): @@ -38,15 +58,28 @@ def get_ilp64(): raise RuntimeError("NPY_USE_BLAS_ILP64 set on 32-bit arch") return "64_" +def get_manylinux(arch): + if arch in ('x86_64', 'i686'): + default = '2010' + else: + default = '2014' + ret = os.environ.get("MB_ML_VER", default) + # XXX For PEP 600 this can be a glibc version + assert ret in ('1', '2010', '2014'), f'invalid MB_ML_VER {ret}' + return ret + + def download_openblas(target, arch, ilp64): - import urllib3 + ml_ver = get_manylinux(arch) fnsuffix = {None: "", "64_": "64_"}[ilp64] filename = '' - if arch in ('aarch64', 'ppc64le', 's390x'): - suffix = f'manylinux2014_{arch}.tar.gz' + headers = {'User-Agent': ('Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 ; ' + '(KHTML, like Gecko) Chrome/41.0.2228.0 ' + 'Safari/537.3')} + if arch in ('aarch64', 'ppc64le', 's390x', 'x86_64', 'i686'): + suffix = f'manylinux{ml_ver}_{arch}.tar.gz' filename = f'{BASEURL}/openblas{fnsuffix}-{OPENBLAS_LONG}-{suffix}' typ = 'tar.gz' - typ = 'tar.gz' elif arch == 'darwin': suffix = 'macosx_10_9_x86_64-gf_1becaaa.tar.gz' filename = f'{BASEURL}/openblas{fnsuffix}-{OPENBLAS_LONG}-{suffix}' @@ -58,24 +91,28 @@ def download_openblas(target, arch, ilp64): suffix = 'win_amd64-gcc_7_1_0.zip' filename = f'{BASEURL}/openblas{fnsuffix}-{OPENBLAS_LONG}-{suffix}' typ = 'zip' - elif 'x86' in arch: - if IS_32BIT: - suffix = 'manylinux2010_i686.tar.gz' - else: - suffix = 'manylinux2010_x86_64.tar.gz' - filename = f'{BASEURL}/openblas{fnsuffix}-{OPENBLAS_LONG}-{suffix}' - typ = 'tar.gz' if not filename: return None - print("Downloading:", filename, file=sys.stderr) - http = urllib3.PoolManager() - response = http.request('GET', filename) + req = Request(url=filename, headers=headers) + response = urlopen(req) + length = response.getheader('content-length') if response.status != 200: print(f'Could not download "{filename}"', file=sys.stderr) return None + print(f"Downloading {length} from {filename}", file=sys.stderr) + data = response.read() + # Verify hash + key = os.path.basename(filename) + sha256_returned = hashlib.sha256(data).hexdigest() + if key not in sha256_vals: + raise ValueError( + f'key "{key}" with hash "{sha256_returned}" not in sha256_vals') + sha256_expected = sha256_vals[key] + if sha256_returned != sha256_expected: + raise ValueError(f'sha256 hash mismatch for filename {filename}') print("Saving to file", file=sys.stderr) with open(target, 'wb') as fid: - fid.write(response.data) + fid.write(data) return typ def setup_openblas(arch=get_arch(), ilp64=get_ilp64()): @@ -197,9 +234,10 @@ def test_setup(arches): def items(): for arch in arches: yield arch, None - if arch in ('x86', 'darwin', 'windows'): + if arch not in ('i686'): yield arch, '64_' + errs = [] for arch, ilp64 in items(): if arch == '': continue @@ -208,9 +246,11 @@ def test_setup(arches): try: try: target = setup_openblas(arch, ilp64) - except: - print(f'Could not setup {arch}') - raise + except Exception as e: + print(f'Could not setup {arch}:') + print(str(e)) + errs.append(e) + continue if not target: raise RuntimeError(f'Could not setup {arch}') print(target) @@ -227,6 +267,9 @@ def test_setup(arches): os.unlink(target) else: shutil.rmtree(target) + if errs: + raise errs[0] + def test_version(expected_version, ilp64=get_ilp64()): """ diff --git a/tools/pypy-test.sh b/tools/pypy-test.sh index e98c12587..e24d7a99d 100755 --- a/tools/pypy-test.sh +++ b/tools/pypy-test.sh @@ -7,8 +7,9 @@ set -o pipefail set -x sudo apt-get -yq update -sudo apt-get -yq install libatlas-base-dev liblapack-dev gfortran-5 python3-urllib3 -F77=gfortran-5 F90=gfortran-5 \ +sudo apt-get -yq install gfortran-5 +export F77=gfortran-5 +export F90=gfortran-5 # Download the proper OpenBLAS x64 precompiled library target=$(python3 tools/openblas_support.py) @@ -27,8 +28,8 @@ include_dirs = $target/lib:$LIB runtime_library_dirs = $target/lib EOF -echo getting PyPy 3.6 nightly -wget -q http://buildbot.pypy.org/nightly/py3.6/pypy-c-jit-latest-linux64.tar.bz2 -O pypy.tar.bz2 +echo getting PyPy 3.6-v7.3.1 +wget -q https://downloads.python.org/pypy/pypy3.6-v7.3.1-linux64.tar.bz2 -O pypy.tar.bz2 mkdir -p pypy3 (cd pypy3; tar --strip-components=1 -xf ../pypy.tar.bz2) pypy3/bin/pypy3 -mensurepip diff --git a/tools/refguide_check.py b/tools/refguide_check.py index e6cfc8b77..31d2997d3 100644 --- a/tools/refguide_check.py +++ b/tools/refguide_check.py @@ -450,7 +450,7 @@ def validate_rst_syntax(text, name, dots=True): return False, "ERROR: %s: no documentation" % (name,) ok_unknown_items = set([ - 'mod', 'currentmodule', 'autosummary', 'data', 'attr', + 'mod', 'doc', 'currentmodule', 'autosummary', 'data', 'attr', 'obj', 'versionadded', 'versionchanged', 'module', 'class', 'ref', 'func', 'toctree', 'moduleauthor', 'term', 'c:member', 'sectionauthor', 'codeauthor', 'eq', 'doi', 'DOI', 'arXiv', 'arxiv' diff --git a/tools/travis-before-install.sh b/tools/travis-before-install.sh index dbe2f6ea2..e468dd932 100755 --- a/tools/travis-before-install.sh +++ b/tools/travis-before-install.sh @@ -41,8 +41,7 @@ pip install --upgrade pip # A specific version of cython is required, so we read the cython package # requirement using `grep cython test_requirements.txt` instead of simply # writing 'pip install setuptools wheel cython'. -# urllib3 is needed for openblas_support -pip install setuptools wheel urllib3 `grep cython test_requirements.txt` +pip install setuptools wheel `grep cython test_requirements.txt` if [ -n "$DOWNLOAD_OPENBLAS" ]; then pwd |