diff options
| -rw-r--r-- | azure-pipelines.yml | 2 | ||||
| -rw-r--r-- | azure-steps-windows.yml | 2 | ||||
| -rw-r--r-- | shippable.yml | 1 | ||||
| -rw-r--r-- | tools/openblas_support.py | 64 | ||||
| -rwxr-xr-x | tools/pypy-test.sh | 7 | ||||
| -rwxr-xr-x | tools/travis-before-install.sh | 3 |
6 files changed, 57 insertions, 22 deletions
diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7b23fef61..564f5d8e8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -40,7 +40,6 @@ stages: /bin/bash -xc "cd numpy && \ /opt/python/cp38-cp38/bin/python -mvenv venv &&\ source venv/bin/activate && \ - python3 -m pip install urllib3 && \ target=\$(python3 tools/openblas_support.py) && \ cp -r \$target/lib/* /usr/lib && \ cp \$target/include/* /usr/include && \ @@ -122,7 +121,6 @@ stages: # primarily on file size / name details - script: | set -xe - python -m pip install urllib3 target=$(python tools/openblas_support.py) ls -lR $target # manually link to appropriate system paths diff --git a/azure-steps-windows.yml b/azure-steps-windows.yml index 46b8c1ff9..c39faddf9 100644 --- a/azure-steps-windows.yml +++ b/azure-steps-windows.yml @@ -14,7 +14,7 @@ steps: # to $PYTHON_EXE's directory since that is on a different drive which # mingw does not like. Instead copy it to a directory and set OPENBLAS, # since OPENBLAS will be picked up by the openblas discovery - python -m pip install urllib3 + python -m pip install $target = $(python tools/openblas_support.py) mkdir openblas echo Copying $target to openblas/openblas$env:OPENBLAS_SUFFIX.a diff --git a/shippable.yml b/shippable.yml index 6985f38ef..dc3617e12 100644 --- a/shippable.yml +++ b/shippable.yml @@ -25,7 +25,6 @@ build: - sudo add-apt-repository ppa:ubuntu-toolchain-r/test - sudo apt-get update - sudo apt-get install gcc gfortran libgfortran5 - - python -mpip install urllib3 - target=$(python tools/openblas_support.py) - ls -lR "${target}" - sudo cp -r "${target}"/lib/* /usr/lib diff --git a/tools/openblas_support.py b/tools/openblas_support.py index a729662c5..a4b63c800 100644 --- a/tools/openblas_support.py +++ b/tools/openblas_support.py @@ -1,13 +1,15 @@ +import glob +import hashlib import os +import platform import sys -import glob import shutil +import tarfile import textwrap -import platform +import zipfile from tempfile import mkstemp, gettempdir -import zipfile -import tarfile +from urllib.request import urlopen, Request OPENBLAS_V = '0.3.9' # Temporary build of OpenBLAS to test a fix for dynamic detection of CPU @@ -15,8 +17,25 @@ OPENBLAS_LONG = 'v0.3.7-527-g79fd006c' # the 0.3.7 is misleading BASE_LOC = 'https://anaconda.org/multibuild-wheels-staging/openblas-libs' BASEURL = f'{BASE_LOC}/{OPENBLAS_LONG}/download' ARCHITECTURES = ['', 'windows', 'darwin', 'aarch64', 'x86_64', 'i686', 'ppc64le', 's390x'] +sha256_vals = { +"openblas-v0.3.7-527-g79fd006c-win_amd64-gcc_7_1_0.zip": "7249d68c02e6b6339e06edfeab1fecddf29ee1e67a3afaa77917c320c43de840", +"openblas64_-v0.3.7-527-g79fd006c-win_amd64-gcc_7_1_0.zip": "6488e0961a5926e47242f63b63b41cfdd661e6f1d267e8e313e397cde4775c17", +"openblas-v0.3.7-527-g79fd006c-macosx_10_9_x86_64-gf_1becaaa.tar.gz": "69434bd626bbc495da9ce8c36b005d140c75e3c47f94e88c764a199e820f9259", +"openblas64_-v0.3.7-527-g79fd006c-macosx_10_9_x86_64-gf_1becaaa.tar.gz": "093f6d953e3fa76a86809be67bd1f0b27656671b5a55b233169cfaa43fd63e22", +"openblas-v0.3.7-527-g79fd006c-manylinux2014_aarch64.tar.gz": "42676c69dc48cd6e412251b39da6b955a5a0e00323ddd77f9137f7c259d35319", +"openblas64_-v0.3.7-527-g79fd006c-manylinux2014_aarch64.tar.gz": "5aec167af4052cf5e9e3e416c522d9794efabf03a2aea78b9bb3adc94f0b73d8", +"openblas-v0.3.7-527-g79fd006c-manylinux2010_x86_64.tar.gz": "fa67c6cc29d4cc5c70a147c80526243239a6f95fc3feadcf83a78176cd9c526b", +"openblas64_-v0.3.7-527-g79fd006c-manylinux2010_x86_64.tar.gz": "9ad34e89a5307dcf5823bf5c020580d0559a0c155fe85b44fc219752e61852b0", +"openblas-v0.3.7-527-g79fd006c-manylinux2010_i686.tar.gz": "0b8595d316c8b7be84ab1f1d5a0c89c1b35f7c987cdaf61d441bcba7ab4c7439", +"openblas-v0.3.7-527-g79fd006c-manylinux2014_ppc64le.tar.gz": "3e1c7d6472c34e7210e3605be4bac9ddd32f613d44297dc50cf2d067e720c4a9", +"openblas64_-v0.3.7-527-g79fd006c-manylinux2014_ppc64le.tar.gz": "a0885873298e21297a04be6cb7355a585df4fa4873e436b4c16c0a18fc9073ea", +"openblas-v0.3.7-527-g79fd006c-manylinux2014_s390x.tar.gz": "79b454320817574e20499d58f05259ed35213bea0158953992b910607b17f240", +"openblas64_-v0.3.7-527-g79fd006c-manylinux2014_s390x.tar.gz": "9fddbebf5301518fc4a5d2022a61886544a0566868c8c014359a1ee6b17f2814", +} + IS_32BIT = sys.maxsize < 2**32 + def get_arch(): if platform.system() == 'Windows': ret = 'windows' @@ -50,10 +69,12 @@ def get_manylinux(arch): def download_openblas(target, arch, ilp64): - import urllib3 ml_ver = get_manylinux(arch) fnsuffix = {None: "", "64_": "64_"}[ilp64] filename = '' + headers = {'User-Agent': ('Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 ; ' + '(KHTML, like Gecko) Chrome/41.0.2228.0 ' + 'Safari/537.3')} if arch in ('aarch64', 'ppc64le', 's390x', 'x86_64', 'i686'): suffix = f'manylinux{ml_ver}_{arch}.tar.gz' filename = f'{BASEURL}/openblas{fnsuffix}-{OPENBLAS_LONG}-{suffix}' @@ -71,15 +92,26 @@ def download_openblas(target, arch, ilp64): typ = 'zip' if not filename: return None - print("Downloading:", filename, file=sys.stderr) - http = urllib3.PoolManager() - response = http.request('GET', filename) + req = Request(url=filename, headers=headers) + response = urlopen(req) + length = response.getheader('content-length') if response.status != 200: print(f'Could not download "{filename}"', file=sys.stderr) return None + print(f"Downloading {length} from {filename}", file=sys.stderr) + data = response.read() + # Verify hash + key = os.path.basename(filename) + sha256_returned = hashlib.sha256(data).hexdigest() + if key not in sha256_vals: + raise ValueError( + f'key "{key}" with hash "{sha256_returned}" not in sha256_vals') + sha256_expected = sha256_vals[key] + if sha256_returned != sha256_expected: + raise ValueError(f'sha256 hash mismatch for filename {filename}') print("Saving to file", file=sys.stderr) with open(target, 'wb') as fid: - fid.write(response.data) + fid.write(data) return typ def setup_openblas(arch=get_arch(), ilp64=get_ilp64()): @@ -201,9 +233,10 @@ def test_setup(arches): def items(): for arch in arches: yield arch, None - if arch in ('x86', 'darwin', 'windows'): + if arch not in ('i686'): yield arch, '64_' + errs = [] for arch, ilp64 in items(): if arch == '': continue @@ -212,9 +245,11 @@ def test_setup(arches): try: try: target = setup_openblas(arch, ilp64) - except: - print(f'Could not setup {arch}') - raise + except Exception as e: + print(f'Could not setup {arch}:') + print(str(e)) + errs.append(e) + continue if not target: raise RuntimeError(f'Could not setup {arch}') print(target) @@ -231,6 +266,9 @@ def test_setup(arches): os.unlink(target) else: shutil.rmtree(target) + if errs: + raise errs[0] + def test_version(expected_version, ilp64=get_ilp64()): """ diff --git a/tools/pypy-test.sh b/tools/pypy-test.sh index 9da1ba5f8..e24d7a99d 100755 --- a/tools/pypy-test.sh +++ b/tools/pypy-test.sh @@ -6,9 +6,10 @@ set -o pipefail # Print expanded commands set -x -#sudo apt-get -yq update -#sudo apt-get -yq install libatlas-base-dev liblapack-dev gfortran-5 python3-urllib3 -F77=gfortran-5 F90=gfortran-5 \ +sudo apt-get -yq update +sudo apt-get -yq install gfortran-5 +export F77=gfortran-5 +export F90=gfortran-5 # Download the proper OpenBLAS x64 precompiled library target=$(python3 tools/openblas_support.py) diff --git a/tools/travis-before-install.sh b/tools/travis-before-install.sh index dbe2f6ea2..e468dd932 100755 --- a/tools/travis-before-install.sh +++ b/tools/travis-before-install.sh @@ -41,8 +41,7 @@ pip install --upgrade pip # A specific version of cython is required, so we read the cython package # requirement using `grep cython test_requirements.txt` instead of simply # writing 'pip install setuptools wheel cython'. -# urllib3 is needed for openblas_support -pip install setuptools wheel urllib3 `grep cython test_requirements.txt` +pip install setuptools wheel `grep cython test_requirements.txt` if [ -n "$DOWNLOAD_OPENBLAS" ]; then pwd |