From c9422ffb532fba0c0717bde07649de8e2a021c10 Mon Sep 17 00:00:00 2001 From: mattip Date: Tue, 12 May 2020 15:15:34 +0300 Subject: ENH: backport scipy changes to openblas download script --- tools/openblas_support.py | 64 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 51 insertions(+), 13 deletions(-) (limited to 'tools/openblas_support.py') diff --git a/tools/openblas_support.py b/tools/openblas_support.py index a729662c5..a4b63c800 100644 --- a/tools/openblas_support.py +++ b/tools/openblas_support.py @@ -1,13 +1,15 @@ +import glob +import hashlib import os +import platform import sys -import glob import shutil +import tarfile import textwrap -import platform +import zipfile from tempfile import mkstemp, gettempdir -import zipfile -import tarfile +from urllib.request import urlopen, Request OPENBLAS_V = '0.3.9' # Temporary build of OpenBLAS to test a fix for dynamic detection of CPU @@ -15,8 +17,25 @@ OPENBLAS_LONG = 'v0.3.7-527-g79fd006c' # the 0.3.7 is misleading BASE_LOC = 'https://anaconda.org/multibuild-wheels-staging/openblas-libs' BASEURL = f'{BASE_LOC}/{OPENBLAS_LONG}/download' ARCHITECTURES = ['', 'windows', 'darwin', 'aarch64', 'x86_64', 'i686', 'ppc64le', 's390x'] +sha256_vals = { +"openblas-v0.3.7-527-g79fd006c-win_amd64-gcc_7_1_0.zip": "7249d68c02e6b6339e06edfeab1fecddf29ee1e67a3afaa77917c320c43de840", +"openblas64_-v0.3.7-527-g79fd006c-win_amd64-gcc_7_1_0.zip": "6488e0961a5926e47242f63b63b41cfdd661e6f1d267e8e313e397cde4775c17", +"openblas-v0.3.7-527-g79fd006c-macosx_10_9_x86_64-gf_1becaaa.tar.gz": "69434bd626bbc495da9ce8c36b005d140c75e3c47f94e88c764a199e820f9259", +"openblas64_-v0.3.7-527-g79fd006c-macosx_10_9_x86_64-gf_1becaaa.tar.gz": "093f6d953e3fa76a86809be67bd1f0b27656671b5a55b233169cfaa43fd63e22", +"openblas-v0.3.7-527-g79fd006c-manylinux2014_aarch64.tar.gz": "42676c69dc48cd6e412251b39da6b955a5a0e00323ddd77f9137f7c259d35319", +"openblas64_-v0.3.7-527-g79fd006c-manylinux2014_aarch64.tar.gz": "5aec167af4052cf5e9e3e416c522d9794efabf03a2aea78b9bb3adc94f0b73d8", +"openblas-v0.3.7-527-g79fd006c-manylinux2010_x86_64.tar.gz": "fa67c6cc29d4cc5c70a147c80526243239a6f95fc3feadcf83a78176cd9c526b", +"openblas64_-v0.3.7-527-g79fd006c-manylinux2010_x86_64.tar.gz": "9ad34e89a5307dcf5823bf5c020580d0559a0c155fe85b44fc219752e61852b0", +"openblas-v0.3.7-527-g79fd006c-manylinux2010_i686.tar.gz": "0b8595d316c8b7be84ab1f1d5a0c89c1b35f7c987cdaf61d441bcba7ab4c7439", +"openblas-v0.3.7-527-g79fd006c-manylinux2014_ppc64le.tar.gz": "3e1c7d6472c34e7210e3605be4bac9ddd32f613d44297dc50cf2d067e720c4a9", +"openblas64_-v0.3.7-527-g79fd006c-manylinux2014_ppc64le.tar.gz": "a0885873298e21297a04be6cb7355a585df4fa4873e436b4c16c0a18fc9073ea", +"openblas-v0.3.7-527-g79fd006c-manylinux2014_s390x.tar.gz": "79b454320817574e20499d58f05259ed35213bea0158953992b910607b17f240", +"openblas64_-v0.3.7-527-g79fd006c-manylinux2014_s390x.tar.gz": "9fddbebf5301518fc4a5d2022a61886544a0566868c8c014359a1ee6b17f2814", +} + IS_32BIT = sys.maxsize < 2**32 + def get_arch(): if platform.system() == 'Windows': ret = 'windows' @@ -50,10 +69,12 @@ def get_manylinux(arch): def download_openblas(target, arch, ilp64): - import urllib3 ml_ver = get_manylinux(arch) fnsuffix = {None: "", "64_": "64_"}[ilp64] filename = '' + headers = {'User-Agent': ('Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 ; ' + '(KHTML, like Gecko) Chrome/41.0.2228.0 ' + 'Safari/537.3')} if arch in ('aarch64', 'ppc64le', 's390x', 'x86_64', 'i686'): suffix = f'manylinux{ml_ver}_{arch}.tar.gz' filename = f'{BASEURL}/openblas{fnsuffix}-{OPENBLAS_LONG}-{suffix}' @@ -71,15 +92,26 @@ def download_openblas(target, arch, ilp64): typ = 'zip' if not filename: return None - print("Downloading:", filename, file=sys.stderr) - http = urllib3.PoolManager() - response = http.request('GET', filename) + req = Request(url=filename, headers=headers) + response = urlopen(req) + length = response.getheader('content-length') if response.status != 200: print(f'Could not download "{filename}"', file=sys.stderr) return None + print(f"Downloading {length} from {filename}", file=sys.stderr) + data = response.read() + # Verify hash + key = os.path.basename(filename) + sha256_returned = hashlib.sha256(data).hexdigest() + if key not in sha256_vals: + raise ValueError( + f'key "{key}" with hash "{sha256_returned}" not in sha256_vals') + sha256_expected = sha256_vals[key] + if sha256_returned != sha256_expected: + raise ValueError(f'sha256 hash mismatch for filename {filename}') print("Saving to file", file=sys.stderr) with open(target, 'wb') as fid: - fid.write(response.data) + fid.write(data) return typ def setup_openblas(arch=get_arch(), ilp64=get_ilp64()): @@ -201,9 +233,10 @@ def test_setup(arches): def items(): for arch in arches: yield arch, None - if arch in ('x86', 'darwin', 'windows'): + if arch not in ('i686'): yield arch, '64_' + errs = [] for arch, ilp64 in items(): if arch == '': continue @@ -212,9 +245,11 @@ def test_setup(arches): try: try: target = setup_openblas(arch, ilp64) - except: - print(f'Could not setup {arch}') - raise + except Exception as e: + print(f'Could not setup {arch}:') + print(str(e)) + errs.append(e) + continue if not target: raise RuntimeError(f'Could not setup {arch}') print(target) @@ -231,6 +266,9 @@ def test_setup(arches): os.unlink(target) else: shutil.rmtree(target) + if errs: + raise errs[0] + def test_version(expected_version, ilp64=get_ilp64()): """ -- cgit v1.2.1