ENH: backport scipy changes to openblas download script

1 files changed, 51 insertions, 13 deletions

diff --git a/tools/openblas_support.py b/tools/openblas_support.py
index a729662c5..a4b63c800 100644
--- a/tools/openblas_support.py
+++ b/tools/openblas_support.py
@@ -1,13 +1,15 @@
+import glob
+import hashlib
 import os
+import platform
 import sys
-import glob
 import shutil
+import tarfile
 import textwrap
-import platform
+import zipfile
 
 from tempfile import mkstemp, gettempdir
-import zipfile
-import tarfile
+from urllib.request import urlopen, Request
 
 OPENBLAS_V = '0.3.9'
 # Temporary build of OpenBLAS to test a fix for dynamic detection of CPU
@@ -15,8 +17,25 @@ OPENBLAS_LONG = 'v0.3.7-527-g79fd006c'  # the 0.3.7 is misleading
 BASE_LOC = 'https://anaconda.org/multibuild-wheels-staging/openblas-libs'
 BASEURL = f'{BASE_LOC}/{OPENBLAS_LONG}/download'
 ARCHITECTURES = ['', 'windows', 'darwin', 'aarch64', 'x86_64', 'i686', 'ppc64le', 's390x']
+sha256_vals = {
+"openblas-v0.3.7-527-g79fd006c-win_amd64-gcc_7_1_0.zip": "7249d68c02e6b6339e06edfeab1fecddf29ee1e67a3afaa77917c320c43de840",
+"openblas64_-v0.3.7-527-g79fd006c-win_amd64-gcc_7_1_0.zip": "6488e0961a5926e47242f63b63b41cfdd661e6f1d267e8e313e397cde4775c17",
+"openblas-v0.3.7-527-g79fd006c-macosx_10_9_x86_64-gf_1becaaa.tar.gz": "69434bd626bbc495da9ce8c36b005d140c75e3c47f94e88c764a199e820f9259",
+"openblas64_-v0.3.7-527-g79fd006c-macosx_10_9_x86_64-gf_1becaaa.tar.gz": "093f6d953e3fa76a86809be67bd1f0b27656671b5a55b233169cfaa43fd63e22",
+"openblas-v0.3.7-527-g79fd006c-manylinux2014_aarch64.tar.gz": "42676c69dc48cd6e412251b39da6b955a5a0e00323ddd77f9137f7c259d35319",
+"openblas64_-v0.3.7-527-g79fd006c-manylinux2014_aarch64.tar.gz": "5aec167af4052cf5e9e3e416c522d9794efabf03a2aea78b9bb3adc94f0b73d8",
+"openblas-v0.3.7-527-g79fd006c-manylinux2010_x86_64.tar.gz": "fa67c6cc29d4cc5c70a147c80526243239a6f95fc3feadcf83a78176cd9c526b",
+"openblas64_-v0.3.7-527-g79fd006c-manylinux2010_x86_64.tar.gz": "9ad34e89a5307dcf5823bf5c020580d0559a0c155fe85b44fc219752e61852b0",
+"openblas-v0.3.7-527-g79fd006c-manylinux2010_i686.tar.gz": "0b8595d316c8b7be84ab1f1d5a0c89c1b35f7c987cdaf61d441bcba7ab4c7439",
+"openblas-v0.3.7-527-g79fd006c-manylinux2014_ppc64le.tar.gz": "3e1c7d6472c34e7210e3605be4bac9ddd32f613d44297dc50cf2d067e720c4a9",
+"openblas64_-v0.3.7-527-g79fd006c-manylinux2014_ppc64le.tar.gz": "a0885873298e21297a04be6cb7355a585df4fa4873e436b4c16c0a18fc9073ea",
+"openblas-v0.3.7-527-g79fd006c-manylinux2014_s390x.tar.gz": "79b454320817574e20499d58f05259ed35213bea0158953992b910607b17f240",
+"openblas64_-v0.3.7-527-g79fd006c-manylinux2014_s390x.tar.gz": "9fddbebf5301518fc4a5d2022a61886544a0566868c8c014359a1ee6b17f2814",
+}
+
 
 IS_32BIT = sys.maxsize < 2**32
+
 def get_arch():
     if platform.system() == 'Windows':
         ret = 'windows'
@@ -50,10 +69,12 @@ def get_manylinux(arch):
 
 
 def download_openblas(target, arch, ilp64):
-    import urllib3
     ml_ver = get_manylinux(arch)
     fnsuffix = {None: "", "64_": "64_"}[ilp64]
     filename = ''
+    headers = {'User-Agent': ('Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 ; '
+                              '(KHTML, like Gecko) Chrome/41.0.2228.0 '
+                              'Safari/537.3')}
     if arch in ('aarch64', 'ppc64le', 's390x', 'x86_64', 'i686'):
         suffix = f'manylinux{ml_ver}_{arch}.tar.gz'
         filename = f'{BASEURL}/openblas{fnsuffix}-{OPENBLAS_LONG}-{suffix}'
@@ -71,15 +92,26 @@ def download_openblas(target, arch, ilp64):
         typ = 'zip'
     if not filename:
         return None
-    print("Downloading:", filename, file=sys.stderr)
-    http = urllib3.PoolManager()
-    response = http.request('GET', filename)
+    req = Request(url=filename, headers=headers)
+    response = urlopen(req)
+    length = response.getheader('content-length')
     if response.status != 200:
         print(f'Could not download "{filename}"', file=sys.stderr)
         return None
+    print(f"Downloading {length} from {filename}", file=sys.stderr)
+    data = response.read()
+    # Verify hash
+    key = os.path.basename(filename)
+    sha256_returned = hashlib.sha256(data).hexdigest()
+    if key not in sha256_vals:
+        raise ValueError(
+            f'key "{key}" with hash "{sha256_returned}" not in sha256_vals')
+    sha256_expected = sha256_vals[key]
+    if sha256_returned != sha256_expected:
+        raise ValueError(f'sha256 hash mismatch for filename {filename}')
     print("Saving to file", file=sys.stderr)
     with open(target, 'wb') as fid:
-        fid.write(response.data)
+        fid.write(data)
     return typ
 
 def setup_openblas(arch=get_arch(), ilp64=get_ilp64()):
@@ -201,9 +233,10 @@ def test_setup(arches):
     def items():
         for arch in arches:
             yield arch, None
-            if arch in ('x86', 'darwin', 'windows'):
+            if arch not in ('i686'):
                 yield arch, '64_'
 
+    errs = []
     for arch, ilp64 in items():
         if arch == '':
             continue
@@ -212,9 +245,11 @@ def test_setup(arches):
         try:
             try:
                 target = setup_openblas(arch, ilp64)
-            except:
-                print(f'Could not setup {arch}')
-                raise
+            except Exception as e:
+                print(f'Could not setup {arch}:')
+                print(str(e))
+                errs.append(e)
+                continue
             if not target:
                 raise RuntimeError(f'Could not setup {arch}')
             print(target)
@@ -231,6 +266,9 @@ def test_setup(arches):
                     os.unlink(target)
                 else:
                     shutil.rmtree(target)
+    if errs:
+        raise errs[0]
+
 
 def test_version(expected_version, ilp64=get_ilp64()):
     """