Merge pull request #15639 from mattip/update-openblas

BLD: update openblas download to new location, use manylinux2010-base

6 files changed, 31 insertions, 26 deletions

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index f1af9a333..46d39ca1d 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -39,10 +39,10 @@ stages:
             /bin/bash -xc "cd numpy && \
             /opt/python/cp38-cp38/bin/python -mvenv venv &&\
             source venv/bin/activate && \
+            python3 -m pip install urllib3 && \
             target=\$(python3 tools/openblas_support.py) && \
             cp -r \$target/lib/* /usr/lib && \
             cp \$target/include/* /usr/include && \
-            yum install -y libgfortran-4.4.7 && \
             python3 -m pip install -r test_requirements.txt && \
             echo CFLAGS \$CFLAGS && \
             python3 -m pip install -v . && \
@@ -105,6 +105,7 @@ stages:
     # matches our MacOS wheel builds -- currently based
     # primarily on file size / name details
     - script: |
+        python -mpip install urllib3
         target=$(python tools/openblas_support.py)
         ls -lR $target
         # manually link to appropriate system paths
diff --git a/azure-steps-windows.yml b/azure-steps-windows.yml
index fdc969890..eddc6a9b9 100644
--- a/azure-steps-windows.yml
+++ b/azure-steps-windows.yml
@@ -13,6 +13,7 @@ steps:
     Write-Host "Python Version: $pyversion"
     $target = "C:\\hostedtoolcache\\windows\\Python\\$pyversion\\$(PYTHON_ARCH)\\lib\\openblas$env:OPENBLAS_SUFFIX.a"
     Write-Host "target path: $target"
+    python -mpip install urllib3
     $openblas = python tools/openblas_support.py
     cp $openblas $target
   displayName: 'Download / Install OpenBLAS'
diff --git a/shippable.yml b/shippable.yml
index bea2a2206..513ea3069 100644
--- a/shippable.yml
+++ b/shippable.yml
@@ -25,6 +25,7 @@ build:
     - sudo add-apt-repository ppa:ubuntu-toolchain-r/test
     - sudo apt-get update
     - sudo apt-get install gcc gfortran libgfortran5
+    - python -mpip install urllib3
     - target=$(python tools/openblas_support.py)
     - ls -lR "${target}"
     - sudo cp -r "${target}"/lib/* /usr/lib
diff --git a/tools/openblas_support.py b/tools/openblas_support.py
index cee76022f..7a88fe2ca 100644
--- a/tools/openblas_support.py
+++ b/tools/openblas_support.py
@@ -6,15 +6,13 @@ import textwrap
 import platform
 
 from tempfile import mkstemp, gettempdir
-from urllib.request import urlopen
-from urllib.error import HTTPError
 import zipfile
 import tarfile
 
 OPENBLAS_V = 'v0.3.8'
 OPENBLAS_LONG = 'v0.3.5-605-gc815b8fb'  # the 0.3.5 is misleading
-BASE_LOC = ''
-RACKSPACE = 'https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com'
+BASE_LOC = 'https://anaconda.org/multibuild-wheels-staging/openblas-libs'
+BASEURL = f'{BASE_LOC}/{OPENBLAS_LONG}/download'
 ARCHITECTURES = ['', 'windows', 'darwin', 'aarch64', 'x86', 'ppc64le', 's390x']
 
 IS_32BIT = sys.maxsize < 2**32
@@ -40,40 +38,43 @@ def get_ilp64():
     return "64_"
 
 def download_openblas(target, arch, ilp64):
+    import urllib3
     fnsuffix = {None: "", "64_": "64_"}[ilp64]
     filename = ''
     if arch in ('aarch64', 'ppc64le', 's390x'):
         suffix = f'manylinux2014_{arch}.tar.gz'
-        filename = f'{RACKSPACE}/openblas{fnsuffix}-{OPENBLAS_LONG}-{suffix}'
+        filename = f'{BASEURL}/openblas{fnsuffix}-{OPENBLAS_LONG}-{suffix}'
         typ = 'tar.gz'
         typ = 'tar.gz'
     elif arch == 'darwin':
         suffix = 'macosx_10_9_x86_64-gf_1becaaa.tar.gz'
-        filename = f'{RACKSPACE}/openblas{fnsuffix}-{OPENBLAS_LONG}-{suffix}'
+        filename = f'{BASEURL}/openblas{fnsuffix}-{OPENBLAS_LONG}-{suffix}'
         typ = 'tar.gz'
     elif arch == 'windows':
         if IS_32BIT:
             suffix = 'win32-gcc_7_1_0.zip'
         else:
             suffix = 'win_amd64-gcc_7_1_0.zip'
-        filename = f'{RACKSPACE}/openblas{fnsuffix}-{OPENBLAS_LONG}-{suffix}'
+        filename = f'{BASEURL}/openblas{fnsuffix}-{OPENBLAS_LONG}-{suffix}'
         typ = 'zip'
     elif 'x86' in arch:
         if IS_32BIT:
-            suffix = 'manylinux1_i686.tar.gz'
+            suffix = 'manylinux2010_i686.tar.gz'
         else:
-            suffix = 'manylinux1_x86_64.tar.gz'
-        filename = f'{RACKSPACE}/openblas{fnsuffix}-{OPENBLAS_LONG}-{suffix}'
+            suffix = 'manylinux2010_x86_64.tar.gz'
+        filename = f'{BASEURL}/openblas{fnsuffix}-{OPENBLAS_LONG}-{suffix}'
         typ = 'tar.gz'
     if not filename:
         return None
     print("Downloading:", filename, file=sys.stderr)
-    try:
-        with open(target, 'wb') as fid:
-            fid.write(urlopen(filename).read())
-    except HTTPError:
-        print(f'Could not download "{filename}"')
+    http = urllib3.PoolManager()
+    response = http.request('GET', filename)
+    if response.status != 200:
+        print(f'Could not download "{filename}"', file=sys.stderr)
         return None
+    print("Saving to file", file=sys.stderr)
+    with open(target, 'wb') as fid:
+        fid.write(response.data)
     return typ
 
 def setup_openblas(arch=get_arch(), ilp64=get_ilp64()):
diff --git a/tools/pypy-test.sh b/tools/pypy-test.sh
index 68e6c1cfa..5a7b28f9a 100755
--- a/tools/pypy-test.sh
+++ b/tools/pypy-test.sh
@@ -6,7 +6,7 @@ set -o pipefail
 # Print expanded commands
 set -x
 
-sudo apt-get -yq install libatlas-base-dev liblapack-dev gfortran-5
+sudo apt-get -yq install libatlas-base-dev liblapack-dev gfortran-5 python3-urllib3
 F77=gfortran-5 F90=gfortran-5 \
 
 # Download the proper OpenBLAS x64 precompiled library
diff --git a/tools/travis-before-install.sh b/tools/travis-before-install.sh
index ab11e73f9..dbe2f6ea2 100755
--- a/tools/travis-before-install.sh
+++ b/tools/travis-before-install.sh
@@ -9,14 +9,6 @@ free -m
 df -h
 ulimit -a
 
-if [ -n "$DOWNLOAD_OPENBLAS" ]; then
-  pwd
-  ls -ltrh
-  target=$(python tools/openblas_support.py)
-  sudo cp -r $target/lib/* /usr/lib
-  sudo cp $target/include/* /usr/include
-fi
-
 mkdir builds
 pushd builds
 
@@ -49,6 +41,15 @@ pip install --upgrade pip
 # A specific version of cython is required, so we read the cython package
 # requirement using `grep cython test_requirements.txt` instead of simply
 # writing 'pip install setuptools wheel cython'.
-pip install setuptools wheel `grep cython test_requirements.txt`
+# urllib3 is needed for openblas_support
+pip install setuptools wheel urllib3 `grep cython test_requirements.txt`
+
+if [ -n "$DOWNLOAD_OPENBLAS" ]; then
+  pwd
+  target=$(python tools/openblas_support.py)
+  sudo cp -r $target/lib/* /usr/lib
+  sudo cp $target/include/* /usr/include
+fi
+
 
 if [ -n "$USE_ASV" ]; then pip install asv; fi