diff options
| -rw-r--r-- | azure-pipelines.yml | 40 | ||||
| -rw-r--r-- | shippable.yml | 12 | ||||
| -rw-r--r-- | tools/openblas_support.py | 167 | ||||
| -rwxr-xr-x | tools/pypy-test.sh | 19 | ||||
| -rwxr-xr-x | tools/travis-before-install.sh | 11 |
5 files changed, 194 insertions, 55 deletions
diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9e9001611..86aed8dab 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -20,21 +20,17 @@ jobs: docker pull i386/ubuntu:bionic docker run -v $(pwd):/numpy i386/ubuntu:bionic /bin/bash -c "cd numpy && \ apt-get -y update && \ - apt-get -y install python3.6-dev python3-pip locales && \ + apt-get -y install python3.6-dev python3-pip locales python3-certifi && \ locale-gen fr_FR && update-locale && \ pip3 install setuptools nose cython==0.29.0 pytest pytz pickle5 && \ apt-get -y install gfortran-5 wget && \ - cd .. && \ - mkdir openblas && cd openblas && \ - wget https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-v0.3.5-274-g6a8b4269-manylinux1_i686.tar.gz && \ - tar zxvf openblas-v0.3.5-274-g6a8b4269-manylinux1_i686.tar.gz && \ - cp -r ./usr/local/lib/* /usr/lib && \ - cp ./usr/local/include/* /usr/include && \ - cd ../numpy && \ + target=\$(python3 tools/openblas_support.py) && \ + cp -r \$target/usr/local/lib/* /usr/lib && \ + cp \$target/usr/local/include/* /usr/include && \ python3 -m pip install . && \ F77=gfortran-5 F90=gfortran-5 \ CFLAGS='-UNDEBUG -std=c99' python3 runtests.py -n --mode=full -- -rsx --junitxml=junit/test-results.xml && \ - cd ../openblas && python3 -c \"$(TEST_GET_CONFIG)\"" + cd .. && python3 -c \"$(TEST_GET_CONFIG)\"" displayName: 'Run 32-bit Ubuntu Docker Build / Tests' - task: PublishTestResults@2 condition: succeededOrFailed() @@ -83,11 +79,10 @@ jobs: # matches our MacOS wheel builds -- currently based # primarily on file size / name details - script: | - wget "https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-v0.3.5-274-g6a8b4269-macosx_10_9_x86_64-gf_1becaaa.tar.gz" - tar -zxvf openblas-v0.3.5-274-g6a8b4269-macosx_10_9_x86_64-gf_1becaaa.tar.gz + target=$(python tools/openblas_support.py) # manually link to appropriate system paths - cp ./usr/local/lib/* /usr/local/lib/ - cp ./usr/local/include/* /usr/local/include/ + cp $target/usr/local/lib/* /usr/local/lib/ + cp $target/usr/local/include/* /usr/local/include/ displayName: 'install pre-built openblas' - script: python -m pip install --upgrade pip setuptools wheel displayName: 'Install tools' @@ -124,11 +119,6 @@ jobs: - job: Windows pool: vmImage: 'VS2017-Win2016' - variables: - # openblas URLs from numpy-wheels - # appveyor / Windows config - OPENBLAS_32: "https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-v0.3.5-274-g6a8b4269-win32-gcc_7_1_0.zip" - OPENBLAS_64: "https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-v0.3.5-274-g6a8b4269-win_amd64-gcc_7_1_0.zip" strategy: maxParallel: 6 matrix: @@ -136,33 +126,28 @@ jobs: PYTHON_VERSION: '3.6' PYTHON_ARCH: 'x86' TEST_MODE: fast - OPENBLAS: $(OPENBLAS_32) BITS: 32 Python37-32bit-fast: PYTHON_VERSION: '3.7' PYTHON_ARCH: 'x86' TEST_MODE: fast - OPENBLAS: $(OPENBLAS_32) BITS: 32 Python35-64bit-full: PYTHON_VERSION: '3.5' PYTHON_ARCH: 'x64' TEST_MODE: full - OPENBLAS: $(OPENBLAS_64) BITS: 64 Python36-64bit-full: PYTHON_VERSION: '3.6' PYTHON_ARCH: 'x64' TEST_MODE: full INSTALL_PICKLE5: 1 - OPENBLAS: $(OPENBLAS_64) BITS: 64 Python37-64bit-full: PYTHON_VERSION: '3.7' PYTHON_ARCH: 'x64' TEST_MODE: full INSTALL_PICKLE5: 1 - OPENBLAS: $(OPENBLAS_64) BITS: 64 steps: - task: UsePythonVersion@0 @@ -176,17 +161,16 @@ jobs: displayName: 'Install dependencies; some are optional to avoid test skips' - script: if [%INSTALL_PICKLE5%]==[1] python -m pip install pickle5 displayName: 'Install optional pickle5 backport (only for python3.6 and 3.7)' + - powershell: | - $wc = New-Object net.webclient - $wc.Downloadfile("$(OPENBLAS)", "openblas.zip") - $tmpdir = New-TemporaryFile | %{ rm $_; mkdir $_ } - Expand-Archive "openblas.zip" $tmpdir $pyversion = python -c "from __future__ import print_function; import sys; print(sys.version.split()[0])" Write-Host "Python Version: $pyversion" $target = "C:\\hostedtoolcache\\windows\\Python\\$pyversion\\$(PYTHON_ARCH)\\lib\\openblas.a" Write-Host "target path: $target" - cp $tmpdir\$(BITS)\lib\libopenblas_v0.3.5-274-g6a8b4269-gcc_7_1_0.a $target + $openblas = python tools/openblas_support.py + cp $openblas $target displayName: 'Download / Install OpenBLAS' + - powershell: | choco install -y mingw --forcex86 --force --version=5.3.0 displayName: 'Install 32-bit mingw for 32-bit builds' diff --git a/shippable.yml b/shippable.yml index 9fbd17d9a..2f4856525 100644 --- a/shippable.yml +++ b/shippable.yml @@ -23,15 +23,11 @@ build: ci: # install dependencies - sudo apt-get install gcc gfortran - # ARMv8 OpenBLAS built using script available here: - # https://github.com/tylerjereddy/openblas-static-gcc/tree/master/ARMv8 - # build done on GCC compile farm machine named gcc115 - # tarball uploaded manually to an unshared Dropbox location - - wget -O openblas-v0.3.5-armv8.tar.gz https://www.dropbox.com/s/pbqkxzlmih4cky1/openblas-v0.3.5-armv8.tar.gz?dl=0 - - tar zxvf openblas-v0.3.5-armv8.tar.gz - - sudo cp -r ./64/lib/* /usr/lib - - sudo cp ./64/include/* /usr/include + - target=$(python tools/openblas_support.py) + - sudo cp -r "${target}"/64/lib/* /usr/lib + - sudo cp "${target}"/64/include/* /usr/include - pip install --upgrade pip + # we will pay the ~13 minute cost of compiling Cython only when a new # version is scraped in by pip; otherwise, use the cached # wheel shippable places on Amazon S3 after we build it once diff --git a/tools/openblas_support.py b/tools/openblas_support.py index 52d283a6c..460f36de5 100644 --- a/tools/openblas_support.py +++ b/tools/openblas_support.py @@ -1,5 +1,141 @@ +from __future__ import division, absolute_import, print_function import os +import sys import textwrap +import platform +try: + from urllib.request import urlopen + from urllib.error import HTTPError +except: + #Python2 + from urllib2 import urlopen, HTTPError + +from tempfile import mkstemp, gettempdir +import zipfile +import tarfile + +OPENBLAS_V = 'v0.3.5' +OPENBLAS_LONG = 'v0.3.5-274-g6a8b4269' +BASE_LOC = '' +RACKSPACE = 'https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com' +ARCHITECTURES = ['', 'windows', 'darwin', 'arm', 'x86', 'ppc64'] + +IS_32BIT = sys.maxsize < 2**32 +def get_arch(): + if platform.system() == 'Windows': + ret = 'windows' + elif platform.system() == 'Darwin': + ret = 'darwin' + # Python3 returns a named tuple, but Python2 does not, so we are stuck + elif 'arm' in os.uname()[-1]: + ret = 'arm'; + elif 'aarch64' in os.uname()[-1]: + ret = 'arm'; + elif 'x86' in os.uname()[-1]: + ret = 'x86' + elif 'ppc64' in os.uname()[-1]: + ret = 'ppc64' + else: + ret = '' + assert ret in ARCHITECTURES + return ret + +def download_openblas(target, arch): + filename = '' + if arch == 'arm': + # ARMv8 OpenBLAS built using script available here: + # https://github.com/tylerjereddy/openblas-static-gcc/tree/master/ARMv8 + # build done on GCC compile farm machine named gcc115 + # tarball uploaded manually to an unshared Dropbox location + filename = ('https://www.dropbox.com/s/pbqkxzlmih4cky1/' + 'openblas-{}-armv8.tar.gz?dl=1'.format(OPENBLAS_V)) + typ = 'tar.gz' + elif arch == 'ppc64': + # build script for POWER8 OpenBLAS available here: + # https://github.com/tylerjereddy/openblas-static-gcc/blob/master/power8 + # built on GCC compile farm machine named gcc112 + # manually uploaded tarball to an unshared Dropbox location + filename = ('https://www.dropbox.com/s/zcwhk7c2zptwy0s/' + 'openblas-{}-ppc64le-power8.tar.gz?dl=1'.format(OPENBLAS_V)) + typ = 'tar.gz' + elif arch == 'darwin': + filename = '{0}/openblas-{1}-macosx_10_9_x86_64-gf_1becaaa.tar.gz'.format( + RACKSPACE, OPENBLAS_LONG) + typ = 'tar.gz' + elif arch == 'windows': + if IS_32BIT: + suffix = 'win32-gcc_7_1_0.zip' + else: + suffix = 'win_amd64-gcc_7_1_0.zip' + filename = '{0}/openblas-{1}-{2}'.format(RACKSPACE, OPENBLAS_LONG, suffix) + typ = 'zip' + elif arch == 'x86': + if IS_32BIT: + suffix = 'manylinux1_i686.tar.gz' + else: + suffix = 'manylinux1_x86_64.tar.gz' + filename = '{0}/openblas-{1}-{2}'.format(RACKSPACE, OPENBLAS_LONG, suffix) + typ = 'tar.gz' + if not filename: + return None + try: + with open(target, 'wb') as fid: + fid.write(urlopen(filename).read()) + except HTTPError: + print('Could not download "%s"' % filename) + return None + return typ + +def setup_openblas(arch=get_arch()): + ''' + Download and setup an openblas library for building. If successful, + the configuration script will find it automatically. + + Returns + ------- + msg : str + path to extracted files on success, otherwise indicates what went wrong + To determine success, do ``os.path.exists(msg)`` + ''' + _, tmp = mkstemp() + if not arch: + raise ValueError('unknown architecture') + typ = download_openblas(tmp, arch) + if not typ: + return '' + if arch == 'windows': + if not typ == 'zip': + return 'expecting to download zipfile on windows, not %s' % str(typ) + return unpack_windows_zip(tmp) + else: + if not typ == 'tar.gz': + return 'expecting to download tar.gz, not %s' % str(typ) + return unpack_targz(tmp) + +def unpack_windows_zip(fname): + import sysconfig + with zipfile.ZipFile(fname, 'r') as zf: + # Get the openblas.a file, but not openblas.dll.a nor openblas.dev.a + lib = [x for x in zf.namelist() if OPENBLAS_LONG in x and + x.endswith('a') and not x.endswith('dll.a') and + not x.endswith('dev.a')] + if not lib: + return 'could not find libopenblas_%s*.a ' \ + 'in downloaded zipfile' % OPENBLAS_LONG + target = os.path.join(gettempdir(), 'openblas.a') + with open(target, 'wb') as fid: + fid.write(zf.read(lib[0])) + return target + +def unpack_targz(fname): + target = os.path.join(gettempdir(), 'openblas') + if not os.path.exists(target): + os.mkdir(target) + with tarfile.open(fname, 'r') as zf: + # TODO: check that all the zf.getnames() files do not escape the + # extract directory (no leading '../', '/') + zf.extractall(target) + return target def make_init(dirname): ''' @@ -40,3 +176,34 @@ def make_init(dirname): stacklevel=1) """)) +def test_setup(arches): + ''' + Make sure all the downloadable files exist and can be opened + ''' + for arch in arches: + if arch == '': + continue + try: + target = setup_openblas(arch) + except: + print('Could not setup %s' % arch) + raise + if not target: + raise RuntimeError('Could not setup %s' % arch) + print(target) + +if __name__ == '__main__': + import argparse + parser = argparse.ArgumentParser( + description='Download and expand an OpenBLAS archive for this ' \ + 'architecture') + parser.add_argument('--test', nargs='*', default=None, + help='Test different architectures. "all", or any of %s' % ARCHITECTURES) + args = parser.parse_args() + if args.test is None: + print(setup_openblas()) + else: + if len(args.test) == 0 or 'all' in args.test: + test_setup(ARCHITECTURES) + else: + test_setup(args.test) diff --git a/tools/pypy-test.sh b/tools/pypy-test.sh index 28afdea5d..314ebbb36 100755 --- a/tools/pypy-test.sh +++ b/tools/pypy-test.sh @@ -11,22 +11,19 @@ sudo apt-get -yq install libatlas-base-dev liblapack-dev gfortran-5 F77=gfortran-5 F90=gfortran-5 \ # Download the proper OpenBLAS x64 precompiled library -OPENBLAS=openblas-v0.3.5-274-g6a8b4269-manylinux1_x86_64.tar.gz -echo getting $OPENBLAS -wget -q https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/$OPENBLAS -O openblas.tar.gz -mkdir -p openblas -(cd openblas; tar -xf ../openblas.tar.gz) -export LD_LIBRARY_PATH=$PWD/openblas/usr/local/lib -export LIB=$PWD/openblas/usr/local/lib -export INCLUDE=$PWD/openblas/usr/local/include +target=$(python tools/openblas_support.py) +echo getting OpenBLAS into $target +export LD_LIBRARY_PATH=$target/usr/local/lib +export LIB=$target/usr/local/lib +export INCLUDE=$target/usr/local/include # Use a site.cfg to build with local openblas cat << EOF > site.cfg [openblas] libraries = openblas -library_dirs = $PWD/openblas/usr/local/lib:$LIB -include_dirs = $PWD/openblas/usr/local/lib:$LIB -runtime_library_dirs = $PWD/openblas/usr/local/lib +library_dirs = $target/usr/local/lib:$LIB +include_dirs = $target/usr/local/lib:$LIB +runtime_library_dirs = $target/usr/local/lib EOF echo getting PyPy 3.6 nightly diff --git a/tools/travis-before-install.sh b/tools/travis-before-install.sh index db1f0bc5c..7e5131dcf 100755 --- a/tools/travis-before-install.sh +++ b/tools/travis-before-install.sh @@ -26,14 +26,9 @@ if [ -n "$INSTALL_PICKLE5" ]; then fi if [ -n "$PPC64_LE" ]; then - # build script for POWER8 OpenBLAS available here: - # https://github.com/tylerjereddy/openblas-static-gcc/blob/master/power8 - # built on GCC compile farm machine named gcc112 - # manually uploaded tarball to an unshared Dropbox location - wget -O openblas-power8.tar.gz https://www.dropbox.com/s/zcwhk7c2zptwy0s/openblas-v0.3.5-ppc64le-power8.tar.gz?dl=0 - tar zxvf openblas-power8.tar.gz - sudo cp -r ./64/lib/* /usr/lib - sudo cp ./64/include/* /usr/include + target=$(python tools/openblas_support.py) + sudo cp -r $target/64/lib/* /usr/lib + sudo cp $target/64/include/* /usr/include fi pip install --upgrade pip setuptools |