Merge pull request #9645 from xoviat/appveyor

ENH: enable OpenBLAS on windows.

9 files changed, 220 insertions, 36 deletions

diff --git a/appveyor.yml b/appveyor.yml
index 9f4aacc29..fcd79def6 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -1,30 +1,195 @@
-skip_tags: true
-clone_depth: 1
+# As config was originally based on an example by Olivier Grisel. Thanks!
+# https://github.com/ogrisel/python-appveyor-demo/blob/master/appveyor.yml
+clone_depth: 50
 
-os: Visual Studio 2015
+# No reason for us to restrict the number concurrent jobs
+max_jobs: 100
+
+cache:
+  - '%LOCALAPPDATA%\pip\Cache'
 
 environment:
+  global:
+      MINGW_32: C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin
+      MINGW_64: C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin
+      OPENBLAS_32: https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-5f998ef_gcc7_1_0_win32.zip
+      OPENBLAS_64: https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-5f998ef_gcc7_1_0_win64.zip
+      APPVEYOR_SAVE_CACHE_ON_ERROR: true
+      APPVEYOR_SKIP_FINALIZE_ON_EXIT: true
+      TEST_TIMEOUT: 1000
+
   matrix:
-    - PY_MAJOR_VER: 2
-      PYTHON_ARCH: "x86"
-    - PY_MAJOR_VER: 3
-      PYTHON_ARCH: "x86_64"
-    - PY_MAJOR_VER: 3
-      PYTHON_ARCH: "x86"
+    - PYTHON: C:\Python36
+      PYTHON_VERSION: 3.6
+      PYTHON_ARCH: 32
+      TEST_MODE: fast
 
-build_script:
-# If there's a newer build queued for the same PR, cancel this one
+    - PYTHON: C:\Python27-x64
+      PYTHON_VERSION: 2.7
+      PYTHON_ARCH: 64
+      TEST_MODE: fast
+
+    - PYTHON: C:\Python34-x64
+      PYTHON_VERSION: 3.4
+      PYTHON_ARCH: 64
+      TEST_MODE: fast
+
+    - PYTHON: C:\Python36-x64
+      PYTHON_VERSION: 3.6
+      PYTHON_ARCH: 64
+      TEST_MODE: full
+
+    - PYTHON: C:\Python27
+      PYTHON_VERSION: 2.7
+      PYTHON_ARCH: 32
+      SKIP_NOTAG: true
+      TEST_MODE: full
+
+    - PYTHON: C:\Python34
+      PYTHON_VERSION: 3.4
+      PYTHON_ARCH: 32
+      SKIP_NOTAG: true
+      TEST_MODE: full
+
+    - PYTHON: C:\Python35-x64
+      PYTHON_VERSION: 3.5
+      PYTHON_ARCH: 64
+      SKIP_NOTAG: true
+      TEST_MODE: full
+
+    - PYTHON: C:\Python35
+      PYTHON_VERSION: 3.5
+      PYTHON_ARCH: 32
+      SKIP_NOTAG: true
+      TEST_MODE: full
+
+init:
+  - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%"
+  - "ECHO \"%APPVEYOR_SCHEDULED_BUILD%\""
+  # If there is a newer build queued for the same PR, cancel this one.
+  # The AppVeyor 'rollout builds' option is supposed to serve the same
+  # purpose but it is problematic because it tends to cancel builds pushed
+  # directly to master instead of just PR builds (or the converse).
+  # credits: JuliaLang developers.
   - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod `
         https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | `
         Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { `
-        throw "There are newer queued builds for this pull request, failing early." }
-  - ps: Start-FileDownload "https://repo.continuum.io/miniconda/Miniconda$env:PY_MAJOR_VER-latest-Windows-$env:PYTHON_ARCH.exe" C:\Miniconda.exe; echo "Finished downloading miniconda"
-  - cmd: C:\Miniconda.exe /S /D=C:\Py
-  - SET PATH=C:\Py;C:\Py\Scripts;C:\Py\Library\bin;%PATH%
-  - conda config --set always_yes yes
-  - conda update conda
-  - conda install cython nose pytz
-  - pip install . -vvv
+          Write-Host "There are newer queued builds for this pull request, skipping build."
+          Exit-AppveyorBuild
+        }
+  - ps: |
+      If (($env:SKIP_NOTAG -eq "true") -and ($env:APPVEYOR_REPO_TAG -ne "true")) {
+          Write-Host "Skipping build, not at a tag."
+          Exit-AppveyorBuild
+      }
+
+install:
+  # Show size of cache
+  - C:\cygwin\bin\du -hs "%LOCALAPPDATA%\pip\Cache"
+  # Prepend newly installed Python to the PATH of this build (this cannot be
+  # done from inside the powershell script as it would require to restart
+  # the parent CMD process).
+  - SET PATH=%PYTHON%;%PYTHON%\Scripts;%PATH%
+
+  # Check that we have the expected version and architecture for Python
+  - python --version
+  - >-
+     %CMD_IN_ENV%
+     python -c "import sys,platform,struct;
+     print(sys.platform, platform.machine(), struct.calcsize('P') * 8, )"
+
+  # Install "openblas.a" to PYTHON\lib
+  # Library provided by Matthew Brett at https://github.com/matthew-brett/build-openblas
+  - ps: |
+      $PYTHON_ARCH = $env:PYTHON_ARCH
+      $PYTHON = $env:PYTHON
+      If ($PYTHON_ARCH -eq 32) {
+          $OPENBLAS = $env:OPENBLAS_32
+      } Else {
+          $OPENBLAS = $env:OPENBLAS_64
+      }
+      $clnt = new-object System.Net.WebClient
+      $file = "$(New-TemporaryFile).zip"
+      $tmpdir = New-TemporaryFile | %{ rm $_; mkdir $_ }
+      $destination = "$PYTHON\lib\openblas.a"
+
+      echo $file
+      echo $tmpdir
+      echo $OPENBLAS
+
+      $clnt.DownloadFile($OPENBLAS,$file)
+      Get-FileHash $file | Format-List
+
+      Expand-Archive $file $tmpdir      
+
+      rm $tmpdir\$PYTHON_ARCH\lib\*.dll.a
+      $lib = ls $tmpdir\$PYTHON_ARCH\lib\*.a | ForEach { ls $_ } | Select-Object -first 1
+      echo $lib
+
+      cp $lib $destination
+      ls $destination
+
+  # Upgrade to the latest pip.
+  - '%CMD_IN_ENV% python -m pip install -U pip setuptools wheel'
+
+  # Install the numpy test dependencies.
+  - '%CMD_IN_ENV% pip install -U --timeout 5 --retries 2 -r tools/ci/appveyor/requirements.txt'
+
+build_script:
+  # Here, we add MinGW to the path to be able to link an OpenBLAS.dll
+  # We then use the import library from the DLL to compile with MSVC
+  - ps: |
+      $PYTHON_ARCH = $env:PYTHON_ARCH
+      If ($PYTHON_ARCH -eq 32) {
+          $MINGW = $env:MINGW_32 
+      } Else {
+          $MINGW = $env:MINGW_64
+      }
+      $env:Path += ";$MINGW"
+      $env:NPY_NUM_BUILD_JOBS = "4"
+      mkdir dist
+      pip wheel -v -v -v --wheel-dir=dist .
+
+      # For each wheel that pip has placed in the "dist" directory
+      # First, upload the wheel to the "artifacts" tab and then
+      # install the wheel. If we have only built numpy (as is the case here),
+      # then there will be one wheel to install.
+      
+      # This method is more representative of what will be distributed,
+      # because it actually tests what the built wheels will be rather than
+      # what 'setup.py install' will do and at it uploads the wheels so that
+      # they can be inspected.
+
+      ls dist -r | Foreach-Object {
+          appveyor PushArtifact $_.FullName
+          pip install $_.FullName
+      }
 
 test_script:
   - python runtests.py -v -n
+
+after_build:
+  # Remove old or huge cache files to hopefully not exceed the 1GB cache limit.
+  #
+  # If the cache limit is reached, the cache will not be updated (of not even
+  # created in the first run). So this is a trade of between keeping the cache
+  # current and having a cache at all.
+  # NB: This is done only `on_success` since the cache in uploaded only on
+  # success anyway.
+  - C:\cygwin\bin\find "%LOCALAPPDATA%\pip" -type f -mtime +360 -delete
+  - C:\cygwin\bin\find "%LOCALAPPDATA%\pip" -type f -size +10M -delete
+  - C:\cygwin\bin\find "%LOCALAPPDATA%\pip" -empty -delete
+  # Show size of cache
+  - C:\cygwin\bin\du -hs "%LOCALAPPDATA%\pip\Cache"
+
+on_finish:
+  # We can get a nice display of test results in the "test" tab with py.test
+  # For now, this does nothing.
+  - ps: |
+      If (Test-Path .\junit-results.xml) {
+        (new-object net.webclient).UploadFile(
+          "https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)",
+          (Resolve-Path .\junit-results.xml)
+        )
+      }
+      $LastExitCode = 0
diff --git a/numpy/distutils/tests/test_system_info.py b/numpy/distutils/tests/test_system_info.py
index 026179d37..de680298c 100644
--- a/numpy/distutils/tests/test_system_info.py
+++ b/numpy/distutils/tests/test_system_info.py
@@ -68,7 +68,7 @@ def have_compiler():
         try:
             if not compiler.initialized:
                 compiler.initialize()  # MSVC is different
-        except DistutilsError:
+        except (DistutilsError, ValueError):
             return False
         cmd = [compiler.cc]
     try:
diff --git a/numpy/f2py/tests/test_block_docstring.py b/numpy/f2py/tests/test_block_docstring.py
index c3f9dc856..eb11201ef 100644
--- a/numpy/f2py/tests/test_block_docstring.py
+++ b/numpy/f2py/tests/test_block_docstring.py
@@ -1,9 +1,10 @@
 from __future__ import division, absolute_import, print_function
 
 import textwrap
+import sys
 from . import util
 
-from numpy.testing import run_module_suite, assert_equal
+from numpy.testing import run_module_suite, assert_equal, dec
 
 class TestBlockDocString(util.F2PyTest):
     code = """
@@ -15,6 +16,7 @@ class TestBlockDocString(util.F2PyTest):
       END
     """
 
+    @dec.knownfailureif(sys.platform=='win32', msg='Fails with MinGW64 Gfortran (Issue #9673)')
     def test_block_docstring(self):
         expected = "'i'-array(2,3)\n"
         assert_equal(self.module.block.__doc__, expected)
diff --git a/numpy/f2py/tests/test_callback.py b/numpy/f2py/tests/test_callback.py
index ea29043ed..cf7427d20 100644
--- a/numpy/f2py/tests/test_callback.py
+++ b/numpy/f2py/tests/test_callback.py
@@ -2,6 +2,7 @@ from __future__ import division, absolute_import, print_function
 
 import math
 import textwrap
+import sys
 
 from numpy import array
 from numpy.testing import run_module_suite, assert_, assert_equal, dec
@@ -119,6 +120,7 @@ cf2py  intent(out) a
         r = t(a.mth)
         assert_(r == 9, repr(r))
 
+    @dec.knownfailureif(sys.platform=='win32', msg='Fails with MinGW64 Gfortran (Issue #9673)')
     def test_string_callback(self):
 
         def callback(code):
diff --git a/numpy/f2py/tests/test_common.py b/numpy/f2py/tests/test_common.py
index aaa35b678..81082e575 100644
--- a/numpy/f2py/tests/test_common.py
+++ b/numpy/f2py/tests/test_common.py
@@ -1,11 +1,11 @@
 from __future__ import division, absolute_import, print_function
 
 import os
-
-from numpy.testing import run_module_suite, assert_array_equal, dec
+import sys
 import numpy as np
 from . import util
 
+from numpy.testing import run_module_suite, assert_array_equal, dec
 
 def _path(*a):
     return os.path.join(*((os.path.dirname(__file__),) + a))
@@ -13,6 +13,7 @@ def _path(*a):
 class TestCommonBlock(util.F2PyTest):
     sources = [_path('src', 'common', 'block.f')]
 
+    @dec.knownfailureif(sys.platform=='win32', msg='Fails with MinGW64 Gfortran (Issue #9673)')
     def test_common_block(self):
         self.module.initcb()
         assert_array_equal(self.module.block.long_bn,
diff --git a/numpy/f2py/tests/util.py b/numpy/f2py/tests/util.py
index 55716a2eb..881b32810 100644
--- a/numpy/f2py/tests/util.py
+++ b/numpy/f2py/tests/util.py
@@ -16,10 +16,11 @@ import atexit
 import textwrap
 import re
 import random
+import numpy.f2py
 
 from numpy.compat import asbytes, asstr
-import numpy.f2py
-from numpy.testing import SkipTest, temppath
+from numpy.testing import SkipTest, temppath, dec
+from importlib import import_module
 
 try:
     from hashlib import md5
@@ -146,8 +147,7 @@ def build_module(source_files, options=[], skip=[], only=[], module_name=None):
             os.unlink(fn)
 
     # Import
-    __import__(module_name)
-    return sys.modules[module_name]
+    return import_module(module_name)
 
 
 @_memoize
@@ -319,6 +319,7 @@ class F2PyTest(object):
     module = None
     module_name = None
 
+    @dec.knownfailureif(sys.platform=='win32', msg='Fails with MinGW64 Gfortran (Issue #9673)')
     def setup(self):
         if self.module is not None:
             return
diff --git a/numpy/ma/tests/test_core.py b/numpy/ma/tests/test_core.py
index 8d4284140..41c56ca1e 100644
--- a/numpy/ma/tests/test_core.py
+++ b/numpy/ma/tests/test_core.py
@@ -13,6 +13,7 @@ import warnings
 import pickle
 import operator
 import itertools
+import sys
 from functools import reduce
 
 
@@ -47,6 +48,7 @@ from numpy.ma.core import (
     ravel, repeat, reshape, resize, shape, sin, sinh, sometrue, sort, sqrt,
     subtract, sum, take, tan, tanh, transpose, where, zeros,
     )
+from numpy.testing import dec
 
 pi = np.pi
 
@@ -3632,6 +3634,8 @@ class TestMaskedArrayMathMethods(object):
             assert_almost_equal(np.sqrt(mXvar0[k]),
                                 mX[:, k].compressed().std())
 
+    @dec.knownfailureif(sys.platform=='win32' and sys.version_info < (3, 6),
+                        msg='Fails on Python < 3.6 (Issue #9671)')
     @suppress_copy_mask_on_assignment
     def test_varstd_specialcases(self):
         # Test a special case for var
diff --git a/numpy/testing/tests/test_utils.py b/numpy/testing/tests/test_utils.py
index 1a2a621ea..c440d8eca 100644
--- a/numpy/testing/tests/test_utils.py
+++ b/numpy/testing/tests/test_utils.py
@@ -267,15 +267,18 @@ class TestEqual(TestArrayEqual):
         try:
             self._assert_func(np.array([1, 2]), np.matrix([1, 2]))
         except AssertionError as e:
-            self.assertEqual(
-                str(e),
-                "\nArrays are not equal\n\n"
-                "(shapes (2,), (1, 2) mismatch)\n"
-                " x: array([1, 2])\n"
-                " y: [repr failed for <matrix>: The truth value of an array "
-                "with more than one element is ambiguous. Use a.any() or "
-                "a.all()]")
-
+            msg = str(e)
+            msg2 = msg.replace("shapes (2L,), (1L, 2L)", "shapes (2,), (1, 2)")
+            msg_reference = "\nArrays are not equal\n\n" \
+                "(shapes (2,), (1, 2) mismatch)\n" \
+                " x: array([1, 2])\n" \
+                " y: [repr failed for <matrix>: The truth value of an array " \
+                "with more than one element is ambiguous. Use a.any() or " \
+                "a.all()]"
+            try:
+                self.assertEqual(msg, msg_reference)
+            except AssertionError:
+                self.assertEqual(msg2, msg_reference)
 
 class TestArrayAlmostEqual(_GenericTest, unittest.TestCase):
 
diff --git a/tools/ci/appveyor/requirements.txt b/tools/ci/appveyor/requirements.txt
new file mode 100644
index 000000000..fba8260da
--- /dev/null
+++ b/tools/ci/appveyor/requirements.txt
@@ -0,0 +1,6 @@
+cython
+nose
+pytest-timeout
+pytest-xdist
+pytest-env
+pytest-faulthandler