TST: Clean up travis-test and make it work with current travis ci.

Travis ci is migrating to GCI and the 32 bit tests broke in the process.
This cleans up the tools/travis-test script, fixes it for current
travis, and changes the 32 bit test to use python 2.7 in order to turn
up errors involving python long integers.

In preparation for dropping Python 3.2 and 3.3, the USE_DEBUG test is
run in the travis ci trusty beta so that python3 defaults to 3.4.

2 files changed, 85 insertions, 56 deletions

diff --git a/.travis.yml b/.travis.yml
index f14b9c912..589d7a9e6 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -29,16 +29,18 @@ python:
   - 3.5
 matrix:
   include:
-    - python: 3.3
-      env: USE_CHROOT=1 ARCH=i386 DIST=trusty PYTHON=3.4
+    - python: 2.7
+      env: USE_CHROOT=1 ARCH=i386 DIST=trusty PYTHON=2.7
       sudo: true
+      dist: trusty
       addons:
         apt:
           packages:
-            - *common_packages
             - debootstrap
-    - python: 3.2
+    - python: 3.4
       env: USE_DEBUG=1
+      sudo: true
+      dist: trusty
       addons:
         apt:
           packages:
diff --git a/tools/travis-test.sh b/tools/travis-test.sh
index 3591c36df..939594d8c 100755
--- a/tools/travis-test.sh
+++ b/tools/travis-test.sh
@@ -1,4 +1,5 @@
 #!/bin/bash
+
 set -ex
 
 # Travis legacy boxes give you 1.5 CPUs, container-based boxes give you 2 CPUs
@@ -10,54 +11,94 @@ if [ -r /usr/lib/libeatmydata/libeatmydata.so ]; then
   export LD_PRELOAD=/usr/lib/libeatmydata/libeatmydata.so
 fi
 
+# travis venv tests override python
+PYTHON=${PYTHON:-python}
+PIP=${PIP:-pip}
+
+# explicit python version needed here
+if [ -n "$USE_DEBUG" ]; then
+  PYTHON="python3-dbg"
+fi
+
+if [ -n "$PYTHON_OO" ]; then
+  PYTHON="${PYTHON} -OO"
+fi
+
 # make some warnings fatal, mostly to match windows compilers
 werrors="-Werror=declaration-after-statement -Werror=vla -Werror=nonnull"
 
 setup_base()
 {
   # We used to use 'setup.py install' here, but that has the terrible
-  # behaviour that if a copy of the package is already installed in
-  # the install location, then the new copy just gets dropped on top
-  # of it. Travis typically has a stable numpy release pre-installed,
-  # and if we don't remove it, then we can accidentally end up
-  # e.g. running old test modules that were in the stable release but
-  # have been removed from master. (See gh-2765, gh-2768.)  Using 'pip
-  # install' also has the advantage that it tests that numpy is 'pip
-  # install' compatible, see e.g. gh-2766...
-if [ -z "$USE_DEBUG" ]; then
-  if [ -z "$IN_CHROOT" ]; then
-    $PIP install .
+  # behaviour that if a copy of the package is already installed in the
+  # install location, then the new copy just gets dropped on top of it.
+  # Travis typically has a stable numpy release pre-installed, and if we
+  # don't remove it, then we can accidentally end up e.g. running old
+  # test modules that were in the stable release but have been removed
+  # from master. (See gh-2765, gh-2768.)  Using 'pip install' also has
+  # the advantage that it tests that numpy is 'pip install' compatible,
+  # see e.g. gh-2766...
+  if [ -z "$USE_DEBUG" ]; then
+    if [ -z "$IN_CHROOT" ]; then
+      $PIP install .
+    else
+      sysflags="$($PYTHON -c "from distutils import sysconfig; \
+        print (sysconfig.get_config_var('CFLAGS'))")"
+      CFLAGS="$sysflags $werrors -Wlogical-op" $PIP install . 2>&1 | tee log
+      grep -v "_configtest" log \
+        | grep -vE "ld returned 1|no previously-included files matching" \
+        | grep -E "warning\>" \
+        | tee warnings
+      # Check for an acceptable number of warnings. Some warnings are out of
+      # our control, so adjust the number as needed.
+      [[ $(wc -l < warnings) -lt 1 ]]
+    fi
   else
-    sysflags="$($PYTHON -c "from distutils import sysconfig; print (sysconfig.get_config_var('CFLAGS'))")"
-    CFLAGS="$sysflags $werrors -Wlogical-op" $PIP install . 2>&1 | tee log
-    grep -v "_configtest" log | grep -vE "ld returned 1|no previously-included files matching" | grep -E "warning\>";
-    # accept a mysterious memset warning that shows with -flto
-    test $(grep -v "_configtest" log | grep -vE "ld returned 1|no previously-included files matching" | grep -E "warning\>" -c) -lt 2;
+    sysflags="$($PYTHON -c "from distutils import sysconfig; \
+      print (sysconfig.get_config_var('CFLAGS'))")"
+    CFLAGS="$sysflags $werrors" $PYTHON setup.py build_ext --inplace
   fi
-else
-  sysflags="$($PYTHON -c "from distutils import sysconfig; print (sysconfig.get_config_var('CFLAGS'))")"
-  CFLAGS="$sysflags $werrors" $PYTHON setup.py build_ext --inplace
-fi
 }
 
 setup_chroot()
 {
   # this can all be replaced with:
   # apt-get install libpython2.7-dev:i386
-  # CC="gcc -m32" LDSHARED="gcc -m32 -shared" LDFLAGS="-m32 -shared" linux32 python setup.py build
+  # CC="gcc -m32" LDSHARED="gcc -m32 -shared" LDFLAGS="-m32 -shared" \
+  #   linux32 python setup.py build
   # when travis updates to ubuntu 14.04
+  #
+  # Numpy may not distinquish between 64 and 32 bit atlas in the
+  # configuration stage.
   DIR=$1
   set -u
-  sudo debootstrap --variant=buildd --include=fakeroot,build-essential --arch=$ARCH --foreign $DIST $DIR
+  sudo debootstrap --variant=buildd --include=fakeroot,build-essential \
+    --arch=$ARCH --foreign $DIST $DIR
   sudo chroot $DIR ./debootstrap/debootstrap --second-stage
+
+  # put the numpy repo in the chroot directory
   sudo rsync -a $TRAVIS_BUILD_DIR $DIR/
-  echo deb http://archive.ubuntu.com/ubuntu/ $DIST main restricted universe multiverse | sudo tee -a $DIR/etc/apt/sources.list
-  echo deb http://archive.ubuntu.com/ubuntu/ $DIST-updates main restricted universe multiverse | sudo tee -a $DIR/etc/apt/sources.list
-  echo deb http://security.ubuntu.com/ubuntu $DIST-security  main restricted universe multiverse | sudo tee -a $DIR/etc/apt/sources.list
+
+  # set up repos in the chroot directory for installing packages
+  echo deb http://archive.ubuntu.com/ubuntu/ \
+    $DIST main restricted universe multiverse \
+    | sudo tee -a $DIR/etc/apt/sources.list
+  echo deb http://archive.ubuntu.com/ubuntu/ \
+    $DIST-updates main restricted universe multiverse \
+    | sudo tee -a $DIR/etc/apt/sources.list
+  echo deb http://security.ubuntu.com/ubuntu \
+    $DIST-security  main restricted universe multiverse \
+    | sudo tee -a $DIR/etc/apt/sources.list
+
+  # install needed packages
   sudo chroot $DIR bash -c "apt-get update"
-  sudo chroot $DIR bash -c "apt-get install -qq -y --force-yes eatmydata"
-  echo /usr/lib/libeatmydata/libeatmydata.so | sudo tee -a $DIR/etc/ld.so.preload
-  sudo chroot $DIR bash -c "apt-get install -qq -y --force-yes libatlas-dev libatlas-base-dev gfortran python3-dev python3-nose python3-pip cython3 cython"
+  sudo chroot $DIR bash -c "apt-get install -qq -y --force-yes \
+    eatmydata libatlas-dev libatlas-base-dev gfortran \
+    python-dev python-nose python-pip cython"
+
+  # faster operation with preloaded eatmydata
+  echo /usr/lib/libeatmydata/libeatmydata.so | \
+    sudo tee -a $DIR/etc/ld.so.preload
 }
 
 run_test()
@@ -70,42 +111,29 @@ run_test()
   # of numpy in the source directory.
   mkdir -p empty
   cd empty
-  INSTALLDIR=$($PYTHON -c "import os; import numpy; print(os.path.dirname(numpy.__file__))")
+  INSTALLDIR=$($PYTHON -c \
+    "import os; import numpy; print(os.path.dirname(numpy.__file__))")
   export PYTHONWARNINGS=default
-  $PYTHON ../tools/test-installed-numpy.py # --mode=full
-  # - coverage run --source=$INSTALLDIR --rcfile=../.coveragerc $(which $PYTHON) ../tools/test-installed-numpy.py
-  # - coverage report --rcfile=../.coveragerc --show-missing
-
+  $PYTHON ../tools/test-installed-numpy.py
   if [ -n "$USE_ASV" ]; then
     pushd ../benchmarks
     $PYTHON `which asv` machine --machine travis
     $PYTHON `which asv` dev 2>&1| tee asv-output.log
     if grep -q Traceback asv-output.log; then
-        echo "Some benchmarks have errors!"
-        exit 1
+      echo "Some benchmarks have errors!"
+      exit 1
     fi
     popd
   fi
 }
 
-# travis venv tests override python
-PYTHON=${PYTHON:-python}
-PIP=${PIP:-pip}
-
-if [ -n "$USE_DEBUG" ]; then
-  PYTHON=python3-dbg
-fi
-
-if [ -n "$PYTHON_OO" ]; then
-  PYTHON="$PYTHON -OO"
-fi
-
 export PYTHON
 export PIP
 if [ -n "$USE_WHEEL" ] && [ $# -eq 0 ]; then
   # Build wheel
   $PIP install wheel
-  # ensure that the pip / setuptools versions deployed inside the venv are recent enough
+  # ensure that the pip / setuptools versions deployed inside
+  # the venv are recent enough
   $PIP install -U virtualenv
   $PYTHON setup.py bdist_wheel
   # Make another virtualenv to install into
@@ -117,15 +145,14 @@ if [ -n "$USE_WHEEL" ] && [ $# -eq 0 ]; then
   pip install nose
   popd
   run_test
-elif [ "$USE_CHROOT" != "1" ]; then
-  setup_base
-  run_test
 elif [ -n "$USE_CHROOT" ] && [ $# -eq 0 ]; then
   DIR=/chroot
   setup_chroot $DIR
   # run again in chroot with this time testing
-  sudo linux32 chroot $DIR bash -c "cd numpy && PYTHON=python3 PIP=pip3 IN_CHROOT=1 $0 test"
+  sudo linux32 chroot $DIR bash -c \
+    "cd numpy && PYTHON=python PIP=pip IN_CHROOT=1 $0 test"
 else
+  setup_base
   run_test
 fi