Merge pull request #3910 from pv/runtests-fix

MAINT: update runtests.py from scipy

1 files changed, 112 insertions, 28 deletions

diff --git a/runtests.py b/runtests.py
index 97d37ab11..5f7239240 100755
--- a/runtests.py
+++ b/runtests.py
@@ -10,6 +10,7 @@ Examples::
     $ python runtests.py -s {SAMPLE_SUBMODULE}
     $ python runtests.py -t {SAMPLE_TEST}
     $ python runtests.py --ipython
+    $ python runtests.py --python somescript.py
 
 """
 
@@ -23,9 +24,17 @@ PROJECT_ROOT_FILES = ['numpy', 'LICENSE.txt', 'setup.py']
 SAMPLE_TEST = "numpy/linalg/tests/test_linalg.py:test_byteorder_check"
 SAMPLE_SUBMODULE = "linalg"
 
+EXTRA_PATH = ['/usr/lib/ccache', '/usr/lib/f90cache',
+              '/usr/local/lib/ccache', '/usr/local/lib/f90cache']
+
 # ---------------------------------------------------------------------
 
-__doc__ = __doc__.format(**globals())
+
+if __doc__ is None:
+    __doc__ = "Run without -OO if you want usage info"
+else:
+    __doc__ = __doc__.format(**globals())
+
 
 import sys
 import os
@@ -36,8 +45,12 @@ sys.path.pop(0)
 
 import shutil
 import subprocess
+import time
+import imp
 from argparse import ArgumentParser, REMAINDER
 
+ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__)))
+
 def main(argv):
     parser = ArgumentParser(usage=__doc__.lstrip())
     parser.add_argument("--verbose", "-v", action="count", default=1,
@@ -68,8 +81,10 @@ def main(argv):
                         help="Start Unix shell with PYTHONPATH set")
     parser.add_argument("--debug", "-g", action="store_true",
                         help="Debug build")
+    parser.add_argument("--show-build-log", action="store_true",
+                        help="Show build output rather than using a log file")
     parser.add_argument("args", metavar="ARGS", default=[], nargs=REMAINDER,
-                        help="Arguments to pass to Nose")
+                        help="Arguments to pass to Nose, Python or shell")
     args = parser.parse_args(argv)
 
     if args.pythonpath:
@@ -81,26 +96,40 @@ def main(argv):
         sys.path.insert(0, site_dir)
         os.environ['PYTHONPATH'] = site_dir
 
+    extra_argv = args.args[:]
+    if extra_argv and extra_argv[0] == '--':
+        extra_argv = extra_argv[1:]
+
     if args.python:
-        import code
-        code.interact()
-        sys.exit(0)
+        if extra_argv:
+            # Don't use subprocess, since we don't want to include the
+            # current path in PYTHONPATH.
+            sys.argv = extra_argv
+            with open(extra_argv[0], 'r') as f:
+                script = f.read()
+            sys.modules['__main__'] = imp.new_module('__main__')
+            ns = dict(__name__='__main__',
+                      __file__=extra_argv[0])
+            exec_(script, ns)
+            sys.exit(0)
+        else:
+            import code
+            code.interact()
+            sys.exit(0)
 
     if args.ipython:
         import IPython
-        IPython.embed()
+        IPython.embed(user_ns={})
         sys.exit(0)
 
     if args.shell:
         shell = os.environ.get('SHELL', 'sh')
         print("Spawning a Unix shell...")
-        os.execv(shell, [shell])
+        os.execv(shell, [shell] + extra_argv)
         sys.exit(1)
 
-    extra_argv = args.args
-
     if args.coverage:
-        dst_dir = os.path.join('build', 'coverage')
+        dst_dir = os.path.join(ROOT_DIR, 'build', 'coverage')
         fn = os.path.join(dst_dir, 'coverage_html.js')
         if os.path.isdir(dst_dir) and os.path.isfile(fn):
             shutil.rmtree(dst_dir)
@@ -128,11 +157,28 @@ def main(argv):
         __import__(PROJECT_MODULE)
         test = sys.modules[PROJECT_MODULE].test
 
-    result = test(args.mode,
-                  verbose=args.verbose,
-                  extra_argv=args.args,
-                  doctests=args.doctests,
-                  coverage=args.coverage)
+    # Run the tests under build/test
+    test_dir = os.path.join(ROOT_DIR, 'build', 'test')
+
+    try:
+        shutil.rmtree(test_dir)
+    except OSError:
+        pass
+    try:
+        os.makedirs(test_dir)
+    except OSError:
+        pass
+
+    cwd = os.getcwd()
+    try:
+        os.chdir(test_dir)
+        result = test(args.mode,
+                      verbose=args.verbose,
+                      extra_argv=extra_argv,
+                      doctests=args.doctests,
+                      coverage=args.coverage)
+    finally:
+        os.chdir(cwd)
 
     if result.wasSuccessful():
         sys.exit(0)
@@ -150,42 +196,64 @@ def build_project(args):
 
     """
 
-    root_dir = os.path.abspath(os.path.join(os.path.dirname(__file__)))
-    root_ok = [os.path.exists(os.path.join(root_dir, fn))
+    root_ok = [os.path.exists(os.path.join(ROOT_DIR, fn))
                for fn in PROJECT_ROOT_FILES]
     if not all(root_ok):
         print("To build the project, run runtests.py in "
               "git checkout or unpacked source")
         sys.exit(1)
 
-    dst_dir = os.path.join(root_dir, 'build', 'testenv')
+    dst_dir = os.path.join(ROOT_DIR, 'build', 'testenv')
 
     env = dict(os.environ)
     cmd = [sys.executable, 'setup.py']
 
     # Always use ccache, if installed
-    env['PATH'] = os.pathsep.join(['/usr/lib/ccache']
-                                  + env.get('PATH', '').split(os.pathsep))
+    env['PATH'] = os.pathsep.join(EXTRA_PATH + env.get('PATH', '').split(os.pathsep))
 
     if args.debug:
         # assume everyone uses gcc/gfortran
         env['OPT'] = '-O0 -ggdb'
         env['FOPT'] = '-O0 -ggdb'
-        cmd += ["build", "--debug"]
+        cmd += ["build"]
 
     cmd += ['install', '--prefix=' + dst_dir]
 
-    print("Building, see build.log...")
-    with open('build.log', 'w') as log:
-        ret = subprocess.call(cmd, env=env, stdout=log, stderr=log,
-                              cwd=root_dir)
+    log_filename = os.path.join(ROOT_DIR, 'build.log')
+
+    if args.show_build_log:
+        ret = subprocess.call(cmd, env=env, cwd=ROOT_DIR)
+    else:
+        log_filename = os.path.join(ROOT_DIR, 'build.log')
+        print("Building, see build.log...")
+        with open(log_filename, 'w') as log:
+            p = subprocess.Popen(cmd, env=env, stdout=log, stderr=log,
+                                 cwd=ROOT_DIR)
+
+        # Wait for it to finish, and print something to indicate the
+        # process is alive, but only if the log file has grown (to
+        # allow continuous integration environments kill a hanging
+        # process accurately if it produces no output)
+        last_blip = time.time()
+        last_log_size = os.stat(log_filename).st_size
+        while p.poll() is None:
+            time.sleep(0.5)
+            if time.time() - last_blip > 60:
+                log_size = os.stat(log_filename).st_size
+                if log_size > last_log_size:
+                    print("    ... build in progress")
+                    last_blip = time.time()
+                    last_log_size = log_size
+
+        ret = p.wait()
 
     if ret == 0:
         print("Build OK")
     else:
-        with open('build.log', 'r') as f:
-            print(f.read())
-        print("Build failed!")
+        if not args.show_build_log:
+            with open(log_filename, 'r') as f:
+                print(f.read())
+            print("Build failed!")
         sys.exit(1)
 
     from distutils.sysconfig import get_python_lib
@@ -193,5 +261,21 @@ def build_project(args):
 
     return site_dir
 
+if sys.version_info[0] >= 3:
+    import builtins
+    exec_ = getattr(builtins, "exec")
+else:
+    def exec_(code, globs=None, locs=None):
+        """Execute code in a namespace."""
+        if globs is None:
+            frame = sys._getframe(1)
+            globs = frame.f_globals
+            if locs is None:
+                locs = frame.f_locals
+            del frame
+        elif locs is None:
+            locs = globs
+        exec("""exec code in globs, locs""")
+
 if __name__ == "__main__":
     main(argv=sys.argv[1:])