ENH: automatically determine compile dependencies

Use these dependencies to avoid unnecessary recompilations of unchanged
files.

3 files changed, 77 insertions, 4 deletions

diff --git a/numpy/distutils/ccompiler.py b/numpy/distutils/ccompiler.py
index af48d1d63..5f9bd4a84 100644
--- a/numpy/distutils/ccompiler.py
+++ b/numpy/distutils/ccompiler.py
@@ -4,11 +4,12 @@ import os
 import re
 import sys
 import types
+import shlex
 from copy import copy
 from distutils import ccompiler
 from distutils.ccompiler import *
 from distutils.errors import DistutilsExecError, DistutilsModuleError, \
-                             DistutilsPlatformError
+                             DistutilsPlatformError, CompileError
 from distutils.sysconfig import customize_compiler
 from distutils.version import LooseVersion
 
@@ -19,6 +20,44 @@ from numpy.distutils.misc_util import cyg2win32, is_sequence, mingw32, \
                                       quote_args, get_num_build_jobs
 
 
+def _needs_build(obj):
+    """
+    Check if an objects needs to be rebuild based on its dependencies
+
+    Parameters
+    ----------
+    obj : str
+        object file
+
+    Returns
+    -------
+    bool
+    """
+    # defined in unixcompiler.py
+    dep_file = obj + '.d'
+    if not os.path.exists(dep_file):
+        return True
+
+    # dep_file is a makefile containing 'object: dependencies'
+    # formated like posix shell (spaces escaped, \ line continuations)
+    with open(dep_file, "r") as f:
+        deps = [x for x in shlex.split(f.read(), posix=True)
+                if x != "\n" and not x.endswith(":")]
+
+    try:
+        t_obj = os.stat(obj).st_mtime
+
+        # check if any of the dependencies is newer than the object
+        # the dependencies includes the source used to create the object
+        for f in deps:
+            if os.stat(f).st_mtime > t_obj:
+                return True
+    except OSError:
+        # no object counts as newer (shouldn't happen if dep_file exists)
+        return True
+
+    return False
+
 def replace_method(klass, method_name, func):
     if sys.version_info[0] < 3:
         m = types.MethodType(func, None, klass)
@@ -191,7 +230,8 @@ def CCompiler_compile(self, sources, output_dir=None, macros=None,
 
     def single_compile(args):
         obj, (src, ext) = args
-        self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
+        if _needs_build(obj):
+            self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
 
     if isinstance(self, FCompiler):
         objects_to_build = list(build.keys())
@@ -389,6 +429,30 @@ def CCompiler_customize(self, dist, need_cxx=0):
                 log.warn("#### %s #######" % (self.compiler,))
             if not hasattr(self, 'compiler_cxx'):
                 log.warn('Missing compiler_cxx fix for ' + self.__class__.__name__)
+
+
+    # check if compiler supports gcc style automatic dependencies
+    # run on every extension so skip for known good compilers
+    if hasattr(self, 'compiler') and ('gcc' in self.compiler[0] or
+                                      'g++' in self.compiler[0] or
+                                      'clang' in self.compiler[0]):
+        self._auto_depends = True
+    elif os.name == 'posix':
+        import tempfile
+        import shutil
+        tmpdir = tempfile.mkdtemp()
+        try:
+            fn = os.path.join(tmpdir, "file.c")
+            with open(fn, "w") as f:
+                f.write("int a;\n")
+            self.compile([fn], output_dir=tmpdir,
+                         extra_preargs=['-MMD', '-MF', fn + '.d'])
+            self._auto_depends = True
+        except CompileError:
+            self._auto_depends = False
+        finally:
+            shutil.rmtree(tmpdir)
+
     return
 
 replace_method(CCompiler, 'customize', CCompiler_customize)
diff --git a/numpy/distutils/tests/test_system_info.py b/numpy/distutils/tests/test_system_info.py
index 3576de814..73b841692 100644
--- a/numpy/distutils/tests/test_system_info.py
+++ b/numpy/distutils/tests/test_system_info.py
@@ -65,7 +65,8 @@ def have_compiler():
         cmd = compiler.compiler  # Unix compilers
     except AttributeError:
         try:
-            compiler.initialize()  # MSVC is different
+            if not compiler.initialized:
+                compiler.initialize()  # MSVC is different
         except DistutilsError:
             return False
         cmd = [compiler.cc]
diff --git a/numpy/distutils/unixccompiler.py b/numpy/distutils/unixccompiler.py
index a92ccd3e7..307b56ce4 100644
--- a/numpy/distutils/unixccompiler.py
+++ b/numpy/distutils/unixccompiler.py
@@ -44,8 +44,16 @@ def UnixCCompiler__compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts
             self.linker_so = llink_s.split() + opt.split()
 
     display = '%s: %s' % (os.path.basename(self.compiler_so[0]), src)
+
+    # gcc style automatic dependencies, outputs a makefile (-MF) that lists
+    # all headers needed by a c file as a side effect of compilation (-MMD)
+    if getattr(self, '_auto_depends', False):
+        deps = ['-MMD', '-MF', obj + '.d']
+    else:
+        deps = []
+
     try:
-        self.spawn(self.compiler_so + cc_args + [src, '-o', obj] +
+        self.spawn(self.compiler_so + cc_args + [src, '-o', obj] + deps +
                    extra_postargs, display = display)
     except DistutilsExecError:
         msg = str(get_exception())