ENH: support parallel compilation of extensions

Allow extensions using numpy.distutils to compile in parallel.
By passing `--jobs=n` or `-j n` to `setup.py build` the compilation of
extensions is now performed in `n` parallel processes.
Additionally the environment variable NPY_NUM_BUILD_JOBS is used as
the default value, if its unset the default is serial compilation.

The parallelization is limited to within the files of an extension, so
only numpy multiarraymodule really profits but its still a nice
improvement when you have 2-4 cores.
Unfortunately Cython will not profit at all as it tends to build one
module per file.

3 files changed, 29 insertions, 1 deletions

diff --git a/numpy/distutils/command/build.py b/numpy/distutils/command/build.py
index b6912be15..f7249ae81 100644
--- a/numpy/distutils/command/build.py
+++ b/numpy/distutils/command/build.py
@@ -16,6 +16,8 @@ class build(old_build):
     user_options = old_build.user_options + [
         ('fcompiler=', None,
          "specify the Fortran compiler type"),
+        ('jobs=', 'j',
+         "number of parallel jobs"),
         ]
 
     help_options = old_build.help_options + [
@@ -26,8 +28,14 @@ class build(old_build):
     def initialize_options(self):
         old_build.initialize_options(self)
         self.fcompiler = None
+        self.jobs = None
 
     def finalize_options(self):
+        if self.jobs:
+            try:
+                self.jobs = int(self.jobs)
+            except ValueError:
+                raise ValueError("--jobs/-j argument must be an integer")
         build_scripts = self.build_scripts
         old_build.finalize_options(self)
         plat_specifier = ".%s-%s" % (get_platform(), sys.version[0:3])
diff --git a/numpy/distutils/command/build_clib.py b/numpy/distutils/command/build_clib.py
index 84ca87250..6e65a3bfb 100644
--- a/numpy/distutils/command/build_clib.py
+++ b/numpy/distutils/command/build_clib.py
@@ -30,6 +30,8 @@ class build_clib(old_build_clib):
         ('fcompiler=', None,
          "specify the Fortran compiler type"),
         ('inplace', 'i', 'Build in-place'),
+        ('jobs=', 'j',
+         "number of parallel jobs"),
         ]
 
     boolean_options = old_build_clib.boolean_options + ['inplace']
@@ -38,7 +40,16 @@ class build_clib(old_build_clib):
         old_build_clib.initialize_options(self)
         self.fcompiler = None
         self.inplace = 0
-        return
+        self.jobs = None
+
+    def finalize_options(self):
+        if self.jobs:
+            try:
+                self.jobs = int(self.jobs)
+            except ValueError:
+                raise ValueError("--jobs/-j argument must be an integer")
+        old_build_clib.finalize_options(self)
+        self.set_undefined_options('build', ('jobs', 'jobs'))
 
     def have_f_sources(self):
         for (lib_name, build_info) in self.libraries:
diff --git a/numpy/distutils/command/build_ext.py b/numpy/distutils/command/build_ext.py
index 68aab21eb..59c453607 100644
--- a/numpy/distutils/command/build_ext.py
+++ b/numpy/distutils/command/build_ext.py
@@ -34,6 +34,8 @@ class build_ext (old_build_ext):
     user_options = old_build_ext.user_options + [
         ('fcompiler=', None,
          "specify the Fortran compiler type"),
+        ('jobs=', 'j',
+         "number of parallel jobs"),
         ]
 
     help_options = old_build_ext.help_options + [
@@ -44,12 +46,19 @@ class build_ext (old_build_ext):
     def initialize_options(self):
         old_build_ext.initialize_options(self)
         self.fcompiler = None
+        self.jobs = None
 
     def finalize_options(self):
+        if self.jobs:
+            try:
+                self.jobs = int(self.jobs)
+            except ValueError:
+                raise ValueError("--jobs/-j argument must be an integer")
         incl_dirs = self.include_dirs
         old_build_ext.finalize_options(self)
         if incl_dirs is not None:
             self.include_dirs.extend(self.distribution.include_dirs or [])
+        self.set_undefined_options('build', ('jobs', 'jobs'))
 
     def run(self):
         if not self.extensions: