From 23d546176e2fe0b4d7d2e9100032bbf8107927e1 Mon Sep 17 00:00:00 2001 From: Julian Taylor Date: Wed, 8 Oct 2014 20:23:32 +0200 Subject: ENH: support parallel compilation of extensions Allow extensions using numpy.distutils to compile in parallel. By passing `--jobs=n` or `-j n` to `setup.py build` the compilation of extensions is now performed in `n` parallel processes. Additionally the environment variable NPY_NUM_BUILD_JOBS is used as the default value, if its unset the default is serial compilation. The parallelization is limited to within the files of an extension, so only numpy multiarraymodule really profits but its still a nice improvement when you have 2-4 cores. Unfortunately Cython will not profit at all as it tends to build one module per file. --- numpy/distutils/command/build.py | 8 ++++++++ numpy/distutils/command/build_clib.py | 13 ++++++++++++- numpy/distutils/command/build_ext.py | 9 +++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) (limited to 'numpy/distutils/command') diff --git a/numpy/distutils/command/build.py b/numpy/distutils/command/build.py index b6912be15..f7249ae81 100644 --- a/numpy/distutils/command/build.py +++ b/numpy/distutils/command/build.py @@ -16,6 +16,8 @@ class build(old_build): user_options = old_build.user_options + [ ('fcompiler=', None, "specify the Fortran compiler type"), + ('jobs=', 'j', + "number of parallel jobs"), ] help_options = old_build.help_options + [ @@ -26,8 +28,14 @@ class build(old_build): def initialize_options(self): old_build.initialize_options(self) self.fcompiler = None + self.jobs = None def finalize_options(self): + if self.jobs: + try: + self.jobs = int(self.jobs) + except ValueError: + raise ValueError("--jobs/-j argument must be an integer") build_scripts = self.build_scripts old_build.finalize_options(self) plat_specifier = ".%s-%s" % (get_platform(), sys.version[0:3]) diff --git a/numpy/distutils/command/build_clib.py b/numpy/distutils/command/build_clib.py index 84ca87250..6e65a3bfb 100644 --- a/numpy/distutils/command/build_clib.py +++ b/numpy/distutils/command/build_clib.py @@ -30,6 +30,8 @@ class build_clib(old_build_clib): ('fcompiler=', None, "specify the Fortran compiler type"), ('inplace', 'i', 'Build in-place'), + ('jobs=', 'j', + "number of parallel jobs"), ] boolean_options = old_build_clib.boolean_options + ['inplace'] @@ -38,7 +40,16 @@ class build_clib(old_build_clib): old_build_clib.initialize_options(self) self.fcompiler = None self.inplace = 0 - return + self.jobs = None + + def finalize_options(self): + if self.jobs: + try: + self.jobs = int(self.jobs) + except ValueError: + raise ValueError("--jobs/-j argument must be an integer") + old_build_clib.finalize_options(self) + self.set_undefined_options('build', ('jobs', 'jobs')) def have_f_sources(self): for (lib_name, build_info) in self.libraries: diff --git a/numpy/distutils/command/build_ext.py b/numpy/distutils/command/build_ext.py index 68aab21eb..59c453607 100644 --- a/numpy/distutils/command/build_ext.py +++ b/numpy/distutils/command/build_ext.py @@ -34,6 +34,8 @@ class build_ext (old_build_ext): user_options = old_build_ext.user_options + [ ('fcompiler=', None, "specify the Fortran compiler type"), + ('jobs=', 'j', + "number of parallel jobs"), ] help_options = old_build_ext.help_options + [ @@ -44,12 +46,19 @@ class build_ext (old_build_ext): def initialize_options(self): old_build_ext.initialize_options(self) self.fcompiler = None + self.jobs = None def finalize_options(self): + if self.jobs: + try: + self.jobs = int(self.jobs) + except ValueError: + raise ValueError("--jobs/-j argument must be an integer") incl_dirs = self.include_dirs old_build_ext.finalize_options(self) if incl_dirs is not None: self.include_dirs.extend(self.distribution.include_dirs or []) + self.set_undefined_options('build', ('jobs', 'jobs')) def run(self): if not self.extensions: -- cgit v1.2.1