[demo] how-to replacing numpy custom generation engine by raw C++

This is just a technical prototype to measure and discuss the impact and
implication of moving to C++ for kernel code generation.

6 files changed, 96 insertions, 52 deletions

diff --git a/numpy/distutils/ccompiler.py b/numpy/distutils/ccompiler.py
index 4495c8fee..9c85d28b9 100644
--- a/numpy/distutils/ccompiler.py
+++ b/numpy/distutils/ccompiler.py
@@ -23,7 +23,8 @@ from numpy.distutils.exec_command import (
 )
 from numpy.distutils.misc_util import cyg2win32, is_sequence, mingw32, \
                                       get_num_build_jobs, \
-                                      _commandline_dep_string
+                                      _commandline_dep_string, \
+                                      sanitize_cxx_flags
 
 # globals for parallel build management
 import threading
@@ -674,7 +675,9 @@ def CCompiler_cxx_compiler(self):
         return self
 
     cxx = copy(self)
-    cxx.compiler_so = [cxx.compiler_cxx[0]] + cxx.compiler_so[1:]
+    cxx.compiler_cxx = cxx.compiler_cxx
+    cxx.compiler_so = [cxx.compiler_cxx[0]] + \
+                      sanitize_cxx_flags(cxx.compiler_so[1:])
     if sys.platform.startswith('aix') and 'ld_so_aix' in cxx.linker_so[0]:
         # AIX needs the ld_so_aix script included with Python
         cxx.linker_so = [cxx.linker_so[0], cxx.compiler_cxx[0]] \
diff --git a/numpy/distutils/ccompiler_opt.py b/numpy/distutils/ccompiler_opt.py
index e7fd494d3..9c386b3ac 100644
--- a/numpy/distutils/ccompiler_opt.py
+++ b/numpy/distutils/ccompiler_opt.py
@@ -188,7 +188,8 @@ class _Config:
             # native usually works only with x86
             native = '-march=native',
             opt = '-O3',
-            werror = '-Werror'
+            werror = '-Werror',
+            cxx = '-std=c++11',
         ),
         clang = dict(
             native = '-march=native',
@@ -198,22 +199,26 @@ class _Config:
             # cases `-Werror` gets skipped during the availability test due to
             # "unused arguments" warnings.
             # see https://github.com/numpy/numpy/issues/19624
-            werror = '-Werror-implicit-function-declaration -Werror'
+            werror = '-Werror=switch -Werror',
+            cxx = '-std=c++11',
         ),
         icc = dict(
             native = '-xHost',
             opt = '-O3',
-            werror = '-Werror'
+            werror = '-Werror',
+            cxx = '-std=c++11',
         ),
         iccw = dict(
             native = '/QxHost',
             opt = '/O3',
-            werror = '/Werror'
+            werror = '/Werror',
+            cxx = '-std=c++11',
         ),
         msvc = dict(
             native = None,
             opt = '/O2',
-            werror = '/WX'
+            werror = '/WX',
+            cxx = '-std=c++11',
         )
     )
     conf_min_features = dict(
@@ -555,6 +560,7 @@ class _Distutils:
         flags = kwargs.pop("extra_postargs", []) + flags
         if not ccompiler:
             ccompiler = self._ccompiler
+
         return ccompiler.compile(sources, extra_postargs=flags, **kwargs)
 
     def dist_test(self, source, flags, macros=[]):
diff --git a/numpy/distutils/command/build_clib.py b/numpy/distutils/command/build_clib.py
index 0e31a7dee..a481758c1 100644
--- a/numpy/distutils/command/build_clib.py
+++ b/numpy/distutils/command/build_clib.py
@@ -264,6 +264,8 @@ class build_clib(old_build_clib):
         if include_dirs is None:
             include_dirs = []
         extra_postargs = build_info.get('extra_compiler_args') or []
+        extra_cflags = build_info.get('extra_cflags') or []
+        extra_cxxflags = build_info.get('extra_cxxflags') or []
 
         include_dirs.extend(get_numpy_include_dirs())
         # where compiled F90 module files are:
@@ -315,38 +317,45 @@ class build_clib(old_build_clib):
                 macros=macros + copt_macros,
                 include_dirs=include_dirs,
                 debug=self.debug,
-                extra_postargs=extra_postargs,
+                extra_postargs=extra_postargs + extra_cxxflags,
                 ccompiler=cxx_compiler
             )
 
         if copt_c_sources:
             log.info("compiling C dispatch-able sources")
-            objects += self.compiler_opt.try_dispatch(copt_c_sources,
-                                                      output_dir=self.build_temp,
-                                                      src_dir=copt_build_src,
-                                                      macros=macros + copt_macros,
-                                                      include_dirs=include_dirs,
-                                                      debug=self.debug,
-                                                      extra_postargs=extra_postargs)
+            objects += self.compiler_opt.try_dispatch(
+                copt_c_sources,
+                output_dir=self.build_temp,
+                src_dir=copt_build_src,
+                macros=macros + copt_macros,
+                include_dirs=include_dirs,
+                debug=self.debug,
+                extra_postargs=extra_postargs + extra_cflags)
 
         if c_sources:
             log.info("compiling C sources")
-            objects += compiler.compile(c_sources,
-                                        output_dir=self.build_temp,
-                                        macros=macros + copt_macros,
-                                        include_dirs=include_dirs,
-                                        debug=self.debug,
-                                        extra_postargs=extra_postargs + copt_baseline_flags)
+            objects += compiler.compile(
+                c_sources,
+                output_dir=self.build_temp,
+                macros=macros + copt_macros,
+                include_dirs=include_dirs,
+                debug=self.debug,
+                extra_postargs=(extra_postargs +
+                                copt_baseline_flags +
+                                extra_cflags))
 
         if cxx_sources:
             log.info("compiling C++ sources")
             cxx_compiler = compiler.cxx_compiler()
-            cxx_objects = cxx_compiler.compile(cxx_sources,
-                                               output_dir=self.build_temp,
-                                               macros=macros + copt_macros,
-                                               include_dirs=include_dirs,
-                                               debug=self.debug,
-                                               extra_postargs=extra_postargs + copt_baseline_flags)
+            cxx_objects = cxx_compiler.compile(
+                cxx_sources,
+                output_dir=self.build_temp,
+                macros=macros + copt_macros,
+                include_dirs=include_dirs,
+                debug=self.debug,
+                extra_postargs=(extra_postargs +
+                                copt_baseline_flags +
+                                extra_cxxflags))
             objects.extend(cxx_objects)
 
         if f_sources or fmodule_sources:
diff --git a/numpy/distutils/command/build_ext.py b/numpy/distutils/command/build_ext.py
index b8378d473..7040a2411 100644
--- a/numpy/distutils/command/build_ext.py
+++ b/numpy/distutils/command/build_ext.py
@@ -243,7 +243,8 @@ class build_ext (old_build_ext):
             if l and l != ext_language and ext.language:
                 log.warn('resetting extension %r language from %r to %r.' %
                          (ext.name, l, ext_language))
-            ext.language = ext_language
+            if not ext.language:
+                ext.language = ext_language
             # global language
             all_languages.update(ext_languages)
 
@@ -376,6 +377,9 @@ class build_ext (old_build_ext):
             log.info("building '%s' extension", ext.name)
 
         extra_args = ext.extra_compile_args or []
+        extra_cflags = ext.extra_c_compile_args or []
+        extra_cxxflags = ext.extra_cxx_compile_args or []
+
         macros = ext.define_macros[:]
         for undef in ext.undef_macros:
             macros.append((undef,))
@@ -462,38 +466,43 @@ class build_ext (old_build_ext):
                 macros=macros + copt_macros,
                 include_dirs=include_dirs,
                 debug=self.debug,
-                extra_postargs=extra_args,
+                extra_postargs=extra_args + extra_cxxflags,
                 ccompiler=cxx_compiler,
                 **kws
             )
         if copt_c_sources:
             log.info("compiling C dispatch-able sources")
-            c_objects += self.compiler_opt.try_dispatch(copt_c_sources,
-                                                        output_dir=output_dir,
-                                                        src_dir=copt_build_src,
-                                                        macros=macros + copt_macros,
-                                                        include_dirs=include_dirs,
-                                                        debug=self.debug,
-                                                        extra_postargs=extra_args,
-                                                        **kws)
+            c_objects += self.compiler_opt.try_dispatch(
+                copt_c_sources,
+                output_dir=output_dir,
+                src_dir=copt_build_src,
+                macros=macros + copt_macros,
+                include_dirs=include_dirs,
+                debug=self.debug,
+                extra_postargs=extra_args + extra_cflags,
+                **kws)
         if c_sources:
             log.info("compiling C sources")
-            c_objects += self.compiler.compile(c_sources,
-                                               output_dir=output_dir,
-                                               macros=macros + copt_macros,
-                                               include_dirs=include_dirs,
-                                               debug=self.debug,
-                                               extra_postargs=extra_args + copt_baseline_flags,
-                                               **kws)
+            c_objects += self.compiler.compile(
+                c_sources,
+                output_dir=output_dir,
+                macros=macros + copt_macros,
+                include_dirs=include_dirs,
+                debug=self.debug,
+                extra_postargs=(extra_args + copt_baseline_flags +
+                                extra_cflags),
+                **kws)
         if cxx_sources:
             log.info("compiling C++ sources")
-            c_objects += cxx_compiler.compile(cxx_sources,
-                                              output_dir=output_dir,
-                                              macros=macros + copt_macros,
-                                              include_dirs=include_dirs,
-                                              debug=self.debug,
-                                              extra_postargs=extra_args + copt_baseline_flags,
-                                              **kws)
+            c_objects += cxx_compiler.compile(
+                cxx_sources,
+                output_dir=output_dir,
+                macros=macros + copt_macros,
+                include_dirs=include_dirs,
+                debug=self.debug,
+                extra_postargs=(extra_args + copt_baseline_flags +
+                                extra_cxxflags),
+                **kws)
 
         extra_postargs = []
         f_objects = []
diff --git a/numpy/distutils/extension.py b/numpy/distutils/extension.py
index c90b5d725..3ede013e0 100644
--- a/numpy/distutils/extension.py
+++ b/numpy/distutils/extension.py
@@ -47,6 +47,8 @@ class Extension(old_Extension):
             language=None,
             f2py_options=None,
             module_dirs=None,
+            extra_c_compile_args=None,
+            extra_cxx_compile_args=None,
             extra_f77_compile_args=None,
             extra_f90_compile_args=None,):
 
@@ -83,6 +85,8 @@ class Extension(old_Extension):
         # numpy_distutils features
         self.f2py_options = f2py_options or []
         self.module_dirs = module_dirs or []
+        self.extra_c_compile_args = extra_c_compile_args or []
+        self.extra_cxx_compile_args = extra_cxx_compile_args or []
         self.extra_f77_compile_args = extra_f77_compile_args or []
         self.extra_f90_compile_args = extra_f90_compile_args or []
 
diff --git a/numpy/distutils/misc_util.py b/numpy/distutils/misc_util.py
index c9e051237..f0f9b4bd7 100644
--- a/numpy/distutils/misc_util.py
+++ b/numpy/distutils/misc_util.py
@@ -11,6 +11,7 @@ import multiprocessing
 import textwrap
 import importlib.util
 from threading import local as tlocal
+from functools import reduce
 
 import distutils
 from distutils.errors import DistutilsError
@@ -43,7 +44,7 @@ __all__ = ['Configuration', 'get_numpy_include_dirs', 'default_config_dict',
            'dot_join', 'get_frame', 'minrelpath', 'njoin',
            'is_sequence', 'is_string', 'as_list', 'gpaths', 'get_language',
            'get_build_architecture', 'get_info', 'get_pkg_info',
-           'get_num_build_jobs']
+           'get_num_build_jobs', 'sanitize_cxx_flags']
 
 class InstallableLib:
     """
@@ -2478,3 +2479,15 @@ def get_build_architecture():
     # systems, so delay the import to here.
     from distutils.msvccompiler import get_build_architecture
     return get_build_architecture()
+
+
+_cxx_ignore_flags = {'-Werror=implicit-function-declaration'}
+
+
+def sanitize_cxx_flags(cxxflags):
+    '''
+    Some flags are valid for C but not C++. Prune them.
+    '''
+    return [flag for flag in cxxflags if flag not in _cxx_ignore_flags]
+
+