Merge pull request #23528 from seberg/allow-backcomp-builds

ENH: Allow, and default to, downstream building with old API

10 files changed, 155 insertions, 57 deletions

diff --git a/numpy/core/code_generators/cversions.txt b/numpy/core/code_generators/cversions.txt
index bd4b71180..e52193d7a 100644
--- a/numpy/core/code_generators/cversions.txt
+++ b/numpy/core/code_generators/cversions.txt
@@ -67,5 +67,7 @@
 # Version 16 (NumPy 1.23)
 # NonNull attributes removed from numpy_api.py
 # Version 16 (NumPy 1.24) No change.
-# Version 16 (NumPy 1.25) No change.
 0x00000010 = 04a7bf1e65350926a0e528798da263c0
+
+# Version 17 (NumPy 1.25) No actual change.
+0x00000011 = ca1aebdad799358149567d9d93cbca09
diff --git a/numpy/core/code_generators/genapi.py b/numpy/core/code_generators/genapi.py
index 655d5ed67..2cdaba52d 100644
--- a/numpy/core/code_generators/genapi.py
+++ b/numpy/core/code_generators/genapi.py
@@ -12,6 +12,7 @@ import os
 import re
 import sys
 import importlib.util
+import textwrap
 
 from os.path import join
 
@@ -98,6 +99,27 @@ def _repl(str):
     return str.replace('Bool', 'npy_bool')
 
 
+class MinVersion:
+    def __init__(self, version):
+        """ Version should be the normal NumPy version, e.g. "1.25" """
+        major, minor = version.split(".")
+        self.version = f"NPY_{major}_{minor}_API_VERSION"
+
+    def __str__(self):
+        # Used by version hashing:
+        return self.version
+
+    def add_guard(self, name, normal_define):
+        """Wrap a definition behind a version guard"""
+        wrap = textwrap.dedent(f"""
+            #if NPY_FEATURE_VERSION >= {self.version}
+            {{define}}
+            #endif""")
+
+        # we only insert `define` later to avoid confusing dedent:
+        return wrap.format(define=normal_define)
+
+
 class StealRef:
     def __init__(self, arg):
         self.arg = arg # counting from 1
@@ -391,7 +413,20 @@ class FunctionApi:
     def __init__(self, name, index, annotations, return_type, args, api_name):
         self.name = name
         self.index = index
-        self.annotations = annotations
+
+        self.min_version = None
+        self.annotations = []
+        for annotation in annotations:
+            # String checks, because manual import breaks isinstance
+            if type(annotation).__name__ == "StealRef":
+                self.annotations.append(annotation)
+            elif type(annotation).__name__ == "MinVersion":
+                if self.min_version is not None:
+                    raise ValueError("Two minimum versions specified!")
+                self.min_version = annotation
+            else:
+                raise ValueError(f"unknown annotation {annotation}")
+
         self.return_type = return_type
         self.args = args
         self.api_name = api_name
@@ -403,13 +438,14 @@ class FunctionApi:
         return argstr
 
     def define_from_array_api_string(self):
-        define = """\
-#define %s \\\n        (*(%s (*)(%s)) \\
-         %s[%d])""" % (self.name,
-                                self.return_type,
-                                self._argtypes_string(),
-                                self.api_name,
-                                self.index)
+        arguments = self._argtypes_string()
+        define = textwrap.dedent(f"""\
+            #define {self.name} \\
+                    (*({self.return_type} (*)({arguments})) \\
+                {self.api_name}[{self.index}])""")
+
+        if self.min_version is not None:
+            define = self.min_version.add_guard(self.name, define)
         return define
 
     def array_api_define(self):
diff --git a/numpy/core/code_generators/numpy_api.py b/numpy/core/code_generators/numpy_api.py
index 6b9080403..bb3b15806 100644
--- a/numpy/core/code_generators/numpy_api.py
+++ b/numpy/core/code_generators/numpy_api.py
@@ -18,17 +18,17 @@ import os
 import importlib.util
 
 
-def get_StealRef():
+def get_annotations():
     # Convoluted because we can't import from numpy.distutils
     # (numpy is not yet built)
     genapi_py = os.path.join(os.path.dirname(__file__), 'genapi.py')
     spec = importlib.util.spec_from_file_location('conv_template', genapi_py)
     mod = importlib.util.module_from_spec(spec)
     spec.loader.exec_module(mod)
-    return mod.StealRef
+    return mod.StealRef, mod.MinVersion
 
 
-StealRef = get_StealRef()
+StealRef, MinVersion = get_annotations()
 #from code_generators.genapi import StealRef
 
 # index, type
@@ -367,8 +367,8 @@ multiarray_funcs_api = {
     'PyArray_ResolveWritebackIfCopy':       (302,),
     'PyArray_SetWritebackIfCopyBase':       (303,),
     # End 1.14 API
-    'PyDataMem_SetHandler':                 (304,),
-    'PyDataMem_GetHandler':                 (305,),
+    'PyDataMem_SetHandler':                 (304, MinVersion("1.22")),
+    'PyDataMem_GetHandler':                 (305, MinVersion("1.22")),
     # End 1.22 API
 }
 
@@ -422,7 +422,7 @@ ufunc_funcs_api = {
     # End 1.7 API
     'PyUFunc_RegisterLoopForDescr':             (41,),
     # End 1.8 API
-    'PyUFunc_FromFuncAndDataAndSignatureAndIdentity': (42,),
+    'PyUFunc_FromFuncAndDataAndSignatureAndIdentity': (42, MinVersion("1.16")),
     # End 1.16 API
 }
 
diff --git a/numpy/core/include/numpy/arrayscalars.h b/numpy/core/include/numpy/arrayscalars.h
index a20a68016..258bf95b6 100644
--- a/numpy/core/include/numpy/arrayscalars.h
+++ b/numpy/core/include/numpy/arrayscalars.h
@@ -139,7 +139,9 @@ typedef struct {
         /* note that the PyObject_HEAD macro lives right here */
         PyUnicodeObject base;
         Py_UCS4 *obval;
+    #if NPY_FEATURE_VERSION >= NPY_1_20_API_VERSION
         char *buffer_fmt;
+    #endif
 } PyUnicodeScalarObject;
 
 
@@ -149,7 +151,9 @@ typedef struct {
         PyArray_Descr *descr;
         int flags;
         PyObject *base;
+    #if NPY_FEATURE_VERSION >= NPY_1_20_API_VERSION
         void *_buffer_info;  /* private buffer info, tagged to allow warning */
+    #endif
 } PyVoidScalarObject;
 
 /* Macros
diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h
index 45ecb6955..742ba5261 100644
--- a/numpy/core/include/numpy/ndarraytypes.h
+++ b/numpy/core/include/numpy/ndarraytypes.h
@@ -44,23 +44,6 @@
 #define NPY_FAIL 0
 #define NPY_SUCCEED 1
 
-/*
- * Binary compatibility version number.  This number is increased
- * whenever the C-API is changed such that binary compatibility is
- * broken, i.e. whenever a recompile of extension modules is needed.
- */
-#define NPY_VERSION NPY_ABI_VERSION
-
-/*
- * Minor API version.  This number is increased whenever a change is
- * made to the C-API -- whether it breaks binary compatibility or not.
- * Some changes, such as adding a function pointer to the end of the
- * function table, can be made without breaking binary compatibility.
- * In this case, only the NPY_FEATURE_VERSION (*not* NPY_VERSION)
- * would be increased.  Whenever binary compatibility is broken, both
- * NPY_VERSION and NPY_FEATURE_VERSION should be increased.
- */
-#define NPY_FEATURE_VERSION NPY_API_VERSION
 
 enum NPY_TYPES {    NPY_BOOL=0,
                     NPY_BYTE, NPY_UBYTE,
@@ -729,11 +712,15 @@ typedef struct tagPyArrayObject_fields {
     int flags;
     /* For weak references */
     PyObject *weakreflist;
+#if NPY_FEATURE_VERSION >= NPY_1_20_API_VERSION
     void *_buffer_info;  /* private buffer info, tagged to allow warning */
+#endif
     /*
      * For malloc/calloc/realloc/free per object
      */
+#if NPY_FEATURE_VERSION >= NPY_1_22_API_VERSION
     PyObject *mem_handler;
+#endif
 } PyArrayObject_fields;
 
 /*
@@ -1671,11 +1658,13 @@ PyArray_CLEARFLAGS(PyArrayObject *arr, int flags)
     ((PyArrayObject_fields *)arr)->flags &= ~flags;
 }
 
-static inline NPY_RETURNS_BORROWED_REF PyObject *
-PyArray_HANDLER(PyArrayObject *arr)
-{
-    return ((PyArrayObject_fields *)arr)->mem_handler;
-}
+#if NPY_FEATURE_VERSION >= NPY_1_22_API_VERSION
+    static inline NPY_RETURNS_BORROWED_REF PyObject *
+    PyArray_HANDLER(PyArrayObject *arr)
+    {
+        return ((PyArrayObject_fields *)arr)->mem_handler;
+    }
+#endif
 
 #define PyTypeNum_ISBOOL(type) ((type) == NPY_BOOL)
 
diff --git a/numpy/core/include/numpy/numpyconfig.h b/numpy/core/include/numpy/numpyconfig.h
index b7d7e2ca4..1c25aa5fc 100644
--- a/numpy/core/include/numpy/numpyconfig.h
+++ b/numpy/core/include/numpy/numpyconfig.h
@@ -56,30 +56,83 @@
 
 
 /**
- * To help with the NPY_NO_DEPRECATED_API macro, we include API version
- * numbers for specific versions of NumPy. To exclude all API that was
- * deprecated as of 1.7, add the following before #including any NumPy
- * headers:
+ * To help with both NPY_TARGET_VERSION and the NPY_NO_DEPRECATED_API macro,
+ * we include API version numbers for specific versions of NumPy.
+ * To exclude all API that was deprecated as of 1.7, add the following before
+ * #including any NumPy headers:
  *   #define NPY_NO_DEPRECATED_API  NPY_1_7_API_VERSION
+ * The same is true for NPY_TARGET_VERSION, although NumPy will default to
+ * a backwards compatible build anyway.
  */
 #define NPY_1_7_API_VERSION 0x00000007
 #define NPY_1_8_API_VERSION 0x00000008
-#define NPY_1_9_API_VERSION 0x00000008
-#define NPY_1_10_API_VERSION 0x00000008
-#define NPY_1_11_API_VERSION 0x00000008
-#define NPY_1_12_API_VERSION 0x00000008
-#define NPY_1_13_API_VERSION 0x00000008
-#define NPY_1_14_API_VERSION 0x00000008
-#define NPY_1_15_API_VERSION 0x00000008
-#define NPY_1_16_API_VERSION 0x00000008
-#define NPY_1_17_API_VERSION 0x00000008
-#define NPY_1_18_API_VERSION 0x00000008
-#define NPY_1_19_API_VERSION 0x00000008
+#define NPY_1_9_API_VERSION 0x00000009
+#define NPY_1_10_API_VERSION 0x0000000a
+#define NPY_1_11_API_VERSION 0x0000000a
+#define NPY_1_12_API_VERSION 0x0000000a
+#define NPY_1_13_API_VERSION 0x0000000b
+#define NPY_1_14_API_VERSION 0x0000000c
+#define NPY_1_15_API_VERSION 0x0000000c
+#define NPY_1_16_API_VERSION 0x0000000d
+#define NPY_1_17_API_VERSION 0x0000000d
+#define NPY_1_18_API_VERSION 0x0000000d
+#define NPY_1_19_API_VERSION 0x0000000d
 #define NPY_1_20_API_VERSION 0x0000000e
 #define NPY_1_21_API_VERSION 0x0000000e
 #define NPY_1_22_API_VERSION 0x0000000f
 #define NPY_1_23_API_VERSION 0x00000010
 #define NPY_1_24_API_VERSION 0x00000010
-#define NPY_1_25_API_VERSION 0x00000010
+#define NPY_1_25_API_VERSION 0x00000011
+
+
+/*
+ * Binary compatibility version number.  This number is increased
+ * whenever the C-API is changed such that binary compatibility is
+ * broken, i.e. whenever a recompile of extension modules is needed.
+ */
+#define NPY_VERSION NPY_ABI_VERSION
+
+/*
+ * Minor API version we are compiling to be compatible with.  The version
+ * Number is always increased when the API changes via: `NPY_API_VERSION`
+ * (and should maybe just track the NumPy version).
+ *
+ * If we have an internal build, we always target the current version of
+ * course.
+ *
+ * For downstream users, we default to an older version to provide them with
+ * maximum compatibility by default.  Downstream can choose to extend that
+ * default, or narrow it down if they wish to use newer API.  If you adjust
+ * this, consider the Python version support (example for 1.25.x):
+ *
+ * NumPy 1.25.x supports Python:                     3.9  3.10  3.11  (3.12)
+ * NumPy 1.19.x supports Python:      3.6  3.7  3.8  3.9
+ * NumPy 1.17.x supports Python: 3.5  3.6  3.7  3.8
+ * NumPy 1.15.x supports Python: ...  3.6  3.7
+ *
+ * Users of the stable ABI may wish to target the last Python that is not
+ * end of life.  This would be 3.8 at NumPy 1.25 release time.
+ * 1.17 as default was the choice of oldest-support-numpy at the time and
+ * has in practice no limit (comapared to 1.19).  Even earlier becomes legacy.
+ */
+#if defined(NPY_INTERNAL_BUILD) && NPY_INTERNAL_BUILD
+    /* NumPy internal build, always use current version. */
+    #define NPY_FEATURE_VERSION NPY_API_VERSION
+#elif defined(NPY_TARGET_VERSION) && NPY_TARGET_VERSION
+    /* user provided a target version, use it */
+    #define NPY_FEATURE_VERSION NPY_TARGET_VERSION
+#else
+    /* Use the default (increase when dropping Python 3.9 support) */
+    #define NPY_FEATURE_VERSION NPY_1_19_API_VERSION
+#endif
+
+/* Sanity check the (requested) feature version */
+#if NPY_FEATURE_VERSION > NPY_API_VERSION
+    #error "NPY_TARGET_VERSION higher than NumPy headers!"
+#elif NPY_FEATURE_VERSION < NPY_1_15_API_VERSION
+    /* No support for irrelevant old targets, no need for error, but warn. */
+    #warning "Requested NumPy target lower than supported NumPy 1.15."
+#endif
+
 
 #endif  /* NUMPY_CORE_INCLUDE_NUMPY_NPY_NUMPYCONFIG_H_ */
diff --git a/numpy/core/include/numpy/ufuncobject.h b/numpy/core/include/numpy/ufuncobject.h
index bb0633100..9e00f2e56 100644
--- a/numpy/core/include/numpy/ufuncobject.h
+++ b/numpy/core/include/numpy/ufuncobject.h
@@ -197,7 +197,7 @@ typedef struct _tagPyUFuncObject {
         npy_uint32 iter_flags;
 
         /* New in NPY_API_VERSION 0x0000000D and above */
-
+    #if NPY_FEATURE_VERSION >= NPY_1_16_API_VERSION
         /*
          * for each core_num_dim_ix distinct dimension names,
          * the possible "frozen" size (-1 if not frozen).
@@ -211,13 +211,15 @@ typedef struct _tagPyUFuncObject {
 
         /* Identity for reduction, when identity == PyUFunc_IdentityValue */
         PyObject *identity_value;
+    #endif  /* NPY_FEATURE_VERSION >= NPY_1_16_API_VERSION */
 
         /* New in NPY_API_VERSION 0x0000000F and above */
-
+    #if NPY_FEATURE_VERSION >= NPY_1_22_API_VERSION
         /* New private fields related to dispatching */
         void *_dispatch_cache;
         /* A PyListObject of `(tuple of DTypes, ArrayMethod/Promoter)` */
         PyObject *_loops;
+    #endif
 } PyUFuncObject;
 
 #include "arrayobject.h"
diff --git a/numpy/core/meson.build b/numpy/core/meson.build
index e51a7ba3b..f375e9647 100644
--- a/numpy/core/meson.build
+++ b/numpy/core/meson.build
@@ -44,7 +44,8 @@ C_ABI_VERSION = '0x01000009'
 # 0x0000000f - 1.22.x
 # 0x00000010 - 1.23.x
 # 0x00000010 - 1.24.x
-C_API_VERSION = '0x00000010'
+# 0x00000011 - 1.25.x
+C_API_VERSION = '0x00000011'
 
 # Check whether we have a mismatch between the set C API VERSION and the
 # actual C API VERSION. Will raise a MismatchCAPIError if so.
diff --git a/numpy/core/setup_common.py b/numpy/core/setup_common.py
index ef8d21fa7..b5bc0dec1 100644
--- a/numpy/core/setup_common.py
+++ b/numpy/core/setup_common.py
@@ -48,7 +48,13 @@ C_ABI_VERSION = 0x01000009
 # 0x0000000f - 1.22.x
 # 0x00000010 - 1.23.x
 # 0x00000010 - 1.24.x
-C_API_VERSION = 0x00000010
+# 0x00000011 - 1.25.x
+C_API_VERSION = 0x00000011
+
+# By default, when compiling downstream libraries against NumPy,```
+# pick an older feature version.  For example, for 1.25.x we default to the
+# 1.19 API and support going back all the way to 1.15.x (if so desired).
+# This is set up in `numpyconfig.h`.
 
 class MismatchCAPIError(ValueError):
     pass
@@ -91,6 +97,10 @@ def check_api_version(apiversion, codegen_dir):
                f"checksum in core/codegen_dir/cversions.txt is {api_hash}. If "
                "functions were added in the C API, you have to update "
                f"C_API_VERSION in {__file__}."
+               "\n"
+               "Please make sure that new additions are guarded with "
+               "MinVersion() to make them unavailable when wishing support "
+               "for older NumPy versions."
                )
         raise MismatchCAPIError(msg)
 
diff --git a/numpy/core/tests/test_mem_policy.py b/numpy/core/tests/test_mem_policy.py
index d5dfbc38b..479f702ea 100644
--- a/numpy/core/tests/test_mem_policy.py
+++ b/numpy/core/tests/test_mem_policy.py
@@ -89,6 +89,7 @@ def get_module(tmp_path):
          """),
     ]
     prologue = '''
+        #define NPY_TARGET_VERSION NPY_1_22_API_VERSION
         #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
         #include <numpy/arrayobject.h>
         /*