Merge pull request #23020 from seberg/faster-array-function

ENH: Improve array function overhead by using vectorcall

3 files changed, 402 insertions, 235 deletions

diff --git a/numpy/core/src/multiarray/arrayfunction_override.c b/numpy/core/src/multiarray/arrayfunction_override.c
index 2bb3fbe28..c9b579ffe 100644
--- a/numpy/core/src/multiarray/arrayfunction_override.c
+++ b/numpy/core/src/multiarray/arrayfunction_override.c
@@ -1,11 +1,15 @@
 #define NPY_NO_DEPRECATED_API NPY_API_VERSION
 #define _MULTIARRAYMODULE
 
+#include <Python.h>
+#include "structmember.h"
+
 #include "npy_pycompat.h"
 #include "get_attr_string.h"
 #include "npy_import.h"
 #include "multiarraymodule.h"
 
+#include "arrayfunction_override.h"
 
 /* Return the ndarray.__array_function__ method. */
 static PyObject *
@@ -200,183 +204,67 @@ call_array_function(PyObject* argument, PyObject* method,
 }
 
 
-/**
- * Internal handler for the array-function dispatching. The helper returns
- * either the result, or NotImplemented (as a borrowed reference).
- *
- * @param public_api The public API symbol used for dispatching
- * @param relevant_args Arguments which may implement __array_function__
- * @param args Original arguments
- * @param kwargs Original keyword arguments
- *
- * @returns The result of the dispatched version, or a borrowed reference
- *          to NotImplemented to indicate the default implementation should
- *          be used.
+
+/*
+ * Helper to convert from vectorcall convention, since the protocol requires
+ * args and kwargs to be passed as tuple and dict explicitly.
+ * We always pass a dict, so always returns it.
  */
-static PyObject *
-array_implement_array_function_internal(
-    PyObject *public_api, PyObject *relevant_args,
-    PyObject *args, PyObject *kwargs)
+static int
+get_args_and_kwargs(
+        PyObject *const *fast_args, Py_ssize_t len_args, PyObject *kwnames,
+        PyObject **out_args, PyObject **out_kwargs)
 {
-    PyObject *implementing_args[NPY_MAXARGS];
-    PyObject *array_function_methods[NPY_MAXARGS];
-    PyObject *types = NULL;
-
-    PyObject *result = NULL;
-
-    static PyObject *errmsg_formatter = NULL;
+    len_args = PyVectorcall_NARGS(len_args);
+    PyObject *args = PyTuple_New(len_args);
+    PyObject *kwargs = NULL;
 
-    relevant_args = PySequence_Fast(
-        relevant_args,
-        "dispatcher for __array_function__ did not return an iterable");
-    if (relevant_args == NULL) {
-        return NULL;
-    }
-
-    /* Collect __array_function__ implementations */
-    int num_implementing_args = get_implementing_args_and_methods(
-        relevant_args, implementing_args, array_function_methods);
-    if (num_implementing_args == -1) {
-        goto cleanup;
-    }
-
-    /*
-     * Handle the typical case of no overrides. This is merely an optimization
-     * if some arguments are ndarray objects, but is also necessary if no
-     * arguments implement __array_function__ at all (e.g., if they are all
-     * built-in types).
-     */
-    int any_overrides = 0;
-    for (int j = 0; j < num_implementing_args; j++) {
-        if (!is_default_array_function(array_function_methods[j])) {
-            any_overrides = 1;
-            break;
-        }
+    if (args == NULL) {
+        return -1;
     }
-    if (!any_overrides) {
-        /*
-         * When the default implementation should be called, return
-         * `Py_NotImplemented` to indicate this.
-         */
-        result = Py_NotImplemented;
-        goto cleanup;
+    for (Py_ssize_t i = 0; i < len_args; i++) {
+        Py_INCREF(fast_args[i]);
+        PyTuple_SET_ITEM(args, i, fast_args[i]);
     }
-
-    /*
-     * Create a Python object for types.
-     * We use a tuple, because it's the fastest Python collection to create
-     * and has the bonus of being immutable.
-     */
-    types = PyTuple_New(num_implementing_args);
-    if (types == NULL) {
-        goto cleanup;
+    kwargs = PyDict_New();
+    if (kwargs == NULL) {
+        Py_DECREF(args);
+        return -1;
     }
-    for (int j = 0; j < num_implementing_args; j++) {
-        PyObject *arg_type = (PyObject *)Py_TYPE(implementing_args[j]);
-        Py_INCREF(arg_type);
-        PyTuple_SET_ITEM(types, j, arg_type);
-    }
-
-    /* Call __array_function__ methods */
-    for (int j = 0; j < num_implementing_args; j++) {
-        PyObject *argument = implementing_args[j];
-        PyObject *method = array_function_methods[j];
-
-        /*
-         * We use `public_api` instead of `implementation` here so
-         * __array_function__ implementations can do equality/identity
-         * comparisons.
-         */
-        result = call_array_function(
-            argument, method, public_api, types, args, kwargs);
-
-        if (result == Py_NotImplemented) {
-            /* Try the next one */
-            Py_DECREF(result);
-            result = NULL;
-        }
-        else {
-            /* Either a good result, or an exception was raised. */
-            goto cleanup;
+    if (kwnames != NULL) {
+        Py_ssize_t nkwargs = PyTuple_GET_SIZE(kwnames);
+        for (Py_ssize_t i = 0; i < nkwargs; i++) {
+            PyObject *key = PyTuple_GET_ITEM(kwnames, i);
+            PyObject *value = fast_args[i+len_args];
+            if (PyDict_SetItem(kwargs, key, value) < 0) {
+                Py_DECREF(args);
+                Py_DECREF(kwargs);
+                return -1;
+            }
         }
     }
+    *out_args = args;
+    *out_kwargs = kwargs;
+    return 0;
+}
 
+
+static void
+set_no_matching_types_error(PyObject *public_api, PyObject *types)
+{
+    static PyObject *errmsg_formatter = NULL;
     /* No acceptable override found, raise TypeError. */
     npy_cache_import("numpy.core._internal",
                      "array_function_errmsg_formatter",
                      &errmsg_formatter);
     if (errmsg_formatter != NULL) {
         PyObject *errmsg = PyObject_CallFunctionObjArgs(
-            errmsg_formatter, public_api, types, NULL);
+                errmsg_formatter, public_api, types, NULL);
         if (errmsg != NULL) {
             PyErr_SetObject(PyExc_TypeError, errmsg);
             Py_DECREF(errmsg);
         }
     }
-
-cleanup:
-    for (int j = 0; j < num_implementing_args; j++) {
-        Py_DECREF(implementing_args[j]);
-        Py_DECREF(array_function_methods[j]);
-    }
-    Py_XDECREF(types);
-    Py_DECREF(relevant_args);
-    return result;
-}
-
-
-/*
- * Implements the __array_function__ protocol for a Python function, as described in
- * in NEP-18. See numpy.core.overrides for a full docstring.
- */
-NPY_NO_EXPORT PyObject *
-array_implement_array_function(
-    PyObject *NPY_UNUSED(dummy), PyObject *positional_args)
-{
-    PyObject *res, *implementation, *public_api, *relevant_args, *args, *kwargs;
-
-    if (!PyArg_UnpackTuple(
-            positional_args, "implement_array_function", 5, 5,
-            &implementation, &public_api, &relevant_args, &args, &kwargs)) {
-        return NULL;
-    }
-
-    /*
-     * Remove `like=` kwarg, which is NumPy-exclusive and thus not present
-     * in downstream libraries. If `like=` is specified but doesn't
-     * implement `__array_function__`, raise a `TypeError`.
-     */
-    if (kwargs != NULL && PyDict_Contains(kwargs, npy_ma_str_like)) {
-        PyObject *like_arg = PyDict_GetItem(kwargs, npy_ma_str_like);
-        if (like_arg != NULL) {
-            PyObject *tmp_has_override = get_array_function(like_arg);
-            if (tmp_has_override == NULL) {
-                return PyErr_Format(PyExc_TypeError,
-                        "The `like` argument must be an array-like that "
-                        "implements the `__array_function__` protocol.");
-            }
-            Py_DECREF(tmp_has_override);
-            PyDict_DelItem(kwargs, npy_ma_str_like);
-
-            /*
-             * If `like=` kwarg was removed, `implementation` points to the NumPy
-             * public API, as `public_api` is in that case the wrapper dispatcher
-             * function. For example, in the `np.full` case, `implementation` is
-             * `np.full`, whereas `public_api` is `_full_with_like`. This is done
-             * to ensure `__array_function__` implementations can do
-             * equality/identity comparisons when `like=` is present.
-             */
-            public_api = implementation;
-        }
-    }
-
-    res = array_implement_array_function_internal(
-        public_api, relevant_args, args, kwargs);
-
-    if (res == Py_NotImplemented) {
-        return PyObject_Call(implementation, args, kwargs);
-    }
-    return res;
 }
 
 /*
@@ -392,64 +280,52 @@ array_implement_c_array_function_creation(
     PyObject *args, PyObject *kwargs,
     PyObject *const *fast_args, Py_ssize_t len_args, PyObject *kwnames)
 {
-    PyObject *relevant_args = NULL;
+    PyObject *dispatch_types = NULL;
     PyObject *numpy_module = NULL;
     PyObject *public_api = NULL;
     PyObject *result = NULL;
 
     /* If `like` doesn't implement `__array_function__`, raise a `TypeError` */
-    PyObject *tmp_has_override = get_array_function(like);
-    if (tmp_has_override == NULL) {
+    PyObject *method = get_array_function(like);
+    if (method == NULL) {
         return PyErr_Format(PyExc_TypeError,
                 "The `like` argument must be an array-like that "
                 "implements the `__array_function__` protocol.");
     }
-    Py_DECREF(tmp_has_override);
-
-    if (fast_args != NULL) {
+    if (is_default_array_function(method)) {
         /*
-         * Convert from vectorcall convention, since the protocol requires
-         * the normal convention.  We have to do this late to ensure the
-         * normal path where NotImplemented is returned is fast.
+         * Return a borrowed reference of Py_NotImplemented to defer back to
+         * the original function.
          */
+        Py_DECREF(method);
+        return Py_NotImplemented;
+    }
+
+    /* We needs args and kwargs for __array_function__ (when not using it). */
+    if (fast_args != NULL) {
         assert(args == NULL);
         assert(kwargs == NULL);
-        args = PyTuple_New(len_args);
-        if (args == NULL) {
-            return NULL;
-        }
-        for (Py_ssize_t i = 0; i < len_args; i++) {
-            Py_INCREF(fast_args[i]);
-            PyTuple_SET_ITEM(args, i, fast_args[i]);
-        }
-        if (kwnames != NULL) {
-            kwargs = PyDict_New();
-            if (kwargs == NULL) {
-                Py_DECREF(args);
-                return NULL;
-            }
-            Py_ssize_t nkwargs = PyTuple_GET_SIZE(kwnames);
-            for (Py_ssize_t i = 0; i < nkwargs; i++) {
-                PyObject *key = PyTuple_GET_ITEM(kwnames, i);
-                PyObject *value = fast_args[i+len_args];
-                if (PyDict_SetItem(kwargs, key, value) < 0) {
-                    Py_DECREF(args);
-                    Py_DECREF(kwargs);
-                    return NULL;
-                }
-            }
+        if (get_args_and_kwargs(
+                fast_args, len_args, kwnames, &args, &kwargs) < 0) {
+            goto finish;
         }
     }
+    else {
+        Py_INCREF(args);
+        Py_INCREF(kwargs);
+    }
 
-    relevant_args = PyTuple_Pack(1, like);
-    if (relevant_args == NULL) {
+    dispatch_types = PyTuple_Pack(1, Py_TYPE(like));
+    if (dispatch_types == NULL) {
         goto finish;
     }
+
     /* The like argument must be present in the keyword arguments, remove it */
     if (PyDict_DelItem(kwargs, npy_ma_str_like) < 0) {
         goto finish;
     }
 
+    /* Fetch the actual symbol (the long way right now) */
     numpy_module = PyImport_Import(npy_ma_str_numpy);
     if (numpy_module == NULL) {
         goto finish;
@@ -466,16 +342,21 @@ array_implement_c_array_function_creation(
         goto finish;
     }
 
-    result = array_implement_array_function_internal(
-            public_api, relevant_args, args, kwargs);
+    result = call_array_function(like, method,
+            public_api, dispatch_types, args, kwargs);
 
-  finish:
-    if (kwnames != NULL) {
-        /* args and kwargs were converted from vectorcall convention */
-        Py_XDECREF(args);
-        Py_XDECREF(kwargs);
+    if (result == Py_NotImplemented) {
+        /* This shouldn't really happen as there is only one type, but... */
+        Py_DECREF(result);
+        result = NULL;
+        set_no_matching_types_error(public_api, dispatch_types);
     }
-    Py_XDECREF(relevant_args);
+
+  finish:
+    Py_DECREF(method);
+    Py_XDECREF(args);
+    Py_XDECREF(kwargs);
+    Py_XDECREF(dispatch_types);
     Py_XDECREF(public_api);
     return result;
 }
@@ -530,3 +411,275 @@ cleanup:
     Py_DECREF(relevant_args);
     return result;
 }
+
+
+typedef struct {
+    PyObject_HEAD
+    vectorcallfunc vectorcall;
+    PyObject *dict;
+    PyObject *relevant_arg_func;
+    PyObject *default_impl;
+} PyArray_ArrayFunctionDispatcherObject;
+
+
+static void
+dispatcher_dealloc(PyArray_ArrayFunctionDispatcherObject *self)
+{
+    Py_CLEAR(self->relevant_arg_func);
+    Py_CLEAR(self->default_impl);
+    Py_CLEAR(self->dict);
+    PyObject_FREE(self);
+}
+
+
+static PyObject *
+dispatcher_vectorcall(PyArray_ArrayFunctionDispatcherObject *self,
+        PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames)
+{
+    PyObject *result = NULL;
+    PyObject *types = NULL;
+    PyObject *relevant_args = NULL;
+
+    PyObject *public_api;
+
+    /* __array_function__ passes args, kwargs.  These may be filled: */
+    PyObject *packed_args = NULL;
+    PyObject *packed_kwargs = NULL;
+
+    PyObject *implementing_args[NPY_MAXARGS];
+    PyObject *array_function_methods[NPY_MAXARGS];
+
+    int num_implementing_args;
+
+    if (self->relevant_arg_func != NULL) {
+        public_api = (PyObject *)self;
+
+        /* Typical path, need to call the relevant_arg_func and unpack them */
+        relevant_args = PyObject_Vectorcall(
+                self->relevant_arg_func, args, len_args, kwnames);
+        if (relevant_args == NULL) {
+            return NULL;
+        }
+        Py_SETREF(relevant_args, PySequence_Fast(relevant_args,
+                "dispatcher for __array_function__ did not return an iterable"));
+        if (relevant_args == NULL) {
+            return NULL;
+        }
+
+        num_implementing_args = get_implementing_args_and_methods(
+                relevant_args, implementing_args, array_function_methods);
+        if (num_implementing_args < 0) {
+            Py_DECREF(relevant_args);
+            return NULL;
+        }
+    }
+    else {
+        /* For like= dispatching from Python, the public_symbol is the impl */
+        public_api = self->default_impl;
+
+        /*
+         * We are dealing with `like=` from Python.  For simplicity, the
+         * Python code passes it on as the first argument.
+         */
+        if (PyVectorcall_NARGS(len_args) == 0) {
+            PyErr_Format(PyExc_TypeError,
+                    "`like` argument dispatching, but first argument is not "
+                    "positional in call to %S.", self->default_impl);
+            return NULL;
+        }
+
+        array_function_methods[0] = get_array_function(args[0]);
+        if (array_function_methods[0] == NULL) {
+            return PyErr_Format(PyExc_TypeError,
+                    "The `like` argument must be an array-like that "
+                    "implements the `__array_function__` protocol.");
+        }
+        num_implementing_args = 1;
+        implementing_args[0] = args[0];
+        Py_INCREF(implementing_args[0]);
+
+        /* do not pass the like argument */
+        len_args = PyVectorcall_NARGS(len_args) - 1;
+        len_args |= PY_VECTORCALL_ARGUMENTS_OFFSET;
+        args++;
+    }
+
+    /*
+     * Handle the typical case of no overrides. This is merely an optimization
+     * if some arguments are ndarray objects, but is also necessary if no
+     * arguments implement __array_function__ at all (e.g., if they are all
+     * built-in types).
+     */
+    int any_overrides = 0;
+    for (int j = 0; j < num_implementing_args; j++) {
+        if (!is_default_array_function(array_function_methods[j])) {
+            any_overrides = 1;
+            break;
+        }
+    }
+    if (!any_overrides) {
+        /* Directly call the actual implementation. */
+        result = PyObject_Vectorcall(self->default_impl, args, len_args, kwnames);
+        goto cleanup;
+    }
+
+    /* Find args and kwargs as tuple and dict, as we pass them out: */
+    if (get_args_and_kwargs(
+            args, len_args, kwnames, &packed_args, &packed_kwargs) < 0) {
+        goto cleanup;
+    }
+
+    /*
+     * Create a Python object for types.
+     * We use a tuple, because it's the fastest Python collection to create
+     * and has the bonus of being immutable.
+     */
+    types = PyTuple_New(num_implementing_args);
+    if (types == NULL) {
+        goto cleanup;
+    }
+    for (int j = 0; j < num_implementing_args; j++) {
+        PyObject *arg_type = (PyObject *)Py_TYPE(implementing_args[j]);
+        Py_INCREF(arg_type);
+        PyTuple_SET_ITEM(types, j, arg_type);
+    }
+
+    /* Call __array_function__ methods */
+    for (int j = 0; j < num_implementing_args; j++) {
+        PyObject *argument = implementing_args[j];
+        PyObject *method = array_function_methods[j];
+
+        result = call_array_function(
+                argument, method, public_api, types,
+                packed_args, packed_kwargs);
+
+        if (result == Py_NotImplemented) {
+            /* Try the next one */
+            Py_DECREF(result);
+            result = NULL;
+        }
+        else {
+            /* Either a good result, or an exception was raised. */
+            goto cleanup;
+        }
+    }
+
+    set_no_matching_types_error(public_api, types);
+
+cleanup:
+    for (int j = 0; j < num_implementing_args; j++) {
+        Py_DECREF(implementing_args[j]);
+        Py_DECREF(array_function_methods[j]);
+    }
+    Py_XDECREF(packed_args);
+    Py_XDECREF(packed_kwargs);
+    Py_XDECREF(types);
+    Py_XDECREF(relevant_args);
+    return result;
+}
+
+
+static PyObject *
+dispatcher_new(PyTypeObject *NPY_UNUSED(cls), PyObject *args, PyObject *kwargs)
+{
+    PyArray_ArrayFunctionDispatcherObject *self;
+
+    self = PyObject_New(
+            PyArray_ArrayFunctionDispatcherObject,
+            &PyArrayFunctionDispatcher_Type);
+    if (self == NULL) {
+        return PyErr_NoMemory();
+    }
+
+    char *kwlist[] = {"", "", NULL};
+    if (!PyArg_ParseTupleAndKeywords(
+            args, kwargs, "OO:_ArrayFunctionDispatcher", kwlist,
+            &self->relevant_arg_func, &self->default_impl)) {
+        Py_DECREF(self);
+        return NULL;
+    }
+
+    self->vectorcall = (vectorcallfunc)dispatcher_vectorcall;
+    if (self->relevant_arg_func == Py_None) {
+        /* NULL in the relevant arg function means we use `like=` */
+        Py_CLEAR(self->relevant_arg_func);
+    }
+    else {
+        Py_INCREF(self->relevant_arg_func);
+    }
+    Py_INCREF(self->default_impl);
+
+    /* Need to be like a Python function that has arbitrary attributes */
+    self->dict = PyDict_New();
+    if (self->dict == NULL) {
+        Py_DECREF(self);
+        return NULL;
+    }
+    return (PyObject *)self;
+}
+
+
+static PyObject *
+dispatcher_str(PyArray_ArrayFunctionDispatcherObject *self)
+{
+    return PyObject_Str(self->default_impl);
+}
+
+
+static PyObject *
+dispatcher_repr(PyObject *self)
+{
+    PyObject *name = PyObject_GetAttrString(self, "__name__");
+    if (name == NULL) {
+        return NULL;
+    }
+    /* Print like a normal function */
+    return PyUnicode_FromFormat("<function %S at %p>", name, self);
+}
+
+static PyObject *
+dispatcher_get_implementation(
+        PyArray_ArrayFunctionDispatcherObject *self, void *NPY_UNUSED(closure))
+{
+    Py_INCREF(self->default_impl);
+    return self->default_impl;
+}
+
+
+static PyObject *
+dispatcher_reduce(PyObject *self, PyObject *NPY_UNUSED(args))
+{
+    return PyObject_GetAttrString(self, "__qualname__");
+}
+
+
+static struct PyMethodDef func_dispatcher_methods[] = {
+    {"__reduce__",
+        (PyCFunction)dispatcher_reduce, METH_NOARGS, NULL},
+    {NULL, NULL, 0, NULL}
+};
+
+
+static struct PyGetSetDef func_dispatcher_getset[] = {
+    {"__dict__", &PyObject_GenericGetDict, 0, NULL, 0},
+    {"_implementation", (getter)&dispatcher_get_implementation, 0, NULL, 0},
+    {0, 0, 0, 0, 0}
+};
+
+
+NPY_NO_EXPORT PyTypeObject PyArrayFunctionDispatcher_Type = {
+     PyVarObject_HEAD_INIT(NULL, 0)
+     .tp_name = "numpy._ArrayFunctionDispatcher",
+     .tp_basicsize = sizeof(PyArray_ArrayFunctionDispatcherObject),
+     /* We have a dict, so in theory could traverse, but in practice... */
+     .tp_dictoffset = offsetof(PyArray_ArrayFunctionDispatcherObject, dict),
+     .tp_dealloc = (destructor)dispatcher_dealloc,
+     .tp_new = (newfunc)dispatcher_new,
+     .tp_str = (reprfunc)dispatcher_str,
+     .tp_repr = (reprfunc)dispatcher_repr,
+     .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_VECTORCALL,
+     .tp_methods = func_dispatcher_methods,
+     .tp_getset = func_dispatcher_getset,
+     .tp_call = &PyVectorcall_Call,
+     .tp_vectorcall_offset = offsetof(PyArray_ArrayFunctionDispatcherObject, vectorcall),
+};
diff --git a/numpy/core/src/multiarray/arrayfunction_override.h b/numpy/core/src/multiarray/arrayfunction_override.h
index 09f7ee548..3b8b88bac 100644
--- a/numpy/core/src/multiarray/arrayfunction_override.h
+++ b/numpy/core/src/multiarray/arrayfunction_override.h
@@ -1,9 +1,7 @@
 #ifndef NUMPY_CORE_SRC_MULTIARRAY_ARRAYFUNCTION_OVERRIDE_H_
 #define NUMPY_CORE_SRC_MULTIARRAY_ARRAYFUNCTION_OVERRIDE_H_
 
-NPY_NO_EXPORT PyObject *
-array_implement_array_function(
-    PyObject *NPY_UNUSED(dummy), PyObject *positional_args);
+extern NPY_NO_EXPORT PyTypeObject PyArrayFunctionDispatcher_Type;
 
 NPY_NO_EXPORT PyObject *
 array__get_implementing_args(
diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c
index 94fa2a909..db9931e64 100644
--- a/numpy/core/src/multiarray/multiarraymodule.c
+++ b/numpy/core/src/multiarray/multiarraymodule.c
@@ -2046,10 +2046,9 @@ fail:
 }
 
 static PyObject *
-array_empty_like(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds)
+array_empty_like(PyObject *NPY_UNUSED(ignored),
+        PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames)
 {
-
-    static char *kwlist[] = {"prototype", "dtype", "order", "subok", "shape", NULL};
     PyArrayObject *prototype = NULL;
     PyArray_Descr *dtype = NULL;
     NPY_ORDER order = NPY_KEEPORDER;
@@ -2058,12 +2057,15 @@ array_empty_like(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds)
     /* -1 is a special value meaning "not specified" */
     PyArray_Dims shape = {NULL, -1};
 
-    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&|O&O&iO&:empty_like", kwlist,
-                &PyArray_Converter, &prototype,
-                &PyArray_DescrConverter2, &dtype,
-                &PyArray_OrderConverter, &order,
-                &subok,
-                &PyArray_OptionalIntpConverter, &shape)) {
+    NPY_PREPARE_ARGPARSER;
+
+    if (npy_parse_arguments("empty_like", args, len_args, kwnames,
+            "prototype", &PyArray_Converter, &prototype,
+            "|dtype", &PyArray_DescrConverter2, &dtype,
+            "|order", &PyArray_OrderConverter, &order,
+            "|subok", &PyArray_PythonPyIntFromInt, &subok,
+            "|shape", &PyArray_OptionalIntpConverter, &shape,
+            NULL, NULL, NULL) < 0) {
         goto fail;
     }
     /* steals the reference to dtype if it's not NULL */
@@ -2460,7 +2462,8 @@ array_frombuffer(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds
 }
 
 static PyObject *
-array_concatenate(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject *kwds)
+array_concatenate(PyObject *NPY_UNUSED(dummy),
+        PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames)
 {
     PyObject *a0;
     PyObject *out = NULL;
@@ -2469,10 +2472,15 @@ array_concatenate(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject *kwds)
     PyObject *casting_obj = NULL;
     PyObject *res;
     int axis = 0;
-    static char *kwlist[] = {"seq", "axis", "out", "dtype", "casting", NULL};
-    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O&O$O&O:concatenate", kwlist,
-                &a0, PyArray_AxisConverter, &axis, &out,
-                PyArray_DescrConverter2, &dtype, &casting_obj)) {
+
+    NPY_PREPARE_ARGPARSER;
+    if (npy_parse_arguments("concatenate", args, len_args, kwnames,
+            "seq", NULL, &a0,
+            "|axis", &PyArray_AxisConverter, &axis,
+            "|out", NULL, &out,
+            "$dtype", &PyArray_DescrConverter2, &dtype,
+            "$casting", NULL, &casting_obj,
+            NULL, NULL, NULL) < 0) {
         return NULL;
     }
     int casting_not_passed = 0;
@@ -2515,14 +2523,18 @@ array_innerproduct(PyObject *NPY_UNUSED(dummy), PyObject *args)
 }
 
 static PyObject *
-array_matrixproduct(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject* kwds)
+array_matrixproduct(PyObject *NPY_UNUSED(dummy),
+        PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames)
 {
     PyObject *v, *a, *o = NULL;
     PyArrayObject *ret;
-    static char* kwlist[] = {"a", "b", "out", NULL};
 
-    if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|O:matrixproduct",
-                                     kwlist, &a, &v, &o)) {
+    NPY_PREPARE_ARGPARSER;
+    if (npy_parse_arguments("dot", args, len_args, kwnames,
+            "a", NULL, &a,
+            "b", NULL, &v,
+            "|out", NULL, &o,
+            NULL, NULL, NULL) < 0) {
         return NULL;
     }
     if (o != NULL) {
@@ -3455,8 +3467,8 @@ array_lexsort(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds)
 }
 
 static PyObject *
-array_can_cast_safely(PyObject *NPY_UNUSED(self), PyObject *args,
-        PyObject *kwds)
+array_can_cast_safely(PyObject *NPY_UNUSED(self),
+        PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames)
 {
     PyObject *from_obj = NULL;
     PyArray_Descr *d1 = NULL;
@@ -3464,12 +3476,13 @@ array_can_cast_safely(PyObject *NPY_UNUSED(self), PyObject *args,
     int ret;
     PyObject *retobj = NULL;
     NPY_CASTING casting = NPY_SAFE_CASTING;
-    static char *kwlist[] = {"from_", "to", "casting", NULL};
 
-    if(!PyArg_ParseTupleAndKeywords(args, kwds, "OO&|O&:can_cast", kwlist,
-                &from_obj,
-                PyArray_DescrConverter2, &d2,
-                PyArray_CastingConverter, &casting)) {
+    NPY_PREPARE_ARGPARSER;
+    if (npy_parse_arguments("can_cast", args, len_args, kwnames,
+            "from_", NULL, &from_obj,
+            "to", &PyArray_DescrConverter2, &d2,
+            "|casting", &PyArray_CastingConverter, &casting,
+            NULL, NULL, NULL) < 0) {
         goto finish;
     }
     if (d2 == NULL) {
@@ -4432,7 +4445,7 @@ static struct PyMethodDef array_module_methods[] = {
         METH_FASTCALL | METH_KEYWORDS, NULL},
     {"empty_like",
         (PyCFunction)array_empty_like,
-        METH_VARARGS|METH_KEYWORDS, NULL},
+        METH_FASTCALL|METH_KEYWORDS, NULL},
     {"scalar",
         (PyCFunction)array_scalar,
         METH_VARARGS|METH_KEYWORDS, NULL},
@@ -4453,13 +4466,13 @@ static struct PyMethodDef array_module_methods[] = {
         METH_VARARGS|METH_KEYWORDS, NULL},
     {"concatenate",
         (PyCFunction)array_concatenate,
-        METH_VARARGS|METH_KEYWORDS, NULL},
+        METH_FASTCALL|METH_KEYWORDS, NULL},
     {"inner",
         (PyCFunction)array_innerproduct,
         METH_VARARGS, NULL},
     {"dot",
         (PyCFunction)array_matrixproduct,
-        METH_VARARGS | METH_KEYWORDS, NULL},
+        METH_FASTCALL | METH_KEYWORDS, NULL},
     {"vdot",
         (PyCFunction)array_vdot,
         METH_VARARGS | METH_KEYWORDS, NULL},
@@ -4483,7 +4496,7 @@ static struct PyMethodDef array_module_methods[] = {
         METH_VARARGS | METH_KEYWORDS, NULL},
     {"can_cast",
         (PyCFunction)array_can_cast_safely,
-        METH_VARARGS | METH_KEYWORDS, NULL},
+        METH_FASTCALL | METH_KEYWORDS, NULL},
     {"promote_types",
         (PyCFunction)array_promote_types,
         METH_VARARGS, NULL},
@@ -4539,9 +4552,6 @@ static struct PyMethodDef array_module_methods[] = {
         METH_VARARGS | METH_KEYWORDS, NULL},
     {"_monotonicity", (PyCFunction)arr__monotonicity,
         METH_VARARGS | METH_KEYWORDS, NULL},
-    {"implement_array_function",
-        (PyCFunction)array_implement_array_function,
-        METH_VARARGS, NULL},
     {"interp", (PyCFunction)arr_interp,
         METH_VARARGS | METH_KEYWORDS, NULL},
     {"interp_complex", (PyCFunction)arr_interp_complex,
@@ -5112,6 +5122,12 @@ PyMODINIT_FUNC PyInit__multiarray_umath(void) {
     if (set_typeinfo(d) != 0) {
         goto err;
     }
+    if (PyType_Ready(&PyArrayFunctionDispatcher_Type) < 0) {
+        goto err;
+    }
+    PyDict_SetItemString(
+            d, "_ArrayFunctionDispatcher",
+            (PyObject *)&PyArrayFunctionDispatcher_Type);
     if (PyType_Ready(&PyArrayMethod_Type) < 0) {
         goto err;
     }