summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--doc/release/upcoming_changes/20000.deprecation.rst5
-rw-r--r--numpy/core/_add_newdocs.py3
-rw-r--r--numpy/core/fromnumeric.py6
-rw-r--r--numpy/core/src/multiarray/datetime.c22
-rw-r--r--numpy/core/src/multiarray/item_selection.c10
-rw-r--r--numpy/core/src/multiarray/usertypes.c67
-rw-r--r--numpy/core/tests/test_datetime.py37
-rw-r--r--numpy/core/tests/test_deprecations.py23
-rw-r--r--numpy/core/tests/test_multiarray.py68
-rw-r--r--numpy/typing/mypy_plugin.py5
10 files changed, 202 insertions, 44 deletions
diff --git a/doc/release/upcoming_changes/20000.deprecation.rst b/doc/release/upcoming_changes/20000.deprecation.rst
new file mode 100644
index 000000000..e0a56cd47
--- /dev/null
+++ b/doc/release/upcoming_changes/20000.deprecation.rst
@@ -0,0 +1,5 @@
+Passing boolean ``kth`` values to (arg-)partition has been deprecated
+---------------------------------------------------------------------
+`~numpy.partition` and `~numpy.argpartition` would previously accept boolean
+values for the ``kth`` parameter, which would subsequently be converted into
+integers. This behavior has now been deprecated.
diff --git a/numpy/core/_add_newdocs.py b/numpy/core/_add_newdocs.py
index bb0c2ea12..37f21211f 100644
--- a/numpy/core/_add_newdocs.py
+++ b/numpy/core/_add_newdocs.py
@@ -4044,6 +4044,9 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('partition',
The order of all elements in the partitions is undefined.
If provided with a sequence of kth it will partition all elements
indexed by kth of them into their sorted position at once.
+
+ .. deprecated:: 1.22.0
+ Passing booleans as index is deprecated.
axis : int, optional
Axis along which to sort. Default is -1, which means sort along the
last axis.
diff --git a/numpy/core/fromnumeric.py b/numpy/core/fromnumeric.py
index 5ecb1e666..29d215ea0 100644
--- a/numpy/core/fromnumeric.py
+++ b/numpy/core/fromnumeric.py
@@ -689,6 +689,9 @@ def partition(a, kth, axis=-1, kind='introselect', order=None):
it. The order of all elements in the partitions is undefined. If
provided with a sequence of k-th it will partition all elements
indexed by k-th of them into their sorted position at once.
+
+ .. deprecated:: 1.22.0
+ Passing booleans as index is deprecated.
axis : int or None, optional
Axis along which to sort. If None, the array is flattened before
sorting. The default is -1, which sorts along the last axis.
@@ -781,6 +784,9 @@ def argpartition(a, kth, axis=-1, kind='introselect', order=None):
elements in the partitions is undefined. If provided with a
sequence of k-th it will partition all of them into their sorted
position at once.
+
+ .. deprecated:: 1.22.0
+ Passing booleans as index is deprecated.
axis : int or None, optional
Axis along which to sort. The default is -1 (the last axis). If
None, the flattened array is used.
diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c
index b24bc0356..e0064c017 100644
--- a/numpy/core/src/multiarray/datetime.c
+++ b/numpy/core/src/multiarray/datetime.c
@@ -13,6 +13,7 @@
#include <Python.h>
#include "numpy/arrayobject.h"
+#include "numpyos.h"
#include "npy_config.h"
#include "npy_pycompat.h"
@@ -723,12 +724,21 @@ parse_datetime_extended_unit_from_string(char const *str, Py_ssize_t len,
{
char const *substr = str, *substrend = NULL;
int den = 1;
+ npy_longlong true_meta_val;
/* First comes an optional integer multiplier */
out_meta->num = (int)strtol_const(substr, &substrend, 10);
if (substr == substrend) {
out_meta->num = 1;
}
+ else {
+ // check for 32-bit integer overflow
+ char *endptr = NULL;
+ true_meta_val = NumPyOS_strtoll(substr, &endptr, 10);
+ if (true_meta_val > INT_MAX || true_meta_val < 0) {
+ goto bad_input;
+ }
+ }
substr = substrend;
/* Next comes the unit itself, followed by either '/' or the string end */
@@ -3776,7 +3786,17 @@ time_to_time_resolve_descriptors(
meta2 = get_datetime_metadata_from_dtype(loop_descrs[1]);
assert(meta2 != NULL);
- if (meta1->base == meta2->base && meta1->num == meta2->num) {
+ if ((meta1->base == meta2->base && meta1->num == meta2->num) ||
+ // handle some common metric prefix conversions
+ // 1000 fold conversions
+ ((meta2->base >= 7) && (meta1->base - meta2->base == 1)
+ && ((meta1->num / meta2->num) == 1000)) ||
+ // 10^6 fold conversions
+ ((meta2->base >= 7) && (meta1->base - meta2->base == 2)
+ && ((meta1->num / meta2->num) == 1000000)) ||
+ // 10^9 fold conversions
+ ((meta2->base >= 7) && (meta1->base - meta2->base == 3)
+ && ((meta1->num / meta2->num) == 1000000000))) {
if (byteorder_may_allow_view) {
return NPY_NO_CASTING | byteorder_may_allow_view;
}
diff --git a/numpy/core/src/multiarray/item_selection.c b/numpy/core/src/multiarray/item_selection.c
index ad5478bbf..ee66378a9 100644
--- a/numpy/core/src/multiarray/item_selection.c
+++ b/numpy/core/src/multiarray/item_selection.c
@@ -1292,7 +1292,15 @@ partition_prep_kth_array(PyArrayObject * ktharray,
npy_intp * kth;
npy_intp nkth, i;
- if (!PyArray_CanCastSafely(PyArray_TYPE(ktharray), NPY_INTP)) {
+ if (PyArray_ISBOOL(ktharray)) {
+ /* 2021-09-29, NumPy 1.22 */
+ if (DEPRECATE(
+ "Passing booleans as partition index is deprecated"
+ " (warning added in NumPy 1.22)") < 0) {
+ return NULL;
+ }
+ }
+ else if (!PyArray_ISINTEGER(ktharray)) {
PyErr_Format(PyExc_TypeError, "Partition index must be integer");
return NULL;
}
diff --git a/numpy/core/src/multiarray/usertypes.c b/numpy/core/src/multiarray/usertypes.c
index d0cf53576..a338d712d 100644
--- a/numpy/core/src/multiarray/usertypes.c
+++ b/numpy/core/src/multiarray/usertypes.c
@@ -268,6 +268,56 @@ PyArray_RegisterDataType(PyArray_Descr *descr)
return typenum;
}
+
+/*
+ * Checks that there is no cast already cached using the new casting-impl
+ * mechanism.
+ * In that case, we do not clear out the cache (but otherwise silently
+ * continue). Users should not modify casts after they have been used,
+ * but this may also happen accidentally during setup (and may never have
+ * mattered). See https://github.com/numpy/numpy/issues/20009
+ */
+static int _warn_if_cast_exists_already(
+ PyArray_Descr *descr, int totype, char *funcname)
+{
+ PyArray_DTypeMeta *to_DType = PyArray_DTypeFromTypeNum(totype);
+ if (to_DType == NULL) {
+ return -1;
+ }
+ PyObject *cast_impl = PyDict_GetItemWithError(
+ NPY_DT_SLOTS(NPY_DTYPE(descr))->castingimpls, (PyObject *)to_DType);
+ Py_DECREF(to_DType);
+ if (cast_impl == NULL) {
+ if (PyErr_Occurred()) {
+ return -1;
+ }
+ }
+ else {
+ char *extra_msg;
+ if (cast_impl == Py_None) {
+ extra_msg = "the cast will continue to be considered impossible.";
+ }
+ else {
+ extra_msg = "the previous definition will continue to be used.";
+ }
+ Py_DECREF(cast_impl);
+ PyArray_Descr *to_descr = PyArray_DescrFromType(totype);
+ int ret = PyErr_WarnFormat(PyExc_RuntimeWarning, 1,
+ "A cast from %R to %R was registered/modified using `%s` "
+ "after the cast had been used. "
+ "This registration will have (mostly) no effect: %s\n"
+ "The most likely fix is to ensure that casts are the first "
+ "thing initialized after dtype registration. "
+ "Please contact the NumPy developers with any questions!",
+ descr, to_descr, funcname, extra_msg);
+ Py_DECREF(to_descr);
+ if (ret < 0) {
+ return -1;
+ }
+ }
+ return 0;
+}
+
/*NUMPY_API
Register Casting Function
Replaces any function currently stored.
@@ -279,14 +329,19 @@ PyArray_RegisterCastFunc(PyArray_Descr *descr, int totype,
PyObject *cobj, *key;
int ret;
- if (totype < NPY_NTYPES_ABI_COMPATIBLE) {
- descr->f->cast[totype] = castfunc;
- return 0;
- }
if (totype >= NPY_NTYPES && !PyTypeNum_ISUSERDEF(totype)) {
PyErr_SetString(PyExc_TypeError, "invalid type number.");
return -1;
}
+ if (_warn_if_cast_exists_already(
+ descr, totype, "PyArray_RegisterCastFunc") < 0) {
+ return -1;
+ }
+
+ if (totype < NPY_NTYPES_ABI_COMPATIBLE) {
+ descr->f->cast[totype] = castfunc;
+ return 0;
+ }
if (descr->f->castdict == NULL) {
descr->f->castdict = PyDict_New();
if (descr->f->castdict == NULL) {
@@ -328,6 +383,10 @@ PyArray_RegisterCanCast(PyArray_Descr *descr, int totype,
"RegisterCanCast must be user-defined.");
return -1;
}
+ if (_warn_if_cast_exists_already(
+ descr, totype, "PyArray_RegisterCanCast") < 0) {
+ return -1;
+ }
if (scalar == NPY_NOSCALAR) {
/*
diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py
index 5a490646e..69eba7ba0 100644
--- a/numpy/core/tests/test_datetime.py
+++ b/numpy/core/tests/test_datetime.py
@@ -63,6 +63,7 @@ class TestDateTime:
assert_raises(TypeError, np.dtype, 'm7')
assert_raises(TypeError, np.dtype, 'M16')
assert_raises(TypeError, np.dtype, 'm16')
+ assert_raises(TypeError, np.dtype, 'M8[3000000000ps]')
def test_datetime_casting_rules(self):
# Cannot cast safely/same_kind between timedelta and datetime
@@ -137,6 +138,42 @@ class TestDateTime:
assert_(not np.can_cast('M8[h]', 'M8', casting='same_kind'))
assert_(not np.can_cast('M8[h]', 'M8', casting='safe'))
+ def test_datetime_prefix_conversions(self):
+ # regression tests related to gh-19631;
+ # test metric prefixes from seconds down to
+ # attoseconds for bidirectional conversions
+ smaller_units = ['M8[7000ms]',
+ 'M8[2000us]',
+ 'M8[1000ns]',
+ 'M8[5000ns]',
+ 'M8[2000ps]',
+ 'M8[9000fs]',
+ 'M8[1000as]',
+ 'M8[2000000ps]',
+ 'M8[1000000as]',
+ 'M8[2000000000ps]',
+ 'M8[1000000000as]']
+ larger_units = ['M8[7s]',
+ 'M8[2ms]',
+ 'M8[us]',
+ 'M8[5us]',
+ 'M8[2ns]',
+ 'M8[9ps]',
+ 'M8[1fs]',
+ 'M8[2us]',
+ 'M8[1ps]',
+ 'M8[2ms]',
+ 'M8[1ns]']
+ for larger_unit, smaller_unit in zip(larger_units, smaller_units):
+ assert np.can_cast(larger_unit, smaller_unit, casting='safe')
+ assert np.can_cast(smaller_unit, larger_unit, casting='safe')
+
+ @pytest.mark.parametrize("unit", [
+ "s", "ms", "us", "ns", "ps", "fs", "as"])
+ def test_prohibit_negative_datetime(self, unit):
+ with assert_raises(TypeError):
+ np.array([1], dtype=f"M8[-1{unit}]")
+
def test_compare_generic_nat(self):
# regression tests for gh-6452
assert_(np.datetime64('NaT') !=
diff --git a/numpy/core/tests/test_deprecations.py b/numpy/core/tests/test_deprecations.py
index 1d0c5dfac..898ff8075 100644
--- a/numpy/core/tests/test_deprecations.py
+++ b/numpy/core/tests/test_deprecations.py
@@ -1192,3 +1192,26 @@ class TestUFuncForcedDTypeWarning(_DeprecationTestCase):
np.maximum(arr, arr, dtype="m8[ns]") # previously used the "ns"
with pytest.warns(DeprecationWarning, match=self.message):
np.maximum.reduce(arr, dtype="m8[ns]") # never preserved the "ns"
+
+
+PARTITION_DICT = {
+ "partition method": np.arange(10).partition,
+ "argpartition method": np.arange(10).argpartition,
+ "partition function": lambda kth: np.partition(np.arange(10), kth),
+ "argpartition function": lambda kth: np.argpartition(np.arange(10), kth),
+}
+
+
+@pytest.mark.parametrize("func", PARTITION_DICT.values(), ids=PARTITION_DICT)
+class TestPartitionBoolIndex(_DeprecationTestCase):
+ # Deprecated 2021-09-29, NumPy 1.22
+ warning_cls = DeprecationWarning
+ message = "Passing booleans as partition index is deprecated"
+
+ def test_deprecated(self, func):
+ self.assert_deprecated(lambda: func(True))
+ self.assert_deprecated(lambda: func([False, True]))
+
+ def test_not_deprecated(self, func):
+ self.assert_not_deprecated(lambda: func(1))
+ self.assert_not_deprecated(lambda: func([0, 1]))
diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py
index b5f9f8af3..0da36bbea 100644
--- a/numpy/core/tests/test_multiarray.py
+++ b/numpy/core/tests/test_multiarray.py
@@ -2511,27 +2511,19 @@ class TestMethods:
assert_(not isinstance(a.searchsorted(b, 'left', s), A))
assert_(not isinstance(a.searchsorted(b, 'right', s), A))
- def test_argpartition_out_of_range(self):
+ @pytest.mark.parametrize("dtype", np.typecodes["All"])
+ def test_argpartition_out_of_range(self, dtype):
# Test out of range values in kth raise an error, gh-5469
- d = np.arange(10)
+ d = np.arange(10).astype(dtype=dtype)
assert_raises(ValueError, d.argpartition, 10)
assert_raises(ValueError, d.argpartition, -11)
- # Test also for generic type argpartition, which uses sorting
- # and used to not bound check kth
- d_obj = np.arange(10, dtype=object)
- assert_raises(ValueError, d_obj.argpartition, 10)
- assert_raises(ValueError, d_obj.argpartition, -11)
- def test_partition_out_of_range(self):
+ @pytest.mark.parametrize("dtype", np.typecodes["All"])
+ def test_partition_out_of_range(self, dtype):
# Test out of range values in kth raise an error, gh-5469
- d = np.arange(10)
+ d = np.arange(10).astype(dtype=dtype)
assert_raises(ValueError, d.partition, 10)
assert_raises(ValueError, d.partition, -11)
- # Test also for generic type partition, which uses sorting
- # and used to not bound check kth
- d_obj = np.arange(10, dtype=object)
- assert_raises(ValueError, d_obj.partition, 10)
- assert_raises(ValueError, d_obj.partition, -11)
def test_argpartition_integer(self):
# Test non-integer values in kth raise an error/
@@ -2551,26 +2543,30 @@ class TestMethods:
d_obj = np.arange(10, dtype=object)
assert_raises(TypeError, d_obj.partition, 9.)
- def test_partition_empty_array(self):
+ @pytest.mark.parametrize("kth_dtype", np.typecodes["AllInteger"])
+ def test_partition_empty_array(self, kth_dtype):
# check axis handling for multidimensional empty arrays
+ kth = np.array(0, dtype=kth_dtype)[()]
a = np.array([])
a.shape = (3, 2, 1, 0)
for axis in range(-a.ndim, a.ndim):
msg = 'test empty array partition with axis={0}'.format(axis)
- assert_equal(np.partition(a, 0, axis=axis), a, msg)
+ assert_equal(np.partition(a, kth, axis=axis), a, msg)
msg = 'test empty array partition with axis=None'
- assert_equal(np.partition(a, 0, axis=None), a.ravel(), msg)
+ assert_equal(np.partition(a, kth, axis=None), a.ravel(), msg)
- def test_argpartition_empty_array(self):
+ @pytest.mark.parametrize("kth_dtype", np.typecodes["AllInteger"])
+ def test_argpartition_empty_array(self, kth_dtype):
# check axis handling for multidimensional empty arrays
+ kth = np.array(0, dtype=kth_dtype)[()]
a = np.array([])
a.shape = (3, 2, 1, 0)
for axis in range(-a.ndim, a.ndim):
msg = 'test empty array argpartition with axis={0}'.format(axis)
- assert_equal(np.partition(a, 0, axis=axis),
+ assert_equal(np.partition(a, kth, axis=axis),
np.zeros_like(a, dtype=np.intp), msg)
msg = 'test empty array argpartition with axis=None'
- assert_equal(np.partition(a, 0, axis=None),
+ assert_equal(np.partition(a, kth, axis=None),
np.zeros_like(a.ravel(), dtype=np.intp), msg)
def test_partition(self):
@@ -2901,10 +2897,12 @@ class TestMethods:
assert_array_equal(np.partition(d, kth)[kth], tgt,
err_msg="data: %r\n kth: %r" % (d, kth))
- def test_argpartition_gh5524(self):
+ @pytest.mark.parametrize("kth_dtype", np.typecodes["AllInteger"])
+ def test_argpartition_gh5524(self, kth_dtype):
# A test for functionality of argpartition on lists.
- d = [6,7,3,2,9,0]
- p = np.argpartition(d,1)
+ kth = np.array(1, dtype=kth_dtype)[()]
+ d = [6, 7, 3, 2, 9, 0]
+ p = np.argpartition(d, kth)
self.assert_partitioned(np.array(d)[p],[1])
def test_flatten(self):
@@ -4200,7 +4198,7 @@ class TestArgmaxArgminCommon:
(3, 4, 1, 2), (4, 1, 2, 3)]
@pytest.mark.parametrize("size, axis", itertools.chain(*[[(size, axis)
- for axis in list(range(-len(size), len(size))) + [None]]
+ for axis in list(range(-len(size), len(size))) + [None]]
for size in sizes]))
@pytest.mark.parametrize('method', [np.argmax, np.argmin])
def test_np_argmin_argmax_keepdims(self, size, axis, method):
@@ -4221,7 +4219,7 @@ class TestArgmaxArgminCommon:
assert_equal(res, res_orig)
assert_(res.shape == new_shape)
outarray = np.empty(res.shape, dtype=res.dtype)
- res1 = method(arr, axis=axis, out=outarray,
+ res1 = method(arr, axis=axis, out=outarray,
keepdims=True)
assert_(res1 is outarray)
assert_equal(res, outarray)
@@ -4234,7 +4232,7 @@ class TestArgmaxArgminCommon:
wrong_shape[0] = 2
wrong_outarray = np.empty(wrong_shape, dtype=res.dtype)
with pytest.raises(ValueError):
- method(arr.T, axis=axis,
+ method(arr.T, axis=axis,
out=wrong_outarray, keepdims=True)
# non-contiguous arrays
@@ -4252,18 +4250,18 @@ class TestArgmaxArgminCommon:
assert_(res.shape == new_shape)
outarray = np.empty(new_shape[::-1], dtype=res.dtype)
outarray = outarray.T
- res1 = method(arr.T, axis=axis, out=outarray,
+ res1 = method(arr.T, axis=axis, out=outarray,
keepdims=True)
assert_(res1 is outarray)
assert_equal(res, outarray)
if len(size) > 0:
- # one dimension lesser for non-zero sized
+ # one dimension lesser for non-zero sized
# array should raise an error
with pytest.raises(ValueError):
- method(arr[0], axis=axis,
+ method(arr[0], axis=axis,
out=outarray, keepdims=True)
-
+
if len(size) > 0:
wrong_shape = list(new_shape)
if axis is not None:
@@ -4272,7 +4270,7 @@ class TestArgmaxArgminCommon:
wrong_shape[0] = 2
wrong_outarray = np.empty(wrong_shape, dtype=res.dtype)
with pytest.raises(ValueError):
- method(arr.T, axis=axis,
+ method(arr.T, axis=axis,
out=wrong_outarray, keepdims=True)
@pytest.mark.parametrize('method', ['max', 'min'])
@@ -4287,7 +4285,7 @@ class TestArgmaxArgminCommon:
axes.remove(i)
assert_(np.all(a_maxmin == aarg_maxmin.choose(
*a.transpose(i, *axes))))
-
+
@pytest.mark.parametrize('method', ['argmax', 'argmin'])
def test_output_shape(self, method):
# see also gh-616
@@ -4330,7 +4328,7 @@ class TestArgmaxArgminCommon:
[('argmax', np.argmax),
('argmin', np.argmin)])
def test_np_vs_ndarray(self, arr_method, np_method):
- # make sure both ndarray.argmax/argmin and
+ # make sure both ndarray.argmax/argmin and
# numpy.argmax/argmin support out/axis args
a = np.random.normal(size=(2, 3))
arg_method = getattr(a, arr_method)
@@ -4344,7 +4342,7 @@ class TestArgmaxArgminCommon:
# check keyword args
out1 = np.zeros(3, dtype=int)
out2 = np.zeros(3, dtype=int)
- assert_equal(arg_method(out=out1, axis=0),
+ assert_equal(arg_method(out=out1, axis=0),
np_method(a, out=out2, axis=0))
assert_equal(out1, out2)
@@ -4438,7 +4436,7 @@ class TestArgmax:
assert_equal(np.argmax(arr), pos, err_msg="%r" % arr)
assert_equal(arr[np.argmax(arr)], val, err_msg="%r" % arr)
-
+
def test_maximum_signed_integers(self):
a = np.array([1, 2**7 - 1, -2**7], dtype=np.int8)
diff --git a/numpy/typing/mypy_plugin.py b/numpy/typing/mypy_plugin.py
index 5421d6bfa..5ac75f94d 100644
--- a/numpy/typing/mypy_plugin.py
+++ b/numpy/typing/mypy_plugin.py
@@ -131,9 +131,8 @@ if TYPE_CHECKING or MYPY_EX is None:
for i, value in enumerate(iterable):
if getattr(value, "id", None) == id:
return i
- else:
- raise ValueError("Failed to identify a `ImportFrom` instance "
- f"with the following id: {id!r}")
+ raise ValueError("Failed to identify a `ImportFrom` instance "
+ f"with the following id: {id!r}")
def _override_imports(
file: MypyFile,
View Lorry Controller Status