From 6363bb7cbb1d0e292e8af87f92a28faf781d3d95 Mon Sep 17 00:00:00 2001 From: jayvius Date: Thu, 12 Jul 2012 13:35:52 -0500 Subject: Add ability to get view when selecting subset of fields. Add ability to get view when selecting subset of fields in a struct array, for numpy 1.8.0. Currently a copy is still returned - set WARN_ON_WRITE flag which will be removed in a future version of numpy. --- numpy/core/_internal.py | 7 +++++++ numpy/core/src/multiarray/mapping.c | 1 + 2 files changed, 8 insertions(+) (limited to 'numpy') diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py index 309b53c44..29d21a9bc 100644 --- a/numpy/core/_internal.py +++ b/numpy/core/_internal.py @@ -298,6 +298,13 @@ def _index_fields(ary, fields): for name in fields: newarray[name] = ary[name] + names = [name for name in fields if name in dt.names] + formats = dt.fields[name][0] for name in fields if name in dt.names] + offsets = [dt.fields[name][1] for name in fields if name in dt.names] + + view_dtype = {'names':names, 'formats':formats, 'offsets':offsets} + view = ary.view(dtype=view_dtype) + return newarray # Given a string containing a PEP 3118 format specifier, diff --git a/numpy/core/src/multiarray/mapping.c b/numpy/core/src/multiarray/mapping.c index cdefb9982..b2643f89f 100644 --- a/numpy/core/src/multiarray/mapping.c +++ b/numpy/core/src/multiarray/mapping.c @@ -975,6 +975,7 @@ array_subscript(PyArrayObject *self, PyObject *op) } obj = PyObject_CallMethod(_numpy_internal, "_index_fields", "OO", self, op); + PyArray_ENABLEFLAGS((PyArrayObject*)obj, NPY_ARRAY_WARN_ON_WRITE); Py_DECREF(_numpy_internal); return obj; } -- cgit v1.2.1 From 0c09e1f184fde335f3533b0b9ef6ea46379309db Mon Sep 17 00:00:00 2001 From: jayvius Date: Thu, 12 Jul 2012 13:37:28 -0500 Subject: fix previous commit to return copy of view instead of view --- numpy/core/_internal.py | 17 ++++------------- numpy/core/src/multiarray/mapping.c | 5 ++++- 2 files changed, 8 insertions(+), 14 deletions(-) (limited to 'numpy') diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py index 29d21a9bc..88a41fd0d 100644 --- a/numpy/core/_internal.py +++ b/numpy/core/_internal.py @@ -286,26 +286,17 @@ def _newnames(datatype, order): # construct a new array with just those fields copied over def _index_fields(ary, fields): from multiarray import empty, dtype + from numpy import copy dt = ary.dtype - new_dtype = [(name, dt[name]) for name in fields if name in dt.names] - if ary.flags.f_contiguous: - order = 'F' - else: - order = 'C' - - newarray = empty(ary.shape, dtype=new_dtype, order=order) - - for name in fields: - newarray[name] = ary[name] names = [name for name in fields if name in dt.names] - formats = dt.fields[name][0] for name in fields if name in dt.names] + formats = [dt.fields[name][0] for name in fields if name in dt.names] offsets = [dt.fields[name][1] for name in fields if name in dt.names] - view_dtype = {'names':names, 'formats':formats, 'offsets':offsets} + view_dtype = {'names':names, 'formats':formats, 'offsets':offsets, 'itemsize':dt.itemsize} view = ary.view(dtype=view_dtype) - return newarray + return copy(view) # Given a string containing a PEP 3118 format specifier, # construct a Numpy dtype diff --git a/numpy/core/src/multiarray/mapping.c b/numpy/core/src/multiarray/mapping.c index b2643f89f..663a3ef7f 100644 --- a/numpy/core/src/multiarray/mapping.c +++ b/numpy/core/src/multiarray/mapping.c @@ -975,8 +975,11 @@ array_subscript(PyArrayObject *self, PyObject *op) } obj = PyObject_CallMethod(_numpy_internal, "_index_fields", "OO", self, op); - PyArray_ENABLEFLAGS((PyArrayObject*)obj, NPY_ARRAY_WARN_ON_WRITE); Py_DECREF(_numpy_internal); + if (obj == NULL) { + return NULL; + } + PyArray_ENABLEFLAGS((PyArrayObject*)obj, NPY_ARRAY_WARN_ON_WRITE); return obj; } } -- cgit v1.2.1 From 91cb088047bd357a1b266fa8cb9d0deefea5820c Mon Sep 17 00:00:00 2001 From: jayvius Date: Thu, 12 Jul 2012 13:38:17 -0500 Subject: Change WARN_ON_WRITE Deprecation Warning Change WARN_ON_WRITE Deprecation Warning to include returning a copy when selecting muultiple fields of a structured array. --- numpy/core/src/multiarray/arrayobject.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'numpy') diff --git a/numpy/core/src/multiarray/arrayobject.c b/numpy/core/src/multiarray/arrayobject.c index e8bc6b7b6..4a219e70e 100644 --- a/numpy/core/src/multiarray/arrayobject.c +++ b/numpy/core/src/multiarray/arrayobject.c @@ -694,9 +694,10 @@ array_might_be_written(PyArrayObject *obj) { const char *msg = "Numpy has detected that you (may be) writing to an array returned\n" - "by numpy.diagonal. This code will likely break in the next numpy\n" - "release -- see numpy.diagonal docs for details. The quick fix is\n" - "to make an explicit copy (e.g., do arr.diagonal().copy())."; + "by numpy.diagonal or by selecting multiple fields in a structured\n" + "array. This code will likely break in the next numpy release --\n" + "see numpy.diagonal or structured array docs for details. The quick\n" + "fix is to make an explicit copy (e.g., do arr.diagonal().copy())."; if (PyArray_FLAGS(obj) & NPY_ARRAY_WARN_ON_WRITE) { if (DEPRECATE_FUTUREWARNING(msg) < 0) { return -1; -- cgit v1.2.1 From a32325d1d5fc5e23622db5027a6dd35d42193e95 Mon Sep 17 00:00:00 2001 From: jayvius Date: Thu, 12 Jul 2012 13:39:19 -0500 Subject: Call view object's copy method --- numpy/core/_internal.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'numpy') diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py index 88a41fd0d..92ab0c8b0 100644 --- a/numpy/core/_internal.py +++ b/numpy/core/_internal.py @@ -286,7 +286,6 @@ def _newnames(datatype, order): # construct a new array with just those fields copied over def _index_fields(ary, fields): from multiarray import empty, dtype - from numpy import copy dt = ary.dtype names = [name for name in fields if name in dt.names] @@ -296,7 +295,7 @@ def _index_fields(ary, fields): view_dtype = {'names':names, 'formats':formats, 'offsets':offsets, 'itemsize':dt.itemsize} view = ary.view(dtype=view_dtype) - return copy(view) + return view.copy() # Given a string containing a PEP 3118 format specifier, # construct a Numpy dtype -- cgit v1.2.1 From 781468b5938ce1a79804613b222063d81e99963d Mon Sep 17 00:00:00 2001 From: Jay Bourque Date: Tue, 17 Jul 2012 11:59:56 -0500 Subject: Updated reference docs for DeprecationWarning --- numpy/core/src/multiarray/arrayobject.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'numpy') diff --git a/numpy/core/src/multiarray/arrayobject.c b/numpy/core/src/multiarray/arrayobject.c index 4a219e70e..f0e9e36a5 100644 --- a/numpy/core/src/multiarray/arrayobject.c +++ b/numpy/core/src/multiarray/arrayobject.c @@ -694,10 +694,11 @@ array_might_be_written(PyArrayObject *obj) { const char *msg = "Numpy has detected that you (may be) writing to an array returned\n" - "by numpy.diagonal or by selecting multiple fields in a structured\n" + "by numpy.diagonal or by selecting multiple fields in a record\n" "array. This code will likely break in the next numpy release --\n" - "see numpy.diagonal or structured array docs for details. The quick\n" - "fix is to make an explicit copy (e.g., do arr.diagonal().copy())."; + "see numpy.diagonal or arrays.indexing reference docs for details.\n" + "The quick fix is to make an explicit copy (e.g., do\n" + "arr.diagonal().copy() or arr[['f0','f1']].copy())."; if (PyArray_FLAGS(obj) & NPY_ARRAY_WARN_ON_WRITE) { if (DEPRECATE_FUTUREWARNING(msg) < 0) { return -1; -- cgit v1.2.1 From 2eb9610acab872538742ce7db5cbbae6cb23360e Mon Sep 17 00:00:00 2001 From: Jay Bourque Date: Tue, 17 Jul 2012 15:42:01 -0500 Subject: Add tests for record array indexing - add tests for record array indexing with multiple field names - add tests for DeprecationWarning when writing to array returned by record array indexing with multiple field names --- numpy/core/tests/test_multiarray.py | 60 +++++++++++++++++++++++++++++++++---- 1 file changed, 54 insertions(+), 6 deletions(-) (limited to 'numpy') diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 3427800a2..0fc238a49 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -1909,7 +1909,8 @@ class TestRecord(TestCase): def test_field_names(self): # Test unicode and 8-bit / byte strings can be used a = np.zeros((1,), dtype=[('f1', 'i4'), - ('f2', [('sf1', 'i4')])]) + ('f2', 'i4'), + ('f3', [('sf1', 'i4')])]) is_py3 = sys.version_info[0] >= 3 if is_py3: funcs = (str,) @@ -1934,12 +1935,18 @@ class TestRecord(TestCase): assert_raises(IndexError, b[0].__setitem__, fnn, 1) assert_raises(IndexError, b[0].__getitem__, fnn) # Subfield - fn2 = func('f2') + fn3 = func('f3') sfn1 = func('sf1') - b[fn2][sfn1] = 1 - assert_equal(b[fn2][sfn1], 1) - assert_raises(ValueError, b[fn2].__setitem__, fnn, 1) - assert_raises(ValueError, b[fn2].__getitem__, fnn) + b[fn3][sfn1] = 1 + assert_equal(b[fn3][sfn1], 1) + assert_raises(ValueError, b[fn3].__setitem__, fnn, 1) + assert_raises(ValueError, b[fn3].__getitem__, fnn) + # multiple Subfields + fn2 = func('f2') + b[fn2] = 3 + assert_equal(b[['f1','f2']][0].tolist(), (2, 3)) + assert_equal(b[['f2','f1']][0].tolist(), (3, 2)) + assert_equal(b[['f1','f3']][0].tolist(), (2, (1,))) # non-ascii unicode field indexing is well behaved if not is_py3: raise SkipTest('non ascii unicode field indexing skipped; ' @@ -1948,6 +1955,47 @@ class TestRecord(TestCase): assert_raises(ValueError, a.__setitem__, u'\u03e0', 1) assert_raises(ValueError, a.__getitem__, u'\u03e0') + def test_field_names_deprecation(self): + import warnings + from numpy.testing.utils import WarningManager + def collect_warning_types(f, *args, **kwargs): + ctx = WarningManager(record=True) + warning_log = ctx.__enter__() + warnings.simplefilter("always") + try: + f(*args, **kwargs) + finally: + ctx.__exit__() + return [w.category for w in warning_log] + a = np.zeros((1,), dtype=[('f1', 'i4'), + ('f2', 'i4'), + ('f3', [('sf1', 'i4')])]) + a['f1'][0] = 1 + a['f2'][0] = 2 + a['f3'][0] = (3,) + b = np.zeros((1,), dtype=[('f1', 'i4'), + ('f2', 'i4'), + ('f3', [('sf1', 'i4')])]) + b['f1'][0] = 1 + b['f2'][0] = 2 + b['f3'][0] = (3,) + + # All the different functions raise a warning, but not an error, and + # 'a' is not modified: + assert_equal(collect_warning_types(a[['f1','f2']].__setitem__, 0, (10,20)), + [DeprecationWarning]) + assert_equal(a, b) + # Views also warn + subset = a[['f1','f2']] + subset_view = subset.view() + assert_equal(collect_warning_types(subset_view['f1'].__setitem__, 0, 10), + [DeprecationWarning]) + # But the write goes through: + assert_equal(subset['f1'][0], 10) + # Only one warning per multiple field indexing, though (even if there are + # multiple views involved): + assert_equal(collect_warning_types(subset['f1'].__setitem__, 0, 10), + []) class TestView(TestCase): def test_basic(self): -- cgit v1.2.1 From a03e8b4d286e91ef5823c059dcfb7a52ce420725 Mon Sep 17 00:00:00 2001 From: Jay Bourque Date: Tue, 17 Jul 2012 16:03:41 -0500 Subject: change DeprecationWarning to FutureWarning --- numpy/core/tests/test_multiarray.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'numpy') diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 0fc238a49..e3e24fae1 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -1983,13 +1983,13 @@ class TestRecord(TestCase): # All the different functions raise a warning, but not an error, and # 'a' is not modified: assert_equal(collect_warning_types(a[['f1','f2']].__setitem__, 0, (10,20)), - [DeprecationWarning]) + [FutureWarning]) assert_equal(a, b) # Views also warn subset = a[['f1','f2']] subset_view = subset.view() assert_equal(collect_warning_types(subset_view['f1'].__setitem__, 0, 10), - [DeprecationWarning]) + [FutureWarning]) # But the write goes through: assert_equal(subset['f1'][0], 10) # Only one warning per multiple field indexing, though (even if there are -- cgit v1.2.1