From f1fba70edd1829c64e3290fa6b1a20d01e9d9674 Mon Sep 17 00:00:00 2001 From: Allan Haldane Date: Tue, 30 Jan 2018 19:15:54 -0500 Subject: ENH: add multi-field assignment helpers in np.lib.recfunctions Adds helper functions for the copy->view transition for multi-field indexes. Adds `structured_to_unstructured`, `apply_along_fields`, `assign_fields_by_name`, `require_fields`. --- numpy/lib/tests/test_recfunctions.py | 53 +++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) (limited to 'numpy/lib/tests/test_recfunctions.py') diff --git a/numpy/lib/tests/test_recfunctions.py b/numpy/lib/tests/test_recfunctions.py index 5585a95f9..8b61ba0bf 100644 --- a/numpy/lib/tests/test_recfunctions.py +++ b/numpy/lib/tests/test_recfunctions.py @@ -10,7 +10,8 @@ from numpy.testing import assert_, assert_raises from numpy.lib.recfunctions import ( drop_fields, rename_fields, get_fieldstructure, recursive_fill_fields, find_duplicates, merge_arrays, append_fields, stack_arrays, join_by, - repack_fields) + repack_fields, unstructured_to_structured, structured_to_unstructured, + apply_along_fields, require_fields, assign_fields_by_name) get_names = np.lib.recfunctions.get_names get_names_flat = np.lib.recfunctions.get_names_flat zip_descr = np.lib.recfunctions.zip_descr @@ -204,6 +205,56 @@ class TestRecFunctions(object): dt = np.dtype((np.record, dt)) assert_(repack_fields(dt).type is np.record) + def test_structured_to_unstructured(self): + a = np.zeros(4, dtype=[('a', 'i4'), ('b', 'f4,u2'), ('c', 'f4', 2)]) + out = structured_to_unstructured(a) + assert_equal(out, np.zeros((4,5), dtype='f8')) + + b = np.array([(1, 2, 5), (4, 5, 7), (7, 8 ,11), (10, 11, 12)], + dtype=[('x', 'i4'), ('y', 'f4'), ('z', 'f8')]) + out = np.mean(structured_to_unstructured(b[['x', 'z']]), axis=-1) + assert_equal(out, np.array([ 3. , 5.5, 9. , 11. ])) + + c = np.arange(20).reshape((4,5)) + out = unstructured_to_structured(c, a.dtype) + want = np.array([( 0, ( 1., 2), [ 3., 4.]), + ( 5, ( 6., 7), [ 8., 9.]), + (10, (11., 12), [13., 14.]), + (15, (16., 17), [18., 19.])], + dtype=[('a', ' Date: Wed, 31 Oct 2018 17:23:27 -0400 Subject: ENH: Fixups to multi-field assignment helpers --- numpy/lib/tests/test_recfunctions.py | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'numpy/lib/tests/test_recfunctions.py') diff --git a/numpy/lib/tests/test_recfunctions.py b/numpy/lib/tests/test_recfunctions.py index 8b61ba0bf..7ec33d92a 100644 --- a/numpy/lib/tests/test_recfunctions.py +++ b/numpy/lib/tests/test_recfunctions.py @@ -233,6 +233,14 @@ class TestRecFunctions(object): assert_equal(apply_along_fields(np.mean, d[['x', 'z']]), np.array([ 3. , 5.5, 9. , 11. ])) + # check that for uniform field dtypes we get a view, not a copy: + d = np.array([(1, 2, 5), (4, 5, 7), (7, 8 ,11), (10, 11, 12)], + dtype=[('x', 'i4'), ('y', 'i4'), ('z', 'i4')]) + dd = structured_to_unstructured(d) + ddd = unstructured_to_structured(dd, d.dtype) + assert_(dd.base is d) + assert_(ddd.base is d) + def test_field_assignment_by_name(self): a = np.ones(2, dtype=[('a', 'i4'), ('b', 'f8'), ('c', 'u1')]) newdt = [('b', 'f4'), ('c', 'u1')] -- cgit v1.2.1 From 191d5c78383771e9a4825801062d0f23625410bf Mon Sep 17 00:00:00 2001 From: Allan Haldane Date: Sat, 24 Nov 2018 17:55:55 -0500 Subject: MAINT: Fixups to new functions in np.lib.recfunctions --- numpy/lib/tests/test_recfunctions.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'numpy/lib/tests/test_recfunctions.py') diff --git a/numpy/lib/tests/test_recfunctions.py b/numpy/lib/tests/test_recfunctions.py index 7ec33d92a..11f8a5afa 100644 --- a/numpy/lib/tests/test_recfunctions.py +++ b/numpy/lib/tests/test_recfunctions.py @@ -241,6 +241,14 @@ class TestRecFunctions(object): assert_(dd.base is d) assert_(ddd.base is d) + # test that nested fields with identical names don't break anything + point = np.dtype([('x', int), ('y', int)]) + triangle = np.dtype([('a', point), ('b', point), ('c', point)]) + arr = np.zeros(10, triangle) + res = structured_to_unstructured(arr, dtype=int) + assert_equal(res, np.zeros((10, 6), dtype=int)) + + def test_field_assignment_by_name(self): a = np.ones(2, dtype=[('a', 'i4'), ('b', 'f8'), ('c', 'u1')]) newdt = [('b', 'f4'), ('c', 'u1')] @@ -263,6 +271,11 @@ class TestRecFunctions(object): assign_fields_by_name(a, b) assert_equal(a, np.array([((0,2),), ((0,3),)], dtype=a.dtype)) + # test unstructured code path for 0d arrays + a, b = np.array(3), np.array(0) + assign_fields_by_name(b, a) + assert_equal(b[()], 3) + class TestRecursiveFillFields(object): # Test recursive_fill_fields. -- cgit v1.2.1 From ae6ac2e085635e383a38cc43e85edec39aa6e8d3 Mon Sep 17 00:00:00 2001 From: Allan Haldane Date: Sat, 5 Jan 2019 18:18:01 +0000 Subject: TST: Fix endianness in unstuctured_to_structured test --- numpy/lib/tests/test_recfunctions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'numpy/lib/tests/test_recfunctions.py') diff --git a/numpy/lib/tests/test_recfunctions.py b/numpy/lib/tests/test_recfunctions.py index 11f8a5afa..069693613 100644 --- a/numpy/lib/tests/test_recfunctions.py +++ b/numpy/lib/tests/test_recfunctions.py @@ -221,9 +221,9 @@ class TestRecFunctions(object): ( 5, ( 6., 7), [ 8., 9.]), (10, (11., 12), [13., 14.]), (15, (16., 17), [18., 19.])], - dtype=[('a', ' Date: Sun, 14 Apr 2019 19:26:53 -0700 Subject: BUG: Fix structured_to_unstructured on single-field types Previously a single-field type would decay, which is undesirable. The included test previously did not pass --- numpy/lib/tests/test_recfunctions.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'numpy/lib/tests/test_recfunctions.py') diff --git a/numpy/lib/tests/test_recfunctions.py b/numpy/lib/tests/test_recfunctions.py index 11f8a5afa..d1fcf2153 100644 --- a/numpy/lib/tests/test_recfunctions.py +++ b/numpy/lib/tests/test_recfunctions.py @@ -214,6 +214,8 @@ class TestRecFunctions(object): dtype=[('x', 'i4'), ('y', 'f4'), ('z', 'f8')]) out = np.mean(structured_to_unstructured(b[['x', 'z']]), axis=-1) assert_equal(out, np.array([ 3. , 5.5, 9. , 11. ])) + out = np.mean(structured_to_unstructured(b[['x']]), axis=-1) + assert_equal(out, np.array([ 1. , 4. , 7. , 10. ])) c = np.arange(20).reshape((4,5)) out = unstructured_to_structured(c, a.dtype) -- cgit v1.2.1 From e6a7e05753f8a5f56de9653bfecd85a22f53d7c6 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Sun, 14 Apr 2019 15:49:15 -0700 Subject: BUG: Always return views from structured_to_unstructured when possible Also applies to unstructured_to_structured While producing correct resutls, the test added in this commit would previously make an unecessary copy, causing the assertion to fail. The cause was `astype` was being asked to convert from a subarray of shape `(x, y)` to one of `(x*y,)`, which it cannot do without making a copy. This changes the approach used to skip the step of flattening subarrays to 1d --- numpy/lib/tests/test_recfunctions.py | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'numpy/lib/tests/test_recfunctions.py') diff --git a/numpy/lib/tests/test_recfunctions.py b/numpy/lib/tests/test_recfunctions.py index d1fcf2153..d4d317d30 100644 --- a/numpy/lib/tests/test_recfunctions.py +++ b/numpy/lib/tests/test_recfunctions.py @@ -243,6 +243,15 @@ class TestRecFunctions(object): assert_(dd.base is d) assert_(ddd.base is d) + # including uniform fields with subarrays unpacked + d = np.array([(1, [2, 3], [[ 4, 5], [ 6, 7]]), + (8, [9, 10], [[11, 12], [13, 14]])], + dtype=[('x0', 'i4'), ('x1', ('i4', 2)), ('x2', ('i4', (2, 2)))]) + dd = structured_to_unstructured(d) + ddd = unstructured_to_structured(dd, d.dtype) + assert_(dd.base is d) + assert_(ddd.base is d) + # test that nested fields with identical names don't break anything point = np.dtype([('x', int), ('y', int)]) triangle = np.dtype([('a', point), ('b', point), ('c', point)]) -- cgit v1.2.1 From f16062640e52bcc07cf005f1f26c6f3dab39def4 Mon Sep 17 00:00:00 2001 From: Colin Snyder <8csnyder@gmail.com> Date: Sun, 14 Jul 2019 22:43:13 -0700 Subject: exported correct functions and made private the rest --- numpy/lib/tests/test_recfunctions.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'numpy/lib/tests/test_recfunctions.py') diff --git a/numpy/lib/tests/test_recfunctions.py b/numpy/lib/tests/test_recfunctions.py index f713fb64d..cb357941c 100644 --- a/numpy/lib/tests/test_recfunctions.py +++ b/numpy/lib/tests/test_recfunctions.py @@ -10,11 +10,13 @@ from numpy.testing import assert_, assert_raises from numpy.lib.recfunctions import ( drop_fields, rename_fields, get_fieldstructure, recursive_fill_fields, find_duplicates, merge_arrays, append_fields, stack_arrays, join_by, - repack_fields, unstructured_to_structured, structured_to_unstructured, + repack_fields, _unstructured_to_structured, structured_to_unstructured, apply_along_fields, require_fields, assign_fields_by_name) +get_fieldspec = np.lib.recfunctions._get_fieldspec get_names = np.lib.recfunctions.get_names get_names_flat = np.lib.recfunctions.get_names_flat -zip_descr = np.lib.recfunctions.zip_descr +zip_descr = np.lib.recfunctions._zip_descr +zip_dtype = np.lib.recfunctions._zip_dtype class TestRecFunctions(object): @@ -218,7 +220,7 @@ class TestRecFunctions(object): assert_equal(out, np.array([ 1. , 4. , 7. , 10. ])) c = np.arange(20).reshape((4,5)) - out = unstructured_to_structured(c, a.dtype) + out = _unstructured_to_structured(c, a.dtype) want = np.array([( 0, ( 1., 2), [ 3., 4.]), ( 5, ( 6., 7), [ 8., 9.]), (10, (11., 12), [13., 14.]), @@ -239,7 +241,7 @@ class TestRecFunctions(object): d = np.array([(1, 2, 5), (4, 5, 7), (7, 8 ,11), (10, 11, 12)], dtype=[('x', 'i4'), ('y', 'i4'), ('z', 'i4')]) dd = structured_to_unstructured(d) - ddd = unstructured_to_structured(dd, d.dtype) + ddd = _unstructured_to_structured(dd, d.dtype) assert_(dd.base is d) assert_(ddd.base is d) @@ -248,7 +250,7 @@ class TestRecFunctions(object): (8, [9, 10], [[11, 12], [13, 14]])], dtype=[('x0', 'i4'), ('x1', ('i4', 2)), ('x2', ('i4', (2, 2)))]) dd = structured_to_unstructured(d) - ddd = unstructured_to_structured(dd, d.dtype) + ddd = _unstructured_to_structured(dd, d.dtype) assert_(dd.base is d) assert_(ddd.base is d) -- cgit v1.2.1 From 82bc01143c74f997c0129075bb6f41490bf95bc0 Mon Sep 17 00:00:00 2001 From: Colin Snyder <8csnyder@gmail.com> Date: Sat, 20 Jul 2019 12:28:48 -0700 Subject: fixed unstructured_to_structured in recfunctions --- numpy/lib/tests/test_recfunctions.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'numpy/lib/tests/test_recfunctions.py') diff --git a/numpy/lib/tests/test_recfunctions.py b/numpy/lib/tests/test_recfunctions.py index cb357941c..0126ccaf8 100644 --- a/numpy/lib/tests/test_recfunctions.py +++ b/numpy/lib/tests/test_recfunctions.py @@ -10,7 +10,7 @@ from numpy.testing import assert_, assert_raises from numpy.lib.recfunctions import ( drop_fields, rename_fields, get_fieldstructure, recursive_fill_fields, find_duplicates, merge_arrays, append_fields, stack_arrays, join_by, - repack_fields, _unstructured_to_structured, structured_to_unstructured, + repack_fields, unstructured_to_structured, structured_to_unstructured, apply_along_fields, require_fields, assign_fields_by_name) get_fieldspec = np.lib.recfunctions._get_fieldspec get_names = np.lib.recfunctions.get_names @@ -220,7 +220,7 @@ class TestRecFunctions(object): assert_equal(out, np.array([ 1. , 4. , 7. , 10. ])) c = np.arange(20).reshape((4,5)) - out = _unstructured_to_structured(c, a.dtype) + out = unstructured_to_structured(c, a.dtype) want = np.array([( 0, ( 1., 2), [ 3., 4.]), ( 5, ( 6., 7), [ 8., 9.]), (10, (11., 12), [13., 14.]), @@ -241,7 +241,7 @@ class TestRecFunctions(object): d = np.array([(1, 2, 5), (4, 5, 7), (7, 8 ,11), (10, 11, 12)], dtype=[('x', 'i4'), ('y', 'i4'), ('z', 'i4')]) dd = structured_to_unstructured(d) - ddd = _unstructured_to_structured(dd, d.dtype) + ddd = unstructured_to_structured(dd, d.dtype) assert_(dd.base is d) assert_(ddd.base is d) @@ -250,7 +250,7 @@ class TestRecFunctions(object): (8, [9, 10], [[11, 12], [13, 14]])], dtype=[('x0', 'i4'), ('x1', ('i4', 2)), ('x2', ('i4', (2, 2)))]) dd = structured_to_unstructured(d) - ddd = _unstructured_to_structured(dd, d.dtype) + ddd = unstructured_to_structured(dd, d.dtype) assert_(dd.base is d) assert_(ddd.base is d) -- cgit v1.2.1 From b7a9378b113da2cb27f40c4f485ffeb225f5d01d Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Fri, 16 Aug 2019 18:44:10 -0700 Subject: BUG: recfunctions: Don't return None in place of empty sequences Replacing empty tuples with `None` is a bad idea, and just results in an API that is hard to consume - especially since the behavior was never documented. This affects `get_names`, `get_names_flat`, and `get_fieldstructure`. --- numpy/lib/tests/test_recfunctions.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) (limited to 'numpy/lib/tests/test_recfunctions.py') diff --git a/numpy/lib/tests/test_recfunctions.py b/numpy/lib/tests/test_recfunctions.py index 11f8a5afa..3b972f4b1 100644 --- a/numpy/lib/tests/test_recfunctions.py +++ b/numpy/lib/tests/test_recfunctions.py @@ -113,6 +113,14 @@ class TestRecFunctions(object): test = get_names(ndtype) assert_equal(test, ('a', ('b', ('ba', 'bb')))) + ndtype = np.dtype([('a', int), ('b', [])]) + test = get_names(ndtype) + assert_equal(test, ('a', ('b', ()))) + + ndtype = np.dtype([]) + test = get_names(ndtype) + assert_equal(test, ()) + def test_get_names_flat(self): # Test get_names_flat ndtype = np.dtype([('A', '|S3'), ('B', float)]) @@ -123,6 +131,14 @@ class TestRecFunctions(object): test = get_names_flat(ndtype) assert_equal(test, ('a', 'b', 'ba', 'bb')) + ndtype = np.dtype([('a', int), ('b', [])]) + test = get_names_flat(ndtype) + assert_equal(test, ('a', 'b')) + + ndtype = np.dtype([]) + test = get_names_flat(ndtype) + assert_equal(test, ()) + def test_get_fieldstructure(self): # Test get_fieldstructure @@ -145,6 +161,11 @@ class TestRecFunctions(object): 'BBA': ['B', 'BB'], 'BBB': ['B', 'BB']} assert_equal(test, control) + # 0 fields + ndtype = np.dtype([]) + test = get_fieldstructure(ndtype) + assert_equal(test, {}) + def test_find_duplicates(self): # Test find_duplicates a = ma.array([(2, (2., 'B')), (1, (2., 'B')), (2, (2., 'B')), -- cgit v1.2.1 From 63ecfb884d23a3edcddee55b2bc64582cf8db757 Mon Sep 17 00:00:00 2001 From: Allan Haldane Date: Tue, 20 Aug 2019 16:42:04 -0400 Subject: MAINT: fix behavior of structured_to_unstructured on non-trivial dtypes Fixes #13333 --- numpy/lib/tests/test_recfunctions.py | 37 +++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) (limited to 'numpy/lib/tests/test_recfunctions.py') diff --git a/numpy/lib/tests/test_recfunctions.py b/numpy/lib/tests/test_recfunctions.py index 0126ccaf8..dc4afe077 100644 --- a/numpy/lib/tests/test_recfunctions.py +++ b/numpy/lib/tests/test_recfunctions.py @@ -248,7 +248,8 @@ class TestRecFunctions(object): # including uniform fields with subarrays unpacked d = np.array([(1, [2, 3], [[ 4, 5], [ 6, 7]]), (8, [9, 10], [[11, 12], [13, 14]])], - dtype=[('x0', 'i4'), ('x1', ('i4', 2)), ('x2', ('i4', (2, 2)))]) + dtype=[('x0', 'i4'), ('x1', ('i4', 2)), + ('x2', ('i4', (2, 2)))]) dd = structured_to_unstructured(d) ddd = unstructured_to_structured(dd, d.dtype) assert_(dd.base is d) @@ -262,6 +263,40 @@ class TestRecFunctions(object): assert_equal(res, np.zeros((10, 6), dtype=int)) + # test nested combinations of subarrays and structured arrays, gh-13333 + def subarray(dt, shape): + return np.dtype((dt, shape)) + + def structured(*dts): + return np.dtype([('x{}'.format(i), dt) for i, dt in enumerate(dts)]) + + def inspect(dt, dtype=None): + arr = np.zeros((), dt) + ret = structured_to_unstructured(arr, dtype=dtype) + backarr = unstructured_to_structured(ret, dt) + return ret.shape, ret.dtype, backarr.dtype + + dt = structured(subarray(structured(np.int32, np.int32), 3)) + assert_equal(inspect(dt), ((6,), np.int32, dt)) + + dt = structured(subarray(subarray(np.int32, 2), 2)) + assert_equal(inspect(dt), ((4,), np.int32, dt)) + + dt = structured(np.int32) + assert_equal(inspect(dt), ((1,), np.int32, dt)) + + dt = structured(np.int32, subarray(subarray(np.int32, 2), 2)) + assert_equal(inspect(dt), ((5,), np.int32, dt)) + + dt = structured() + assert_raises(ValueError, structured_to_unstructured, np.zeros(3, dt)) + + # these currently don't work, but we may make it work in the future + assert_raises(NotImplementedError, structured_to_unstructured, + np.zeros(3, dt), dtype=np.int32) + assert_raises(NotImplementedError, unstructured_to_structured, + np.zeros((3,0), dtype=np.int32)) + def test_field_assignment_by_name(self): a = np.ones(2, dtype=[('a', 'i4'), ('b', 'f8'), ('c', 'u1')]) newdt = [('b', 'f4'), ('c', 'u1')] -- cgit v1.2.1 From 49916f66479d7a6b14087f39e06290941e07596d Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Sun, 18 Aug 2019 21:09:32 -0500 Subject: API: Do not return None from recfunctions.drop_fields This return value was not documented and did not make any sense. There's no reason to have a special case here. --- numpy/lib/tests/test_recfunctions.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'numpy/lib/tests/test_recfunctions.py') diff --git a/numpy/lib/tests/test_recfunctions.py b/numpy/lib/tests/test_recfunctions.py index 0c839d486..f9355c2e1 100644 --- a/numpy/lib/tests/test_recfunctions.py +++ b/numpy/lib/tests/test_recfunctions.py @@ -91,8 +91,10 @@ class TestRecFunctions(object): control = np.array([(1,), (4,)], dtype=[('a', int)]) assert_equal(test, control) + # dropping all fields results in an array with no fields test = drop_fields(a, ['a', 'b']) - assert_(test is None) + control = np.array([(), ()], dtype=[]) + assert_equal(test, control) def test_rename_fields(self): # Test rename fields -- cgit v1.2.1 From 003bdc25b75566de64a90e223338b4bd4565155a Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Sun, 18 Aug 2019 20:59:42 -0500 Subject: BUG: Fix flatten_dtype so that nested 0-field structs are flattened correctly This affects the behavior of merge_arrays. --- numpy/lib/tests/test_recfunctions.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'numpy/lib/tests/test_recfunctions.py') diff --git a/numpy/lib/tests/test_recfunctions.py b/numpy/lib/tests/test_recfunctions.py index 0c839d486..38147a85e 100644 --- a/numpy/lib/tests/test_recfunctions.py +++ b/numpy/lib/tests/test_recfunctions.py @@ -378,8 +378,8 @@ class TestMergeArrays(object): z = np.array( [('A', 1.), ('B', 2.)], dtype=[('A', '|S3'), ('B', float)]) w = np.array( - [(1, (2, 3.0)), (4, (5, 6.0))], - dtype=[('a', int), ('b', [('ba', float), ('bb', int)])]) + [(1, (2, 3.0, ())), (4, (5, 6.0, ()))], + dtype=[('a', int), ('b', [('ba', float), ('bb', int), ('bc', [])])]) self.data = (w, x, y, z) def test_solo(self): @@ -450,8 +450,8 @@ class TestMergeArrays(object): test = merge_arrays((x, w), flatten=False) controldtype = [('f0', int), ('f1', [('a', int), - ('b', [('ba', float), ('bb', int)])])] - control = np.array([(1., (1, (2, 3.0))), (2, (4, (5, 6.0)))], + ('b', [('ba', float), ('bb', int), ('bc', [])])])] + control = np.array([(1., (1, (2, 3.0, ()))), (2, (4, (5, 6.0, ())))], dtype=controldtype) assert_equal(test, control) -- cgit v1.2.1