diff options
43 files changed, 619 insertions, 381 deletions
diff --git a/.lgtm.yml b/.lgtm.yml new file mode 100644 index 000000000..c0a9cf59a --- /dev/null +++ b/.lgtm.yml @@ -0,0 +1,9 @@ +path_classifiers: + library: + - tools + generated: + # The exports defined in __init__.py are defined in the Cython module + # np.random.mtrand. By excluding this file we suppress a number of + # "undefined export" alerts + - numpy/random/__init__.py + diff --git a/benchmarks/benchmarks/bench_io.py b/benchmarks/benchmarks/bench_io.py index 782d4ab30..ce30c4345 100644 --- a/benchmarks/benchmarks/bench_io.py +++ b/benchmarks/benchmarks/bench_io.py @@ -3,6 +3,7 @@ from __future__ import absolute_import, division, print_function from .common import Benchmark, get_squares import numpy as np +from io import StringIO class Copy(Benchmark): @@ -62,3 +63,179 @@ class Savez(Benchmark): def time_vb_savez_squares(self): np.savez('tmp.npz', self.squares) + +class LoadtxtCSVComments(Benchmark): + # benchmarks for np.loadtxt comment handling + # when reading in CSV files + + params = [10, int(1e2), int(1e4), int(1e5)] + param_names = ['num_lines'] + + def setup(self, num_lines): + data = [u'1,2,3 # comment'] * num_lines + # unfortunately, timeit will only run setup() + # between repeat events, but not for iterations + # within repeats, so the StringIO object + # will have to be rewinded in the benchmark proper + self.data_comments = StringIO(u'\n'.join(data)) + + def time_comment_loadtxt_csv(self, num_lines): + # benchmark handling of lines with comments + # when loading in from csv files + + # inspired by similar benchmark in pandas + # for read_csv + + # need to rewind StringIO object (unfortunately + # confounding timing result somewhat) for every + # call to timing test proper + np.loadtxt(self.data_comments, + delimiter=u',') + self.data_comments.seek(0) + +class LoadtxtCSVdtypes(Benchmark): + # benchmarks for np.loadtxt operating with + # different dtypes parsed / cast from CSV files + + params = (['float32', 'float64', 'int32', 'int64', + 'complex128', 'str', 'object'], + [10, int(1e2), int(1e4), int(1e5)]) + param_names = ['dtype', 'num_lines'] + + def setup(self, dtype, num_lines): + data = [u'5, 7, 888'] * num_lines + self.csv_data = StringIO(u'\n'.join(data)) + + def time_loadtxt_dtypes_csv(self, dtype, num_lines): + # benchmark loading arrays of various dtypes + # from csv files + + # state-dependent timing benchmark requires + # rewind of StringIO object + + np.loadtxt(self.csv_data, + delimiter=u',', + dtype=dtype) + self.csv_data.seek(0) + +class LoadtxtCSVStructured(Benchmark): + # benchmarks for np.loadtxt operating with + # a structured data type & CSV file + + def setup(self): + num_lines = 50000 + data = [u"M, 21, 72, X, 155"] * num_lines + self.csv_data = StringIO(u'\n'.join(data)) + + def time_loadtxt_csv_struct_dtype(self): + # obligate rewind of StringIO object + # between iterations of a repeat: + + np.loadtxt(self.csv_data, + delimiter=u',', + dtype=[('category_1', 'S1'), + ('category_2', 'i4'), + ('category_3', 'f8'), + ('category_4', 'S1'), + ('category_5', 'f8')]) + self.csv_data.seek(0) + + +class LoadtxtCSVSkipRows(Benchmark): + # benchmarks for loadtxt row skipping when + # reading in csv file data; a similar benchmark + # is present in the pandas asv suite + + params = [0, 500, 10000] + param_names = ['skiprows'] + + def setup(self, skiprows): + np.random.seed(123) + test_array = np.random.rand(100000, 3) + self.fname = 'test_array.csv' + np.savetxt(fname=self.fname, + X=test_array, + delimiter=',') + + def time_skiprows_csv(self, skiprows): + np.loadtxt(self.fname, + delimiter=',', + skiprows=skiprows) + +class LoadtxtReadUint64Integers(Benchmark): + # pandas has a similar CSV reading benchmark + # modified to suit np.loadtxt + + params = [550, 1000, 10000] + param_names = ['size'] + + def setup(self, size): + arr = np.arange(size).astype('uint64') + 2**63 + self.data1 = StringIO(u'\n'.join(arr.astype(str).tolist())) + arr = arr.astype(object) + arr[500] = -1 + self.data2 = StringIO(u'\n'.join(arr.astype(str).tolist())) + + def time_read_uint64(self, size): + # mandatory rewind of StringIO object + # between iterations of a repeat: + np.loadtxt(self.data1) + self.data1.seek(0) + + def time_read_uint64_neg_values(self, size): + # mandatory rewind of StringIO object + # between iterations of a repeat: + np.loadtxt(self.data2) + self.data2.seek(0) + +class LoadtxtUseColsCSV(Benchmark): + # benchmark selective column reading from CSV files + # using np.loadtxt + + params = [2, [1, 3], [1, 3, 5, 7]] + param_names = ['usecols'] + + def setup(self, usecols): + num_lines = 5000 + data = [u'0, 1, 2, 3, 4, 5, 6, 7, 8, 9'] * num_lines + self.csv_data = StringIO(u'\n'.join(data)) + + def time_loadtxt_usecols_csv(self, usecols): + # must rewind StringIO because of state + # dependence of file reading + np.loadtxt(self.csv_data, + delimiter=u',', + usecols=usecols) + self.csv_data.seek(0) + +class LoadtxtCSVDateTime(Benchmark): + # benchmarks for np.loadtxt operating with + # datetime data in a CSV file + + params = [20, 200, 2000, 20000] + param_names = ['num_lines'] + + def setup(self, num_lines): + # create the equivalent of a two-column CSV file + # with date strings in the first column and random + # floating point data in the second column + dates = np.arange('today', 20, dtype=np.datetime64) + np.random.seed(123) + values = np.random.rand(20) + date_line = u'' + + for date, value in zip(dates, values): + date_line += (str(date) + ',' + str(value) + '\n') + + # expand data to specified number of lines + data = date_line * (num_lines // 20) + self.csv_data = StringIO(data) + + def time_loadtxt_csv_datetime(self, num_lines): + # rewind StringIO object -- the timing iterations + # are state-dependent + X = np.loadtxt(self.csv_data, + delimiter=u',', + dtype=([('dates', 'M8[us]'), + ('values', 'float64')])) + self.csv_data.seek(0) diff --git a/doc/release/1.13.0-notes.rst b/doc/release/1.13.0-notes.rst index 4554e53ea..3b719db09 100644 --- a/doc/release/1.13.0-notes.rst +++ b/doc/release/1.13.0-notes.rst @@ -183,11 +183,11 @@ override the behavior of NumPy's ufuncs. This works quite similarly to Python's ``__mul__`` and other binary operation routines. See the documentation for a more detailed description of the implementation and behavior of this new option. The API is provisional, we do not yet guarantee backward compatibility -as modifications may be made pending feedback. See the NEP_ and +as modifications may be made pending feedback. See `NEP 13`_ and documentation_ for more details. -.. _NEP: https://github.com/numpy/numpy/blob/master/doc/neps/ufunc-overrides.rst -.. _documentation: https://github.com/charris/numpy/blob/master/doc/source/reference/arrays.classes.rst +.. _`NEP 13`: http://www.numpy.org/neps/nep-0013-ufunc-overrides.html +.. _documentation: https://github.com/numpy/numpy/blob/master/doc/source/reference/arrays.classes.rst New ``positive`` ufunc ---------------------- diff --git a/doc/release/1.13.1-notes.rst b/doc/release/1.13.1-notes.rst index 807296a85..88a4bc3dd 100644 --- a/doc/release/1.13.1-notes.rst +++ b/doc/release/1.13.1-notes.rst @@ -13,7 +13,7 @@ used with 3.6.0 due to Python bug 29943_. NumPy 1.13.2 will be released shortly after Python 3.6.2 is out to fix that problem. If you are using 3.6.0 the workaround is to upgrade to 3.6.1 or use an earlier Python version. -.. _#29943: https://bugs.python.org/issue29943 +.. _29943: https://bugs.python.org/issue29943 Pull requests merged @@ -21,7 +21,7 @@ Pull requests merged A total of 19 pull requests were merged for this release. * #9240 DOC: BLD: fix lots of Sphinx warnings/errors. -* #9255 Revert "DEP: Raise TypeError for subtract(bool_, bool_)." +* #9255 Revert "DEP: Raise TypeError for subtract(bool, bool)." * #9261 BUG: don't elide into readonly and updateifcopy temporaries for... * #9262 BUG: fix missing keyword rename for common block in numpy.f2py * #9263 BUG: handle resize of 0d array diff --git a/doc/release/1.14.0-notes.rst b/doc/release/1.14.0-notes.rst index 0f14f7703..462631de6 100644 --- a/doc/release/1.14.0-notes.rst +++ b/doc/release/1.14.0-notes.rst @@ -14,11 +14,11 @@ dropping Python 2.7 support in the runup to 2020. The decision has been made to support 2.7 for all releases made in 2018, with the last release being designated a long term release with support for bug fixes extending through 2019. In 2019 support for 2.7 will be dropped in all new releases. More details -can be found in the relevant NEP_. +can be found in `NEP 12`_. This release supports Python 2.7 and 3.4 - 3.6. -.. _NEP: https://github.com/numpy/numpy/blob/master/doc/neps/dropping-python2.7-proposal.rst +.. _`NEP 12`: http://www.numpy.org/neps/nep-0014-dropping-python2.7-proposal.html Highlights @@ -134,8 +134,8 @@ are marked readonly. In the past, it was possible to get away with:: var_arr = np.asarray(val) val_arr += 1 # now errors, previously changed np.ma.masked.data -``np.ma`` functions producing ``fill_value``s have changed ----------------------------------------------------------- +``np.ma`` functions producing ``fill_value`` s have changed +----------------------------------------------------------- Previously, ``np.ma.default_fill_value`` would return a 0d array, but ``np.ma.minimum_fill_value`` and ``np.ma.maximum_fill_value`` would return a tuple of the fields. Instead, all three methods return a structured ``np.void`` diff --git a/doc/release/1.14.1-notes.rst b/doc/release/1.14.1-notes.rst index 2ed4c3e14..7b95c2e28 100644 --- a/doc/release/1.14.1-notes.rst +++ b/doc/release/1.14.1-notes.rst @@ -67,7 +67,7 @@ A total of 36 pull requests were merged for this release. * `#10431 <https://github.com/numpy/numpy/pull/10431>`__: REL: Add 1.14.1 release notes template * `#10435 <https://github.com/numpy/numpy/pull/10435>`__: MAINT: Use ValueError for duplicate field names in lookup (backport) * `#10534 <https://github.com/numpy/numpy/pull/10534>`__: BUG: Provide a better error message for out-of-order fields -* `#10536 <https://github.com/numpy/numpy/pull/10536>`__: BUG: Resize bytes_ columns in genfromtxt (backport of #10401) +* `#10536 <https://github.com/numpy/numpy/pull/10536>`__: BUG: Resize bytes columns in genfromtxt (backport of #10401) * `#10537 <https://github.com/numpy/numpy/pull/10537>`__: BUG: multifield-indexing adds padding bytes: revert for 1.14.1 * `#10539 <https://github.com/numpy/numpy/pull/10539>`__: BUG: fix np.save issue with python 2.7.5 * `#10540 <https://github.com/numpy/numpy/pull/10540>`__: BUG: Add missing DECREF in Py2 int() cast diff --git a/doc/release/1.15.0-notes.rst b/doc/release/1.15.0-notes.rst index 0e3d2a525..7235ca915 100644 --- a/doc/release/1.15.0-notes.rst +++ b/doc/release/1.15.0-notes.rst @@ -99,7 +99,7 @@ Deprecations * Users of ``nditer`` should use the nditer object as a context manager anytime one of the iterator operands is writeable, so that numpy can manage writeback semantics, or should call ``it.close()``. A - `RuntimeWarning` may be emitted otherwise in these cases. + `RuntimeWarning` may be emitted otherwise in these cases. * The ``normed`` argument of ``np.histogram``, deprecated long ago in 1.6.0, now emits a ``DeprecationWarning``. @@ -227,13 +227,6 @@ Changes to ``PyArray_GetDTypeTransferFunction`` significant performance hit, consider implementing ``copyswapn`` to reflect the implementation of ``PyArray_GetStridedCopyFn``. See `#10898 <https://github.com/numpy/numpy/pull/10898>`__. -* Functions ``npy_get_floatstatus_barrier`` and ``npy_clear_floatstatus_barrier`` - have been added and should be used in place of the ``npy_get_floatstatus``and - ``npy_clear_status`` functions. Optimizing compilers like GCC 8.1 and Clang - were rearranging the order of operations when the previous functions were - used in the ufunc SIMD functions, resulting in the floatstatus flags being ' - checked before the operation whose status we wanted to check was run. - See `#10339 <https://github.com/numpy/numpy/issues/10370>`__. New Features diff --git a/doc/release/1.16.0-notes.rst b/doc/release/1.16.0-notes.rst index a93aed7d1..d11eed2d2 100644 --- a/doc/release/1.16.0-notes.rst +++ b/doc/release/1.16.0-notes.rst @@ -15,7 +15,7 @@ Deprecations ============ `typeNA` and `sctypeNA` have been deprecated -------------------------------------------- +-------------------------------------------- The type dictionaries `numpy.core.typeNA` and `numpy.core.sctypeNA` were buggy and not documented. They will be removed in the 1.18 release. Use @@ -47,20 +47,21 @@ Improvements ============ `np.polynomial.Polynomial` classes render in LaTeX in Jupyter notebooks -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +----------------------------------------------------------------------- + When used in a front-end that supports it, `Polynomial` instances are now rendered through LaTeX. The current format is experimental, and is subject to change. ``randint`` and ``choice`` now work on empty distributions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +---------------------------------------------------------- Even when no elements needed to be drawn, ``np.random.randint`` and ``np.random.choice`` raised an error when the arguments described an empty distribution. This has been fixed so that e.g. ``np.random.choice([], 0) == np.array([], dtype=float64)``. ``linalg.lstsq`` and ``linalg.qr`` now work with empty matrices -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +--------------------------------------------------------------- Previously, a ``LinAlgError`` would be raised when an empty matrix/empty matrices (with zero rows and/or columns) is/are passed in. Now outputs of appropriate shapes are returned. @@ -81,7 +82,7 @@ behavior will be appending. This applied to: `LDFLAGS`, `F77FLAGS`, details. ``np.clip`` and the ``clip`` method check for memory overlap -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------------------------------ The ``out`` argument to these functions is now always tested for memory overlap to avoid corrupted results when memory overlap occurs. diff --git a/doc/source/reference/c-api.ufunc.rst b/doc/source/reference/c-api.ufunc.rst index 8c2554a9e..07c7b0c80 100644 --- a/doc/source/reference/c-api.ufunc.rst +++ b/doc/source/reference/c-api.ufunc.rst @@ -85,12 +85,61 @@ Functions Must to an array of length *ntypes* containing :c:type:`PyUFuncGenericFunction` items. These items are pointers to functions that actually implement the underlying - (element-by-element) function :math:`N` times. + (element-by-element) function :math:`N` times with the following + signature: + + .. c:function:: void loopfunc( + char** args, npy_intp* dimensions, npy_intp* steps, void* data) + + *args* + + An array of pointers to the actual data for the input and output + arrays. The input arguments are given first followed by the output + arguments. + + *dimensions* + + A pointer to the size of the dimension over which this function is + looping. + + *steps* + + A pointer to the number of bytes to jump to get to the + next element in this dimension for each of the input and + output arguments. + + *data* + + Arbitrary data (extra arguments, function names, *etc.* ) + that can be stored with the ufunc and will be passed in + when it is called. + + This is an example of a func specialized for addition of doubles + returning doubles. + + .. code-block:: c + + static void + double_add(char **args, npy_intp *dimensions, npy_intp *steps, + void *extra) + { + npy_intp i; + npy_intp is1 = steps[0], is2 = steps[1]; + npy_intp os = steps[2], n = dimensions[0]; + char *i1 = args[0], *i2 = args[1], *op = args[2]; + for (i = 0; i < n; i++) { + *((double *)op) = *((double *)i1) + + *((double *)i2); + i1 += is1; + i2 += is2; + op += os; + } + } :param data: Should be ``NULL`` or a pointer to an array of size *ntypes* . This array may contain arbitrary extra-data to be passed to - the corresponding 1-d loop function in the func array. + the corresponding loop function in the func array. :param types: Length ``(nin + nout) * ntypes`` array of ``char`` encoding the @@ -102,6 +151,9 @@ Functions be ``(char[]) {5, 5, 0, 7, 7, 0}`` since ``NPY_INT32`` is 5, ``NPY_INT64`` is 7, and ``NPY_BOOL`` is 0. + The bit-width names can also be used (e.g. :c:data:`NPY_INT32`, + :c:data:`NPY_COMPLEX128` ) if desired. + :ref:`ufuncs.casting` will be used at runtime to find the first ``func`` callable by the input/output provided. @@ -114,14 +166,19 @@ Functions :param nout: The number of outputs + :param identity: + + Either :c:data:`PyUFunc_One`, :c:data:`PyUFunc_Zero`, + :c:data:`PyUFunc_None`. This specifies what should be returned when + an empty array is passed to the reduce method of the ufunc. + :param name: - The name for the ufunc. Specifying a name of 'add' or - 'multiply' enables a special behavior for integer-typed - reductions when no dtype is given. If the input type is an - integer (or boolean) data type smaller than the size of the - `numpy.int_` data type, it will be internally upcast to the - `numpy.int_` (or `numpy.uint`) - data type. + The name for the ufunc as a ``NULL`` terminated string. Specifying + a name of 'add' or 'multiply' enables a special behavior for + integer-typed reductions when no dtype is given. If the input type is an + integer (or boolean) data type smaller than the size of the `numpy.int_` + data type, it will be internally upcast to the `numpy.int_` (or + `numpy.uint`) data type. :param doc: Allows passing in a documentation string to be stored with the diff --git a/doc/source/user/c-info.beyond-basics.rst b/doc/source/user/c-info.beyond-basics.rst index aee68f6e7..d4d941a5e 100644 --- a/doc/source/user/c-info.beyond-basics.rst +++ b/doc/source/user/c-info.beyond-basics.rst @@ -358,38 +358,6 @@ previously created. Then you call :c:func:`PyUFunc_RegisterLoopForType` this function is ``0`` if the process was successful and ``-1`` with an error condition set if it was not successful. -.. c:function:: int PyUFunc_RegisterLoopForType( \ - PyUFuncObject* ufunc, int usertype, PyUFuncGenericFunction function, \ - int* arg_types, void* data) - - *ufunc* - - The ufunc to attach this loop to. - - *usertype* - - The user-defined type this loop should be indexed under. This number - must be a user-defined type or an error occurs. - - *function* - - The ufunc inner 1-d loop. This function must have the signature as - explained in Section `3 <#sec-creating-a-new>`__ . - - *arg_types* - - (optional) If given, this should contain an array of integers of at - least size ufunc.nargs containing the data-types expected by the loop - function. The data will be copied into a NumPy-managed structure so - the memory for this argument should be deleted after calling this - function. If this is NULL, then it will be assumed that all data-types - are of type usertype. - - *data* - - (optional) Specify any optional data needed by the function which will - be passed when the function is called. - .. index:: pair: dtype; adding new diff --git a/doc/source/user/c-info.ufunc-tutorial.rst b/doc/source/user/c-info.ufunc-tutorial.rst index 788a3429f..96a73f9a6 100644 --- a/doc/source/user/c-info.ufunc-tutorial.rst +++ b/doc/source/user/c-info.ufunc-tutorial.rst @@ -893,9 +893,9 @@ Example NumPy ufunc with structured array dtype arguments This example shows how to create a ufunc for a structured array dtype. For the example we show a trivial ufunc for adding two arrays with dtype 'u8,u8,u8'. The process is a bit different from the other examples since -a call to PyUFunc_FromFuncAndData doesn't fully register ufuncs for +a call to :c:func:`PyUFunc_FromFuncAndData` doesn't fully register ufuncs for custom dtypes and structured array dtypes. We need to also call -PyUFunc_RegisterLoopForDescr to finish setting up the ufunc. +:c:func:`PyUFunc_RegisterLoopForDescr` to finish setting up the ufunc. We only give the C code as the setup.py file is exactly the same as the setup.py file in `Example NumPy ufunc for one dtype`_, except that @@ -1048,133 +1048,6 @@ The C file is given below. #endif } - -.. _`sec:PyUFunc-spec`: - -PyUFunc_FromFuncAndData Specification -===================================== - -What follows is the full specification of PyUFunc_FromFuncAndData, which -automatically generates a ufunc from a C function with the correct signature. - -.. seealso:: :c:func:`PyUFunc_FromFuncAndDataAndSignature` - -.. c:function:: PyObject *PyUFunc_FromFuncAndData( \ - PyUFuncGenericFunction* func, void** data, char* types, int ntypes, \ - int nin, int nout, int identity, char* name, char* doc, int unused) - - *func* - - A pointer to an array of 1-d functions to use. This array must be at - least ntypes long. Each entry in the array must be a - ``PyUFuncGenericFunction`` function. This function has the following - signature. An example of a valid 1d loop function is also given. - - .. c:function:: void loop1d( \ - char** args, npy_intp* dimensions, npy_intp* steps, void* data) - - *args* - - An array of pointers to the actual data for the input and output - arrays. The input arguments are given first followed by the output - arguments. - - *dimensions* - - A pointer to the size of the dimension over which this function is - looping. - - *steps* - - A pointer to the number of bytes to jump to get to the - next element in this dimension for each of the input and - output arguments. - - *data* - - Arbitrary data (extra arguments, function names, *etc.* ) - that can be stored with the ufunc and will be passed in - when it is called. - - .. code-block:: c - - static void - double_add(char **args, npy_intp *dimensions, npy_intp *steps, - void *extra) - { - npy_intp i; - npy_intp is1 = steps[0], is2 = steps[1]; - npy_intp os = steps[2], n = dimensions[0]; - char *i1 = args[0], *i2 = args[1], *op = args[2]; - for (i = 0; i < n; i++) { - *((double *)op) = *((double *)i1) + - *((double *)i2); - i1 += is1; - i2 += is2; - op += os; - } - } - - *data* - - An array of data. There should be ntypes entries (or NULL) --- one for - every loop function defined for this ufunc. This data will be passed - in to the 1-d loop. One common use of this data variable is to pass in - an actual function to call to compute the result when a generic 1-d - loop (e.g. :c:func:`PyUFunc_d_d`) is being used. - - *types* - - An array of type-number signatures (type ``char`` ). This - array should be of size (nin+nout)*ntypes and contain the - data-types for the corresponding 1-d loop. The inputs should - be first followed by the outputs. For example, suppose I have - a ufunc that supports 1 integer and 1 double 1-d loop - (length-2 func and data arrays) that takes 2 inputs and - returns 1 output that is always a complex double, then the - types array would be - - .. code-block:: c - - static char types[3] = {NPY_INT, NPY_DOUBLE, NPY_CDOUBLE} - - The bit-width names can also be used (e.g. :c:data:`NPY_INT32`, - :c:data:`NPY_COMPLEX128` ) if desired. - - *ntypes* - - The number of data-types supported. This is equal to the number of 1-d - loops provided. - - *nin* - - The number of input arguments. - - *nout* - - The number of output arguments. - - *identity* - - Either :c:data:`PyUFunc_One`, :c:data:`PyUFunc_Zero`, - :c:data:`PyUFunc_None`. This specifies what should be returned when - an empty array is passed to the reduce method of the ufunc. - - *name* - - A ``NULL`` -terminated string providing the name of this ufunc - (should be the Python name it will be called). - - *doc* - - A documentation string for this ufunc (will be used in generating the - response to ``{ufunc_name}.__doc__``). Do not include the function - signature or the name as this is generated automatically. - - *unused* - - Unused; kept for compatibility. Just set it to zero. - .. index:: pair: ufunc; adding new diff --git a/numpy/core/_add_newdocs.py b/numpy/core/_add_newdocs.py index b65920fde..9ebd12cbd 100644 --- a/numpy/core/_add_newdocs.py +++ b/numpy/core/_add_newdocs.py @@ -1454,11 +1454,10 @@ add_newdoc('numpy.core.multiarray', 'arange', Values are generated within the half-open interval ``[start, stop)`` (in other words, the interval including `start` but excluding `stop`). For integer arguments the function is equivalent to the Python built-in - `range <https://docs.python.org/library/functions.html#func-range>`_ function, - but returns an ndarray rather than a list. + `range` function, but returns an ndarray rather than a list. When using a non-integer step, such as 0.1, the results will often not - be consistent. It is better to use ``linspace`` for these cases. + be consistent. It is better to use `numpy.linspace` for these cases. Parameters ---------- @@ -2843,40 +2842,19 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('ctypes', ----- Below are the public attributes of this object which were documented in "Guide to NumPy" (we have omitted undocumented public attributes, - as well as documented private attributes): - - * data: A pointer to the memory area of the array as a Python integer. - This memory area may contain data that is not aligned, or not in correct - byte-order. The memory area may not even be writeable. The array - flags and data-type of this array should be respected when passing this - attribute to arbitrary C-code to avoid trouble that can include Python - crashing. User Beware! The value of this attribute is exactly the same - as self._array_interface_['data'][0]. - - * shape (c_intp*self.ndim): A ctypes array of length self.ndim where - the basetype is the C-integer corresponding to dtype('p') on this - platform. This base-type could be c_int, c_long, or c_longlong - depending on the platform. The c_intp type is defined accordingly in - numpy.ctypeslib. The ctypes array contains the shape of the underlying - array. - - * strides (c_intp*self.ndim): A ctypes array of length self.ndim where - the basetype is the same as for the shape attribute. This ctypes array - contains the strides information from the underlying array. This strides - information is important for showing how many bytes must be jumped to - get to the next element in the array. - - * data_as(obj): Return the data pointer cast to a particular c-types object. - For example, calling self._as_parameter_ is equivalent to - self.data_as(ctypes.c_void_p). Perhaps you want to use the data as a - pointer to a ctypes array of floating-point data: - self.data_as(ctypes.POINTER(ctypes.c_double)). - - * shape_as(obj): Return the shape tuple as an array of some other c-types - type. For example: self.shape_as(ctypes.c_short). - - * strides_as(obj): Return the strides tuple as an array of some other - c-types type. For example: self.strides_as(ctypes.c_longlong). + as well as documented private attributes): + + .. autoattribute:: numpy.core._internal._ctypes.data + + .. autoattribute:: numpy.core._internal._ctypes.shape + + .. autoattribute:: numpy.core._internal._ctypes.strides + + .. automethod:: numpy.core._internal._ctypes.data_as + + .. automethod:: numpy.core._internal._ctypes.shape_as + + .. automethod:: numpy.core._internal._ctypes.strides_as Be careful using the ctypes attribute - especially on temporary arrays or arrays constructed on the fly. For example, calling @@ -7158,10 +7136,10 @@ add_newdoc('numpy.core.multiarray', 'datetime_data', array(250, dtype='timedelta64[s]') The result can be used to construct a datetime that uses the same units - as a timedelta:: + as a timedelta >>> np.datetime64('2010', np.datetime_data(dt_25s)) - numpy.datetime64('2010-01-01T00:00:00','25s') + numpy.datetime64('2010-01-01T00:00:00', '25s') """) diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py index 1cf89aab0..48ede14d0 100644 --- a/numpy/core/_internal.py +++ b/numpy/core/_internal.py @@ -9,7 +9,7 @@ from __future__ import division, absolute_import, print_function import re import sys -from numpy.compat import basestring +from numpy.compat import basestring, unicode from .multiarray import dtype, array, ndarray try: import ctypes @@ -257,33 +257,72 @@ class _ctypes(object): self._zerod = False def data_as(self, obj): + """ + Return the data pointer cast to a particular c-types object. + For example, calling ``self._as_parameter_`` is equivalent to + ``self.data_as(ctypes.c_void_p)``. Perhaps you want to use the data as a + pointer to a ctypes array of floating-point data: + ``self.data_as(ctypes.POINTER(ctypes.c_double))``. + """ return self._ctypes.cast(self._data, obj) def shape_as(self, obj): + """ + Return the shape tuple as an array of some other c-types + type. For example: ``self.shape_as(ctypes.c_short)``. + """ if self._zerod: return None return (obj*self._arr.ndim)(*self._arr.shape) def strides_as(self, obj): + """ + Return the strides tuple as an array of some other + c-types type. For example: ``self.strides_as(ctypes.c_longlong)``. + """ if self._zerod: return None return (obj*self._arr.ndim)(*self._arr.strides) def get_data(self): + """ + A pointer to the memory area of the array as a Python integer. + This memory area may contain data that is not aligned, or not in correct + byte-order. The memory area may not even be writeable. The array + flags and data-type of this array should be respected when passing this + attribute to arbitrary C-code to avoid trouble that can include Python + crashing. User Beware! The value of this attribute is exactly the same + as ``self._array_interface_['data'][0]``. + """ return self._data def get_shape(self): + """ + (c_intp*self.ndim): A ctypes array of length self.ndim where + the basetype is the C-integer corresponding to ``dtype('p')`` on this + platform. This base-type could be `ctypes.c_int`, `ctypes.c_long`, or + `ctypes.c_longlong` depending on the platform. + The c_intp type is defined accordingly in `numpy.ctypeslib`. + The ctypes array contains the shape of the underlying array. + """ return self.shape_as(_getintp_ctype()) def get_strides(self): + """ + (c_intp*self.ndim): A ctypes array of length self.ndim where + the basetype is the same as for the shape attribute. This ctypes array + contains the strides information from the underlying array. This strides + information is important for showing how many bytes must be jumped to + get to the next element in the array. + """ return self.strides_as(_getintp_ctype()) def get_as_parameter(self): return self._ctypes.c_void_p(self._data) - data = property(get_data, None, doc="c-types data") - shape = property(get_shape, None, doc="c-types shape") - strides = property(get_strides, None, doc="c-types strides") + data = property(get_data) + shape = property(get_shape) + strides = property(get_strides) _as_parameter_ = property(get_as_parameter, None, doc="_as parameter_") @@ -294,7 +333,7 @@ def _newnames(datatype, order): """ oldnames = datatype.names nameslist = list(oldnames) - if isinstance(order, str): + if isinstance(order, (str, unicode)): order = [order] seen = set() if isinstance(order, (list, tuple)): diff --git a/numpy/core/records.py b/numpy/core/records.py index 612d39322..a483871ba 100644 --- a/numpy/core/records.py +++ b/numpy/core/records.py @@ -42,7 +42,7 @@ import warnings from . import numeric as sb from . import numerictypes as nt -from numpy.compat import isfileobj, bytes, long +from numpy.compat import isfileobj, bytes, long, unicode from .arrayprint import get_printoptions # All of the functions allow formats to be a dtype @@ -174,7 +174,7 @@ class format_parser(object): if (names): if (type(names) in [list, tuple]): pass - elif isinstance(names, str): + elif isinstance(names, (str, unicode)): names = names.split(',') else: raise NameError("illegal input names %s" % repr(names)) diff --git a/numpy/core/src/multiarray/buffer.c b/numpy/core/src/multiarray/buffer.c index 0325f3c6a..c8e3da8bc 100644 --- a/numpy/core/src/multiarray/buffer.c +++ b/numpy/core/src/multiarray/buffer.c @@ -175,6 +175,14 @@ _is_natively_aligned_at(PyArray_Descr *descr, return 1; } +/* + * Fill in str with an appropriate PEP 3118 format string, based on + * descr. For structured dtypes, calls itself recursively. Each call extends + * str at offset then updates offset, and uses descr->byteorder, (and + * possibly the byte order in obj) to determine the byte-order char. + * + * Returns 0 for success, -1 for failure + */ static int _buffer_format_string(PyArray_Descr *descr, _tmp_string_t *str, PyObject* obj, Py_ssize_t *offset, @@ -195,8 +203,8 @@ _buffer_format_string(PyArray_Descr *descr, _tmp_string_t *str, PyObject *item, *subarray_tuple; Py_ssize_t total_count = 1; Py_ssize_t dim_size; + Py_ssize_t old_offset; char buf[128]; - int old_offset; int ret; if (PyTuple_Check(descr->subarray->shape)) { @@ -230,15 +238,15 @@ _buffer_format_string(PyArray_Descr *descr, _tmp_string_t *str, return ret; } else if (PyDataType_HASFIELDS(descr)) { - int base_offset = *offset; + Py_ssize_t base_offset = *offset; _append_str(str, "T{"); for (k = 0; k < PyTuple_GET_SIZE(descr->names); ++k) { PyObject *name, *item, *offset_obj, *tmp; PyArray_Descr *child; char *p; - Py_ssize_t len; - int new_offset; + Py_ssize_t len, new_offset; + int ret; name = PyTuple_GET_ITEM(descr->names, k); item = PyDict_GetItem(descr->fields, name); @@ -266,8 +274,11 @@ _buffer_format_string(PyArray_Descr *descr, _tmp_string_t *str, } /* Insert child item */ - _buffer_format_string(child, str, obj, offset, + ret = _buffer_format_string(child, str, obj, offset, active_byteorder); + if (ret < 0) { + return -1; + } /* Insert field name */ #if defined(NPY_PY3K) @@ -393,8 +404,8 @@ _buffer_format_string(PyArray_Descr *descr, _tmp_string_t *str, case NPY_CFLOAT: if (_append_str(str, "Zf")) return -1; break; case NPY_CDOUBLE: if (_append_str(str, "Zd")) return -1; break; case NPY_CLONGDOUBLE: if (_append_str(str, "Zg")) return -1; break; - /* XXX: datetime */ - /* XXX: timedelta */ + /* XXX NPY_DATETIME */ + /* XXX NPY_TIMEDELTA */ case NPY_OBJECT: if (_append_char(str, 'O')) return -1; break; case NPY_STRING: { char buf[128]; @@ -472,7 +483,29 @@ _buffer_info_new(PyObject *obj) goto fail; } - if (PyArray_IsScalar(obj, Generic)) { + if (PyArray_IsScalar(obj, Datetime) || PyArray_IsScalar(obj, Timedelta)) { + /* + * Special case datetime64 scalars to remain backward compatible. + * This will change in a future version. + * Note arrays of datetime64 and strutured arrays with datetime64 + * fields will not hit this code path and are currently unsupported + * in _buffer_format_string. + */ + _append_char(&fmt, 'B'); + _append_char(&fmt, '\0'); + info->ndim = 1; + info->shape = malloc(sizeof(Py_ssize_t) * 2); + if (info->shape == NULL) { + PyErr_NoMemory(); + goto fail; + } + info->strides = info->shape + info->ndim; + info->shape[0] = 8; + info->strides[0] = 1; + info->format = fmt.s; + return info; + } + else if (PyArray_IsScalar(obj, Generic)) { descr = PyArray_DescrFromScalar(obj); if (descr == NULL) { goto fail; @@ -798,8 +831,6 @@ gentype_getbuffer(PyObject *self, Py_buffer *view, int flags) /* Fill in information */ info = _buffer_get_info(self); if (info == NULL) { - PyErr_SetString(PyExc_BufferError, - "could not get scalar buffer information"); goto fail; } @@ -822,6 +853,9 @@ gentype_getbuffer(PyObject *self, Py_buffer *view, int flags) } #endif view->len = elsize; + if (PyArray_IsScalar(self, Datetime) || PyArray_IsScalar(self, Timedelta)) { + elsize = 1; /* descr->elsize,char is 8,'M', but we return 1,'B' */ + } view->itemsize = elsize; Py_DECREF(descr); diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index a2cf17f4e..f1b8a0209 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1391,10 +1391,12 @@ _array_from_buffer_3118(PyObject *memoryview) if (!is_ctypes) { /* This object has no excuse for a broken PEP3118 buffer */ - PyErr_SetString( + PyErr_Format( PyExc_RuntimeError, - "Item size computed from the PEP 3118 buffer format " - "string does not match the actual item size."); + "Item size %zd for PEP 3118 buffer format " + "string %s does not match the dtype %c item size %d.", + view->itemsize, view->format, descr->type, + descr->elsize); Py_DECREF(descr); return NULL; } diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index 942554cae..8e058d5fb 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -620,6 +620,10 @@ class TestDateTime(object): assert_equal(pickle.loads(pickle.dumps(dt)), dt) dt = np.dtype('M8[W]') assert_equal(pickle.loads(pickle.dumps(dt)), dt) + scalar = np.datetime64('2016-01-01T00:00:00.000000000') + assert_equal(pickle.loads(pickle.dumps(scalar)), scalar) + delta = scalar - np.datetime64('2015-01-01T00:00:00.000000000') + assert_equal(pickle.loads(pickle.dumps(delta)), delta) # Check that loading pickles from 1.6 works pkl = b"cnumpy\ndtype\np0\n(S'M8'\np1\nI0\nI1\ntp2\nRp3\n" + \ @@ -1698,7 +1702,6 @@ class TestDateTime(object): assert_equal(np.busday_offset(np.datetime64('NaT'), 1, roll='preceding'), np.datetime64('NaT')) - def test_datetime_busdaycalendar(self): # Check that it removes NaT, duplicates, and weekends # and sorts the result. diff --git a/numpy/core/tests/test_einsum.py b/numpy/core/tests/test_einsum.py index 8ce374a75..6b5b9c06e 100644 --- a/numpy/core/tests/test_einsum.py +++ b/numpy/core/tests/test_einsum.py @@ -965,7 +965,6 @@ class TestEinsumPath(object): path, path_str = np.einsum_path(*edge_test4, optimize='optimal') self.assert_path_equal(path, ['einsum_path', (0, 1), (0, 1, 2, 3, 4, 5)]) - def test_path_type_input(self): # Test explicit path handeling path_test = self.build_operands('dcc,fce,ea,dbf->ab') diff --git a/numpy/core/tests/test_indexing.py b/numpy/core/tests/test_indexing.py index 276cd9f93..1934d542a 100644 --- a/numpy/core/tests/test_indexing.py +++ b/numpy/core/tests/test_indexing.py @@ -194,7 +194,6 @@ class TestIndexing(object): assert_raises(IndexError, arr.__getitem__, (slice(None), index)) - def test_boolean_indexing_onedim(self): # Indexing a 2-dimensional array with # boolean array of length one diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 1511f5b6b..1c59abaa7 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -108,7 +108,6 @@ class TestFlags(object): assert_equal(self.a.flags['X'], False) assert_equal(self.a.flags['WRITEBACKIFCOPY'], False) - def test_string_align(self): a = np.zeros(4, dtype=np.dtype('|S4')) assert_(a.flags.aligned) @@ -2729,7 +2728,6 @@ class TestMethods(object): # Order of axis argument doesn't matter: assert_equal(b.diagonal(0, 2, 1), [[0, 3], [4, 7]]) - def test_diagonal_view_notwriteable(self): # this test is only for 1.9, the diagonal view will be # writeable in 1.10. @@ -4386,7 +4384,6 @@ class TestIO(object): d.tofile(f) assert_equal(os.path.getsize(self.filename), d.nbytes * 2) - def test_io_open_buffered_fromfile(self): # gh-6632 self.x.tofile(self.filename) @@ -4748,55 +4745,72 @@ class TestRecord(object): # Error raised when multiple fields have the same name assert_raises(ValueError, test_assign) - if sys.version_info[0] >= 3: - def test_bytes_fields(self): - # Bytes are not allowed in field names and not recognized in titles - # on Py3 - assert_raises(TypeError, np.dtype, [(b'a', int)]) - assert_raises(TypeError, np.dtype, [(('b', b'a'), int)]) - - dt = np.dtype([((b'a', 'b'), int)]) - assert_raises(TypeError, dt.__getitem__, b'a') - - x = np.array([(1,), (2,), (3,)], dtype=dt) - assert_raises(IndexError, x.__getitem__, b'a') - - y = x[0] - assert_raises(IndexError, y.__getitem__, b'a') - - def test_multiple_field_name_unicode(self): - def test_assign_unicode(): - dt = np.dtype([("\u20B9", "f8"), - ("B", "f8"), - ("\u20B9", "f8")]) - - # Error raised when multiple fields have the same name(unicode included) - assert_raises(ValueError, test_assign_unicode) - - else: - def test_unicode_field_titles(self): - # Unicode field titles are added to field dict on Py2 - title = u'b' - dt = np.dtype([((title, 'a'), int)]) - dt[title] - dt['a'] - x = np.array([(1,), (2,), (3,)], dtype=dt) - x[title] - x['a'] - y = x[0] - y[title] - y['a'] - - def test_unicode_field_names(self): - # Unicode field names are converted to ascii on Python 2: - encodable_name = u'b' - assert_equal(np.dtype([(encodable_name, int)]).names[0], b'b') - assert_equal(np.dtype([(('a', encodable_name), int)]).names[0], b'b') - - # But raises UnicodeEncodeError if it can't be encoded: - nonencodable_name = u'\uc3bc' - assert_raises(UnicodeEncodeError, np.dtype, [(nonencodable_name, int)]) - assert_raises(UnicodeEncodeError, np.dtype, [(('a', nonencodable_name), int)]) + @pytest.mark.skipif(sys.version_info[0] < 3, reason="Not Python 3") + def test_bytes_fields(self): + # Bytes are not allowed in field names and not recognized in titles + # on Py3 + assert_raises(TypeError, np.dtype, [(b'a', int)]) + assert_raises(TypeError, np.dtype, [(('b', b'a'), int)]) + + dt = np.dtype([((b'a', 'b'), int)]) + assert_raises(TypeError, dt.__getitem__, b'a') + + x = np.array([(1,), (2,), (3,)], dtype=dt) + assert_raises(IndexError, x.__getitem__, b'a') + + y = x[0] + assert_raises(IndexError, y.__getitem__, b'a') + + @pytest.mark.skipif(sys.version_info[0] < 3, reason="Not Python 3") + def test_multiple_field_name_unicode(self): + def test_assign_unicode(): + dt = np.dtype([("\u20B9", "f8"), + ("B", "f8"), + ("\u20B9", "f8")]) + + # Error raised when multiple fields have the same name(unicode included) + assert_raises(ValueError, test_assign_unicode) + + @pytest.mark.skipif(sys.version_info[0] >= 3, reason="Not Python 2") + def test_unicode_field_titles(self): + # Unicode field titles are added to field dict on Py2 + title = u'b' + dt = np.dtype([((title, 'a'), int)]) + dt[title] + dt['a'] + x = np.array([(1,), (2,), (3,)], dtype=dt) + x[title] + x['a'] + y = x[0] + y[title] + y['a'] + + @pytest.mark.skipif(sys.version_info[0] >= 3, reason="Not Python 2") + def test_unicode_field_names(self): + # Unicode field names are converted to ascii on Python 2: + encodable_name = u'b' + assert_equal(np.dtype([(encodable_name, int)]).names[0], b'b') + assert_equal(np.dtype([(('a', encodable_name), int)]).names[0], b'b') + + # But raises UnicodeEncodeError if it can't be encoded: + nonencodable_name = u'\uc3bc' + assert_raises(UnicodeEncodeError, np.dtype, [(nonencodable_name, int)]) + assert_raises(UnicodeEncodeError, np.dtype, [(('a', nonencodable_name), int)]) + + def test_fromarrays_unicode(self): + # A single name string provided to fromarrays() is allowed to be unicode + # on both Python 2 and 3: + x = np.core.records.fromarrays([[0], [1]], names=u'a,b', formats=u'i4,i4') + assert_equal(x['a'][0], 0) + assert_equal(x['b'][0], 1) + + def test_unicode_order(self): + # Test that we can sort with order as a unicode field name in both Python 2 and + # 3: + name = u'b' + x = np.array([1, 3, 2], dtype=[(name, int)]) + x.sort(order=name) + assert_equal(x[u'b'], np.array([1, 2, 3])) def test_field_names(self): # Test unicode and 8-bit / byte strings can be used @@ -4909,7 +4923,6 @@ class TestRecord(object): assert_equal(collect_warnings(c[['f0', 'f2']].view, 'i8,i8'), [FutureWarning]) - def test_record_hash(self): a = np.array([(1, 2), (1, 2)], dtype='i1,i2') a.flags.writeable = False @@ -6470,6 +6483,14 @@ class TestNewBufferProtocol(object): # Issue #4015. self._check_roundtrip(0) + def test_invalid_buffer_format(self): + # datetime64 cannot be used fully in a buffer yet + # Should be fixed in the next Numpy major release + dt = np.dtype([('a', 'uint16'), ('b', 'M8[s]')]) + a = np.empty(3, dt) + assert_raises((ValueError, BufferError), memoryview, a) + assert_raises((ValueError, BufferError), memoryview, np.array((3), 'M8[D]')) + def test_export_simple_1d(self): x = np.array([1, 2, 3, 4, 5], dtype='i') y = memoryview(x) diff --git a/numpy/core/tests/test_nditer.py b/numpy/core/tests/test_nditer.py index 13bc6b34a..5e8165bc5 100644 --- a/numpy/core/tests/test_nditer.py +++ b/numpy/core/tests/test_nditer.py @@ -2358,7 +2358,6 @@ class TestIterNested(object): j.close() assert_equal(a, [[1, 2, 3], [4, 5, 6]]) - def test_dtype_buffered(self): # Test nested iteration with buffering to change dtype diff --git a/numpy/core/tests/test_numeric.py b/numpy/core/tests/test_numeric.py index a1a92ef32..e7181736f 100644 --- a/numpy/core/tests/test_numeric.py +++ b/numpy/core/tests/test_numeric.py @@ -1275,7 +1275,6 @@ class TestArrayComparisons(object): assert_equal(a == None, [False, False, False]) assert_equal(a != None, [True, True, True]) - def test_array_equiv(self): res = np.array_equiv(np.array([1, 2]), np.array([1, 2])) assert_(res) diff --git a/numpy/core/tests/test_regression.py b/numpy/core/tests/test_regression.py index 36478ddb7..8244dfe20 100644 --- a/numpy/core/tests/test_regression.py +++ b/numpy/core/tests/test_regression.py @@ -1828,7 +1828,6 @@ class TestRegression(object): assert_equal(oct(a), oct(0)) assert_equal(hex(a), hex(0)) - def test_object_array_self_copy(self): # An object array being copied into itself DECREF'ed before INCREF'ing # causing segmentation faults (gh-3787) diff --git a/numpy/core/tests/test_scalarbuffer.py b/numpy/core/tests/test_scalarbuffer.py index 6d57a5014..cb6c521e1 100644 --- a/numpy/core/tests/test_scalarbuffer.py +++ b/numpy/core/tests/test_scalarbuffer.py @@ -5,7 +5,7 @@ import sys import numpy as np import pytest -from numpy.testing import assert_, assert_equal +from numpy.testing import assert_, assert_equal, assert_raises # PEP3118 format strings for native (standard alignment and byteorder) types scalars_and_codes = [ @@ -77,3 +77,28 @@ class TestScalarPEP3118(object): mv_a = memoryview(a) assert_equal(mv_x.itemsize, mv_a.itemsize) assert_equal(mv_x.format, mv_a.format) + + def test_datetime_memoryview(self): + # gh-11656 + # Values verified with v1.13.3, shape is not () as in test_scalar_dim + def as_dict(m): + return dict(strides=m.strides, shape=m.shape, itemsize=m.itemsize, + ndim=m.ndim, format=m.format) + + dt1 = np.datetime64('2016-01-01') + dt2 = np.datetime64('2017-01-01') + expected = {'strides': (1,), 'itemsize': 1, 'ndim': 1, + 'shape': (8,), 'format': 'B'} + v = memoryview(dt1) + res = as_dict(v) + assert_equal(res, expected) + + v = memoryview(dt2 - dt1) + res = as_dict(v) + assert_equal(res, expected) + + dt = np.dtype([('a', 'uint16'), ('b', 'M8[s]')]) + a = np.empty(1, dt) + # Fails to create a PEP 3118 valid buffer + assert_raises((ValueError, BufferError), memoryview, a[0]) + diff --git a/numpy/core/tests/test_umath.py b/numpy/core/tests/test_umath.py index 85c9a4929..d4bdb3d4e 100644 --- a/numpy/core/tests/test_umath.py +++ b/numpy/core/tests/test_umath.py @@ -1173,7 +1173,6 @@ class TestBitwiseUFuncs(object): assert_(np.bitwise_xor(zeros, zeros).dtype == dt, msg) assert_(np.bitwise_and(zeros, zeros).dtype == dt, msg) - def test_identity(self): assert_(np.bitwise_or.identity == 0, 'bitwise_or') assert_(np.bitwise_xor.identity == 0, 'bitwise_xor') diff --git a/numpy/lib/arraysetops.py b/numpy/lib/arraysetops.py index 5880ea154..d84455a8f 100644 --- a/numpy/lib/arraysetops.py +++ b/numpy/lib/arraysetops.py @@ -312,12 +312,12 @@ def intersect1d(ar1, ar2, assume_unique=False, return_indices=False): If True, the input arrays are both assumed to be unique, which can speed up the calculation. Default is False. return_indices : bool - If True, the indices which correspond to the intersection of the - two arrays are returned. The first instance of a value is used - if there are multiple. Default is False. - - .. versionadded:: 1.15.0 - + If True, the indices which correspond to the intersection of the two + arrays are returned. The first instance of a value is used if there are + multiple. Default is False. + + .. versionadded:: 1.15.0 + Returns ------- intersect1d : ndarray @@ -326,7 +326,7 @@ def intersect1d(ar1, ar2, assume_unique=False, return_indices=False): The indices of the first occurrences of the common values in `ar1`. Only provided if `return_indices` is True. comm2 : ndarray - The indices of the first occurrences of the common values in `ar2`. + The indices of the first occurrences of the common values in `ar2`. Only provided if `return_indices` is True. @@ -345,7 +345,7 @@ def intersect1d(ar1, ar2, assume_unique=False, return_indices=False): >>> from functools import reduce >>> reduce(np.intersect1d, ([1, 3, 4, 3], [3, 1, 2, 1], [6, 3, 4, 2])) array([3]) - + To return the indices of the values common to the input arrays along with the intersected values: >>> x = np.array([1, 1, 2, 3, 4]) @@ -355,8 +355,11 @@ def intersect1d(ar1, ar2, assume_unique=False, return_indices=False): (array([0, 2, 4]), array([1, 0, 2])) >>> xy, x[x_ind], y[y_ind] (array([1, 2, 4]), array([1, 2, 4]), array([1, 2, 4])) - + """ + ar1 = np.asanyarray(ar1) + ar2 = np.asanyarray(ar2) + if not assume_unique: if return_indices: ar1, ind1 = unique(ar1, return_index=True) @@ -367,7 +370,7 @@ def intersect1d(ar1, ar2, assume_unique=False, return_indices=False): else: ar1 = ar1.ravel() ar2 = ar2.ravel() - + aux = np.concatenate((ar1, ar2)) if return_indices: aux_sort_indices = np.argsort(aux, kind='mergesort') @@ -389,6 +392,7 @@ def intersect1d(ar1, ar2, assume_unique=False, return_indices=False): else: return int1d + def setxor1d(ar1, ar2, assume_unique=False): """ Find the set exclusive-or of two arrays. diff --git a/numpy/lib/histograms.py b/numpy/lib/histograms.py index 422b356f7..f03f30fb0 100644 --- a/numpy/lib/histograms.py +++ b/numpy/lib/histograms.py @@ -260,6 +260,32 @@ def _get_outer_edges(a, range): return first_edge, last_edge +def _unsigned_subtract(a, b): + """ + Subtract two values where a >= b, and produce an unsigned result + + This is needed when finding the difference between the upper and lower + bound of an int16 histogram + """ + # coerce to a single type + signed_to_unsigned = { + np.byte: np.ubyte, + np.short: np.ushort, + np.intc: np.uintc, + np.int_: np.uint, + np.longlong: np.ulonglong + } + dt = np.result_type(a, b) + try: + dt = signed_to_unsigned[dt.type] + except KeyError: + return np.subtract(a, b, dtype=dt) + else: + # we know the inputs are integers, and we are deliberately casting + # signed to unsigned + return np.subtract(a, b, casting='unsafe', dtype=dt) + + def _get_bin_edges(a, bins, range, weights): """ Computes the bins used internally by `histogram`. @@ -311,7 +337,7 @@ def _get_bin_edges(a, bins, range, weights): # Do not call selectors on empty arrays width = _hist_bin_selectors[bin_name](a) if width: - n_equal_bins = int(np.ceil((last_edge - first_edge) / width)) + n_equal_bins = int(np.ceil(_unsigned_subtract(last_edge, first_edge) / width)) else: # Width can be zero for some estimators, e.g. FD when # the IQR of the data is zero. @@ -703,7 +729,7 @@ def histogram(a, bins=10, range=None, normed=None, weights=None, n = np.zeros(n_equal_bins, ntype) # Pre-compute histogram scaling factor - norm = n_equal_bins / (last_edge - first_edge) + norm = n_equal_bins / _unsigned_subtract(last_edge, first_edge) # We iterate over blocks here for two reasons: the first is that for # large arrays, it is actually faster (for example for a 10^8 array it @@ -731,7 +757,7 @@ def histogram(a, bins=10, range=None, normed=None, weights=None, # Compute the bin indices, and for values that lie exactly on # last_edge we need to subtract one - f_indices = (tmp_a - first_edge) * norm + f_indices = _unsigned_subtract(tmp_a, first_edge) * norm indices = f_indices.astype(np.intp) indices[indices == n_equal_bins] -= 1 diff --git a/numpy/lib/nanfunctions.py b/numpy/lib/nanfunctions.py index abd2da1a2..8d6b0f139 100644 --- a/numpy/lib/nanfunctions.py +++ b/numpy/lib/nanfunctions.py @@ -1178,13 +1178,15 @@ def nanquantile(a, q, axis=None, out=None, overwrite_input=False, This optional parameter specifies the interpolation method to use when the desired quantile lies between two data points ``i < j``: - * linear: ``i + (j - i) * fraction``, where ``fraction`` - is the fractional part of the index surrounded by ``i`` - and ``j``. - * lower: ``i``. - * higher: ``j``. - * nearest: ``i`` or ``j``, whichever is nearest. - * midpoint: ``(i + j) / 2``. + + * linear: ``i + (j - i) * fraction``, where ``fraction`` + is the fractional part of the index surrounded by ``i`` + and ``j``. + * lower: ``i``. + * higher: ``j``. + * nearest: ``i`` or ``j``, whichever is nearest. + * midpoint: ``(i + j) / 2``. + keepdims : bool, optional If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, the diff --git a/numpy/lib/stride_tricks.py b/numpy/lib/stride_tricks.py index bc5993802..ca13738c1 100644 --- a/numpy/lib/stride_tricks.py +++ b/numpy/lib/stride_tricks.py @@ -242,7 +242,7 @@ def broadcast_arrays(*args, **kwargs): subok = kwargs.pop('subok', False) if kwargs: raise TypeError('broadcast_arrays() got an unexpected keyword ' - 'argument {!r}'.format(kwargs.keys()[0])) + 'argument {!r}'.format(list(kwargs.keys())[0])) args = [np.array(_m, copy=False, subok=subok) for _m in args] shape = _broadcast_shape(*args) diff --git a/numpy/lib/tests/test__datasource.py b/numpy/lib/tests/test__datasource.py index 70fff3bb0..85788941c 100644 --- a/numpy/lib/tests/test__datasource.py +++ b/numpy/lib/tests/test__datasource.py @@ -33,14 +33,14 @@ def urlopen_stub(url, data=None): old_urlopen = None -def setup(): +def setup_module(): global old_urlopen old_urlopen = urllib_request.urlopen urllib_request.urlopen = urlopen_stub -def teardown(): +def teardown_module(): urllib_request.urlopen = old_urlopen # A valid website for more robust testing diff --git a/numpy/lib/tests/test_arraysetops.py b/numpy/lib/tests/test_arraysetops.py index dace5ade8..c76afb8e5 100644 --- a/numpy/lib/tests/test_arraysetops.py +++ b/numpy/lib/tests/test_arraysetops.py @@ -30,19 +30,30 @@ class TestSetOps(object): ed = np.array([1, 2, 5]) c = intersect1d(a, b) assert_array_equal(c, ed) - assert_array_equal([], intersect1d([], [])) - + + def test_intersect1d_array_like(self): + # See gh-11772 + class Test(object): + def __array__(self): + return np.arange(3) + + a = Test() + res = intersect1d(a, a) + assert_array_equal(res, a) + res = intersect1d([1, 2, 3], [1, 2, 3]) + assert_array_equal(res, [1, 2, 3]) + def test_intersect1d_indices(self): # unique inputs - a = np.array([1, 2, 3, 4]) + a = np.array([1, 2, 3, 4]) b = np.array([2, 1, 4, 6]) c, i1, i2 = intersect1d(a, b, assume_unique=True, return_indices=True) ee = np.array([1, 2, 4]) assert_array_equal(c, ee) assert_array_equal(a[i1], ee) assert_array_equal(b[i2], ee) - + # non-unique inputs a = np.array([1, 2, 2, 3, 4, 3, 2]) b = np.array([1, 8, 4, 2, 2, 3, 2, 3]) @@ -51,7 +62,7 @@ class TestSetOps(object): assert_array_equal(c, ef) assert_array_equal(a[i1], ef) assert_array_equal(b[i2], ef) - + # non1d, unique inputs a = np.array([[2, 4, 5, 6], [7, 8, 1, 15]]) b = np.array([[3, 2, 7, 6], [10, 12, 8, 9]]) @@ -61,7 +72,7 @@ class TestSetOps(object): ea = np.array([2, 6, 7, 8]) assert_array_equal(ea, a[ui1]) assert_array_equal(ea, b[ui2]) - + # non1d, not assumed to be uniqueinputs a = np.array([[2, 4, 5, 6, 6], [4, 7, 8, 7, 2]]) b = np.array([[3, 2, 7, 7], [10, 12, 8, 7]]) @@ -71,7 +82,7 @@ class TestSetOps(object): ea = np.array([2, 7, 8]) assert_array_equal(ea, a[ui1]) assert_array_equal(ea, b[ui2]) - + def test_setxor1d(self): a = np.array([5, 7, 1, 2]) b = np.array([2, 4, 3, 1, 5]) diff --git a/numpy/lib/tests/test_histograms.py b/numpy/lib/tests/test_histograms.py index f136b5c81..561f5f938 100644 --- a/numpy/lib/tests/test_histograms.py +++ b/numpy/lib/tests/test_histograms.py @@ -310,6 +310,20 @@ class TestHistogram(object): assert_equal(d_edge.dtype, dates.dtype) assert_equal(t_edge.dtype, td) + def do_signed_overflow_bounds(self, dtype): + exponent = 8 * np.dtype(dtype).itemsize - 1 + arr = np.array([-2**exponent + 4, 2**exponent - 4], dtype=dtype) + hist, e = histogram(arr, bins=2) + assert_equal(e, [-2**exponent + 4, 0, 2**exponent - 4]) + assert_equal(hist, [1, 1]) + + def test_signed_overflow_bounds(self): + self.do_signed_overflow_bounds(np.byte) + self.do_signed_overflow_bounds(np.short) + self.do_signed_overflow_bounds(np.intc) + self.do_signed_overflow_bounds(np.int_) + self.do_signed_overflow_bounds(np.longlong) + def do_precision_lower_bound(self, float_small, float_large): eps = np.finfo(float_large).eps diff --git a/numpy/lib/tests/test_index_tricks.py b/numpy/lib/tests/test_index_tricks.py index 315251daa..7e9c026e4 100644 --- a/numpy/lib/tests/test_index_tricks.py +++ b/numpy/lib/tests/test_index_tricks.py @@ -113,7 +113,6 @@ class TestRavelUnravelIndex(object): assert_(x.flags.writeable) assert_(y.flags.writeable) - def test_0d(self): # gh-580 x = np.unravel_index(0, ()) diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py index f58c9e33d..1f3664d92 100644 --- a/numpy/lib/tests/test_io.py +++ b/numpy/lib/tests/test_io.py @@ -348,7 +348,6 @@ class TestSaveTxt(object): assert_raises(ValueError, np.savetxt, c, np.array(1)) assert_raises(ValueError, np.savetxt, c, np.array([[[1], [2]]])) - def test_record(self): a = np.array([(1, 2), (3, 4)], dtype=[('x', 'i4'), ('y', 'i4')]) c = BytesIO() @@ -2025,7 +2024,6 @@ M 33 21.99 assert_equal(test['f0'], 0) assert_equal(test['f1'], "testNonethe" + utf8.decode("UTF-8")) - def test_utf8_file_nodtype_unicode(self): # bytes encoding with non-latin1 -> unicode upcast utf8 = u'\u03d6' diff --git a/numpy/lib/tests/test_stride_tricks.py b/numpy/lib/tests/test_stride_tricks.py index 3c2ca8b87..b2bd7da3e 100644 --- a/numpy/lib/tests/test_stride_tricks.py +++ b/numpy/lib/tests/test_stride_tricks.py @@ -3,7 +3,8 @@ from __future__ import division, absolute_import, print_function import numpy as np from numpy.core._rational_tests import rational from numpy.testing import ( - assert_equal, assert_array_equal, assert_raises, assert_ + assert_equal, assert_array_equal, assert_raises, assert_, + assert_raises_regex ) from numpy.lib.stride_tricks import ( as_strided, broadcast_arrays, _broadcast_shape, broadcast_to @@ -57,6 +58,17 @@ def test_same(): assert_array_equal(x, bx) assert_array_equal(y, by) +def test_broadcast_kwargs(): + # ensure that a TypeError is appropriately raised when + # np.broadcast_arrays() is called with any keyword + # argument other than 'subok' + x = np.arange(10) + y = np.arange(10) + + with assert_raises_regex(TypeError, + r'broadcast_arrays\(\) got an unexpected keyword*'): + broadcast_arrays(x, y, dtype='float64') + def test_one_off(): x = np.array([[1, 2, 3]]) diff --git a/numpy/linalg/tests/test_linalg.py b/numpy/linalg/tests/test_linalg.py index ce4a37b09..98a77d8f5 100644 --- a/numpy/linalg/tests/test_linalg.py +++ b/numpy/linalg/tests/test_linalg.py @@ -959,7 +959,6 @@ class TestMatrixPower(object): #FIXME the 'e' dtype might work in future dtnoinv = [object, np.dtype('e'), np.dtype('g'), np.dtype('G')] - def test_large_power(self, dt): power = matrix_power rshft = self.rshft_1.astype(dt) @@ -1022,7 +1021,6 @@ class TestMatrixPower(object): assert_raises(TypeError, matrix_power, mat, 1.5) assert_raises(TypeError, matrix_power, mat, [1]) - def test_exceptions_non_square(self, dt): assert_raises(LinAlgError, matrix_power, np.array([1], dt), 1) assert_raises(LinAlgError, matrix_power, np.array([[1], [2]], dt), 1) diff --git a/numpy/ma/tests/test_core.py b/numpy/ma/tests/test_core.py index 129809b5d..a08a0d956 100644 --- a/numpy/ma/tests/test_core.py +++ b/numpy/ma/tests/test_core.py @@ -514,8 +514,6 @@ class TestMaskedArray(object): fill_value=999999)''') ) - - def test_str_repr_legacy(self): oldopts = np.get_printoptions() np.set_printoptions(legacy='1.13') @@ -788,7 +786,6 @@ class TestMaskedArray(object): control = "(0, [[--, 0.0, --], [0.0, 0.0, --]], 0.0)" assert_equal(str(t_2d0), control) - def test_flatten_structured_array(self): # Test flatten_structured_array on arrays # On ndarray @@ -3174,18 +3171,13 @@ class TestMaskedArrayMethods(object): assert_equal(test.mask, mask_first.mask) # Test sort on dtype with subarray (gh-8069) + # Just check that the sort does not error, structured array subarrays + # are treated as byte strings and that leads to differing behavior + # depending on endianess and `endwith`. dt = np.dtype([('v', int, 2)]) a = a.view(dt) - mask_last = mask_last.view(dt) - mask_first = mask_first.view(dt) - test = sort(a) - assert_equal(test, mask_last) - assert_equal(test.mask, mask_last.mask) - test = sort(a, endwith=False) - assert_equal(test, mask_first) - assert_equal(test.mask, mask_first.mask) def test_argsort(self): # Test argsort diff --git a/numpy/random/tests/test_random.py b/numpy/random/tests/test_random.py index 2e0885024..8328c69c0 100644 --- a/numpy/random/tests/test_random.py +++ b/numpy/random/tests/test_random.py @@ -1454,7 +1454,6 @@ class TestBroadcast(object): assert_raises(ValueError, zipf, np.nan) assert_raises(ValueError, zipf, [0, 0, np.nan]) - def test_geometric(self): p = [0.5] bad_p_one = [-1] diff --git a/numpy/testing/_private/nosetester.py b/numpy/testing/_private/nosetester.py index c2cf58377..1728d9d1f 100644 --- a/numpy/testing/_private/nosetester.py +++ b/numpy/testing/_private/nosetester.py @@ -338,12 +338,14 @@ class NoseTester(object): Identifies the tests to run. This can be a string to pass to the nosetests executable with the '-A' option, or one of several special values. Special values are: + * 'fast' - the default - which corresponds to the ``nosetests -A`` option of 'not slow'. * 'full' - fast (as above) and slow tests as in the 'no -A' option to nosetests - this is the same as ''. * None or '' - run all tests. - attribute_identifier - string passed directly to nosetests as '-A'. + * attribute_identifier - string passed directly to nosetests as '-A'. + verbose : int, optional Verbosity value for test outputs, in the range 1-10. Default is 1. extra_argv : list, optional @@ -352,16 +354,14 @@ class NoseTester(object): If True, run doctests in module. Default is False. coverage : bool, optional If True, report coverage of NumPy code. Default is False. - (This requires the `coverage module: - <http://nedbatchelder.com/code/modules/coverage.html>`_). + (This requires the + `coverage module <https://nedbatchelder.com/code/modules/coveragehtml>`_). raise_warnings : None, str or sequence of warnings, optional This specifies which warnings to configure as 'raise' instead - of being shown once during the test execution. Valid strings are: - - - "develop" : equals ``(Warning,)`` - - "release" : equals ``()``, don't raise on any warnings. + of being shown once during the test execution. Valid strings are: - The default is to use the class initialization value. + * "develop" : equals ``(Warning,)`` + * "release" : equals ``()``, do not raise on any warnings. timer : bool or int, optional Timing of individual tests with ``nose-timer`` (which needs to be installed). If True, time tests and report on all of them. @@ -489,12 +489,14 @@ class NoseTester(object): Identifies the benchmarks to run. This can be a string to pass to the nosetests executable with the '-A' option, or one of several special values. Special values are: + * 'fast' - the default - which corresponds to the ``nosetests -A`` option of 'not slow'. * 'full' - fast (as above) and slow benchmarks as in the 'no -A' option to nosetests - this is the same as ''. * None or '' - run all tests. - attribute_identifier - string passed directly to nosetests as '-A'. + * attribute_identifier - string passed directly to nosetests as '-A'. + verbose : int, optional Verbosity value for benchmark outputs, in the range 1-10. Default is 1. extra_argv : list, optional diff --git a/numpy/testing/_private/utils.py b/numpy/testing/_private/utils.py index 0e2f8ba91..a3832fcde 100644 --- a/numpy/testing/_private/utils.py +++ b/numpy/testing/_private/utils.py @@ -687,6 +687,8 @@ def assert_array_compare(comparison, x, y, err_msg='', verbose=True, equal_inf=True): __tracebackhide__ = True # Hide traceback for py.test from numpy.core import array, isnan, inf, bool_ + from numpy.core.fromnumeric import all as npall + x = array(x, copy=False, subok=True) y = array(y, copy=False, subok=True) @@ -697,14 +699,21 @@ def assert_array_compare(comparison, x, y, err_msg='', verbose=True, return x.dtype.char in "Mm" def func_assert_same_pos(x, y, func=isnan, hasval='nan'): - """Handling nan/inf: combine results of running func on x and y, - checking that they are True at the same locations.""" - # Both the != True comparison here and the cast to bool_ at - # the end are done to deal with `masked`, which cannot be - # compared usefully, and for which .all() yields masked. + """Handling nan/inf. + + Combine results of running func on x and y, checking that they are True + at the same locations. + + """ + # Both the != True comparison here and the cast to bool_ at the end are + # done to deal with `masked`, which cannot be compared usefully, and + # for which np.all yields masked. The use of the function np.all is + # for back compatibility with ndarray subclasses that changed the + # return values of the all method. We are not committed to supporting + # such subclasses, but some used to work. x_id = func(x) y_id = func(y) - if (x_id == y_id).all() != True: + if npall(x_id == y_id) != True: msg = build_err_msg([x, y], err_msg + '\nx and y %s location mismatch:' % (hasval), verbose=verbose, header=header, diff --git a/numpy/testing/tests/test_decorators.py b/numpy/testing/tests/test_decorators.py index ea684140d..d00820b80 100644 --- a/numpy/testing/tests/test_decorators.py +++ b/numpy/testing/tests/test_decorators.py @@ -53,7 +53,6 @@ class TestNoseDecorators(object): assert_(f_istest.__test__) assert_(not f_isnottest.__test__) - def test_skip_functions_hardcoded(self): @dec.skipif(True) def f1(x): diff --git a/numpy/testing/tests/test_utils.py b/numpy/testing/tests/test_utils.py index 84d310992..2c60e2867 100644 --- a/numpy/testing/tests/test_utils.py +++ b/numpy/testing/tests/test_utils.py @@ -1391,7 +1391,6 @@ class TestAssertNoGcCycles(object): assert_no_gc_cycles(no_cycle) - def test_asserts(self): def make_cycle(): a = [] @@ -1406,7 +1405,6 @@ class TestAssertNoGcCycles(object): with assert_raises(AssertionError): assert_no_gc_cycles(make_cycle) - def test_fails(self): """ Test that in cases where the garbage cannot be collected, we raise an diff --git a/runtests.py b/runtests.py index 388b911a1..81c7c103f 100755 --- a/runtests.py +++ b/runtests.py @@ -328,9 +328,10 @@ def build_project(args): # Always use ccache, if installed env['PATH'] = os.pathsep.join(EXTRA_PATH + env.get('PATH', '').split(os.pathsep)) cvars = distutils.sysconfig.get_config_vars() - if 'gcc' in cvars.get('CC', ''): + compiler = env.get('CC') or cvars.get('CC', '') + if 'gcc' in compiler: # Check that this isn't clang masquerading as gcc. - if sys.platform != 'darwin' or 'gnu-gcc' in cvars.get('CC', ''): + if sys.platform != 'darwin' or 'gnu-gcc' in compiler: # add flags used as werrors warnings_as_errors = ' '.join([ # from tools/travis-test.sh |