diff options
27 files changed, 929 insertions, 178 deletions
@@ -9,6 +9,8 @@ # gives no duplicates. Aaron Baecker <abaecker@localhost> abaecker <abaecker@localhost> +Alan Fontenot <logeaux@yahoo.com> logeaux <logeaux@yahoo.com> +Alan Fontenot <logeaux@yahoo.com> logeaux <36168460+logeaux@users.noreply.github.com> Abdul Muneer <abdulmuneer@gmail.com> abdulmuneer <abdulmuneer@gmail.com> Adam Ginsburg <adam.g.ginsburg@gmail.com> Adam Ginsburg <keflavich@gmail.com> Albert Jornet Puig <albert.jornet@ic3.cat> jurnix <albert.jornet@ic3.cat> @@ -19,10 +21,12 @@ Alex Griffing <argriffi@ncsu.edu> argriffing <argriffing@users.noreply.github.co Alex Thomas <alexthomas93@users.noreply.github.com> alexthomas93 <alexthomas93@users.noreply.github.com> Alexander Belopolsky <abalkin@enlnt.com> Alexander Belopolsky <a@enlnt.com> Alexander Belopolsky <abalkin@enlnt.com> Alexander Belopolsky <a@enlnt.com> +Alexander Belopolsky <abalkin@enlnt.com> sasha <sasha@localhost> Alexander Shadchin <alexandr.shadchin@gmail.com> Alexandr Shadchin <alexandr.shadchin@gmail.com> Alexander Shadchin <alexandr.shadchin@gmail.com> shadchin <alexandr.shadchin@gmail.com> Allan Haldane <allan.haldane@gmail.com> ahaldane <ealloc@gmail.com> Alok Singhal <gandalf013@gmail.com> Alok Singhal <alok@merfinllc.com> +Alyssa Quek <alyssaquek@gmail.com> alyssaq <alyssaquek@gmail.com> Amir Sarabadani <ladsgroup@gmail.com> amir <ladsgroup@gmail.com> Anatoly Techtonik <techtonik@gmail.com> anatoly techtonik <techtonik@gmail.com> Andrei Kucharavy <ank@andreikucharavy.com> chiffa <ank@andreikucharavy.com> @@ -47,10 +51,12 @@ Bryan Van de Ven <bryanv@continuum.io> Bryan Van de Ven <bryan@laptop.local> Carl Kleffner <cmkleffner@gmail.com> carlkl <cmkleffner@gmail.com> Chris Burns <chris.burns@localhost> chris.burns <chris.burns@localhost> Chris Kerr <debdepba@dasganma.tk> Chris Kerr <cjk34@cam.ac.uk> +Christian Clauss <cclauss@bluewin.ch> cclauss <cclauss@bluewin.ch> Christopher Hanley <chanley@gmail.com> chanley <chanley@gmail.com> Christoph Gohlke <cgohlke@uci.edu> cgholke <?@?> Christoph Gohlke <cgohlke@uci.edu> cgohlke <cgohlke@uci.edu> Christoph Gohlke <cgohlke@uci.edu> Christolph Gohlke <cgohlke@uci.edu> +Daniel B Allan <daniel.b.allan@gmail.com> danielballan <daniel.b.allan@gmail.com> Daniel da Silva <mail@danieldasilva.org> Daniel da Silva <daniel@meltingwax.net> Daniel da Silva <mail@danieldasilva.org> Daniel da Silva <var.mail.daniel@gmail.com> Daniel J Farrell <danieljfarrel@me.com> danieljfarrell <danieljfarrel@me.com> @@ -61,11 +67,13 @@ David Huard <david.huard@gmail.com> dhuard <dhuard@localhost> David M Cooke <cookedm@localhost> cookedm <cookedm@localhost> David Nicholson <davidjn@google.com> davidjn <dnic12345@gmail.com> David Ochoa <ochoadavid@gmail.com> ochoadavid <ochoadavid@gmail.com> +Dawid Zych <dawid.zych@yandex.com> silenc3r <dawid.zych@yandex.com> Derek Homeier <derek@astro.physik.uni-goettingen.de> Derek Homeier <dhomeie@gwdg.de> Derek Homeier <derek@astro.physik.uni-goettingen.de> Derek Homeir <derek@astro.phsik.uni-goettingen.de> Derek Homeier <derek@astro.physik.uni-goettingen.de> Derek Homier <derek@astro.physik.uni-goettingen.de> Derrick Williams <myutat@gmail.com> derrick <myutat@gmail.com> Dmitriy Shalyga <zuko3d@gmail.com> zuko3d <zuko3d@gmail.com> +Ed Schofield <edschofield@localhost> edschofield <edschofield@localhost> Egor Zindy <ezindy@gmail.com> zindy <ezindy@gmail.com> Endolith <endolith@gmail.com> Eric Fode <ericfode@gmail.com> Eric Fode <ericfode@linuxlaptop.(none)> @@ -97,15 +105,19 @@ Irvin Probst <irvin.probst@ensta-bretagne.fr> I--P <irvin.probst@ensta-bretagne. Jaime Fernandez <jaime.frio@gmail.com> Jaime Fernandez <jaime.fernandez@hp.com> Jaime Fernandez <jaime.frio@gmail.com> jaimefrio <jaime.frio@gmail.com> Jaime Fernandez <jaime.frio@gmail.com> Jaime <jaime.frio@gmail.com> +James Webber <jamestwebber@gmail.com> jamestwebber <jamestwebber@gmail.com> Jarrod Millman <millman@berkeley.edu> Jarrod Millman <jarrod.millman@gmail.com> Jason Grout <jason-github@creativetrax.com> Jason Grout <jason.grout@drake.edu> Jason King <pizza@netspace.net.au> jason king <pizza@netspace.net.au> Jay Bourque <jay.bourque@continuum.io> jayvius <jay.bourque@continuum.io> Jean Utke <jutke@allstate.com> jutke <jutke@allstate.com> +Jeffrey Yancey <jeffrey@octane5.com> Jeff <3820914+jeffyancey@users.noreply.github.com> Jerome Kelleher <jerome.kelleher@ed.ac.uk> jeromekelleher <jerome.kelleher@ed.ac.uk> Johannes Schönberger <hannesschoenberger@gmail.com> Johannes Schönberger <jschoenberger@demuc.de> +John Darbyshire <24256554+attack68@users.noreply.github.com> attack68 <24256554+attack68@users.noreply.github.com> Joseph Fox-Rabinovitz <jfoxrabinovitz@gmail.com> Joseph Fox-Rabinovitz <joseph.r.fox-rabinovitz@nasa.gov> -Joseph Fox-Rabinovitz <jfoxrabinovitz@gmail.com> Mad Physicist <madphysicist@users.noreply.github.com> +Joseph Fox-Rabinovitz <jfoxrabinovitz@gmail.com> Joseph Fox-Rabinovitz <madphysicist@users.noreply.github.com> +Joseph Fox-Rabinovitz <jfoxrabinovitz@gmail.com> Mad Physicist <madphysicist@users.noreply.github.com> Joseph Martinot-Lagarde <contrebasse@gmail.com> Joseph Martinot-Lagarde <joseph.martinot-lagarde@onera.fr> Julian Taylor <juliantaylor108@gmail.com> Julian Taylor <jtaylor.debian@googlemail.com> Julian Taylor <juliantaylor108@gmail.com> Julian Taylor <juliantaylor108@googlemail.com> @@ -115,10 +127,12 @@ Khaled Ben Abdallah Okuda <khaled.ben.okuda@gmail.com> KhaledTo <khaled.ben.okud Konrad Kapp <k_kapp@yahoo.com> k_kapp@yahoo.com <k_kapp@yahoo.com> Lars Buitinck <larsmans@gmail.com> Lars Buitinck <l.buitinck@esciencecenter.nl> Lars Buitinck <larsmans@gmail.com> Lars Buitinck <L.J.Buitinck@uva.nl> +Lars Grüter <lagru@mailbox.org> Lars G <lagru@mailbox.org> Luis Pedro Coelho <luis@luispedro.org> Luis Pedro Coelho <lpc@cmu.edu> Luke Zoltan Kelley <lkelley@cfa.harvard.edu> lzkelley <lkelley@cfa.harvard.edu> Manoj Kumar <manojkumarsivaraj334@gmail.com> MechCoder <manojkumarsivaraj334@gmail.com> Mark DePristo <mdepristo@synapdx.com> markdepristo <mdepristo@synapdx.com> +Mark Weissman <mw9050@gmail.com> m-d-w <mw9050@gmail.com> Mark Wiebe <mwwiebe@gmail.com> Mark <mwwiebe@gmail.com> Mark Wiebe <mwwiebe@gmail.com> Mark Wiebe <mwiebe@continuum.io> Mark Wiebe <mwwiebe@gmail.com> Mark Wiebe <mwiebe@enthought.com> @@ -134,6 +148,7 @@ Michael Droettboom <mdboom@gmail.com> mdroe <mdroe@localhost> Michael K. Tran <trankmichael@gmail.com> mtran <trankmichael@gmail.com> Michael Martin <mmartin4242@gmail.com> mmartin <mmartin4242@gmail.com> Michael Schnaitter <schnaitterm@knights.ucf.edu> schnaitterm <schnaitterm@users.noreply.github.com> +Muhammad Kasim <firman.kasim@gmail.com> mfkasim91 <firman.kasim@gmail.com> Nathaniel J. Smith <njs@pobox.com> njsmith <njs@pobox.com> Naveen Arunachalam <notatroll.troll@gmail.com> naveenarun <notatroll.troll@gmail.com> Nicolas Scheffer <nicolas.scheffer@sri.com> Nicolas Scheffer <scheffer@speech.sri.com> @@ -175,15 +190,22 @@ Stephan Hoyer <shoyer@gmail.com> Stephan Hoyer <shoyer@climate.com> Steven J Kern <kern.steven0@gmail.com> Thomas A Caswell <tcaswell@gmail.com> Thomas A Caswell <tcaswell@bnl.gov> Tim Cera <tim@cerazone.net> tim cera <tcera@sjrwmd.com> +Tim Teichmann <t.teichmann@dashdos.com> tteichmann <t.teichmann@dashdos.com> +Tim Teichmann <t.teichmann@dashdos.com> tteichmann <44259103+tteichmann@users.noreply.github.com> Tom Boyd <pezcore@users.noreply.github.com> pezcore <pezcore@users.noreply.github.com> Tom Poole <t.b.poole@gmail.com> tpoole <t.b.poole@gmail.com> Travis Oliphant <travis@continuum.io> Travis E. Oliphant <teoliphant@gmail.com> Travis Oliphant <travis@continuum.io> Travis Oliphant <oliphant@enthought.com> Valentin Haenel <valentin@haenel.co> Valentin Haenel <valentin.haenel@gmx.de> Warren Weckesser <warren.weckesser@enthought.com> Warren Weckesser <warren.weckesser@gmail.com> +Weitang Li <liwt31@163.com> wtli@Dirac <liwt31@163.com> +Weitang Li <liwt31@163.com> wtli <liwt31@163.com> Wendell Smith <wendellwsmith@gmail.com> Wendell Smith <wackywendell@gmail.com> William Spotz <wfspotz@sandia.gov@localhost> wfspotz@sandia.gov <wfspotz@sandia.gov@localhost> +Wim Glenn <wim.glenn@melbourneit.com.au> wim glenn <wim.glenn@melbourneit.com.au> Wojtek Ruszczewski <git@wr.waw.pl> wrwrwr <git@wr.waw.pl> +Yuji Kanagawa <yuji.kngw.80s.revive@gmail.com> kngwyu <yuji.kngw.80s.revive@gmail.com> Zixu Zhao <zixu.zhao.tireless@gmail.com> ZZhaoTireless <zixu.zhao.tireless@gmail.com> Ziyan Zhou<ziyan.zhou@mujin.co.jp> Ziyan <ziyan.zhou@mujin.co.jp> -luzpaz <luzpaz@users.noreply.github.com> luz.paz <luzpaz@users.noreply.github.com> +luzpaz <kunda@scribus.net> luz.paz <luzpaz@users.noreply.github.com> +luzpaz <kunda@scribus.net> luzpaz <luzpaz@users.noreply.github.com> diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d1e792c83..a05da2e5b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -6,6 +6,25 @@ trigger: - master - maintenance/* jobs: +- job: Linux_Python_36_32bit_full + pool: + vmIMage: 'ubuntu-16.04' + steps: + - script: | + docker pull i386/ubuntu:bionic + docker run -v $(pwd):/numpy i386/ubuntu:bionic /bin/bash -c "cd numpy && \ + apt-get -y update && \ + apt-get -y install python3.6-dev python3-pip locales && \ + locale-gen fr_FR && update-locale && \ + pip3 install setuptools nose cython==0.29.0 pytest pytz pickle5 && \ + apt-get -y install libopenblas-dev gfortran && \ + NUMPY_EXPERIMENTAL_ARRAY_FUNCTION=1 \ + python3 runtests.py --mode=full -- -rsx --junitxml=junit/test-results.xml" + displayName: 'Run 32-bit Ubuntu Docker Build / Tests' + - task: PublishTestResults@2 + inputs: + testResultsFiles: '**/test-*.xml' + testRunTitle: 'Publish test results for Python 3.6-32 bit' - job: macOS pool: # NOTE: at time of writing, there is a danger diff --git a/doc/release/1.16.0-notes.rst b/doc/release/1.16.0-notes.rst index 3f7b7be32..8ab2cab15 100644 --- a/doc/release/1.16.0-notes.rst +++ b/doc/release/1.16.0-notes.rst @@ -19,6 +19,11 @@ Highlights New functions ============= + * New functions in the `numpy.lib.recfunctions` module to ease the structured + assignment changes: `assign_fields_by_name`, `structured_to_unstructured`, + `unstructured_to_structured`, `apply_along_fields`, and `require_fields`. + See the user guide at <https://docs.scipy.org/doc/numpy/user/basics.rec.html> + for more info. Deprecations ============ @@ -123,6 +128,13 @@ of: New Features ============ +Integrated squared error (ISE) estimator added to ``histogram`` +--------------------------------------------------------------- +This method (``bins='stone'``) for optimizing the bin number is a generalization of the +Scott's rule. The Scott's rule assumes the distribution is approximately +Normal, while the ISE is a nonparametric method based on cross-validation. +https://en.wikipedia.org/wiki/Histogram#Minimizing_cross-validation_estimated_squared_error + ``max_rows`` keyword added for ``np.loadtxt`` --------------------------------------------- New keyword ``max_rows`` in `numpy.loadtxt` sets the maximum rows of the @@ -329,6 +341,16 @@ types. As of this release, this caveat is lifted - now: * Bitfields are no longer interpreted as sub-arrays * Pointers are no longer replaced with the type that they point to +A new ``ndpointer.contents`` member +----------------------------------- +This matches the ``.contents`` member of normal ctypes arrays, and can be used +to construct an ``np.array`` around the pointers contents. + +This replaces ``np.array(some_nd_pointer)``, which stopped working in 1.15. + +As a side effect of this change, ``ndpointer`` now supports dtypes with +overlapping fields and padding. + Changes ======= diff --git a/numpy/core/_add_newdocs.py b/numpy/core/_add_newdocs.py index bfb832dfc..bc034c3e9 100644 --- a/numpy/core/_add_newdocs.py +++ b/numpy/core/_add_newdocs.py @@ -5579,13 +5579,13 @@ add_newdoc('numpy.core.multiarray', 'dtype', ('char', add_newdoc('numpy.core.multiarray', 'dtype', ('descr', """ - PEP3118 interface description of the data-type. + `__array_interface__` description of the data-type. The format is that required by the 'descr' key in the - PEP3118 `__array_interface__` attribute. + `__array_interface__` attribute. - Warning: This attribute exists specifically for PEP3118 compliance, and - is not a datatype description compatible with `np.dtype`. + Warning: This attribute exists specifically for `__array_interface__`, + and is not a datatype description compatible with `np.dtype`. """)) add_newdoc('numpy.core.multiarray', 'dtype', ('fields', diff --git a/numpy/core/_dtype_ctypes.py b/numpy/core/_dtype_ctypes.py index ca365d2cb..0852b1ef2 100644 --- a/numpy/core/_dtype_ctypes.py +++ b/numpy/core/_dtype_ctypes.py @@ -66,18 +66,34 @@ def _from_ctypes_structure(t): return np.dtype(fields, align=True) -def dtype_from_ctypes_scalar(t): +def _from_ctypes_scalar(t): """ Return the dtype type with endianness included if it's the case """ - if t.__ctype_be__ is t: + if getattr(t, '__ctype_be__', None) is t: return np.dtype('>' + t._type_) - elif t.__ctype_le__ is t: + elif getattr(t, '__ctype_le__', None) is t: return np.dtype('<' + t._type_) else: return np.dtype(t._type_) +def _from_ctypes_union(t): + formats = [] + offsets = [] + names = [] + for fname, ftyp in t._fields_: + names.append(fname) + formats.append(dtype_from_ctypes_type(ftyp)) + offsets.append(0) # Union fields are offset to 0 + + return np.dtype(dict( + formats=formats, + offsets=offsets, + names=names, + itemsize=ctypes.sizeof(t))) + + def dtype_from_ctypes_type(t): """ Construct a dtype object from a ctypes type @@ -89,12 +105,9 @@ def dtype_from_ctypes_type(t): elif issubclass(t, _ctypes.Structure): return _from_ctypes_structure(t) elif issubclass(t, _ctypes.Union): - # TODO - raise NotImplementedError( - "conversion from ctypes.Union types like {} to dtype" - .format(t.__name__)) - elif isinstance(t._type_, str): - return dtype_from_ctypes_scalar(t) + return _from_ctypes_union(t) + elif isinstance(getattr(t, '_type_', None), str): + return _from_ctypes_scalar(t) else: raise NotImplementedError( "Unknown ctypes type {}".format(t.__name__)) diff --git a/numpy/core/code_generators/cversions.txt b/numpy/core/code_generators/cversions.txt index 73cd8ddd6..00f10df57 100644 --- a/numpy/core/code_generators/cversions.txt +++ b/numpy/core/code_generators/cversions.txt @@ -43,7 +43,8 @@ # Version 12 (NumPy 1.15) No change. 0x0000000c = a1bc756c5782853ec2e3616cf66869d8 -# Version 13 (NumPy 1.16) Deprecate PyArray_SetNumericOps and -# PyArray_GetNumericOps, Added fields core_dim_flags and core_dim_sizes -# to PyUFuncObject -0x0000000d = a1bc756c5782853ec2e3616cf66869d8 +# Version 13 (NumPy 1.16) +# Deprecate PyArray_SetNumericOps and PyArray_GetNumericOps, +# Add fields core_dim_flags and core_dim_sizes to PyUFuncObject. +# Add PyUFunc_FromFuncAndDataAndSignatureAndIdentity to ufunc_funcs_api. +0x0000000d = 5b0e8bbded00b166125974fc71e80a33 diff --git a/numpy/core/code_generators/numpy_api.py b/numpy/core/code_generators/numpy_api.py index fdf97ac00..a71c236fd 100644 --- a/numpy/core/code_generators/numpy_api.py +++ b/numpy/core/code_generators/numpy_api.py @@ -402,8 +402,7 @@ ufunc_funcs_api = { # End 1.7 API 'PyUFunc_RegisterLoopForDescr': (41,), # End 1.8 API - 'PyUFunc_FromFuncAndDataAndSignatureAndIdentity': - (42,), + 'PyUFunc_FromFuncAndDataAndSignatureAndIdentity': (42,), # End 1.16 API } diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index da006909a..b0b749c80 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -505,7 +505,8 @@ typedef struct { PyArray_NonzeroFunc *nonzero; /* - * Used for arange. + * Used for arange. Should return 0 on success + * and -1 on failure. * Can be NULL. */ PyArray_FillFunc *fill; diff --git a/numpy/core/include/numpy/npy_1_7_deprecated_api.h b/numpy/core/include/numpy/npy_1_7_deprecated_api.h index 76b57b748..a6ee21219 100644 --- a/numpy/core/include/numpy/npy_1_7_deprecated_api.h +++ b/numpy/core/include/numpy/npy_1_7_deprecated_api.h @@ -5,6 +5,8 @@ #error "Should never include npy_*_*_deprecated_api directly." #endif +/* Emit a warning if the user did not specifically request the old API */ +#ifndef NPY_NO_DEPRECATED_API #if defined(_WIN32) #define _WARN___STR2__(x) #x #define _WARN___STR1__(x) _WARN___STR2__(x) @@ -16,6 +18,7 @@ "#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION" #endif /* TODO: How to do this warning message for other compilers? */ +#endif /* * This header exists to collect all dangerous/deprecated NumPy API diff --git a/numpy/core/setup.py b/numpy/core/setup.py index efcacfb8e..23a9e268b 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -379,8 +379,9 @@ def check_mathlib(config_cmd): def visibility_define(config): """Return the define value to use for NPY_VISIBILITY_HIDDEN (may be empty string).""" - if config.check_compiler_gcc4(): - return '__attribute__((visibility("hidden")))' + hide = '__attribute__((visibility("hidden")))' + if config.check_gcc_function_attribute(hide, 'hideme'): + return hide else: return '' diff --git a/numpy/core/src/multiarray/_multiarray_tests.c.src b/numpy/core/src/multiarray/_multiarray_tests.c.src index b98c3afb3..2a8275572 100644 --- a/numpy/core/src/multiarray/_multiarray_tests.c.src +++ b/numpy/core/src/multiarray/_multiarray_tests.c.src @@ -11,6 +11,13 @@ #include "npy_extint128.h" #include "common.h" + +#if defined(MS_WIN32) || defined(__CYGWIN__) +#define EXPORT(x) __declspec(dllexport) x +#else +#define EXPORT(x) x +#endif + #define ARRAY_SIZE(a) (sizeof(a)/sizeof(a[0])) /* test PyArray_IsPythonScalar, before including private py3 compat header */ @@ -31,6 +38,12 @@ IsPythonScalar(PyObject * dummy, PyObject *args) #include "npy_pycompat.h" +/** Function to test calling via ctypes */ +EXPORT(void*) forward_pointer(void *x) +{ + return x; +} + /* * TODO: * - Handle mode @@ -2053,3 +2066,9 @@ init_multiarray_tests(void) } return RETVAL; } + +NPY_NO_EXPORT int +test_not_exported(void) +{ + return 1; +} diff --git a/numpy/core/src/multiarray/arraytypes.c.src b/numpy/core/src/multiarray/arraytypes.c.src index d3aa1bd92..823ee7115 100644 --- a/numpy/core/src/multiarray/arraytypes.c.src +++ b/numpy/core/src/multiarray/arraytypes.c.src @@ -3594,9 +3594,10 @@ OBJECT_dot(char *ip1, npy_intp is1, char *ip2, npy_intp is2, char *op, npy_intp #define BOOL_fill NULL /* this requires buffer to be filled with objects or NULL */ -static void +static int OBJECT_fill(PyObject **buffer, npy_intp length, void *NPY_UNUSED(ignored)) { + int retval = 0; npy_intp i; PyObject *start = buffer[0]; PyObject *delta = buffer[1]; @@ -3604,27 +3605,31 @@ OBJECT_fill(PyObject **buffer, npy_intp length, void *NPY_UNUSED(ignored)) delta = PyNumber_Subtract(delta, start); if (!delta) { - return; + return -1; } second = start = PyNumber_Add(start, delta); if (!start) { - goto finish; + goto error; } buffer += 2; for (i = 2; i < length; i++, buffer++) { start = PyNumber_Add(start, delta); if (!start) { - goto finish; + goto error; } Py_XDECREF(*buffer); *buffer = start; } + goto finish; + +error: + retval = -1; finish: Py_XDECREF(second); Py_DECREF(delta); - return; + return retval; } /**begin repeat @@ -3638,7 +3643,7 @@ finish: * npy_float, npy_double, npy_longdouble, * npy_datetime, npy_timedelta# */ -static void +static int @NAME@_fill(@type@ *buffer, npy_intp length, void *NPY_UNUSED(ignored)) { npy_intp i; @@ -3649,10 +3654,11 @@ static void for (i = 2; i < length; ++i) { buffer[i] = start + i*delta; } + return 0; } /**end repeat**/ -static void +static int HALF_fill(npy_half *buffer, npy_intp length, void *NPY_UNUSED(ignored)) { npy_intp i; @@ -3663,6 +3669,7 @@ HALF_fill(npy_half *buffer, npy_intp length, void *NPY_UNUSED(ignored)) for (i = 2; i < length; ++i) { buffer[i] = npy_float_to_half(start + i*delta); } + return 0; } /**begin repeat @@ -3670,7 +3677,7 @@ HALF_fill(npy_half *buffer, npy_intp length, void *NPY_UNUSED(ignored)) * #NAME = CFLOAT, CDOUBLE, CLONGDOUBLE# * #type = npy_cfloat, npy_cdouble, npy_clongdouble# */ -static void +static int @NAME@_fill(@type@ *buffer, npy_intp length, void *NPY_UNUSED(ignore)) { npy_intp i; @@ -3688,6 +3695,7 @@ static void buffer->real = start.real + i*delta.real; buffer->imag = start.imag + i*delta.imag; } + return 0; } /**end repeat**/ @@ -4167,6 +4175,53 @@ small_correlate(const char * d_, npy_intp dstride, } /* +*/ + +/* A clone function for the datetime dtype c_metadata */ +static NpyAuxData * +_datetime_dtype_metadata_clone(NpyAuxData *data) +{ + PyArray_DatetimeDTypeMetaData *newdata = + (PyArray_DatetimeDTypeMetaData *)PyArray_malloc( + sizeof(*newdata)); + if (newdata == NULL) { + PyErr_NoMemory(); + return NULL; + } + + memcpy(newdata, data, sizeof(*newdata)); + + return (NpyAuxData *)newdata; +} + +/* + * Allcoate and initialize a PyArray_DatetimeDTypeMetaData object + */ +static NpyAuxData* +_create_datetime_metadata(NPY_DATETIMEUNIT base, int num) +{ + PyArray_DatetimeDTypeMetaData *data; + + /* Allocate memory for the metadata */ + data = PyArray_malloc(sizeof(*data)); + if (data == NULL) { + PyErr_NoMemory(); + return NULL; + } + + /* Initialize the base aux data */ + memset(data, 0, sizeof(PyArray_DatetimeDTypeMetaData)); + data->base.free = (NpyAuxData_FreeFunc *)PyArray_free; + data->base.clone = _datetime_dtype_metadata_clone; + + data->meta.base = base; + data->meta.num = num; + + return (NpyAuxData*)data; +} + + +/* ***************************************************************************** ** SETUP FUNCTION POINTERS ** ***************************************************************************** @@ -4522,66 +4577,6 @@ PyArray_DescrFromType(int type) return ret; } -/* A clone function for the datetime dtype metadata */ -static NpyAuxData * -datetime_dtype_metadata_clone(NpyAuxData *data) -{ - PyArray_DatetimeDTypeMetaData *newdata = - (PyArray_DatetimeDTypeMetaData *)PyArray_malloc( - sizeof(PyArray_DatetimeDTypeMetaData)); - if (newdata == NULL) { - return NULL; - } - - memcpy(newdata, data, sizeof(PyArray_DatetimeDTypeMetaData)); - - return (NpyAuxData *)newdata; -} - -/* - * Initializes the c_metadata field for the _builtin_descrs DATETIME - * and TIMEDELTA. - * - * must not be static, gcc 4.1.2 on redhat 5 then miscompiles this function - * see gh-5163 - * - */ -NPY_NO_EXPORT int -initialize_builtin_datetime_metadata(void) -{ - PyArray_DatetimeDTypeMetaData *data1, *data2; - - /* Allocate memory for the metadata */ - data1 = PyArray_malloc(sizeof(PyArray_DatetimeDTypeMetaData)); - if (data1 == NULL) { - return -1; - } - data2 = PyArray_malloc(sizeof(PyArray_DatetimeDTypeMetaData)); - if (data2 == NULL) { - PyArray_free(data1); - return -1; - } - - /* Initialize the base aux data */ - memset(data1, 0, sizeof(PyArray_DatetimeDTypeMetaData)); - memset(data2, 0, sizeof(PyArray_DatetimeDTypeMetaData)); - data1->base.free = (NpyAuxData_FreeFunc *)PyArray_free; - data2->base.free = (NpyAuxData_FreeFunc *)PyArray_free; - data1->base.clone = datetime_dtype_metadata_clone; - data2->base.clone = datetime_dtype_metadata_clone; - - /* Set to the default metadata */ - data1->meta.base = NPY_DATETIME_DEFAULTUNIT; - data1->meta.num = 1; - data2->meta.base = NPY_DATETIME_DEFAULTUNIT; - data2->meta.num = 1; - - _builtin_descrs[NPY_DATETIME]->c_metadata = (NpyAuxData *)data1; - _builtin_descrs[NPY_TIMEDELTA]->c_metadata = (NpyAuxData *)data2; - - return 0; -} - /* ***************************************************************************** ** SETUP TYPE INFO ** @@ -4650,7 +4645,14 @@ set_typeinfo(PyObject *dict) /**end repeat**/ - if (initialize_builtin_datetime_metadata() < 0) { + _builtin_descrs[NPY_DATETIME]->c_metadata = _create_datetime_metadata( + NPY_DATETIME_DEFAULTUNIT, 1); + if (_builtin_descrs[NPY_DATETIME]->c_metadata == NULL) { + return -1; + } + _builtin_descrs[NPY_TIMEDELTA]->c_metadata = _create_datetime_metadata( + NPY_DATETIME_DEFAULTUNIT, 1); + if (_builtin_descrs[NPY_DATETIME]->c_metadata == NULL) { return -1; } diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 9e42c115c..909a24359 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -833,7 +833,10 @@ PyArray_InnerProduct(PyObject *op1, PyObject *op2) typenum = PyArray_ObjectType(op2, typenum); typec = PyArray_DescrFromType(typenum); if (typec == NULL) { - PyErr_SetString(PyExc_TypeError, "Cannot find a common data type."); + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "Cannot find a common data type."); + } goto fail; } @@ -919,7 +922,10 @@ PyArray_MatrixProduct2(PyObject *op1, PyObject *op2, PyArrayObject* out) typenum = PyArray_ObjectType(op2, typenum); typec = PyArray_DescrFromType(typenum); if (typec == NULL) { - PyErr_SetString(PyExc_TypeError, "Cannot find a common data type."); + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "Cannot find a common data type."); + } return NULL; } @@ -2358,7 +2364,10 @@ array_matmul(PyObject *NPY_UNUSED(m), PyObject *args, PyObject* kwds) dtype = PyArray_DescrFromObject(in1, NULL); dtype = PyArray_DescrFromObject(in2, dtype); if (dtype == NULL) { - PyErr_SetString(PyExc_ValueError, "Cannot find a common data type."); + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, + "Cannot find a common data type."); + } return NULL; } typenum = dtype->type_num; diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py index f2e7f8f50..8cde19612 100644 --- a/numpy/core/tests/test_dtype.py +++ b/numpy/core/tests/test_dtype.py @@ -807,9 +807,9 @@ class TestFromCTypes(object): p_uint8 = ctypes.POINTER(ctypes.c_uint8) assert_raises(TypeError, np.dtype, p_uint8) - @pytest.mark.xfail( - reason="Unions are not implemented", - raises=NotImplementedError) + def test_void_pointer(self): + self.check(ctypes.c_void_p, np.uintp) + def test_union(self): class Union(ctypes.Union): _fields_ = [ @@ -824,6 +824,52 @@ class TestFromCTypes(object): )) self.check(Union, expected) + def test_union_with_struct_packed(self): + class Struct(ctypes.Structure): + _pack_ = 1 + _fields_ = [ + ('one', ctypes.c_uint8), + ('two', ctypes.c_uint32) + ] + + class Union(ctypes.Union): + _fields_ = [ + ('a', ctypes.c_uint8), + ('b', ctypes.c_uint16), + ('c', ctypes.c_uint32), + ('d', Struct), + ] + expected = np.dtype(dict( + names=['a', 'b', 'c', 'd'], + formats=['u1', np.uint16, np.uint32, [('one', 'u1'), ('two', np.uint32)]], + offsets=[0, 0, 0, 0], + itemsize=ctypes.sizeof(Union) + )) + self.check(Union, expected) + + def test_union_packed(self): + class Struct(ctypes.Structure): + _fields_ = [ + ('one', ctypes.c_uint8), + ('two', ctypes.c_uint32) + ] + _pack_ = 1 + class Union(ctypes.Union): + _pack_ = 1 + _fields_ = [ + ('a', ctypes.c_uint8), + ('b', ctypes.c_uint16), + ('c', ctypes.c_uint32), + ('d', Struct), + ] + expected = np.dtype(dict( + names=['a', 'b', 'c', 'd'], + formats=['u1', np.uint16, np.uint32, [('one', 'u1'), ('two', np.uint32)]], + offsets=[0, 0, 0, 0], + itemsize=ctypes.sizeof(Union) + )) + self.check(Union, expected) + def test_packed_structure(self): class PackedStructure(ctypes.Structure): _pack_ = 1 diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index d1b306ef0..51fe6e9ef 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -2727,6 +2727,17 @@ class TestMethods(object): np.dot(a, b, out=out) np.matmul(a, b, out=out) + def test_dot_matmul_inner_array_casting_fails(self): + + class A(object): + def __array__(self, *args, **kwargs): + raise NotImplementedError + + # Don't override the error from calling __array__() + assert_raises(NotImplementedError, np.dot, A(), A()) + assert_raises(NotImplementedError, np.matmul, A(), A()) + assert_raises(NotImplementedError, np.inner, A(), A()) + def test_diagonal(self): a = np.arange(12).reshape((3, 4)) assert_equal(a.diagonal(), [0, 5, 10]) diff --git a/numpy/ctypeslib.py b/numpy/ctypeslib.py index 24cfc6762..11368587f 100644 --- a/numpy/ctypeslib.py +++ b/numpy/ctypeslib.py @@ -55,7 +55,9 @@ __all__ = ['load_library', 'ndpointer', 'test', 'ctypes_load_library', 'c_intp', 'as_ctypes', 'as_array'] import os -from numpy import integer, ndarray, dtype as _dtype, deprecate, array +from numpy import ( + integer, ndarray, dtype as _dtype, deprecate, array, frombuffer +) from numpy.core.multiarray import _flagdict, flagsobj try: @@ -175,24 +177,6 @@ def _flags_fromnum(num): class _ndptr(_ndptr_base): - - def _check_retval_(self): - """This method is called when this class is used as the .restype - attribute for a shared-library function. It constructs a numpy - array from a void pointer.""" - return array(self) - - @property - def __array_interface__(self): - return {'descr': self._dtype_.descr, - '__ref': self, - 'strides': None, - 'shape': self._shape_, - 'version': 3, - 'typestr': self._dtype_.descr[0][1], - 'data': (self.value, False), - } - @classmethod def from_param(cls, obj): if not isinstance(obj, ndarray): @@ -213,6 +197,34 @@ class _ndptr(_ndptr_base): return obj.ctypes +class _concrete_ndptr(_ndptr): + """ + Like _ndptr, but with `_shape_` and `_dtype_` specified. + + Notably, this means the pointer has enough information to reconstruct + the array, which is not generally true. + """ + def _check_retval_(self): + """ + This method is called when this class is used as the .restype + attribute for a shared-library function, to automatically wrap the + pointer into an array. + """ + return self.contents + + @property + def contents(self): + """ + Get an ndarray viewing the data pointed to by this pointer. + + This mirrors the `contents` attribute of a normal ctypes pointer + """ + full_dtype = _dtype((self._dtype_, self._shape_)) + full_ctype = ctypes.c_char * full_dtype.itemsize + buffer = ctypes.cast(self, ctypes.POINTER(full_ctype)).contents + return frombuffer(buffer, dtype=full_dtype).squeeze(axis=0) + + # Factory for an array-checking class with from_param defined for # use with ctypes argtypes mechanism _pointer_type_cache = {} @@ -269,8 +281,11 @@ def ndpointer(dtype=None, ndim=None, shape=None, flags=None): """ + # normalize dtype to an Optional[dtype] if dtype is not None: dtype = _dtype(dtype) + + # normalize flags to an Optional[int] num = None if flags is not None: if isinstance(flags, str): @@ -287,10 +302,23 @@ def ndpointer(dtype=None, ndim=None, shape=None, flags=None): except Exception: raise TypeError("invalid flags specification") num = _num_fromflags(flags) + + # normalize shape to an Optional[tuple] + if shape is not None: + try: + shape = tuple(shape) + except TypeError: + # single integer -> 1-tuple + shape = (shape,) + + cache_key = (dtype, ndim, shape, num) + try: - return _pointer_type_cache[(dtype, ndim, shape, num)] + return _pointer_type_cache[cache_key] except KeyError: pass + + # produce a name for the new type if dtype is None: name = 'any' elif dtype.names: @@ -300,23 +328,21 @@ def ndpointer(dtype=None, ndim=None, shape=None, flags=None): if ndim is not None: name += "_%dd" % ndim if shape is not None: - try: - strshape = [str(x) for x in shape] - except TypeError: - strshape = [str(shape)] - shape = (shape,) - shape = tuple(shape) - name += "_"+"x".join(strshape) + name += "_"+"x".join(str(x) for x in shape) if flags is not None: name += "_"+"_".join(flags) + + if dtype is not None and shape is not None: + base = _concrete_ndptr else: - flags = [] - klass = type("ndpointer_%s"%name, (_ndptr,), + base = _ndptr + + klass = type("ndpointer_%s"%name, (base,), {"_dtype_": dtype, "_shape_" : shape, "_ndim_" : ndim, "_flags_" : num}) - _pointer_type_cache[(dtype, shape, ndim, num)] = klass + _pointer_type_cache[cache_key] = klass return klass diff --git a/numpy/distutils/ccompiler.py b/numpy/distutils/ccompiler.py index 4b5b96aad..2c80685c3 100644 --- a/numpy/distutils/ccompiler.py +++ b/numpy/distutils/ccompiler.py @@ -19,7 +19,7 @@ from numpy.distutils import log from numpy.distutils.compat import get_exception from numpy.distutils.exec_command import filepath_from_subprocess_output from numpy.distutils.misc_util import cyg2win32, is_sequence, mingw32, \ - quote_args, get_num_build_jobs, \ + get_num_build_jobs, \ _commandline_dep_string # globals for parallel build management @@ -772,8 +772,13 @@ ccompiler.new_compiler = new_compiler _distutils_gen_lib_options = gen_lib_options def gen_lib_options(compiler, library_dirs, runtime_library_dirs, libraries): - library_dirs = quote_args(library_dirs) - runtime_library_dirs = quote_args(runtime_library_dirs) + # the version of this function provided by CPython allows the following + # to return lists, which are unpacked automatically: + # - compiler.runtime_library_dir_option + # our version extends the behavior to: + # - compiler.library_dir_option + # - compiler.library_option + # - compiler.find_library_file r = _distutils_gen_lib_options(compiler, library_dirs, runtime_library_dirs, libraries) lib_opts = [] @@ -793,11 +798,6 @@ for _cc in ['msvc9', 'msvc', '_msvc', 'bcpp', 'cygwinc', 'emxc', 'unixc']: if _m is not None: setattr(_m, 'gen_lib_options', gen_lib_options) -_distutils_gen_preprocess_options = gen_preprocess_options -def gen_preprocess_options (macros, include_dirs): - include_dirs = quote_args(include_dirs) - return _distutils_gen_preprocess_options(macros, include_dirs) -ccompiler.gen_preprocess_options = gen_preprocess_options ##Fix distutils.util.split_quoted: # NOTE: I removed this fix in revision 4481 (see ticket #619), but it appears diff --git a/numpy/distutils/exec_command.py b/numpy/distutils/exec_command.py index af7810d75..f2916d24f 100644 --- a/numpy/distutils/exec_command.py +++ b/numpy/distutils/exec_command.py @@ -67,8 +67,10 @@ def filepath_from_subprocess_output(output): Inherited from `exec_command`, and possibly incorrect. """ - output = output.decode(locale.getpreferredencoding(False), - errors='replace') + mylocale = locale.getpreferredencoding(False) + if mylocale is None: + mylocale = 'ascii' + output = output.decode(mylocale, errors='replace') output = output.replace('\r\n', '\n') # Another historical oddity if output[-1:] == '\n': @@ -278,9 +280,10 @@ def _exec_command(command, use_shell=None, use_tee = None, **env): return 127, '' text, err = proc.communicate() - text = text.decode(locale.getpreferredencoding(False), - errors='replace') - + mylocale = locale.getpreferredencoding(False) + if mylocale is None: + mylocale = 'ascii' + text = text.decode(mylocale, errors='replace') text = text.replace('\r\n', '\n') # Another historical oddity if text[-1:] == '\n': diff --git a/numpy/distutils/fcompiler/gnu.py b/numpy/distutils/fcompiler/gnu.py index 0f7e48152..81769e562 100644 --- a/numpy/distutils/fcompiler/gnu.py +++ b/numpy/distutils/fcompiler/gnu.py @@ -315,6 +315,12 @@ class Gnu95FCompiler(GnuFCompiler): module_dir_switch = '-J' module_include_switch = '-I' + if sys.platform[:3] == 'aix': + executables['linker_so'].append('-lpthread') + if platform.architecture()[0][:2] == '64': + for key in ['compiler_f77', 'compiler_f90','compiler_fix','linker_so', 'linker_exe']: + executables[key].append('-maix64') + g2c = 'gfortran' def _universal_flags(self, cmd): diff --git a/numpy/doc/structured_arrays.py b/numpy/doc/structured_arrays.py index ab97c5df6..42711a7c0 100644 --- a/numpy/doc/structured_arrays.py +++ b/numpy/doc/structured_arrays.py @@ -443,6 +443,15 @@ behavior since Numpy 1.7. >>> repack_fields(a[['a','c']]).view('i8') # supported 1.15 and 1.16 array([0, 0, 0]) + The :module:`numpy.lib.recfunctions` module has other new methods + introduced in numpy 1.16 to help users account for this change. These are + :func:`numpy.lib.recfunctions.structured_to_unstructured`, + :func:`numpy.lib.recfunctions.unstructured_to_structured`, + :func:`numpy.lib.recfunctions.apply_along_fields`, + :func:`numpy.lib.recfunctions.assign_fields_by_name`, and + :func:`numpy.lib.recfunctions.require_fields`. + + Assigning to an array with a multi-field index will behave the same in Numpy 1.15 and Numpy 1.16. In both versions the assignment will modify the original array:: diff --git a/numpy/lib/histograms.py b/numpy/lib/histograms.py index 06b30f978..482eabe14 100644 --- a/numpy/lib/histograms.py +++ b/numpy/lib/histograms.py @@ -21,7 +21,7 @@ array_function_dispatch = functools.partial( _range = range -def _hist_bin_sqrt(x): +def _hist_bin_sqrt(x, range): """ Square root histogram bin estimator. @@ -38,10 +38,11 @@ def _hist_bin_sqrt(x): ------- h : An estimate of the optimal bin width for the given data. """ + del range # unused return x.ptp() / np.sqrt(x.size) -def _hist_bin_sturges(x): +def _hist_bin_sturges(x, range): """ Sturges histogram bin estimator. @@ -60,10 +61,11 @@ def _hist_bin_sturges(x): ------- h : An estimate of the optimal bin width for the given data. """ + del range # unused return x.ptp() / (np.log2(x.size) + 1.0) -def _hist_bin_rice(x): +def _hist_bin_rice(x, range): """ Rice histogram bin estimator. @@ -83,10 +85,11 @@ def _hist_bin_rice(x): ------- h : An estimate of the optimal bin width for the given data. """ + del range # unused return x.ptp() / (2.0 * x.size ** (1.0 / 3)) -def _hist_bin_scott(x): +def _hist_bin_scott(x, range): """ Scott histogram bin estimator. @@ -104,10 +107,52 @@ def _hist_bin_scott(x): ------- h : An estimate of the optimal bin width for the given data. """ + del range # unused return (24.0 * np.pi**0.5 / x.size)**(1.0 / 3.0) * np.std(x) -def _hist_bin_doane(x): +def _hist_bin_stone(x, range): + """ + Histogram bin estimator based on minimizing the estimated integrated squared error (ISE). + + The number of bins is chosen by minimizing the estimated ISE against the unknown true distribution. + The ISE is estimated using cross-validation and can be regarded as a generalization of Scott's rule. + https://en.wikipedia.org/wiki/Histogram#Scott.27s_normal_reference_rule + + This paper by Stone appears to be the origination of this rule. + http://digitalassets.lib.berkeley.edu/sdtr/ucb/text/34.pdf + + Parameters + ---------- + x : array_like + Input data that is to be histogrammed, trimmed to range. May not + be empty. + range : (float, float) + The lower and upper range of the bins. + + Returns + ------- + h : An estimate of the optimal bin width for the given data. + """ + + n = x.size + ptp_x = np.ptp(x) + if n <= 1 or ptp_x == 0: + return 0 + + def jhat(nbins): + hh = ptp_x / nbins + p_k = np.histogram(x, bins=nbins, range=range)[0] / n + return (2 - (n + 1) * p_k.dot(p_k)) / hh + + nbins_upper_bound = max(100, int(np.sqrt(n))) + nbins = min(_range(1, nbins_upper_bound + 1), key=jhat) + if nbins == nbins_upper_bound: + warnings.warn("The number of bins estimated may be suboptimal.", RuntimeWarning, stacklevel=2) + return ptp_x / nbins + + +def _hist_bin_doane(x, range): """ Doane's histogram bin estimator. @@ -125,6 +170,7 @@ def _hist_bin_doane(x): ------- h : An estimate of the optimal bin width for the given data. """ + del range # unused if x.size > 2: sg1 = np.sqrt(6.0 * (x.size - 2) / ((x.size + 1.0) * (x.size + 3))) sigma = np.std(x) @@ -141,7 +187,7 @@ def _hist_bin_doane(x): return 0.0 -def _hist_bin_fd(x): +def _hist_bin_fd(x, range): """ The Freedman-Diaconis histogram bin estimator. @@ -166,11 +212,12 @@ def _hist_bin_fd(x): ------- h : An estimate of the optimal bin width for the given data. """ + del range # unused iqr = np.subtract(*np.percentile(x, [75, 25])) return 2.0 * iqr * x.size ** (-1.0 / 3.0) -def _hist_bin_auto(x): +def _hist_bin_auto(x, range): """ Histogram bin estimator that uses the minimum width of the Freedman-Diaconis and Sturges estimators if the FD bandwidth is non zero @@ -204,8 +251,9 @@ def _hist_bin_auto(x): -------- _hist_bin_fd, _hist_bin_sturges """ - fd_bw = _hist_bin_fd(x) - sturges_bw = _hist_bin_sturges(x) + fd_bw = _hist_bin_fd(x, range) + sturges_bw = _hist_bin_sturges(x, range) + del range # unused if fd_bw: return min(fd_bw, sturges_bw) else: @@ -213,7 +261,8 @@ def _hist_bin_auto(x): return sturges_bw # Private dict initialized at module load time -_hist_bin_selectors = {'auto': _hist_bin_auto, +_hist_bin_selectors = {'stone': _hist_bin_stone, + 'auto': _hist_bin_auto, 'doane': _hist_bin_doane, 'fd': _hist_bin_fd, 'rice': _hist_bin_rice, @@ -348,7 +397,7 @@ def _get_bin_edges(a, bins, range, weights): n_equal_bins = 1 else: # Do not call selectors on empty arrays - width = _hist_bin_selectors[bin_name](a) + width = _hist_bin_selectors[bin_name](a, (first_edge, last_edge)) if width: n_equal_bins = int(np.ceil(_unsigned_subtract(last_edge, first_edge) / width)) else: @@ -450,6 +499,11 @@ def histogram_bin_edges(a, bins=10, range=None, weights=None): Less robust estimator that that takes into account data variability and data size. + 'stone' + Estimator based on leave-one-out cross-validation estimate of + the integrated squared error. Can be regarded as a generalization + of Scott's rule. + 'rice' Estimator does not take variability into account, only data size. Commonly overestimates number of bins required. diff --git a/numpy/lib/npyio.py b/numpy/lib/npyio.py index 1da5b0a25..f623c58e7 100644 --- a/numpy/lib/npyio.py +++ b/numpy/lib/npyio.py @@ -441,8 +441,8 @@ def load(file, mmap_mode=None, allow_pickle=True, fix_imports=True, else: # Try a pickle if not allow_pickle: - raise ValueError("allow_pickle=False, but file does not contain " - "non-pickled data") + raise ValueError("Cannot load file containing pickled data " + "when allow_pickle=False") try: return pickle.load(fid, **pickle_kwargs) except Exception: diff --git a/numpy/lib/recfunctions.py b/numpy/lib/recfunctions.py index 53a586f56..20e91af5f 100644 --- a/numpy/lib/recfunctions.py +++ b/numpy/lib/recfunctions.py @@ -102,7 +102,7 @@ def get_fieldspec(dtype): fields = ((name, dtype.fields[name]) for name in dtype.names) # keep any titles, if present return [ - (name if len(f) == 2 else (f[2], name), f[0]) + (name if len(f) == 2 else (f[2], name), f[0]) for name, f in fields ] @@ -870,6 +870,330 @@ def repack_fields(a, align=False, recurse=False): dt = np.dtype(fieldinfo, align=align) return np.dtype((a.type, dt)) +def _get_fields_and_offsets(dt, offset=0): + """ + Returns a flat list of (name, dtype, count, offset) tuples of all the + scalar fields in the dtype "dt", including nested fields, in left + to right order. + """ + fields = [] + for name in dt.names: + field = dt.fields[name] + if field[0].names is None: + count = 1 + for size in field[0].shape: + count *= size + fields.append((name, field[0], count, field[1] + offset)) + else: + fields.extend(_get_fields_and_offsets(field[0], field[1] + offset)) + return fields + + +def _structured_to_unstructured_dispatcher(arr, dtype=None, copy=None, + casting=None): + return (arr,) + +@array_function_dispatch(_structured_to_unstructured_dispatcher) +def structured_to_unstructured(arr, dtype=None, copy=False, casting='unsafe'): + """ + Converts and n-D structured array into an (n+1)-D unstructured array. + + The new array will have a new last dimension equal in size to the + number of field-elements of the input array. If not supplied, the output + datatype is determined from the numpy type promotion rules applied to all + the field datatypes. + + Nested fields, as well as each element of any subarray fields, all count + as a single field-elements. + + Parameters + ---------- + arr : ndarray + Structured array or dtype to convert. Cannot contain object datatype. + dtype : dtype, optional + The dtype of the output unstructured array + copy : bool, optional + See copy argument to `ndarray.astype`. If true, always return a copy. + If false, and `dtype` requirements are satisfied, a view is returned. + casting : {'no', 'equiv', 'safe', 'same_kind', 'unsafe'}, optional + See casting argument of `ndarray.astype`. Controls what kind of data + casting may occur. + + Returns + ------- + unstructured : ndarray + Unstructured array with one more dimension. + + Examples + -------- + + >>> a = np.zeros(4, dtype=[('a', 'i4'), ('b', 'f4,u2'), ('c', 'f4', 2)]) + >>> a + array([(0, (0., 0), [0., 0.]), (0, (0., 0), [0., 0.]), + (0, (0., 0), [0., 0.]), (0, (0., 0), [0., 0.])], + dtype=[('a', '<i4'), ('b', [('f0', '<f4'), ('f1', '<u2')]), ('c', '<f4', (2,))]) + >>> structured_to_unstructured(arr) + array([[0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.]]) + + >>> b = np.array([(1, 2, 5), (4, 5, 7), (7, 8 ,11), (10, 11, 12)], + ... dtype=[('x', 'i4'), ('y', 'f4'), ('z', 'f8')]) + >>> np.mean(structured_to_unstructured(b[['x', 'z']]), axis=-1) + array([ 3. , 5.5, 9. , 11. ]) + + """ + if arr.dtype.names is None: + raise ValueError('arr must be a structured array') + + fields = _get_fields_and_offsets(arr.dtype) + names, dts, counts, offsets = zip(*fields) + n_fields = len(names) + + if dtype is None: + out_dtype = np.result_type(*[dt.base for dt in dts]) + else: + out_dtype = dtype + + # Use a series of views and casts to convert to an unstructured array: + + # first view using flattened fields (doesn't work for object arrays) + # Note: dts may include a shape for subarrays + flattened_fields = np.dtype({'names': names, + 'formats': dts, + 'offsets': offsets, + 'itemsize': arr.dtype.itemsize}) + arr = arr.view(flattened_fields) + + # next cast to a packed format with all fields converted to new dtype + packed_fields = np.dtype({'names': names, + 'formats': [(out_dtype, c) for c in counts]}) + arr = arr.astype(packed_fields, copy=copy, casting=casting) + + # finally is it safe to view the packed fields as the unstructured type + return arr.view((out_dtype, sum(counts))) + +def _unstructured_to_structured_dispatcher(arr, dtype=None, names=None, + align=None, copy=None, casting=None): + return (arr,) + +@array_function_dispatch(_unstructured_to_structured_dispatcher) +def unstructured_to_structured(arr, dtype=None, names=None, align=False, + copy=False, casting='unsafe'): + """ + Converts and n-D unstructured array into an (n-1)-D structured array. + + The last dimension of the input array is converted into a structure, with + number of field-elements equal to the size of the last dimension of the + input array. By default all output fields have the input array's dtype, but + an output structured dtype with an equal number of fields-elements can be + supplied instead. + + Nested fields, as well as each element of any subarray fields, all count + towards the number of field-elements. + + Parameters + ---------- + arr : ndarray + Unstructured array or dtype to convert. + dtype : dtype, optional + The structured dtype of the output array + names : list of strings, optional + If dtype is not supplied, this specifies the field names for the output + dtype, in order. The field dtypes will be the same as the input array. + align : boolean, optional + Whether to create an aligned memory layout. + copy : bool, optional + See copy argument to `ndarray.astype`. If true, always return a copy. + If false, and `dtype` requirements are satisfied, a view is returned. + casting : {'no', 'equiv', 'safe', 'same_kind', 'unsafe'}, optional + See casting argument of `ndarray.astype`. Controls what kind of data + casting may occur. + + Returns + ------- + structured : ndarray + Structured array with fewer dimensions. + + Examples + -------- + + >>> dt = np.dtype([('a', 'i4'), ('b', 'f4,u2'), ('c', 'f4', 2)]) + >>> a = np.arange(20).reshape((4,5)) + >>> a + array([[ 0, 1, 2, 3, 4], + [ 5, 6, 7, 8, 9], + [10, 11, 12, 13, 14], + [15, 16, 17, 18, 19]]) + >>> unstructured_to_structured(a, dt) + array([( 0, ( 1., 2), [ 3., 4.]), ( 5, ( 6., 7), [ 8., 9.]), + (10, (11., 12), [13., 14.]), (15, (16., 17), [18., 19.])], + dtype=[('a', '<i4'), ('b', [('f0', '<f4'), ('f1', '<u2')]), ('c', '<f4', (2,))]) + + """ + if arr.shape == (): + raise ValueError('arr must have at least one dimension') + n_elem = arr.shape[-1] + + if dtype is None: + if names is None: + names = ['f{}'.format(n) for n in range(n_elem)] + out_dtype = np.dtype([(n, arr.dtype) for n in names], align=align) + fields = _get_fields_and_offsets(out_dtype) + names, dts, counts, offsets = zip(*fields) + else: + if names is not None: + raise ValueError("don't supply both dtype and names") + # sanity check of the input dtype + fields = _get_fields_and_offsets(dtype) + names, dts, counts, offsets = zip(*fields) + if n_elem != sum(counts): + raise ValueError('The length of the last dimension of arr must ' + 'be equal to the number of fields in dtype') + out_dtype = dtype + if align and not out_dtype.isalignedstruct: + raise ValueError("align was True but dtype is not aligned") + + # Use a series of views and casts to convert to a structured array: + + # first view as a packed structured array of one dtype + packed_fields = np.dtype({'names': names, + 'formats': [(arr.dtype, c) for c in counts]}) + arr = np.ascontiguousarray(arr).view(packed_fields) + + # next cast to an unpacked but flattened format with varied dtypes + flattened_fields = np.dtype({'names': names, + 'formats': dts, + 'offsets': offsets, + 'itemsize': out_dtype.itemsize}) + arr = arr.astype(flattened_fields, copy=copy, casting=casting) + + # finally view as the final nested dtype and remove the last axis + return arr.view(out_dtype)[..., 0] + +def _apply_along_fields_dispatcher(func, arr): + return (arr,) + +@array_function_dispatch(_apply_along_fields_dispatcher) +def apply_along_fields(func, arr): + """ + Apply function 'func' as a reduction across fields of a structured array. + + This is similar to `apply_along_axis`, but treats the fields of a + structured array as an extra axis. + + Parameters + ---------- + func : function + Function to apply on the "field" dimension. This function must + support an `axis` argument, like np.mean, np.sum, etc. + arr : ndarray + Structured array for which to apply func. + + Returns + ------- + out : ndarray + Result of the recution operation + + Examples + -------- + + >>> b = np.array([(1, 2, 5), (4, 5, 7), (7, 8 ,11), (10, 11, 12)], + ... dtype=[('x', 'i4'), ('y', 'f4'), ('z', 'f8')]) + >>> apply_along_fields(np.mean, b) + array([ 2.66666667, 5.33333333, 8.66666667, 11. ]) + >>> apply_along_fields(np.mean, b[['x', 'z']]) + array([ 3. , 5.5, 9. , 11. ]) + + """ + if arr.dtype.names is None: + raise ValueError('arr must be a structured array') + + uarr = structured_to_unstructured(arr) + return func(uarr, axis=-1) + # works and avoids axis requirement, but very, very slow: + #return np.apply_along_axis(func, -1, uarr) + +def _assign_fields_by_name_dispatcher(dst, src, zero_unassigned=None): + return dst, src + +@array_function_dispatch(_assign_fields_by_name_dispatcher) +def assign_fields_by_name(dst, src, zero_unassigned=True): + """ + Assigns values from one structured array to another by field name. + + Normally in numpy >= 1.14, assignment of one structured array to another + copies fields "by position", meaning that the first field from the src is + copied to the first field of the dst, and so on, regardless of field name. + + This function instead copies "by field name", such that fields in the dst + are assigned from the identically named field in the src. This applies + recursively for nested structures. This is how structure assignment worked + in numpy >= 1.6 to <= 1.13. + + Parameters + ---------- + dst : ndarray + src : ndarray + The source and destination arrays during assignment. + zero_unassigned : bool, optional + If True, fields in the dst for which there was no matching + field in the src are filled with the value 0 (zero). This + was the behavior of numpy <= 1.13. If False, those fields + are not modified. + """ + + if dst.dtype.names is None: + dst[:] = src + return + + for name in dst.dtype.names: + if name not in src.dtype.names: + if zero_unassigned: + dst[name] = 0 + else: + assign_fields_by_name(dst[name], src[name], + zero_unassigned) + +def _require_fields_dispatcher(array, required_dtype): + return (array,) + +@array_function_dispatch(_require_fields_dispatcher) +def require_fields(array, required_dtype): + """ + Casts a structured array to a new dtype using assignment by field-name. + + This function assigns to from the old to the new array by name, so the + value of a field in the output array is the value of the field with the + same name in the source array. + + If a field name in the required_dtype does not exist in the + input array, that field is set to 0 in the output array. + + Parameters + ---------- + a : ndarray + array to cast + required_dtype : dtype + datatype for output array + + Returns + ------- + out : ndarray + array with the new dtype, with field values copied from the fields in + the input array with the same name + + Examples + -------- + + >>> a = np.ones(4, dtype=[('a', 'i4'), ('b', 'f8'), ('c', 'u1')]) + >>> require_fields(a, [('b', 'f4'), ('c', 'u1')]) + """ + out = np.empty(array.shape, dtype=required_dtype) + assign_fields_by_name(out, array) + return out + def _stack_arrays_dispatcher(arrays, defaults=None, usemask=None, asrecarray=None, autoconvert=None): diff --git a/numpy/lib/tests/test_histograms.py b/numpy/lib/tests/test_histograms.py index 5b51763b2..49c0d9720 100644 --- a/numpy/lib/tests/test_histograms.py +++ b/numpy/lib/tests/test_histograms.py @@ -431,7 +431,7 @@ class TestHistogramOptimBinNums(object): def test_empty(self): estimator_list = ['fd', 'scott', 'rice', 'sturges', - 'doane', 'sqrt', 'auto'] + 'doane', 'sqrt', 'auto', 'stone'] # check it can deal with empty data for estimator in estimator_list: a, b = histogram([], bins=estimator) @@ -447,11 +447,11 @@ class TestHistogramOptimBinNums(object): # Some basic sanity checking, with some fixed data. # Checking for the correct number of bins basic_test = {50: {'fd': 4, 'scott': 4, 'rice': 8, 'sturges': 7, - 'doane': 8, 'sqrt': 8, 'auto': 7}, + 'doane': 8, 'sqrt': 8, 'auto': 7, 'stone': 2}, 500: {'fd': 8, 'scott': 8, 'rice': 16, 'sturges': 10, - 'doane': 12, 'sqrt': 23, 'auto': 10}, + 'doane': 12, 'sqrt': 23, 'auto': 10, 'stone': 9}, 5000: {'fd': 17, 'scott': 17, 'rice': 35, 'sturges': 14, - 'doane': 17, 'sqrt': 71, 'auto': 17}} + 'doane': 17, 'sqrt': 71, 'auto': 17, 'stone': 20}} for testlen, expectedResults in basic_test.items(): # Create some sort of non uniform data to test with @@ -471,11 +471,11 @@ class TestHistogramOptimBinNums(object): precalculated. """ small_dat = {1: {'fd': 1, 'scott': 1, 'rice': 1, 'sturges': 1, - 'doane': 1, 'sqrt': 1}, + 'doane': 1, 'sqrt': 1, 'stone': 1}, 2: {'fd': 2, 'scott': 1, 'rice': 3, 'sturges': 2, - 'doane': 1, 'sqrt': 2}, + 'doane': 1, 'sqrt': 2, 'stone': 1}, 3: {'fd': 2, 'scott': 2, 'rice': 3, 'sturges': 3, - 'doane': 3, 'sqrt': 2}} + 'doane': 3, 'sqrt': 2, 'stone': 1}} for testlen, expectedResults in small_dat.items(): testdat = np.arange(testlen) @@ -499,7 +499,7 @@ class TestHistogramOptimBinNums(object): """ novar_dataset = np.ones(100) novar_resultdict = {'fd': 1, 'scott': 1, 'rice': 1, 'sturges': 1, - 'doane': 1, 'sqrt': 1, 'auto': 1} + 'doane': 1, 'sqrt': 1, 'auto': 1, 'stone': 1} for estimator, numbins in novar_resultdict.items(): a, b = np.histogram(novar_dataset, estimator) @@ -538,12 +538,32 @@ class TestHistogramOptimBinNums(object): xcenter = np.linspace(-10, 10, 50) outlier_dataset = np.hstack((np.linspace(-110, -100, 5), xcenter)) - outlier_resultdict = {'fd': 21, 'scott': 5, 'doane': 11} + outlier_resultdict = {'fd': 21, 'scott': 5, 'doane': 11, 'stone': 6} for estimator, numbins in outlier_resultdict.items(): a, b = np.histogram(outlier_dataset, estimator) assert_equal(len(a), numbins) + def test_scott_vs_stone(self): + """Verify that Scott's rule and Stone's rule converges for normally distributed data""" + + def nbins_ratio(seed, size): + rng = np.random.RandomState(seed) + x = rng.normal(loc=0, scale=2, size=size) + a, b = len(np.histogram(x, 'stone')[0]), len(np.histogram(x, 'scott')[0]) + return a / (a + b) + + ll = [[nbins_ratio(seed, size) for size in np.geomspace(start=10, stop=100, num=4).round().astype(int)] + for seed in range(256)] + + # the average difference between the two methods decreases as the dataset size increases. + assert_almost_equal(abs(np.mean(ll, axis=0) - 0.5), + [0.1065248, + 0.0968844, + 0.0331818, + 0.0178057], + decimal=3) + def test_simple_range(self): """ Straightforward testing with a mixture of linspace data (for @@ -555,11 +575,11 @@ class TestHistogramOptimBinNums(object): # Checking for the correct number of bins basic_test = { 50: {'fd': 8, 'scott': 8, 'rice': 15, - 'sturges': 14, 'auto': 14}, + 'sturges': 14, 'auto': 14, 'stone': 8}, 500: {'fd': 15, 'scott': 16, 'rice': 32, - 'sturges': 20, 'auto': 20}, + 'sturges': 20, 'auto': 20, 'stone': 80}, 5000: {'fd': 33, 'scott': 33, 'rice': 69, - 'sturges': 27, 'auto': 33} + 'sturges': 27, 'auto': 33, 'stone': 80} } for testlen, expectedResults in basic_test.items(): diff --git a/numpy/lib/tests/test_recfunctions.py b/numpy/lib/tests/test_recfunctions.py index 5585a95f9..7ec33d92a 100644 --- a/numpy/lib/tests/test_recfunctions.py +++ b/numpy/lib/tests/test_recfunctions.py @@ -10,7 +10,8 @@ from numpy.testing import assert_, assert_raises from numpy.lib.recfunctions import ( drop_fields, rename_fields, get_fieldstructure, recursive_fill_fields, find_duplicates, merge_arrays, append_fields, stack_arrays, join_by, - repack_fields) + repack_fields, unstructured_to_structured, structured_to_unstructured, + apply_along_fields, require_fields, assign_fields_by_name) get_names = np.lib.recfunctions.get_names get_names_flat = np.lib.recfunctions.get_names_flat zip_descr = np.lib.recfunctions.zip_descr @@ -204,6 +205,64 @@ class TestRecFunctions(object): dt = np.dtype((np.record, dt)) assert_(repack_fields(dt).type is np.record) + def test_structured_to_unstructured(self): + a = np.zeros(4, dtype=[('a', 'i4'), ('b', 'f4,u2'), ('c', 'f4', 2)]) + out = structured_to_unstructured(a) + assert_equal(out, np.zeros((4,5), dtype='f8')) + + b = np.array([(1, 2, 5), (4, 5, 7), (7, 8 ,11), (10, 11, 12)], + dtype=[('x', 'i4'), ('y', 'f4'), ('z', 'f8')]) + out = np.mean(structured_to_unstructured(b[['x', 'z']]), axis=-1) + assert_equal(out, np.array([ 3. , 5.5, 9. , 11. ])) + + c = np.arange(20).reshape((4,5)) + out = unstructured_to_structured(c, a.dtype) + want = np.array([( 0, ( 1., 2), [ 3., 4.]), + ( 5, ( 6., 7), [ 8., 9.]), + (10, (11., 12), [13., 14.]), + (15, (16., 17), [18., 19.])], + dtype=[('a', '<i4'), + ('b', [('f0', '<f4'), ('f1', '<u2')]), + ('c', '<f4', (2,))]) + assert_equal(out, want) + + d = np.array([(1, 2, 5), (4, 5, 7), (7, 8 ,11), (10, 11, 12)], + dtype=[('x', 'i4'), ('y', 'f4'), ('z', 'f8')]) + assert_equal(apply_along_fields(np.mean, d), + np.array([ 8.0/3, 16.0/3, 26.0/3, 11. ])) + assert_equal(apply_along_fields(np.mean, d[['x', 'z']]), + np.array([ 3. , 5.5, 9. , 11. ])) + + # check that for uniform field dtypes we get a view, not a copy: + d = np.array([(1, 2, 5), (4, 5, 7), (7, 8 ,11), (10, 11, 12)], + dtype=[('x', 'i4'), ('y', 'i4'), ('z', 'i4')]) + dd = structured_to_unstructured(d) + ddd = unstructured_to_structured(dd, d.dtype) + assert_(dd.base is d) + assert_(ddd.base is d) + + def test_field_assignment_by_name(self): + a = np.ones(2, dtype=[('a', 'i4'), ('b', 'f8'), ('c', 'u1')]) + newdt = [('b', 'f4'), ('c', 'u1')] + + assert_equal(require_fields(a, newdt), np.ones(2, newdt)) + + b = np.array([(1,2), (3,4)], dtype=newdt) + assign_fields_by_name(a, b, zero_unassigned=False) + assert_equal(a, np.array([(1,1,2),(1,3,4)], dtype=a.dtype)) + assign_fields_by_name(a, b) + assert_equal(a, np.array([(0,1,2),(0,3,4)], dtype=a.dtype)) + + # test nested fields + a = np.ones(2, dtype=[('a', [('b', 'f8'), ('c', 'u1')])]) + newdt = [('a', [('c', 'u1')])] + assert_equal(require_fields(a, newdt), np.ones(2, newdt)) + b = np.array([((2,),), ((3,),)], dtype=newdt) + assign_fields_by_name(a, b, zero_unassigned=False) + assert_equal(a, np.array([((1,2),), ((1,3),)], dtype=a.dtype)) + assign_fields_by_name(a, b) + assert_equal(a, np.array([((0,2),), ((0,3),)], dtype=a.dtype)) + class TestRecursiveFillFields(object): # Test recursive_fill_fields. diff --git a/numpy/tests/test_ctypeslib.py b/numpy/tests/test_ctypeslib.py index 675f8d242..53b75db07 100644 --- a/numpy/tests/test_ctypeslib.py +++ b/numpy/tests/test_ctypeslib.py @@ -9,20 +9,30 @@ from numpy.distutils.misc_util import get_shared_lib_extension from numpy.testing import assert_, assert_array_equal, assert_raises, assert_equal try: + import ctypes +except ImportError: + ctypes = None +else: cdll = None + test_cdll = None if hasattr(sys, 'gettotalrefcount'): try: cdll = load_library('_multiarray_umath_d', np.core._multiarray_umath.__file__) except OSError: pass + try: + test_cdll = load_library('_multiarray_tests', np.core._multiarray_tests.__file__) + except OSError: + pass if cdll is None: cdll = load_library('_multiarray_umath', np.core._multiarray_umath.__file__) - _HAS_CTYPE = True -except ImportError: - _HAS_CTYPE = False + if test_cdll is None: + test_cdll = load_library('_multiarray_tests', np.core._multiarray_tests.__file__) + c_forward_pointer = test_cdll.forward_pointer -@pytest.mark.skipif(not _HAS_CTYPE, + +@pytest.mark.skipif(ctypes is None, reason="ctypes not available in this python") @pytest.mark.skipif(sys.platform == 'cygwin', reason="Known to fail on cygwin") @@ -108,12 +118,72 @@ class TestNdpointer(object): assert_raises(TypeError, p.from_param, np.array([[1, 2], [3, 4]])) def test_cache(self): - a1 = ndpointer(dtype=np.float64) - a2 = ndpointer(dtype=np.float64) - assert_(a1 == a2) + assert_(ndpointer(dtype=np.float64) is ndpointer(dtype=np.float64)) + + # shapes are normalized + assert_(ndpointer(shape=2) is ndpointer(shape=(2,))) + + # 1.12 <= v < 1.16 had a bug that made these fail + assert_(ndpointer(shape=2) is not ndpointer(ndim=2)) + assert_(ndpointer(ndim=2) is not ndpointer(shape=2)) + +@pytest.mark.skipif(ctypes is None, + reason="ctypes not available on this python installation") +class TestNdpointerCFunc(object): + def test_arguments(self): + """ Test that arguments are coerced from arrays """ + c_forward_pointer.restype = ctypes.c_void_p + c_forward_pointer.argtypes = (ndpointer(ndim=2),) + + c_forward_pointer(np.zeros((2, 3))) + # too many dimensions + assert_raises( + ctypes.ArgumentError, c_forward_pointer, np.zeros((2, 3, 4))) + + @pytest.mark.parametrize( + 'dt', [ + float, + np.dtype(dict( + formats=['<i4', '<i4'], + names=['a', 'b'], + offsets=[0, 2], + itemsize=6 + )) + ], ids=[ + 'float', + 'overlapping-fields' + ] + ) + def test_return(self, dt): + """ Test that return values are coerced to arrays """ + arr = np.zeros((2, 3), dt) + ptr_type = ndpointer(shape=arr.shape, dtype=arr.dtype) + + c_forward_pointer.restype = ptr_type + c_forward_pointer.argtypes = (ptr_type,) + + # check that the arrays are equivalent views on the same data + arr2 = c_forward_pointer(arr) + assert_equal(arr2.dtype, arr.dtype) + assert_equal(arr2.shape, arr.shape) + assert_equal( + arr2.__array_interface__['data'], + arr.__array_interface__['data'] + ) + + def test_vague_return_value(self): + """ Test that vague ndpointer return values do not promote to arrays """ + arr = np.zeros((2, 3)) + ptr_type = ndpointer(dtype=arr.dtype) + + c_forward_pointer.restype = ptr_type + c_forward_pointer.argtypes = (ptr_type,) + + ret = c_forward_pointer(arr) + assert_(isinstance(ret, ptr_type)) -@pytest.mark.skipif(not _HAS_CTYPE, +@pytest.mark.skipif(ctypes is None, reason="ctypes not available on this python installation") class TestAsArray(object): def test_array(self): diff --git a/numpy/tests/test_public_api.py b/numpy/tests/test_public_api.py index 856cca8eb..194f8ecbb 100644 --- a/numpy/tests/test_public_api.py +++ b/numpy/tests/test_public_api.py @@ -4,7 +4,10 @@ import sys import numpy as np import pytest - +try: + import ctypes +except ImportError: + ctypes = None def check_dir(module, module_name=None): """Returns a mapping of all objects with the wrong __module__ attribute.""" @@ -75,3 +78,12 @@ def test_numpy_linalg(): def test_numpy_fft(): bad_results = check_dir(np.fft) assert bad_results == {} + +@pytest.mark.skipif(ctypes is None, + reason="ctypes not available in this python") +def test_NPY_NO_EXPORT(): + cdll = ctypes.CDLL(np.core._multiarray_tests.__file__) + # Make sure an arbitrary NPY_NO_EXPORT function is actually hidden + f = getattr(cdll, 'test_not_exported', None) + assert f is None, ("'test_not_exported' is mistakenly exported, " + "NPY_NO_EXPORT does not work") |