diff options
114 files changed, 2794 insertions, 516 deletions
diff --git a/.travis.yml b/.travis.yml index 73106b4c1..84fdbb8f7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ language: python group: travis_latest os: linux -dist: xenial +dist: bionic # Travis whitelists the installable packages, additions can be requested # https://github.com/travis-ci/apt-package-whitelist @@ -11,6 +11,8 @@ addons: apt: packages: &common_packages - gfortran + - libgfortran5 + - libgfortran3 - libatlas-base-dev # Speedup builds, particularly when USE_CHROOT=1 - eatmydata @@ -27,7 +29,7 @@ stages: env: global: - - OpenBLAS_version=0.3.7 + - OpenBLAS_version=0.3.8 - WHEELHOUSE_UPLOADER_USERNAME=travis.numpy # The following is generated with the command: # travis encrypt -r numpy/numpy WHEELHOUSE_UPLOADER_SECRET=tH3AP1KeY @@ -47,7 +49,6 @@ jobs: - python: 3.7 - python: 3.6 - dist: bionic env: USE_DEBUG=1 addons: apt: @@ -86,6 +87,8 @@ jobs: packages: - gfortran - eatmydata + - libgfortran5 + - libgfortran3 - python: 3.7 env: USE_WHEEL=1 NPY_RELAXED_STRIDES_DEBUG=1 @@ -112,8 +115,18 @@ jobs: arch: s390x env: # use s390x OpenBLAS build, not system ATLAS + - DOWNLOAD_OPENBLAS=1 - ATLAS=None + - python: 3.7 + os: linux + arch: aarch64 + env: + # use ppc64le OpenBLAS build, not system ATLAS + - DOWNLOAD_OPENBLAS=1 + - ATLAS=None + + before_install: - ./tools/travis-before-install.sh diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9a8acc9c1..7bd7a1e1f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -8,7 +8,7 @@ trigger: variables: # OpenBLAS_version should be updated # to match numpy-wheels repo - OpenBLAS_version: 0.3.7 + OpenBLAS_version: 0.3.8 stages: - stage: InitialTests diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json index b3c7f9f20..1046f10f2 100644 --- a/benchmarks/asv.conf.json +++ b/benchmarks/asv.conf.json @@ -42,7 +42,6 @@ // list indicates to just test against the default (latest) // version. "matrix": { - "six": [], "Cython": [], }, diff --git a/benchmarks/benchmarks/bench_app.py b/benchmarks/benchmarks/bench_app.py index 2a649f39b..bee95c201 100644 --- a/benchmarks/benchmarks/bench_app.py +++ b/benchmarks/benchmarks/bench_app.py @@ -2,8 +2,6 @@ from .common import Benchmark import numpy as np -from six.moves import xrange - class LaplaceInplace(Benchmark): params = ['inplace', 'normal'] @@ -59,7 +57,7 @@ class MaxesOfDots(Benchmark): ntime = 200 self.arrays = [np.random.normal(size=(ntime, nfeat)) - for i in xrange(nsubj)] + for i in range(nsubj)] def maxes_of_dots(self, arrays): """ diff --git a/benchmarks/benchmarks/bench_indexing.py b/benchmarks/benchmarks/bench_indexing.py index bb7596d0a..9ee0d1fb5 100644 --- a/benchmarks/benchmarks/bench_indexing.py +++ b/benchmarks/benchmarks/bench_indexing.py @@ -2,8 +2,6 @@ from .common import Benchmark, get_squares_, get_indexes_, get_indexes_rand_ from os.path import join as pjoin import shutil -import sys -import six from numpy import memmap, float32, array import numpy as np from tempfile import mkdtemp @@ -23,13 +21,10 @@ class Indexing(Benchmark): 'indexes_': get_indexes_(), 'indexes_rand_': get_indexes_rand_()} - if sys.version_info[0] >= 3: - code = "def run():\n for a in squares_.values(): a[%s]%s" - else: - code = "def run():\n for a in squares_.itervalues(): a[%s]%s" + code = "def run():\n for a in squares_.values(): a[%s]%s" code = code % (sel, op) - six.exec_(code, ns) + exec(code, ns) self.func = ns['run'] def time_op(self, indexes, sel, op): diff --git a/doc/release/upcoming_changes/15119.deprecation.rst b/doc/release/upcoming_changes/15119.deprecation.rst new file mode 100644 index 000000000..d18e440fe --- /dev/null +++ b/doc/release/upcoming_changes/15119.deprecation.rst @@ -0,0 +1,8 @@ + +Deprecate automatic ``dtype=object`` for ragged input +----------------------------------------------------- +Calling ``np.array([[1, [1, 2, 3]])`` will issue a ``DeprecationWarning`` as +per `NEP 34`_. Users should explicitly use ``dtype=object`` to avoid the +warning. + +.. _`NEP 34`: https://numpy.org/neps/nep-0034.html diff --git a/doc/release/upcoming_changes/template.rst b/doc/release/upcoming_changes/template.rst index 9c8a3b5fc..997b4850e 100644 --- a/doc/release/upcoming_changes/template.rst +++ b/doc/release/upcoming_changes/template.rst @@ -17,6 +17,7 @@ {% if definitions[category]['showcontent'] %} {% for text, values in sections[section][category].items() %} {{ text }} + {{ get_indent(text) }}({{values|join(', ') }}) {% endfor %} diff --git a/doc/source/conf.py b/doc/source/conf.py index 7e3a145f5..957cb17e6 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -179,6 +179,7 @@ latex_elements = { latex_elements['preamble'] = r''' % In the parameters section, place a newline after the Parameters % header +\usepackage{xcolor} \usepackage{expdlist} \let\latexdescription=\description \def\description{\latexdescription{}{} \breaklabel} diff --git a/doc/source/reference/internals.rst b/doc/source/reference/internals.rst index 03d081bf9..aacfabcd3 100644 --- a/doc/source/reference/internals.rst +++ b/doc/source/reference/internals.rst @@ -1,3 +1,5 @@ +.. _numpy-internals: + *************** NumPy internals *************** diff --git a/doc/source/reference/routines.io.rst b/doc/source/reference/routines.io.rst index 8bb29b793..cf66eab49 100644 --- a/doc/source/reference/routines.io.rst +++ b/doc/source/reference/routines.io.rst @@ -1,3 +1,5 @@ +.. _routines.io: + Input and output **************** diff --git a/doc/source/user/absolute_beginners.rst b/doc/source/user/absolute_beginners.rst new file mode 100644 index 000000000..acf9a35b5 --- /dev/null +++ b/doc/source/user/absolute_beginners.rst @@ -0,0 +1,1639 @@ + +**************************************** +NumPy: the absolute basics for beginners +**************************************** + +.. currentmodule:: numpy + +Welcome to the absolute beginner's guide to NumPy! If you have comments or +suggestions, please don’t hesitate to reach out! + + +Welcome to NumPy! +----------------- + +NumPy (**Numerical Python**) is an open source Python library that's used in +almost every field of science and engineering. It's the universal standard for +working with numerical data in Python, and it's at the core of the scientific +Python and PyData ecosystems. NumPy users include everyone from beginning coders +to experienced researchers doing state-of-the-art scientific and industrial +research and development. The NumPy API is used extensively in Pandas, SciPy, +Matplotlib, scikit-learn, scikit-image and most other data science and +scientific Python packages. + +The NumPy library contains multidimensional array and matrix data structures +(you'll find more information about this in later sections). It provides +**ndarray**, a homogeneous n-dimensional array object, with methods to +efficiently operate on it. NumPy can be used to perform a wide variety of +mathematical operations on arrays. It adds powerful data structures to Python +that guarantee efficient calculations with arrays and matrices and it supplies +an enormous library of high-level mathematical functions that operate on these +arrays and matrices. + +Learn more about :ref:`NumPy here <whatisnumpy>`! + +Installing NumPy +---------------- + +To install NumPy, we strongly recommend using a scientific Python distribution. +If you're looking for the full instructions for installing NumPy on your +operating system, you can `find all of the details here +<https://www.scipy.org/install.html>`_. + + + +If you already have Python, you can install NumPy with:: + + conda install numpy + +or :: + + pip install numpy + +If you don't have Python yet, you might want to consider using `Anaconda +<https://www.anaconda.com/>`_. It's the easiest way to get started. The good +thing about getting this distribution is the fact that you don’t need to worry +too much about separately installing NumPy or any of the major packages that +you’ll be using for your data analyses, like pandas, Scikit-Learn, etc. + +You can find all of the installation details in the +`Installation <https://www.scipy.org/install.html>`_ section +at `SciPy <https://www.scipy.org>`_. + +How to import NumPy +------------------- + +Any time you want to use a package or library in your code, you first need to +make it accessible. + +In order to start using NumPy and all of the functions available in NumPy, +you'll need to import it. This can be easily done with this import statement:: + + import numpy as np + +(We shorten ``numpy`` to ``np`` in order to save time and also to keep code +standardized so that anyone working with your code can easily understand and +run it.) + +Reading the example code +------------------------ + +If you aren't already comfortable with reading tutorials that contain a lot of code, +you might not know how to interpret a code block that looks +like this:: + + >>> a2 = a[np.newaxis, :] + >>> a2.shape + (1, 6) + +If you aren't familiar with this style, it's very easy to understand. +If you see ``>>>``, you're looking at **input**, or the code that +you would enter. Everything that doesn't have ``>>>`` in front of it +is **output**, or the results of running your code. This is the style +you see when you run ``python`` on the command line, but if you're using IPython, you might see a different style. + + +What’s the difference between a Python list and a NumPy array? +-------------------------------------------------------------- + +NumPy gives you an enormous range of fast and efficient ways of creating arrays +and manipulating numerical data inside them. While a Python list can contain +different data types within a single list, all of the elements in a NumPy array +should be homogenous. The mathematical operations that are meant to be performed +on arrays would be extremely inefficient if the arrays weren't homogenous. + +**Why use NumPy?** + +NumPy arrays are faster and more compact than Python lists. An array consumes +less memory and is convenient to use. NumPy uses much less memory to store data +and it provides a mechanism of specifying the data types. This allows the code +to be optimized even further. + +What is an array? +----------------- + +An array is a central data structure of the NumPy library. An array is a grid of +values and it contains information about the raw data, how to locate an element, +and how to interpret an element. It has a grid of elements that can be indexed +in :ref:`various ways <quickstart.indexing-slicing-and-iterating>`. +The elements are all of the same type, referred to as the array ``dtype``. + +An array can be indexed by a tuple of nonnegative integers, by booleans, by +another array, or by integers. The ``rank`` of the array is the number of +dimensions. The ``shape`` of the array is a tuple of integers giving the size of +the array along each dimension. + +One way we can initialize NumPy arrays is from Python lists, using nested lists +for two- or higher-dimensional data. + +For example:: + + >>> a = np.array([1, 2, 3, 4, 5, 6]) + +or:: + + >>> a = np.array([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]) + +We can access the elements in the array using square brackets. When you're +accessing elements, remember that indexing in NumPy starts at 0. That means that +if you want to access the first element in your array, you'll be accessing +element "0". + +:: + + >>> print(a[0]) + [1 2 3 4] + + +More information about arrays +----------------------------- + +*This section covers* ``1D array``, ``2D array``, ``ndarray``, ``vector``, ``matrix`` + +------ + +You might occasionally hear an array referred to as a "ndarray," which is +shorthand for "N-dimensional array." An N-dimensional array is simply an array +with any number of dimensions. You might also hear **1-D**, or one-dimensional +array, **2-D**, or two-dimensional array, and so on. The NumPy ``ndarray`` class +is used to represent both matrices and vectors. A **vector** is an array with a +single dimension (there's no difference +between row and column vectors), while a **matrix** refers to an +array with two dimensions. For **3-D** or higher dimensional arrays, the term +**tensor** is also commonly used. + +**What are the attributes of an array?** + +An array is usually a fixed-size container of items of the same type and size. +The number of dimensions and items in an array is defined by its shape. The +shape of an array is a tuple of non-negative integers that specify the sizes of +each dimension. + +In NumPy, dimensions are called **axes**. This means that if you have a 2D array +that looks like this:: + + [[0., 0., 0.], + [1., 1., 1.]] + +Your array has 2 axes. The first axis has a length of 2 and the second axis has +a length of 3. + +Just like in other Python container objects, the contents of an array can be +accessed and modified by indexing or slicing the array. Unlike the typical container +objects, different arrays can share the same data, so changes made on one array might +be visible in another. + +Array **attributes** reflect information intrinsic to the array itself. If you +need to get, or even set, properties of an array without creating a new array, +you can often access an array through its attributes. + +:ref:`Read more about array attributes here <arrays.ndarray>` and learn about +:ref:`array objects here <arrays>`. + + +How to create a basic array +--------------------------- + + +*This section covers* ``np.array()``, ``np.zeros()``, ``np.ones()``, +``np.empty()``, ``np.arange()``, ``np.linspace()``, ``dtype`` + +----- + +To create a NumPy array, you can use the function ``np.array()``. + +All you need to do to create a simple array is pass a list to it. If you choose +to, you can also specify the type of data in your list. +:ref:`You can find more information about data types here <arrays.dtypes>`. :: + + >>> import numpy as np + >>> a = np.array([1, 2, 3]) + +You can visualize your array this way: + +.. image:: images/np_array.png + +*Be aware that these visualizations are meant to simplify ideas and give you a basic understanding of NumPy concepts and mechanics. Arrays and array operations are much more complicated than are captured here!* + +Besides creating an array from a sequence of elements, you can easily create an +array filled with ``0``'s:: + + >>> np.zeros(2) + array([0., 0.]) + +Or an array filled with ``1``'s:: + + >>> np.ones(2) + array([1., 1.]) + +Or even an empty array! The function ``empty`` creates an array whose initial +content is random and depends on the state of the memory. The reason to use +``empty`` over ``zeros`` (or something similar) is speed - just make sure to +fill every element afterwards! :: + + >>> # Create an empty array with 2 elements + >>> np.empty(2) + +You can create an array with a range of elements:: + + >>> np.arange(4) + array([0, 1, 2, 3]) + +And even an array that contains a range of evenly spaced intervals. To do this, +you will specify the **first number**, **last number**, and the **step size**. :: + + >>> np.arange(2, 9, 2) + array([2, 4, 6, 8]) + +You can also use ``np.linspace()`` to create an array with values that are +spaced linearly in a specified interval:: + + >>> np.linspace(0, 10, num=5) + array([ 0. , 2.5, 5. , 7.5, 10. ]) + +**Specifying your data type** + +While the default data type is floating point (``np.float64``), you can explicitly +specify which data type you want using the ``dtype`` keyword. :: + + >>> x = np.ones(2, dtype=np.int64) + >>> x + array([1, 1]) + +:ref:`Learn more about creating arrays here <quickstart.array-creation>` + +Adding, removing, and sorting elements +-------------------------------------- + +*This section covers* ``np.sort()``, ``np.concatenate()`` + +----- + +Sorting an element is simple with ``np.sort()``. You can specify the axis, kind, +and order when you call the function. + +If you start with this array:: + + >>> arr = np.array([2, 1, 5, 3, 7, 4, 6, 8]) + +You can quickly sort the numbers in ascending order with:: + + >>> np.sort(arr) + array([1, 2, 3, 4, 5, 6, 7, 8]) + +In addition to sort, which returns a sorted copy of an array, you can use: + +- `argsort`, which is an indirect sort along a specified axis, +- `lexsort`, which is an indirect stable sort on multiple keys, +- `searchsorted`, which will find elements in a sorted array, and +- `partition`, which is a partial sort. + +To read more about sorting an array, see: `sort`. + +If you start with these arrays:: + + >>> a = np.array([1, 2, 3, 4]) + >>> b = np.array([5, 6, 7, 8]) + +You can concatenate them with ``np.concatenate()``. :: + + >>> np.concatenate((a, b)) + array([1, 2, 3, 4, 5, 6, 7, 8]) + +Or, if you start with these arrays:: + + >>> x = np.array([[1, 2], [3, 4]]) + >>> y = np.array([[5, 6]]) + +You can concatenate them with:: + + >>> np.concatenate((x, y), axis=0) + array([[1, 2], + [3, 4], + [5, 6]]) + +In order to remove elements from an array, it's simple to use indexing to select +the elements that you want to keep. + +To read more about concatenate, see: `concatenate`. + + +How do you know the shape and size of an array? +----------------------------------------------- + +*This section covers* ``ndarray.ndim``, ``ndarray.size``, ``ndarray.shape`` + +----- + +``ndarray.ndim`` will tell you the number of axes, or dimensions, of the array. + +``ndarray.size`` will tell you the total number of elements of the array. This +is the *product* of the elements of the array's shape. + +``ndarray.shape`` will display a tuple of integers that indicate the number of +elements stored along each dimension of the array. If, for example, you have a +2-D array with 2 rows and 3 columns, the shape of your array is ``(2, 3)``. + +For example, if you create this array:: + + >>> array_example = np.array([[[0, 1, 2, 3], + ... [4, 5, 6, 7]], + ... + ... [[0, 1, 2, 3], + ... [4, 5, 6, 7]], + ... + ... [[0 ,1 ,2, 3], + ... [4, 5, 6, 7]]]) + +To find the number of dimensions of the array, run:: + + >>> array_example.ndim + 3 + +To find the total number of elements in the array, run:: + + >>> array_example.size + 24 + +And to find the shape of your array, run:: + + >>> array_example.shape + (3, 2, 4) + + +Can you reshape an array? +------------------------- + +*This section covers* ``arr.reshape()`` + +----- + +**Yes!** + +Using ``arr.reshape()`` will give a new shape to an array without changing the +data. Just remember that when you use the reshape method, the array you want to +produce needs to have the same number of elements as the original array. If you +start with an array with 12 elements, you'll need to make sure that your new +array also has a total of 12 elements. + +If you start with this array:: + + >>> a = np.arange(6) + >>> print(a) + [0 1 2 3 4 5] + +You can use ``reshape()`` to reshape your array. For example, you can reshape +this array to an array with three rows and two columns:: + + >>> b = a.reshape(3, 2) + >>> print(b) + [[0 1] + [2 3] + [4 5]] + +With ``np.reshape``, you can specify a few optional parameters:: + + >>> numpy.reshape(a, newshape, order) + +``a`` is the array to be reshaped. + +``newshape`` is the new shape you want. You can specify an integer or a tuple of +integers. If you specify an integer, the result will be an array of that length. +The shape should be compatible with the original shape. + +``order:`` ``C`` means to read/write the elements using C-like index order, +``F`` means to read/write the elements using Fortran-like index order, ``A`` +means to read/write the elements in Fortran-like index order if a is Fortran +contiguous in memory, C-like order otherwise. (This is an optional parameter and +doesn't need to be specified.) + +If you want to learn more about C and Fortran order, you can +:ref:`read more about the internal organization of NumPy arrays here <numpy-internals>`. +Essentially, C and Fortran orders have to do with how indices correspond +to the order the array is stored in memory. In Fortran, when moving through +the elements of a two-dimensional array as it is stored in memory, the **first** +index is the most rapidly varying index. As the first index moves to the next +row as it changes, the matrix is stored one column at a time. +This is why Fortran is thought of as a **Column-major language**. +In C on the other hand, the **last** index changes +the most rapidly. The matrix is stored by rows, making it a **Row-major +language**. What you do for C or Fortran depends on whether it's more important +to preserve the indexing convention or not reorder the data. + +:ref:`Learn more about shape manipulation here <quickstart.shape-manipulation>`. + + +How to convert a 1D array into a 2D array (how to add a new axis to an array) +----------------------------------------------------------------------------- + +*This section covers* ``np.newaxis``, ``np.expand_dims`` + +----- + +You can use ``np.newaxis`` and ``np.expand_dims`` to increase the dimensions of +your existing array. + +Using ``np.newaxis`` will increase the dimensions of your array by one dimension +when used once. This means that a **1D** array will become a **2D** array, a +**2D** array will become a **3D** array, and so on. + +For example, if you start with this array:: + + >>> a = np.array([1, 2, 3, 4, 5, 6]) + >>> a.shape + (6,) + +You can use ``np.newaxis`` to add a new axis:: + + >>> a2 = a[np.newaxis, :] + >>> a2.shape + (1, 6) + +You can explicitly convert a 1D array with either a row vector or a column +vector using ``np.newaxis``. For example, you can convert a 1D array to a row +vector by inserting an axis along the first dimension:: + + >>> row_vector = a[np.newaxis, :] + >>> row_vector.shape + (1, 6) + +Or, for a column vector, you can insert an axis along the second dimension:: + + >>> col_vector = a[:, np.newaxis] + >>> col_vector.shape + (6, 1) + +You can also expand an array by inserting a new axis at a specified position +with ``np.expand_dims``. + +For example, if you start with this array:: + + >>> a = np.array([1, 2, 3, 4, 5, 6]) + >>> a.shape + (6,) + +You can use ``np.expand_dims`` to add an axis at index position 1 with:: + + >>> b = np.expand_dims(a, axis=1) + >>> b.shape + (6, 1) + +You can add an axis at index position 0 with:: + + >>> c = np.expand_dims(a, axis=0) + >>> c.shape + (1, 6) + +Find more information about :ref:`newaxis here <arrays.indexing>` and +``expand_dims`` at `expand_dims`. + + +Indexing and slicing +-------------------- + +You can index and slice NumPy arrays in the same ways you can slice Python +lists. :: + + >>> data = np.array([1, 2, 3]) + + >>> data[1] + 2 + >>> data[0:2] + array([1, 2]) + >>> data[1:] + array([2, 3]) + >>> data[-2:] + array([2, 3]) + +You can visualize it this way: + +.. image:: images/np_indexing.png + + +You may want to take a section of your array or specific array elements to use +in further analysis or additional operations. To do that, you'll need to subset, +slice, and/or index your arrays. + +If you want to select values from your array that fulfill certain conditions, +it's straightforward with NumPy. + +For example, if you start with this array:: + + >>> a = np.array([[1 , 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]) + +You can easily print all of the values in the array that are less than 5. :: + + >>> print(a[a < 5]) + [1 2 3 4] + +You can also select, for example, numbers that are equal to or greater than 5, +and use that condition to index an array. :: + + >>> five_up = (a >= 5) + >>> print(a[five_up]) + [ 5 6 7 8 9 10 11 12] + +You can select elements that are divisible by 2:: + + >>> divisible_by_2 = a[a%2==0] + >>> print(divisible_by_2) + [ 2 4 6 8 10 12] + +Or you can select elements that satisfy two conditions using the ``&`` and ``|`` +operators:: + + >>> c = a[(a > 2) & (a < 11)] + >>> print(c) + [ 3 4 5 6 7 8 9 10] + +You can also make use of the logical operators **&** and **|** in order to +return boolean values that specify whether or not the values in an array fulfill +a certain condition. This can be useful with arrays that contain names or other +categorical values. :: + + >>> five_up = (a > 5) | (a == 5) + >>> print(five_up) + [[False False False False] + [ True True True True] + [ True True True True]] + +You can also use ``np.nonzero()`` to select elements or indices from an array. + +Starting with this array:: + + >>> a = np.array([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]) + +You can use ``np.nonzero()`` to print the indices of elements that are, for +example, less than 5:: + + >>> b = np.nonzero(a < 5) + >>> print(b) + (array([0, 0, 0, 0]), array([0, 1, 2, 3])) + +In this example, a tuple of arrays was returned: one for each dimension. The +first array represents the row indices where these values are found, and the +second array represents the column indices where the values are found. + +If you want to generate a list of coordinates where the elements exist, you can +zip the arrays, iterate over the list of coordinates, and print them. For +example:: + + >>> list_of_coordinates= list(zip(b[0], b[1])) + + >>> for coord in list_of_coordinates: + ... print(coord) + (0, 0) + (0, 1) + (0, 2) + (0, 3) + +You can also use ``np.nonzero()`` to print the elements in an array that are less +than 5 with:: + + >>> print(a[b]) + [1 2 3 4] + +If the element you're looking for doesn't exist in the array, then the returned +array of indices will be empty. For example:: + + >>> not_there = np.nonzero(a == 42) + >>> print(not_there) + (array([], dtype=int64), array([], dtype=int64)) + +Learn more about :ref:`indexing and slicing here <quickstart.indexing-slicing-and-iterating>` +and :ref:`here <basics.indexing>`. + +Read more about using the nonzero function at: `nonzero`. + + +How to create an array from existing data +----------------------------------------- + +*This section covers* ``slicing and indexing``, ``np.vstack()``, ``np.hstack()``, +``np.hsplit()``, ``.view()``, ``copy()`` + +----- + +You can easily use create a new array from a section of an existing array. + +Let's say you have this array: + +:: + + array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + +You can create a new array from a section of your array any time by specifying +where you want to slice your array. :: + + >>> arr1 = np.array[3:8] + >>> arr1 + array([4, 5, 6, 7, 8]) + +Here, you grabbed a section of your array from index position 3 through index +position 8. + +You can also stack two existing arrays, both vertically and horizontally. Let's +say you have two arrays, ``a1`` and ``a2``:: + + >>> a1 = np.array([[1, 1], + ... [2, 2]]) + + >>> a2 = np.array([[3, 3], + ... [4, 4]]) + +You can stack them vertically with ``vstack``:: + + >>> np.vstack((a1, a2)) + array([[1, 1], + [2, 2], + [3, 3], + [4, 4]]) + +Or stack them horizontally with ``hstack``:: + + >>> np.hstack((a1, a2)) + array([[1, 1, 3, 3], + [2, 2, 4, 4]]) + +You can split an array into several smaller arrays using ``hsplit``. You can +specify either the number of equally shaped arrays to return or the columns +*after* which the division should occur. + +Let's say you have this array:: + + >>> x = np.arange(1, 25).reshape(2, 12) + >>> x + array([[ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], + [13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]]) + +If you wanted to split this array into three equally shaped arrays, you would +run:: + + >>> np.hsplit(x, 3) + [array([[1, 2, 3, 4], + [13, 14, 15, 16]]), array([[ 5, 6, 7, 8], + [17, 18, 19, 20]]), array([[ 9, 10, 11, 12], + [21, 22, 23, 24]])] + +If you wanted to split your array after the third and fourth column, you'd run:: + + >>> np.hsplit(array,(3, 4)) + [array([[1, 2, 3], + [13, 14, 15]]), array([[ 4], + [16]]), array([[ 5, 6, 7, 8, 9, 10, 11, 12], + [17, 18, 19, 20, 21, 22, 23, 24]])] + +:ref:`Learn more about stacking and splitting arrays here <quickstart.stacking-arrays>`. + +You can use the ``view`` method to create a new array object that looks at the +same data as the original array (a *shallow copy*). + +Views are an important NumPy concept! NumPy functions, as well as operations +like indexing and slicing, will return views whenever possible. This saves +memory and is faster (no copy of the data has to be made). However it's +important to be aware of this - modifying data in a view also modifies the +original array! + +Let's say you create this array:: + + >>> a = np.array([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]) + +Now we create an array ``b1`` by slicing ``a`` and modify the first element of +``b1``. This will modify the corresponding element in ``a`` as well! :: + + >>> b1 = a[0, :] + >>> b1 + array([1, 2, 3, 4]) + >>> b1[0] = 99 + >>> b1 + array([99, 2, 3, 4]) + >>> a + array([[99, 2, 3, 4], + [ 5, 6, 7, 8], + [ 9, 10, 11, 12]]) + +Using the ``copy`` method will make a complete copy of the array and its data (a +*deep copy*). To use this on your array, you could run:: + + >>> b2 = a.copy() + +:ref:`Learn more about copies and views here <quickstart.copies-and-views>`. + + +Basic array operations +---------------------- + +*This section covers addition, subtraction, multiplication, division, and more* + +----- + +Once you've created your arrays, you can start to work with them. Let's say, +for example, that you've created two arrays, one called "data" and one called +"ones" + +.. image:: images/np_array_dataones.png + +You can add the arrays together with the plus sign. + +:: + + data + ones + +.. image:: images/np_data_plus_ones.png + +You can, of course, do more than just addition! + +:: + + data - ones + data * data + data / data + +.. image:: images/np_sub_mult_divide.png + +Basic operations are simple with NumPy. If you want to find the sum of the +elements in an array, you'd use ``sum()``. This works for 1D arrays, 2D arrays, +and arrays in higher dimensions. :: + + >>> a = np.array([1, 2, 3, 4]) + + >>> a.sum() + 10 + +To add the rows or the columns in a 2D array, you would specify the axis. + +If you start with this array:: + + >>> b = np.array([[1, 1], [2, 2]]) + +You can sum the rows with:: + + >>> b.sum(axis=0) + array([3, 3]) + +You can sum the columns with:: + + >>> b.sum(axis=1) + array([2, 4]) + +:ref:`Learn more about basic operations here <quickstart.basic-operations>`. + + +Broadcasting +------------ + +There are times when you might want to carry out an operation between an array +and a single number (also called *an operation between a vector and a scalar*) +or between arrays of two different sizes. For example, your array (we'll call it +"data") might contain information about distance in miles but you want to +convert the information to kilometers. You can perform this operation with:: + + >>> data * 1.6 + +.. image:: images/np_multiply_broadcasting.png + +NumPy understands that the multiplication should happen with each cell. That +concept is called **broadcasting**. Broadcasting is a mechanism that allows +NumPy to perform operations on arrays of different shapes. The dimensions of +your array must be compatible, for example, when the dimensions of both arrays +are equal or when one of them is 1. If the dimensions are not compatible, you +will get a ``ValueError``. + +:ref:`Learn more about broadcasting here <basics.broadcasting>`. + + +More useful array operations +---------------------------- + +*This section covers maximum, minimum, sum, mean, product, standard deviation, and more* + +----- + +NumPy also performs aggregation functions. In addition to ``min``, ``max``, and +``sum``, you can easily run ``mean`` to get the average, ``prod`` to get the +result of multiplying the elements together, ``std`` to get the standard +deviation, and more. :: + + >>> data.max() + >>> data.min() + >>> data.sum() + +.. image:: images/np_aggregation.png + +Let's start with this array, called "a" :: + + >>> a = np.array([[0.45053314, 0.17296777, 0.34376245, 0.5510652], + ... [0.54627315, 0.05093587, 0.40067661, 0.55645993], + ... [0.12697628, 0.82485143, 0.26590556, 0.56917101]]) + +It's very common to want to aggregate along a row or column. By default, every +NumPy aggregation function will return the aggregate of the entire array. To +find the sum or the minimum of the elements in your array, run:: + + >>> a.sum() + 4.8595784 + +Or:: + + >>> a.min() + 0.05093587 + +You can specify on which axis you want the aggregation function to be computed. +For example, you can find the minimum value within each column by specifying +``axis=0``. :: + + >>> a.min(axis=0) + array([0.12697628, 0.05093587, 0.26590556, 0.5510652 ]) + +The four values listed above correspond to the number of columns in your array. +With a four-column array, you will get four values as your result. + +Read more about :ref:`array methods here <array.ndarray.methods>`. + + +Creating matrices +----------------- + +You can pass Python lists of lists to create a 2-D array (or "matrix") to +represent them in NumPy. :: + + >>> np.array([[1, 2], [3, 4]]) + +.. image:: images/np_create_matrix.png + +Indexing and slicing operations are useful when you're manipulating matrices:: + + >>> data[0, 1] + >>> data[1 : 3] + >>> data[0 : 2, 0] + +.. image:: images/np_matrix_indexing.png + +You can aggregate matrices the same way you aggregated vectors:: + + >>> data.max() + >>> data.min() + >>> data.sum() + +.. image:: images/np_matrix_aggregation.png + +You can aggregate all the values in a matrix and you can aggregate them across +columns or rows using the ``axis`` parameter:: + + >>> data.max(axis=0) + >>> data.max(axis=1) + +.. image:: images/np_matrix_aggregation_row.png + +Once you've created your matrices, you can add and multiply them using +arithmetic operators if you have two matrices that are the same size. :: + + >>> data + ones + +.. image:: images/np_matrix_arithmetic.png + +You can do these arithmetic operations on matrices of different sizes, but only +if one matrix has only one column or one row. In this case, NumPy will use its +broadcast rules for the operation. :: + + >>> data + ones_row + +.. image:: images/np_matrix_broadcasting.png + +Be aware that when NumPy prints N-dimensional arrays, the last axis is looped +over the fastest while the first axis is the slowest. That means that:: + + >>> np.ones((4, 3, 2)) + +Will print out like this:: + + array([[[1., 1.], + [1., 1.], + [1., 1.]], + + [[1., 1.], + [1., 1.], + [1., 1.]], + + [[1., 1.], + [1., 1.], + [1., 1.]], + + [[1., 1.], + [1., 1.], + [1., 1.]]]) + +There are often instances where we want NumPy to initialize the values of an +array. NumPy offers functions like ``ones()`` and ``zeros()``, and the +``random.Generator`` class for random number generation for that. +All you need to do is pass in the number of elements you want it to generate:: + + >>> np.ones(3) + >>> np.zeros(3) + >>> np.random.random(3) # the simplest way to generate random numbers + +.. image:: images/np_ones_zeros_random.png + +You can also use ``ones()``, ``zeros()``, and ``random()`` to create +an array if you give them a tuple describing the dimensions of the matrix:: + + >>> np.ones((3, 2)) + >>> np.zeros((3, 2)) + >>> rng = np.random.default_rng() # the better way to generate random numbers + >>> rng.random() + +.. image:: images/np_ones_zeros_matrix.png + +Read more about creating arrays, filled with ``0``'s, ``1``'s, other values or +uninitialized, at :ref:`array creation routines <routines.array-creation>`. + + +Generating random numbers +------------------------- + +The use of random number generation is an important part of the configuration +and evaluation of many numerical and machine learning algorithms. Whether you +need to randomly initialize weights in an artificial neural network, split data +into random sets, or randomly shuffle your dataset, being able to generate +random numbers (actually, repeatable pseudo-random numbers) is essential. + +With ``Generator.integers``, you can generate random integers from low (remember +that this is inclusive with NumPy) to high (exclusive). You can set +``endpoint=True`` to make the high number inclusive. + +You can generate a 2 x 4 array of random integers between 0 and 4 with:: + + >>> rng.integers(5, size=(2, 4)) + array([[4, 0, 2, 1], + [3, 2, 2, 0]]) + +:ref:`Read more about random number generation here <numpyrandom>`. + + +How to get unique items and counts +---------------------------------- + +*This section covers* ``np.unique()`` + +----- + +You can find the unique elements in an array easily with ``np.unique``. + +For example, if you start with this array:: + + >>> a = np.array([11, 11, 12, 13, 14, 15, 16, 17, 12, 13, 11, 14, 18, 19, 20]) + +you can use ``np.unique`` to print the unique values in your array:: + + >>> unique_values = np.unique(a) + >>> print(unique_values) + [11 12 13 14 15 16 17 18 19 20] + +To get the indices of unique values in a NumPy array (an array of first index +positions of unique values in the array), just pass the ``return_index`` +argument in ``np.unique()`` as well as your array. :: + + >>> indices_list = np.unique(a, return_index=True) + >>> print(indices_list) + [ 0 2 3 4 5 6 7 12 13 14] + +You can pass the ``return_counts`` argument in ``np.unique()`` along with your +array to get the frequency count of unique values in a NumPy array. :: + + >>> unique_values, occurrence_count = np.unique(a, return_counts=True) + >>> print(occurrence_count) + [3 2 2 2 1 1 1 1 1 1] + +This also works with 2D arrays! +If you start with this array:: + + >>> a_2d = np.array([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [1, 2, 3, 4]]) + +You can find unique values with:: + + >>> unique_values = np.unique(a_2d) + >>> print(unique_values) + [ 1 2 3 4 5 6 7 8 9 10 11 12] + +If the axis argument isn't passed, your 2D array will be flattened. + +If you want to get the unique rows or columns, make sure to pass the ``axis`` +argument. To find the unique rows, specify ``axis=0`` and for columns, specify +``axis=1``. :: + + >>> unique_rows = np.unique(a_2d, axis=0) + >>> print(unique_rows) + [[ 1 2 3 4] + [ 5 6 7 8] + [ 9 10 11 12]] + +To get the unique rows, occurrence count, and index position, you can use:: + + >>> unique_rows, occurence_count, indices = np.unique(a_2d, axis=0, + >>> return_counts=True, return_index=True) + >>> print('Unique Rows: ', '\n', unique_rows) + >>> print('Occurrence Count:', '\n', occurence_count) + >>> print('Indices: ', '\n', indices) + Unique Rows: + [[ 1 2 3 4] + [ 5 6 7 8] + [ 9 10 11 12]] + Occurrence Count: + [0 1 2] + Indices: + [2 1 1] + +To learn more about finding the unique elements in an array, see `unique`. + + +Transposing and reshaping a matrix +---------------------------------- + +*This section covers* ``arr.reshape()``, ``arr.transpose()``, ``arr.T()`` + +----- + +It's common to need to transpose your matrices. NumPy arrays have the property +``T`` that allows you to transpose a matrix. + +.. image:: images/np_transposing_reshaping.png + +You may also need to switch the dimensions of a matrix. This can happen when, +for example, you have a model that expects a certain input shape that is +different from your dataset. This is where the ``reshape`` method can be useful. +You simply need to pass in the new dimensions that you want for the matrix. :: + + >>> data.reshape(2, 3) + >>> data.reshape(3, 2) + +.. image:: images/np_reshape.png + +You can also use ``.transpose`` to reverse or change the axes of an array +according to the values you specify. + +If you start with this array:: + + >>> arr = np.arange(6).reshape((2, 3)) + >>> arr + array([[0, 1, 2], + [3, 4, 5]]) + +You can transpose your array with ``arr.transpose()``. :: + + >>> arr.transpose() + array([[0, 3], + [1, 4], + [2, 5]]) + +To learn more about transposing and reshaping arrays, see `transpose` and +`reshape`. + + +How to reverse an array +----------------------- + +*This section covers* ``np.flip`` + +----- + +NumPy's ``np.flip()`` function allows you to flip, or reverse, the contents of +an array along an axis. When using ``np.flip``, specify the array you would like +to reverse and the axis. If you don't specify the axis, NumPy will reverse the +contents along all of the axes of your input array. + +**Reversing a 1D array** + +If you begin with a 1D array like this one:: + + >>> arr = np.array([1, 2, 3, 4, 5, 6, 7, 8]) + +You can reverse it with:: + + >>> reversed_arr = np.flip(arr) + +If you want to print your reversed array, you can run:: + + >>> print('Reversed Array: ', reversed_arr) + Reversed Array: [8 7 6 5 4 3 2 1] + +**Reversing a 2D array** + +A 2D array works much the same way. + +If you start with this array:: + + >>> arr_2d = np.array([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]) + +You can reverse the content in all of the rows and all of the columns with:: + + >>> reversed_arr = np.flip(arr_2d) + + >>> print('Reversed Array: ') + >>> print(reversed_arr) + Reversed Array: + [[12 11 10 9] + [ 8 7 6 5] + [ 4 3 2 1]] + +You can easily reverse only the *rows* with:: + + >>> reversed_arr_rows = np.flip(arr_2d, axis=0) + + >>> print('Reversed Array: ') + >>> print(reversed_arr_rows) + Reversed Array: + [[ 9 10 11 12] + [ 5 6 7 8] + [ 1 2 3 4]] + +Or reverse only the *columns* with:: + + >>> reversed_arr_columns = np.flip(arr_2d, axis=1) + + >>> print('Reversed Array columns: ') + >>> print(reversed_arr_columns) + [[ 4 3 2 1] + [ 8 7 6 5] + [12 11 10 9]] + +You can also reverse the contents of only one column or row. For example, you +can reverse the contents of the row at index position 1 (the second row):: + + >>> arr_2d[1] = np.flip(arr_2d[1]) + + >>> print('Reversed Array: ') + >>> print(arr_2d) + Reversed Array: + [[ 1 2 3 4] + [ 5 6 7 8] + [ 9 10 11 12]] + +You can also reverse the column at index position 1 (the second column):: + + >>> arr_2d[:,1] = np.flip(arr_2d[:,1]) + + >>> print('Reversed Array: ') + >>> print(arr_2d) + Reversed Array: + [[ 1 10 3 4] + [ 5 6 7 8] + [ 9 2 11 12]] + +Read more about reversing arrays at `flip`. + + +Reshaping and flattening multidimensional arrays +------------------------------------------------ + +*This section covers* ``.flatten()``, ``ravel()`` + +----- + +There are two popular ways to flatten an array: ``.flatten()`` and ``.ravel()``. +The primary difference between the two is that the new array created using +``ravel()`` is actually a reference to the parent array (i.e., a "view"). This +means that any changes to the new array will affect the parent array as well. +Since ``ravel`` does not create a copy, it's memory efficient. + +If you start with this array:: + + >>> x = np.array([[1 , 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]) + +You can use ``flatten`` to flatten your array into a 1D array. :: + + >>> x.flatten() + array([ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]) + +When you use ``flatten``, changes to your new array won't change the parent +array. + +For example:: + + >>> a1 = x.flatten() + >>> a1[0] = 99 + >>> print(x) # Original array + [[ 1 2 3 4] + [ 5 6 7 8] + [ 9 10 11 12]] + >>> print(a1) # New array + [99 2 3 4 5 6 7 8 9 10 11 12] + +But when you use ``ravel``, the changes you make to the new array will affect +the parent array. + +For example:: + + >>> a2 = x.ravel() + >>> a2[0] = 98 + >>> print(x) # Original array + [[98 2 3 4] + [ 5 6 7 8] + [ 9 10 11 12]] + >>> print(a2) # New array + [98 2 3 4 5 6 7 8 9 10 11 12] + +Read more about ``flatten`` at `ndarray.flatten` and ``ravel`` at `ravel`. + + +How to access the docstring for more information +------------------------------------------------ + +*This section covers* ``help()``, ``?``, ``??`` + +----- + +When it comes to the data science ecosystem, Python and NumPy are built with the +user in mind. One of the best examples of this is the built-in access to +documentation. Every object contains the reference to a string, which is known +as the **docstring**. In most cases, this docstring contains a quick and concise +summary of the object and how to use it. Python has a built-in ``help()`` +function that can help you access this information. This means that nearly any +time you need more information, you can use ``help()`` to quickly find the +information that you need. + +For example, :: + + >>> help(max) + +Will return:: + + Help on built-in function max in module builtins: + + max(...) max(iterable, *[, default=obj, key=func]) -> value max(arg1, arg2, + *args, *[, key=func]) -> value + + With a single iterable argument, return its biggest item. The default + keyword-only argument specifies an object to return if the provided + iterable is empty. With two or more arguments, return the largest + argument. + +Because access to additional information is so useful, IPython uses the ``?`` +character as a shorthand for accessing this documentation along with other +relevant information. IPython is a command shell for interactive computing in +multiple languages. +`You can find more information about IPython here <https://ipython.org/>`_. + +For example, :: + + >>> max? + +Will return:: + + Docstring: + max(iterable, *[, default=obj, key=func]) -> value + max(arg1, arg2, *args, *[, key=func]) -> value + + With a single iterable argument, return its biggest item. The + default keyword-only argument specifies an object to return if + the provided iterable is empty. + With two or more arguments, return the largest argument. + Type: builtin_function_or_method + +You can even use this notation for object methods and objects themselves. + +Let's say you create this array:: + + >>> a = np.array([1, 2, 3, 4, 5, 6]) + +Running :: + + >>> a? + +Will return a lot of useful information (first details about ``a`` itself, +followed by the docstring of ``ndarray`` of which ``a`` is an instance):: + + Type: ndarray + String form: [1 2 3 4 5 6] + Length: 6 + File: ~/anaconda3/lib/python3.7/site-packages/numpy/__init__.py + Docstring: <no docstring> + Class docstring: + ndarray(shape, dtype=float, buffer=None, offset=0, + strides=None, order=None) + + An array object represents a multidimensional, homogeneous array + of fixed-size items. An associated data-type object describes the + format of each element in the array (its byte-order, how many bytes it + occupies in memory, whether it is an integer, a floating point number, + or something else, etc.) + + Arrays should be constructed using `array`, `zeros` or `empty` (refer + to the See Also section below). The parameters given here refer to + a low-level method (`ndarray(...)`) for instantiating an array. + + For more information, refer to the `numpy` module and examine the + methods and attributes of an array. + + Parameters + ---------- + (for the __new__ method; see Notes below) + + shape : tuple of ints + Shape of created array. + ... + +This also works for functions and other objects that **you** create. Just +remember to include a docstring with your function using a string literal +(``""" """`` or ``''' '''`` around your documentation). + +For example, if you create this function:: + + >>> def double(a): + >>> '''Return a * 2''' + >>> return a * 2 + +You can run:: + + >>> double? + +Which will return:: + + Signature: double(a) + Docstring: Return a * 2 + File: ~/Desktop/<ipython-input-23-b5adf20be596> + Type: function + +You can reach another level of information by reading the source code of the +object you're interested in. Using a double question mark (``??``) allows you to +access the source code. + +For example, running:: + + >>> double?? + +Will return :: + + Signature: double(a) + Source: def double(a): + '''Return a * 2''' + return a * 2 + File: ~/Desktop/<ipython-input-23-b5adf20be596> + Type: function + +If the object in question is compiled in a language other than Python, using +``??`` will return the same information as ``?``. You'll find this with a lot of +built-in objects and types, for example:: + + >>> len? + Signature: len(obj, /) + Docstring: Return the number of items in a container. + Type: builtin_function_or_method + +and :: + + >>> len?? + Signature: len(obj, /) + Docstring: Return the number of items in a container. + Type: builtin_function_or_method + +have the same output because they were compiled in a programming language other +than Python. + + +Working with mathematical formulas +---------------------------------- + +The ease of implementing mathematical formulas that work on arrays is one of +the things that make NumPy so widely used in the scientific Python community. + +For example, this is the mean square error formula (a central formula used in +supervised machine learning models that deal with regression): + +.. image:: images/np_MSE_formula.png + +Implementing this formula is simple and straightforward in NumPy: + +.. image:: images/np_MSE_implementation.png + +What makes this work so well is that ``predictions`` and ``labels`` can contain +one or a thousand values. They only need to be the same size. + +You can visualize it this way: + +.. image:: images/np_mse_viz1.png + +In this example, both the predictions and labels vectors contain three values, +meaning ``n`` has a value of three. After we carry out subtractions the values +in the vector are squared. Then NumPy sums the values, and your result is the +error value for that prediction and a score for the quality of the model. + +.. image:: images/np_mse_viz2.png + +.. image:: images/np_MSE_explanation2.png + + +How to save and load NumPy objects +---------------------------------- + +*This section covers* ``np.save``, ``np.savez``, ``np.savetxt``, +``np.load``, ``np.loadtxt`` + +----- + +You will, at some point, want to save your arrays to disk and load them back +without having to re-run the code. Fortunately, there are several ways to save +and load objects with NumPy. The ndarray objects can be saved to and loaded from +the disk files with ``loadtxt`` and ``savetxt`` functions that handle normal +text files, ``load`` and ``save`` functions that handle NumPy binary files with +a **.npy** file extension, and a ``savez`` function that handles NumPy files +with a **.npz** file extension. + +The **.npy** and **.npz** files store data, shape, dtype, and other information +required to reconstruct the ndarray in a way that allows the array to be +correctly retrieved, even when the file is on another machine with different +architecture. + +If you want to store a single ndarray object, store it as a .npy file using +``np.save``. If you want to store more than one ndarray object in a single file, +save it as a .npz file using ``np.savez``. You can also save several arrays +into a single file in compressed npz format with `savez_compressed`. + +It's easy to save and load and array with ``np.save()``. Just make sure to +specify the array you want to save and a file name. For example, if you create +this array:: + + >>> a = np.array([1, 2, 3, 4, 5, 6]) + +You can save it as "filename.npy" with:: + + >>> np.save('filename', a) + +You can use ``np.load()`` to reconstruct your array. :: + + >>> b = np.load('filename.npy') + +If you want to check your array, you can run::: + + >>> print(b) + [1 2 3 4 5 6] + +You can save a NumPy array as a plain text file like a **.csv** or **.txt** file +with ``np.savetxt``. + +For example, if you create this array:: + + >>> csv_arr = np.array([1, 2, 3, 4, 5, 6, 7, 8]) + +You can easily save it as a .csv file with the name "new_file.csv" like this:: + + >>> np.savetxt('new_file.csv', csv_arr) + +You can quickly and easily load your saved text file using ``loadtxt()``:: + + >>> np.loadtxt('new_file.csv') + array([1., 2., 3., 4., 5., 6., 7., 8.]) + +The ``savetxt()`` and ``loadtxt()`` functions accept additional optional +parameters such as header, footer, and delimiter. While text files can be easier +for sharing, .npy and .npz files are smaller and faster to read. If you need more +sophisticated handling of your text file (for example, if you need to work with +lines that contain missing values), you will want to use the `genfromtxt` +function. + +With `savetxt`, you can specify headers, footers, comments, and more. + +Learn more about :ref:`input and output routines here <routines.io>`. + + +Importing and exporting a CSV +----------------------------- + +It's simple to read in a CSV that contains existing information. The best and +easiest way to do this is to use +`Pandas <https://pandas.pydata.org/getpandas.html>`_. :: + + >>> import pandas as pd + + >>> # If all of your columns are the same type: + >>> x = pd.read_csv('music.csv').values + + >>> # You can also simply select the columns you need: + >>> x = pd.read_csv('music.csv', columns=['float_colname_1', ...]).values + +.. image:: images/np_pandas.png + +It's simple to use Pandas in order to export your array as well. If you are new +to NumPy, you may want to create a Pandas dataframe from the values in your +array and then write the data frame to a CSV file with Pandas. + +If you created this array "a" :: + + [[-2.58289208, 0.43014843, -1.24082018, 1.59572603], + [ 0.99027828, 1.17150989, 0.94125714, -0.14692469], + [ 0.76989341, 0.81299683, -0.95068423, 0.11769564], + [ 0.20484034, 0.34784527, 1.96979195, 0.51992837]] + +You could create a Pandas dataframe :: + + >>> df = pd.DataFrame(a) + >>> print(df) + +**Output:** + +:: + + 0 1 2 3 + 0 -2.582892 0.430148 -1.240820 1.595726 + 1 0.990278 1.171510 0.941257 -0.146925 + 2 0.769893 0.812997 -0.950684 0.117696 + 3 0.204840 0.347845 1.969792 0.519928 + +You can easily save your dataframe with:: + + >>> df.to_csv('pd.csv') + +And read your CSV with:: + + >>> pd.read_csv('pd.csv') + +.. image:: images/np_readcsv.png + +You can also save your array with the NumPy ``savetxt`` method. :: + + >>> np.savetxt('np.csv', a, fmt='%.2f', delimiter=',', header='1, 2, 3, 4') + +If you're using the command line, you can read your saved CSV any time with a +command such as:: + + >>> cat np.csv + # 1, 2, 3, 4 + -2.58,0.43,-1.24,1.60 + 0.99,1.17,0.94,-0.15 + 0.77,0.81,-0.95,0.12 + 0.20,0.35,1.97,0.52 + +Or you can open the file any time with a text editor! + +If you're interested in learning more about Pandas, take a look at the +`official Pandas documentation <https://pandas.pydata.org/index.html>`_. +Learn how to install Pandas with the +`official Pandas installation information <https://pandas.pydata.org/pandas-docs/stable/install.html>`_. + + +Plotting arrays with Matplotlib +------------------------------- + +If you need to generate a plot for your values, it's very simple with +`Matplotlib <https://matplotlib.org/>`_. + +For example, you may have an array like this one:: + + >>> a = np.array([2, 1, 5, 7, 4, 6, 8, 14, 10, 9, 18, 20, 22]) + +If you already have Matplotlib installed, you can import it with:: + + >>> import matplotlib.pyplot as plt + + >>> # If you're using Jupyter Notebook, you may also want to run the following + >>> line of code to display your code in the notebook: + + >>> %matplotlib inline + +All you need to do to plot your values is run:: + + >>> plt.plot(a) + >>> plt.show() + +.. plot:: user/plots/matplotlib1.py + :align: center + :include-source: 0 + +For example, you can plot a 1D array like this:: + + >>> x = np.linspace(0, 5, 20) + >>> y = np.linspace(0, 10, 20) + >>> plt.plot(x, y, 'purple') # line + >>> plt.plot(x, y, 'o') # dots + +.. plot:: user/plots/matplotlib2.py + :align: center + :include-source: 0 + +With Matplotlib, you have access to an enormous number of visualization options. :: + + >>> from mpl_toolkits.mplot3d import Axes3D + >>> fig = plt.figure() + >>> ax = Axes3D(fig) + >>> X = np.arange(-5, 5, 0.15) + >>> Y = np.arange(-5, 5, 0.15) + >>> X, Y = np.meshgrid(X, Y) + >>> R = np.sqrt(X**2 + Y**2) + >>> Z = np.sin(R) + + >>> ax.plot_surface(X, Y, Z, rstride=1, cstride=1, cmap='viridis') + +.. plot:: user/plots/matplotlib3.py + :align: center + :include-source: 0 + + +To read more about Matplotlib and what it can do, take a look at +`the official documentation <https://matplotlib.org/>`_. +For directions regarding installing Matplotlib, see the official +`installation section <https://matplotlib.org/users/installing.html>`_. + + +------------------------------------------------------- + +*Image credits: Jay Alammar http://jalammar.github.io/* + diff --git a/doc/source/user/basics.broadcasting.rst b/doc/source/user/basics.broadcasting.rst index 4e9016ee0..00bf17a41 100644 --- a/doc/source/user/basics.broadcasting.rst +++ b/doc/source/user/basics.broadcasting.rst @@ -1,3 +1,5 @@ +.. _basics.broadcasting: + ************ Broadcasting ************ diff --git a/doc/source/user/images/np_MSE_explanation.png b/doc/source/user/images/np_MSE_explanation.png Binary files differnew file mode 100644 index 000000000..6e20116f5 --- /dev/null +++ b/doc/source/user/images/np_MSE_explanation.png diff --git a/doc/source/user/images/np_MSE_explanation2.png b/doc/source/user/images/np_MSE_explanation2.png Binary files differnew file mode 100644 index 000000000..578e5022b --- /dev/null +++ b/doc/source/user/images/np_MSE_explanation2.png diff --git a/doc/source/user/images/np_MSE_formula.png b/doc/source/user/images/np_MSE_formula.png Binary files differnew file mode 100644 index 000000000..7e6982995 --- /dev/null +++ b/doc/source/user/images/np_MSE_formula.png diff --git a/doc/source/user/images/np_MSE_implementation.png b/doc/source/user/images/np_MSE_implementation.png Binary files differnew file mode 100644 index 000000000..004e82a1f --- /dev/null +++ b/doc/source/user/images/np_MSE_implementation.png diff --git a/doc/source/user/images/np_aggregation.png b/doc/source/user/images/np_aggregation.png Binary files differnew file mode 100644 index 000000000..4356193eb --- /dev/null +++ b/doc/source/user/images/np_aggregation.png diff --git a/doc/source/user/images/np_array.png b/doc/source/user/images/np_array.png Binary files differnew file mode 100644 index 000000000..24ba41294 --- /dev/null +++ b/doc/source/user/images/np_array.png diff --git a/doc/source/user/images/np_array_data_ones.png b/doc/source/user/images/np_array_data_ones.png Binary files differnew file mode 100644 index 000000000..9b49b6e29 --- /dev/null +++ b/doc/source/user/images/np_array_data_ones.png diff --git a/doc/source/user/images/np_array_dataones.png b/doc/source/user/images/np_array_dataones.png Binary files differnew file mode 100644 index 000000000..d9b132387 --- /dev/null +++ b/doc/source/user/images/np_array_dataones.png diff --git a/doc/source/user/images/np_create_array.png b/doc/source/user/images/np_create_array.png Binary files differnew file mode 100644 index 000000000..878bad95c --- /dev/null +++ b/doc/source/user/images/np_create_array.png diff --git a/doc/source/user/images/np_create_matrix.png b/doc/source/user/images/np_create_matrix.png Binary files differnew file mode 100644 index 000000000..cd685c4f5 --- /dev/null +++ b/doc/source/user/images/np_create_matrix.png diff --git a/doc/source/user/images/np_data_plus_ones.png b/doc/source/user/images/np_data_plus_ones.png Binary files differnew file mode 100644 index 000000000..b80c2648c --- /dev/null +++ b/doc/source/user/images/np_data_plus_ones.png diff --git a/doc/source/user/images/np_indexing.png b/doc/source/user/images/np_indexing.png Binary files differnew file mode 100644 index 000000000..4303ec35b --- /dev/null +++ b/doc/source/user/images/np_indexing.png diff --git a/doc/source/user/images/np_matrix_aggregation.png b/doc/source/user/images/np_matrix_aggregation.png Binary files differnew file mode 100644 index 000000000..9c2fc5110 --- /dev/null +++ b/doc/source/user/images/np_matrix_aggregation.png diff --git a/doc/source/user/images/np_matrix_aggregation_row.png b/doc/source/user/images/np_matrix_aggregation_row.png Binary files differnew file mode 100644 index 000000000..d474c271f --- /dev/null +++ b/doc/source/user/images/np_matrix_aggregation_row.png diff --git a/doc/source/user/images/np_matrix_arithmetic.png b/doc/source/user/images/np_matrix_arithmetic.png Binary files differnew file mode 100644 index 000000000..794702541 --- /dev/null +++ b/doc/source/user/images/np_matrix_arithmetic.png diff --git a/doc/source/user/images/np_matrix_broadcasting.png b/doc/source/user/images/np_matrix_broadcasting.png Binary files differnew file mode 100644 index 000000000..e8102a7d8 --- /dev/null +++ b/doc/source/user/images/np_matrix_broadcasting.png diff --git a/doc/source/user/images/np_matrix_indexing.png b/doc/source/user/images/np_matrix_indexing.png Binary files differnew file mode 100644 index 000000000..97f90f11e --- /dev/null +++ b/doc/source/user/images/np_matrix_indexing.png diff --git a/doc/source/user/images/np_mse_viz1.png b/doc/source/user/images/np_mse_viz1.png Binary files differnew file mode 100644 index 000000000..987a48c79 --- /dev/null +++ b/doc/source/user/images/np_mse_viz1.png diff --git a/doc/source/user/images/np_mse_viz2.png b/doc/source/user/images/np_mse_viz2.png Binary files differnew file mode 100644 index 000000000..5594b03e8 --- /dev/null +++ b/doc/source/user/images/np_mse_viz2.png diff --git a/doc/source/user/images/np_multiply_broadcasting.png b/doc/source/user/images/np_multiply_broadcasting.png Binary files differnew file mode 100644 index 000000000..02337d903 --- /dev/null +++ b/doc/source/user/images/np_multiply_broadcasting.png diff --git a/doc/source/user/images/np_ones_zeros_matrix.png b/doc/source/user/images/np_ones_zeros_matrix.png Binary files differnew file mode 100644 index 000000000..9cb54644f --- /dev/null +++ b/doc/source/user/images/np_ones_zeros_matrix.png diff --git a/doc/source/user/images/np_ones_zeros_random.png b/doc/source/user/images/np_ones_zeros_random.png Binary files differnew file mode 100644 index 000000000..17730713f --- /dev/null +++ b/doc/source/user/images/np_ones_zeros_random.png diff --git a/doc/source/user/images/np_pandas.png b/doc/source/user/images/np_pandas.png Binary files differnew file mode 100644 index 000000000..cc0cd069f --- /dev/null +++ b/doc/source/user/images/np_pandas.png diff --git a/doc/source/user/images/np_readcsv.png b/doc/source/user/images/np_readcsv.png Binary files differnew file mode 100644 index 000000000..9d2b9e0a0 --- /dev/null +++ b/doc/source/user/images/np_readcsv.png diff --git a/doc/source/user/images/np_reshape.png b/doc/source/user/images/np_reshape.png Binary files differnew file mode 100644 index 000000000..7ebb8d69d --- /dev/null +++ b/doc/source/user/images/np_reshape.png diff --git a/doc/source/user/images/np_sub_mult_divide.png b/doc/source/user/images/np_sub_mult_divide.png Binary files differnew file mode 100644 index 000000000..a5df2a687 --- /dev/null +++ b/doc/source/user/images/np_sub_mult_divide.png diff --git a/doc/source/user/images/np_transposing_reshaping.png b/doc/source/user/images/np_transposing_reshaping.png Binary files differnew file mode 100644 index 000000000..5399043c2 --- /dev/null +++ b/doc/source/user/images/np_transposing_reshaping.png diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst index a45fec9ec..b321ee83d 100644 --- a/doc/source/user/index.rst +++ b/doc/source/user/index.rst @@ -14,6 +14,7 @@ classes contained in the package, see the :ref:`reference`. setting-up quickstart + absolute_beginners basics misc numpy-for-matlab-users diff --git a/doc/source/user/plots/matplotlib1.py b/doc/source/user/plots/matplotlib1.py new file mode 100644 index 000000000..2cbf87ffa --- /dev/null +++ b/doc/source/user/plots/matplotlib1.py @@ -0,0 +1,7 @@ +import matplotlib.pyplot as plt +import numpy as np + +a = np.array([2, 1, 5, 7, 4, 6, 8, 14, 10, 9, 18, 20, 22]) + +plt.plot(a) +plt.show()
\ No newline at end of file diff --git a/doc/source/user/plots/matplotlib2.py b/doc/source/user/plots/matplotlib2.py new file mode 100644 index 000000000..e15986c25 --- /dev/null +++ b/doc/source/user/plots/matplotlib2.py @@ -0,0 +1,8 @@ +import matplotlib.pyplot as plt +import numpy as np + +x = np.linspace(0, 5, 20) +y = np.linspace(0, 10, 20) +plt.plot(x, y, 'purple') # line +plt.plot(x, y, 'o') # dots +plt.show()
\ No newline at end of file diff --git a/doc/source/user/plots/matplotlib3.py b/doc/source/user/plots/matplotlib3.py new file mode 100644 index 000000000..af778979b --- /dev/null +++ b/doc/source/user/plots/matplotlib3.py @@ -0,0 +1,16 @@ +import numpy as np +import matplotlib.pyplot as plt +from matplotlib import cm +from mpl_toolkits.mplot3d import Axes3D + +fig = plt.figure() +ax = Axes3D(fig) +X = np.arange(-5, 5, 0.15) +Y = np.arange(-5, 5, 0.15) +X, Y = np.meshgrid(X, Y) +R = np.sqrt(X**2 + Y**2) +Z = np.sin(R) + +ax.plot_surface(X, Y, Z, rstride=1, cstride=1, cmap='viridis') + +plt.show()
\ No newline at end of file diff --git a/doc/source/user/quickstart.rst b/doc/source/user/quickstart.rst index 34e327d75..8a5a863b1 100644 --- a/doc/source/user/quickstart.rst +++ b/doc/source/user/quickstart.rst @@ -20,6 +20,9 @@ If you wish to work the examples in this tutorial, you must also have some software installed on your computer. Please see https://scipy.org/install.html for instructions. + +.. _quickstart.the-basics: + The Basics ========== @@ -95,6 +98,7 @@ An example >>> type(b) <type 'numpy.ndarray'> +.. _quickstart.array-creation: Array Creation -------------- @@ -273,6 +277,8 @@ can change the printing options using ``set_printoptions``. >>> np.set_printoptions(threshold=sys.maxsize) # sys module should be imported +.. _quickstart.basic-operations: + Basic Operations ---------------- @@ -460,6 +466,8 @@ operate elementwise on an array, producing an array as output. `vectorize`, `where` +.. _quickstart.indexing-slicing-and-iterating: + Indexing, Slicing and Iterating ------------------------------- @@ -685,6 +693,9 @@ dimensions are automatically calculated:: `resize`, `ravel` + +.. _quickstart.stacking-arrays: + Stacking together different arrays ---------------------------------- @@ -801,6 +812,9 @@ which the division should occur:: axis, and `array_split` allows one to specify along which axis to split. + +.. _quickstart.copies-and-views: + Copies and Views ================ diff --git a/doc/source/user/whatisnumpy.rst b/doc/source/user/whatisnumpy.rst index abaa2bfed..8478a77c4 100644 --- a/doc/source/user/whatisnumpy.rst +++ b/doc/source/user/whatisnumpy.rst @@ -1,3 +1,5 @@ +.. _whatisnumpy: + ************** What is NumPy? ************** diff --git a/numpy/core/__init__.py b/numpy/core/__init__.py index 815c61924..c2d53fe3e 100644 --- a/numpy/core/__init__.py +++ b/numpy/core/__init__.py @@ -135,16 +135,11 @@ def _ufunc_reduce(func): return _ufunc_reconstruct, (whichmodule(func, name), name) -import sys -if sys.version_info[0] >= 3: - import copyreg -else: - import copy_reg as copyreg +import copyreg copyreg.pickle(ufunc, _ufunc_reduce, _ufunc_reconstruct) # Unclutter namespace (must keep _ufunc_reconstruct for unpickling) del copyreg -del sys del _ufunc_reduce from numpy._pytesttester import PytestTester diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py index 2a750cc24..f21774cb6 100644 --- a/numpy/core/_internal.py +++ b/numpy/core/_internal.py @@ -838,7 +838,7 @@ def npy_ctypes_check(cls): # # (..., _ctypes._CData, object) ctype_base = cls.__mro__[-2] # right now, they're part of the _ctypes module - return 'ctypes' in ctype_base.__module__ + return '_ctypes' in ctype_base.__module__ except Exception: return False diff --git a/numpy/core/arrayprint.py b/numpy/core/arrayprint.py index 918da4a72..ec7e4261f 100644 --- a/numpy/core/arrayprint.py +++ b/numpy/core/arrayprint.py @@ -22,19 +22,12 @@ __docformat__ = 'restructuredtext' # scalars are printed inside an ndarray. Only the latter strs are currently # user-customizable. -import sys import functools import numbers -if sys.version_info[0] >= 3: - try: - from _thread import get_ident - except ImportError: - from _dummy_thread import get_ident -else: - try: - from thread import get_ident - except ImportError: - from dummy_thread import get_ident +try: + from _thread import get_ident +except ImportError: + from _dummy_thread import get_ident import numpy as np from . import numerictypes as _nt diff --git a/numpy/core/code_generators/generate_umath.py b/numpy/core/code_generators/generate_umath.py index f5691d950..7599360f5 100644 --- a/numpy/core/code_generators/generate_umath.py +++ b/numpy/core/code_generators/generate_umath.py @@ -148,12 +148,8 @@ class Ufunc: # String-handling utilities to avoid locale-dependence. import string -if sys.version_info[0] < 3: - UPPER_TABLE = string.maketrans(string.ascii_lowercase, - string.ascii_uppercase) -else: - UPPER_TABLE = bytes.maketrans(bytes(string.ascii_lowercase, "ascii"), - bytes(string.ascii_uppercase, "ascii")) +UPPER_TABLE = bytes.maketrans(bytes(string.ascii_lowercase, "ascii"), + bytes(string.ascii_uppercase, "ascii")) def english_upper(s): """ Apply English case rules to convert ASCII strings to all upper case. @@ -1076,15 +1072,9 @@ def make_ufuncs(funcdict): uf = funcdict[name] mlist = [] docstring = textwrap.dedent(uf.docstring).strip() - if sys.version_info[0] < 3: - docstring = docstring.encode('string-escape') - docstring = docstring.replace(r'"', r'\"') - else: - docstring = docstring.encode('unicode-escape').decode('ascii') - docstring = docstring.replace(r'"', r'\"') - # XXX: I don't understand why the following replace is not - # necessary in the python 2 case. - docstring = docstring.replace(r"'", r"\'") + docstring = docstring.encode('unicode-escape').decode('ascii') + docstring = docstring.replace(r'"', r'\"') + docstring = docstring.replace(r"'", r"\'") # Split the docstring because some compilers (like MS) do not like big # string literal in C code. We split at endlines because textwrap.wrap # do not play well with \n diff --git a/numpy/core/defchararray.py b/numpy/core/defchararray.py index 4d7781317..1cfdc55c0 100644 --- a/numpy/core/defchararray.py +++ b/numpy/core/defchararray.py @@ -1953,8 +1953,8 @@ class chararray(ndarray): # strings in the new array. itemsize = long(itemsize) - if sys.version_info[0] >= 3 and isinstance(buffer, str): - # On Py3, unicode objects do not have the buffer interface + if isinstance(buffer, str): + # unicode objects do not have the buffer interface filler = buffer buffer = None else: diff --git a/numpy/core/numeric.py b/numpy/core/numeric.py index 5a1cbe9fc..72c6089b8 100644 --- a/numpy/core/numeric.py +++ b/numpy/core/numeric.py @@ -35,12 +35,6 @@ bitwise_not = invert ufunc = type(sin) newaxis = None -if sys.version_info[0] >= 3: - import builtins -else: - import __builtin__ as builtins - - array_function_dispatch = functools.partial( overrides.array_function_dispatch, module='numpy') diff --git a/numpy/core/numerictypes.py b/numpy/core/numerictypes.py index c63ea08c7..c06552c4e 100644 --- a/numpy/core/numerictypes.py +++ b/numpy/core/numerictypes.py @@ -80,7 +80,6 @@ Exported symbols include: """ import types as _types -import sys import numbers import warnings @@ -120,11 +119,8 @@ from ._dtype import _kind_name # we don't export these for import *, but we do want them accessible # as numerictypes.bool, etc. -if sys.version_info[0] >= 3: - from builtins import bool, int, float, complex, object, str - unicode = str -else: - from __builtin__ import bool, int, float, complex, object, unicode, str +from builtins import bool, int, float, complex, object, str +unicode = str # We use this later diff --git a/numpy/core/setup_common.py b/numpy/core/setup_common.py index a947f7a3d..6d8e603a7 100644 --- a/numpy/core/setup_common.py +++ b/numpy/core/setup_common.py @@ -244,9 +244,9 @@ def check_long_double_representation(cmd): # Disable multi-file interprocedural optimization in the Intel compiler on Linux # which generates intermediary object files and prevents checking the # float representation. - elif (sys.platform != "win32" - and cmd.compiler.compiler_type.startswith('intel') - and '-ipo' in cmd.compiler.cc_exe): + elif (sys.platform != "win32" + and cmd.compiler.compiler_type.startswith('intel') + and '-ipo' in cmd.compiler.cc_exe): newcompiler = cmd.compiler.cc_exe.replace(' -ipo', '') cmd.compiler.set_executables( compiler=newcompiler, diff --git a/numpy/core/src/multiarray/common.c b/numpy/core/src/multiarray/common.c index 39916a14d..c9ec32268 100644 --- a/numpy/core/src/multiarray/common.c +++ b/numpy/core/src/multiarray/common.c @@ -890,5 +890,3 @@ new_array_for_sum(PyArrayObject *ap1, PyArrayObject *ap2, PyArrayObject* out, } } - - diff --git a/numpy/core/src/multiarray/common.h b/numpy/core/src/multiarray/common.h index e77e51f42..4913eb202 100644 --- a/numpy/core/src/multiarray/common.h +++ b/numpy/core/src/multiarray/common.h @@ -235,7 +235,7 @@ npy_uint_alignment(int itemsize) default: break; } - + return alignment; } @@ -343,3 +343,4 @@ new_array_for_sum(PyArrayObject *ap1, PyArrayObject *ap2, PyArrayObject* out, int nd, npy_intp dimensions[], int typenum, PyArrayObject **result); #endif + diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index bf90142e1..0616bed65 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -678,6 +678,12 @@ discover_itemsize(PyObject *s, int nd, int *itemsize, int string_type) return 0; } +typedef enum { + DISCOVERED_OK = 0, + DISCOVERED_RAGGED = 1, + DISCOVERED_OBJECT = 2 +} discovered_t; + /* * Take an arbitrary object and discover how many dimensions it * has, filling in the dimensions as we go. @@ -685,7 +691,7 @@ discover_itemsize(PyObject *s, int nd, int *itemsize, int string_type) static int discover_dimensions(PyObject *obj, int *maxndim, npy_intp *d, int check_it, int stop_at_string, int stop_at_tuple, - int *out_is_object) + discovered_t *out_is_object) { PyObject *e; npy_intp n, i; @@ -871,7 +877,7 @@ discover_dimensions(PyObject *obj, int *maxndim, npy_intp *d, int check_it, if (PyErr_ExceptionMatches(PyExc_KeyError)) { PyErr_Clear(); *maxndim = 0; - *out_is_object = 1; + *out_is_object = DISCOVERED_OBJECT; return 0; } else { @@ -930,7 +936,7 @@ discover_dimensions(PyObject *obj, int *maxndim, npy_intp *d, int check_it, *maxndim = all_elems_maxndim + 1; if (!all_dimensions_match) { /* typically results in an array containing variable-length lists */ - *out_is_object = 1; + *out_is_object = DISCOVERED_RAGGED; } } @@ -1741,7 +1747,7 @@ PyArray_GetArrayParamsFromObject(PyObject *op, /* Try to treat op as a list of lists */ if (!writeable && PySequence_Check(op)) { - int check_it, stop_at_string, stop_at_tuple, is_object; + int check_it, stop_at_string, stop_at_tuple; int type_num, type; /* @@ -1791,7 +1797,7 @@ PyArray_GetArrayParamsFromObject(PyObject *op, ((*out_dtype)->names || (*out_dtype)->subarray)); *out_ndim = NPY_MAXDIMS; - is_object = 0; + discovered_t is_object = DISCOVERED_OK; if (discover_dimensions( op, out_ndim, out_dims, check_it, stop_at_string, stop_at_tuple, &is_object) < 0) { @@ -1808,7 +1814,27 @@ PyArray_GetArrayParamsFromObject(PyObject *op, return 0; } /* If object arrays are forced */ - if (is_object) { + if (is_object != DISCOVERED_OK) { + static PyObject *visibleDeprecationWarning = NULL; + npy_cache_import( + "numpy", "VisibleDeprecationWarning", + &visibleDeprecationWarning); + if (visibleDeprecationWarning == NULL) { + return -1; + } + if (is_object == DISCOVERED_RAGGED && requested_dtype == NULL) { + /* NumPy 1.19, 2019-11-01 */ + if (PyErr_WarnEx(visibleDeprecationWarning, "Creating an " + "ndarray from ragged nested sequences (which is a " + "list-or-tuple of lists-or-tuples-or ndarrays with " + "different lengths or shapes) is deprecated. If you " + "meant to do this, you must specify 'dtype=object' " + "when creating the ndarray", 1) < 0) + { + return -1; + } + } + /* either DISCOVERED_OBJECT or there is a requested_dtype */ Py_DECREF(*out_dtype); *out_dtype = PyArray_DescrFromType(NPY_OBJECT); if (*out_dtype == NULL) { @@ -2059,6 +2085,8 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, * * NPY_ARRAY_FORCECAST will cause a cast to occur regardless of whether or not * it is safe. + * + * context is passed through to PyArray_GetArrayParamsFromObject */ /*NUMPY_API diff --git a/numpy/core/src/multiarray/einsum.c.src b/numpy/core/src/multiarray/einsum.c.src index 70af3fef9..1cc557825 100644 --- a/numpy/core/src/multiarray/einsum.c.src +++ b/numpy/core/src/multiarray/einsum.c.src @@ -2152,6 +2152,11 @@ get_combined_dims_view(PyArrayObject *op, int iop, char *labels) } /* A repeated label, find the original one and merge them. */ else { +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wuninitialized" +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif int i = icombinemap[idim + label]; icombinemap[idim] = -1; @@ -2164,6 +2169,9 @@ get_combined_dims_view(PyArrayObject *op, int iop, char *labels) return NULL; } new_strides[i] += stride; +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif } } diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index af0cef15b..11e0bc44d 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -1559,7 +1559,6 @@ _prepend_ones(PyArrayObject *arr, int nd, int ndmin, NPY_ORDER order) return ret; } - #define STRIDING_OK(op, order) \ ((order) == NPY_ANYORDER || \ (order) == NPY_KEEPORDER || \ diff --git a/numpy/core/src/umath/simd.inc.src b/numpy/core/src/umath/simd.inc.src index 448aa3e72..121597579 100644 --- a/numpy/core/src/umath/simd.inc.src +++ b/numpy/core/src/umath/simd.inc.src @@ -1134,7 +1134,7 @@ sse2_@kind@_@TYPE@(@type@ * ip, @type@ * op, const npy_intp n) /* Order of operations important for MSVC 2015 */ *op = (*op @OP@ ip[i] || npy_isnan(*op)) ? *op : ip[i]; } - assert((npy_uintp)n < (stride) || npy_is_aligned(&ip[i], VECTOR_SIZE_BYTES)); + assert(n < stride || npy_is_aligned(&ip[i], VECTOR_SIZE_BYTES)); if (i + 3 * stride <= n) { /* load the first elements */ @vtype@ c1 = @vpre@_load_@vsuf@((@type@*)&ip[i]); diff --git a/numpy/core/tests/test_deprecations.py b/numpy/core/tests/test_deprecations.py index 9d68167fa..252133d7b 100644 --- a/numpy/core/tests/test_deprecations.py +++ b/numpy/core/tests/test_deprecations.py @@ -536,3 +536,16 @@ class TestNonZero(_DeprecationTestCase): def test_zerod(self): self.assert_deprecated(lambda: np.nonzero(np.array(0))) self.assert_deprecated(lambda: np.nonzero(np.array(1))) + + +def test_deprecate_ragged_arrays(): + # 2019-11-29 1.19.0 + # + # NEP 34 deprecated automatic object dtype when creating ragged + # arrays. Also see the "ragged" tests in `test_multiarray` + # + # emits a VisibleDeprecationWarning + arg = [1, [2, 3]] + with assert_warns(np.VisibleDeprecationWarning): + np.array(arg) + diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 503566b63..600941cfd 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -424,7 +424,7 @@ class TestArrayConstruction: assert_equal(r, np.ones((2, 6, 6))) d = np.ones((6, )) - r = np.array([[d, d + 1], d + 2]) + r = np.array([[d, d + 1], d + 2], dtype=object) assert_equal(len(r), 2) assert_equal(r[0], [d, d + 1]) assert_equal(r[1], d + 2) @@ -1020,34 +1020,60 @@ class TestCreation: assert_raises(ValueError, np.ndarray, buffer=buf, strides=(0,), shape=(max_bytes//itemsize + 1,), dtype=dtype) - def test_jagged_ndim_object(self): - # Lists of mismatching depths are treated as object arrays - a = np.array([[1], 2, 3]) - assert_equal(a.shape, (3,)) - assert_equal(a.dtype, object) + def _ragged_creation(self, seq): + # without dtype=object, the ragged object should raise + with assert_warns(np.VisibleDeprecationWarning): + a = np.array(seq) + b = np.array(seq, dtype=object) + assert_equal(a, b) + return b - a = np.array([1, [2], 3]) + def test_ragged_ndim_object(self): + # Lists of mismatching depths are treated as object arrays + a = self._ragged_creation([[1], 2, 3]) assert_equal(a.shape, (3,)) assert_equal(a.dtype, object) - a = np.array([1, 2, [3]]) + a = self._ragged_creation([1, [2], 3]) assert_equal(a.shape, (3,)) assert_equal(a.dtype, object) - def test_jagged_shape_object(self): - # The jagged dimension of a list is turned into an object array - a = np.array([[1, 1], [2], [3]]) + a = self._ragged_creation([1, 2, [3]]) assert_equal(a.shape, (3,)) assert_equal(a.dtype, object) - a = np.array([[1], [2, 2], [3]]) + def test_ragged_shape_object(self): + # The ragged dimension of a list is turned into an object array + a = self._ragged_creation([[1, 1], [2], [3]]) assert_equal(a.shape, (3,)) assert_equal(a.dtype, object) - a = np.array([[1], [2], [3, 3]]) + a = self._ragged_creation([[1], [2, 2], [3]]) assert_equal(a.shape, (3,)) assert_equal(a.dtype, object) + a = self._ragged_creation([[1], [2], [3, 3]]) + assert a.shape == (3,) + assert a.dtype == object + + def test_array_of_ragged_array(self): + outer = np.array([None, None]) + outer[0] = outer[1] = np.array([1, 2, 3]) + assert np.array(outer).shape == (2,) + assert np.array([outer]).shape == (1, 2) + + outer_ragged = np.array([None, None]) + outer_ragged[0] = np.array([1, 2, 3]) + outer_ragged[1] = np.array([1, 2, 3, 4]) + # should both of these emit deprecation warnings? + assert np.array(outer_ragged).shape == (2,) + assert np.array([outer_ragged]).shape == (1, 2,) + + def test_deep_nonragged_object(self): + # None of these should raise, even though they are missing dtype=object + a = np.array([[[Decimal(1)]]]) + a = np.array([1, Decimal(1)]) + a = np.array([[1], [Decimal(1)]]) class TestStructured: def test_subarray_field_access(self): diff --git a/numpy/core/tests/test_numeric.py b/numpy/core/tests/test_numeric.py index 79f331121..c4956c298 100644 --- a/numpy/core/tests/test_numeric.py +++ b/numpy/core/tests/test_numeric.py @@ -1209,7 +1209,7 @@ class TestNonzero: def test_nonzero_invalid_object(self): # gh-9295 - a = np.array([np.array([1, 2]), 3]) + a = np.array([np.array([1, 2]), 3], dtype=object) assert_raises(ValueError, np.nonzero, a) class BoolErrors: diff --git a/numpy/core/tests/test_regression.py b/numpy/core/tests/test_regression.py index 1d268d3fc..3a9b96886 100644 --- a/numpy/core/tests/test_regression.py +++ b/numpy/core/tests/test_regression.py @@ -1354,13 +1354,13 @@ class TestRegression: def test_array_from_sequence_scalar_array(self): # Ticket #1078: segfaults when creating an array with a sequence of # 0d arrays. - a = np.array((np.ones(2), np.array(2))) + a = np.array((np.ones(2), np.array(2)), dtype=object) assert_equal(a.shape, (2,)) assert_equal(a.dtype, np.dtype(object)) assert_equal(a[0], np.ones(2)) assert_equal(a[1], np.array(2)) - a = np.array(((1,), np.array(1))) + a = np.array(((1,), np.array(1)), dtype=object) assert_equal(a.shape, (2,)) assert_equal(a.dtype, np.dtype(object)) assert_equal(a[0], (1,)) @@ -1368,7 +1368,7 @@ class TestRegression: def test_array_from_sequence_scalar_array2(self): # Ticket #1081: weird array with strange input... - t = np.array([np.array([]), np.array(0, object)]) + t = np.array([np.array([]), np.array(0, object)], dtype=object) assert_equal(t.shape, (2,)) assert_equal(t.dtype, np.dtype(object)) @@ -2256,9 +2256,10 @@ class TestRegression: x[0], x[-1] = x[-1], x[0] uf = np.frompyfunc(f, 1, 0) - a = np.array([[1, 2, 3], [4, 5], [6, 7, 8, 9]]) + a = np.array([[1, 2, 3], [4, 5], [6, 7, 8, 9]], dtype=object) assert_equal(uf(a), ()) - assert_array_equal(a, [[3, 2, 1], [5, 4], [9, 7, 8, 6]]) + expected = np.array([[3, 2, 1], [5, 4], [9, 7, 8, 6]], dtype=object) + assert_array_equal(a, expected) @pytest.mark.skipif(not HAS_REFCOUNT, reason="Python lacks refcounts") def test_leak_in_structured_dtype_comparison(self): diff --git a/numpy/core/tests/test_ufunc.py b/numpy/core/tests/test_ufunc.py index a299bcf28..abdaeeb93 100644 --- a/numpy/core/tests/test_ufunc.py +++ b/numpy/core/tests/test_ufunc.py @@ -1160,14 +1160,18 @@ class TestUfunc: # Twice reproduced also for tuples: np.add.accumulate(arr, out=arr) np.add.accumulate(arr, out=arr) - assert_array_equal(arr, np.array([[1]*i for i in [1, 3, 6, 10]])) + assert_array_equal(arr, + np.array([[1]*i for i in [1, 3, 6, 10]], dtype=object), + ) # And the same if the axis argument is used arr = np.ones((2, 4), dtype=object) arr[0, :] = [[2] for i in range(4)] np.add.accumulate(arr, out=arr, axis=-1) np.add.accumulate(arr, out=arr, axis=-1) - assert_array_equal(arr[0, :], np.array([[2]*i for i in [1, 3, 6, 10]])) + assert_array_equal(arr[0, :], + np.array([[2]*i for i in [1, 3, 6, 10]], dtype=object), + ) def test_object_array_reduceat_inplace(self): # Checks that in-place reduceats work, see also gh-7465 diff --git a/numpy/distutils/ccompiler.py b/numpy/distutils/ccompiler.py index 35cf9df81..ea7912feb 100644 --- a/numpy/distutils/ccompiler.py +++ b/numpy/distutils/ccompiler.py @@ -271,12 +271,8 @@ def CCompiler_compile(self, sources, output_dir=None, macros=None, if not sources: return [] - # FIXME:RELATIVE_IMPORT - if sys.version_info[0] < 3: - from .fcompiler import FCompiler, is_f_file, has_f90_header - else: - from numpy.distutils.fcompiler import (FCompiler, is_f_file, - has_f90_header) + from numpy.distutils.fcompiler import (FCompiler, is_f_file, + has_f90_header) if isinstance(self, FCompiler): display = [] for fc in ['f77', 'f90', 'fix']: diff --git a/numpy/distutils/cpuinfo.py b/numpy/distutils/cpuinfo.py index efea90113..e066f9888 100644 --- a/numpy/distutils/cpuinfo.py +++ b/numpy/distutils/cpuinfo.py @@ -17,10 +17,7 @@ __all__ = ['cpu'] import sys, re, types import os -if sys.version_info[0] >= 3: - from subprocess import getstatusoutput -else: - from commands import getstatusoutput +from subprocess import getstatusoutput import warnings import platform @@ -484,10 +481,7 @@ class Win32CPUInfo(CPUInfoBase): info = [] try: #XXX: Bad style to use so long `try:...except:...`. Fix it! - if sys.version_info[0] >= 3: - import winreg - else: - import _winreg as winreg + import winreg prgx = re.compile(r"family\s+(?P<FML>\d+)\s+model\s+(?P<MDL>\d+)" r"\s+stepping\s+(?P<STP>\d+)", re.IGNORECASE) diff --git a/numpy/distutils/exec_command.py b/numpy/distutils/exec_command.py index d35b4f898..fb10d2470 100644 --- a/numpy/distutils/exec_command.py +++ b/numpy/distutils/exec_command.py @@ -74,10 +74,6 @@ def filepath_from_subprocess_output(output): # Another historical oddity if output[-1:] == '\n': output = output[:-1] - # stdio uses bytes in python 2, so to avoid issues, we simply - # remove all non-ascii characters - if sys.version_info < (3, 0): - output = output.encode('ascii', errors='replace') return output @@ -89,10 +85,7 @@ def forward_bytes_to_stdout(val): The assumption is that the subprocess call already returned bytes in a suitable encoding. """ - if sys.version_info.major < 3: - # python 2 has binary output anyway - sys.stdout.write(val) - elif hasattr(sys.stdout, 'buffer'): + if hasattr(sys.stdout, 'buffer'): # use the underlying binary output if there is one sys.stdout.buffer.write(val) elif hasattr(sys.stdout, 'encoding'): @@ -305,11 +298,6 @@ def _exec_command(command, use_shell=None, use_tee = None, **env): if text[-1:] == '\n': text = text[:-1] - # stdio uses bytes in python 2, so to avoid issues, we simply - # remove all non-ascii characters - if sys.version_info < (3, 0): - text = text.encode('ascii', errors='replace') - if use_tee and text: print(text) return proc.returncode, text diff --git a/numpy/distutils/fcompiler/gnu.py b/numpy/distutils/fcompiler/gnu.py index 4fc9f33ff..128e54db3 100644 --- a/numpy/distutils/fcompiler/gnu.py +++ b/numpy/distutils/fcompiler/gnu.py @@ -412,8 +412,7 @@ class Gnu95FCompiler(GnuFCompiler): break h.update(block) text = base64.b32encode(h.digest()) - if sys.version_info[0] >= 3: - text = text.decode('ascii') + text = text.decode('ascii') return text.rstrip('=') def _link_wrapper_lib(self, objects, output_dir, extra_dll_dir, diff --git a/numpy/distutils/log.py b/numpy/distutils/log.py index ec1100b1b..79eec00a6 100644 --- a/numpy/distutils/log.py +++ b/numpy/distutils/log.py @@ -4,12 +4,8 @@ from distutils.log import * from distutils.log import Log as old_Log from distutils.log import _global_log -if sys.version_info[0] < 3: - from .misc_util import (red_text, default_text, cyan_text, green_text, - is_sequence, is_string) -else: - from numpy.distutils.misc_util import (red_text, default_text, cyan_text, - green_text, is_sequence, is_string) +from numpy.distutils.misc_util import (red_text, default_text, cyan_text, + green_text, is_sequence, is_string) def _fix_args(args,flag=1): diff --git a/numpy/distutils/mingw32ccompiler.py b/numpy/distutils/mingw32ccompiler.py index e2cd1c19b..475f73718 100644 --- a/numpy/distutils/mingw32ccompiler.py +++ b/numpy/distutils/mingw32ccompiler.py @@ -15,11 +15,7 @@ import textwrap # Overwrite certain distutils.ccompiler functions: import numpy.distutils.ccompiler - -if sys.version_info[0] < 3: - from . import log -else: - from numpy.distutils import log +from numpy.distutils import log # NT stuff # 1. Make sure libpython<version>.a exists for gcc. If not, build it. # 2. Force windows to use gcc (we're struggling with MSVC and g77 support) diff --git a/numpy/distutils/misc_util.py b/numpy/distutils/misc_util.py index eec8d56a3..f9d2be716 100644 --- a/numpy/distutils/misc_util.py +++ b/numpy/distutils/misc_util.py @@ -2242,10 +2242,7 @@ def get_info(pkgname, dirs=None): return info def is_bootstrapping(): - if sys.version_info[0] >= 3: - import builtins - else: - import __builtin__ as builtins + import builtins try: builtins.__NUMPY_SETUP__ diff --git a/numpy/distutils/npy_pkg_config.py b/numpy/distutils/npy_pkg_config.py index 47965b4ae..26a0437fb 100644 --- a/numpy/distutils/npy_pkg_config.py +++ b/numpy/distutils/npy_pkg_config.py @@ -2,10 +2,7 @@ import sys import re import os -if sys.version_info[0] < 3: - from ConfigParser import RawConfigParser -else: - from configparser import RawConfigParser +from configparser import RawConfigParser __all__ = ['FormatError', 'PkgNotFound', 'LibraryInfo', 'VariableSet', 'read_config', 'parse_flags'] diff --git a/numpy/distutils/system_info.py b/numpy/distutils/system_info.py index 6a69fed59..f0641a688 100644 --- a/numpy/distutils/system_info.py +++ b/numpy/distutils/system_info.py @@ -138,12 +138,8 @@ import textwrap from glob import glob from functools import reduce -if sys.version_info[0] < 3: - from ConfigParser import NoOptionError - from ConfigParser import RawConfigParser as ConfigParser -else: - from configparser import NoOptionError - from configparser import RawConfigParser as ConfigParser +from configparser import NoOptionError +from configparser import RawConfigParser as ConfigParser # It seems that some people are importing ConfigParser from here so is # good to keep its class name. Use of RawConfigParser is needed in # order to be able to load path names with percent in them, like @@ -264,32 +260,29 @@ if sys.platform == 'win32': default_include_dirs.extend( os.path.join(library_root, d) for d in _include_dirs) - if sys.version_info >= (3, 3): - # VCpkg is the de-facto package manager on windows for C/C++ - # libraries. If it is on the PATH, then we append its paths here. - # We also don't re-implement shutil.which for Python 2.7 because - # vcpkg doesn't support MSVC 2008. - vcpkg = shutil.which('vcpkg') - if vcpkg: - vcpkg_dir = os.path.dirname(vcpkg) - if platform.architecture() == '32bit': - specifier = 'x86' - else: - specifier = 'x64' - - vcpkg_installed = os.path.join(vcpkg_dir, 'installed') - for vcpkg_root in [ - os.path.join(vcpkg_installed, specifier + '-windows'), - os.path.join(vcpkg_installed, specifier + '-windows-static'), - ]: - add_system_root(vcpkg_root) - - # Conda is another popular package manager that provides libraries - conda = shutil.which('conda') - if conda: - conda_dir = os.path.dirname(conda) - add_system_root(os.path.join(conda_dir, '..', 'Library')) - add_system_root(os.path.join(conda_dir, 'Library')) + # VCpkg is the de-facto package manager on windows for C/C++ + # libraries. If it is on the PATH, then we append its paths here. + vcpkg = shutil.which('vcpkg') + if vcpkg: + vcpkg_dir = os.path.dirname(vcpkg) + if platform.architecture() == '32bit': + specifier = 'x86' + else: + specifier = 'x64' + + vcpkg_installed = os.path.join(vcpkg_dir, 'installed') + for vcpkg_root in [ + os.path.join(vcpkg_installed, specifier + '-windows'), + os.path.join(vcpkg_installed, specifier + '-windows-static'), + ]: + add_system_root(vcpkg_root) + + # Conda is another popular package manager that provides libraries + conda = shutil.which('conda') + if conda: + conda_dir = os.path.dirname(conda) + add_system_root(os.path.join(conda_dir, '..', 'Library')) + add_system_root(os.path.join(conda_dir, 'Library')) else: default_lib_dirs = libpaths(['/usr/local/lib', '/opt/lib', '/usr/lib', @@ -2145,8 +2138,6 @@ class openblas_info(blas_info): extra_args = info['extra_link_args'] except Exception: extra_args = [] - if sys.version_info < (3, 5) and sys.version_info > (3, 0) and c.compiler_type == "msvc": - extra_args.append("/MANIFEST") try: with open(src, 'wt') as f: f.write(s) diff --git a/numpy/distutils/unixccompiler.py b/numpy/distutils/unixccompiler.py index 9a4d3ba52..cf62cb019 100644 --- a/numpy/distutils/unixccompiler.py +++ b/numpy/distutils/unixccompiler.py @@ -8,11 +8,7 @@ from distutils.errors import DistutilsExecError, CompileError from distutils.unixccompiler import * from numpy.distutils.ccompiler import replace_method from numpy.distutils.misc_util import _commandline_dep_string - -if sys.version_info[0] < 3: - from . import log -else: - from numpy.distutils import log +from numpy.distutils import log # Note that UnixCCompiler._compile appeared in Python 2.3 def UnixCCompiler__compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): diff --git a/numpy/fft/_pocketfft.c b/numpy/fft/_pocketfft.c index de86e36d3..764116a84 100644 --- a/numpy/fft/_pocketfft.c +++ b/numpy/fft/_pocketfft.c @@ -10,6 +10,11 @@ * \author Martin Reinecke */ +#define NPY_NO_DEPRECATED_API NPY_API_VERSION + +#include "Python.h" +#include "numpy/arrayobject.h" + #include <math.h> #include <string.h> #include <stdlib.h> @@ -2184,11 +2189,6 @@ WARN_UNUSED_RESULT static int rfft_forward(rfft_plan plan, double c[], double fc return rfftblue_forward(plan->blueplan,c,fct); } -#define NPY_NO_DEPRECATED_API NPY_API_VERSION - -#include "Python.h" -#include "numpy/arrayobject.h" - static PyObject * execute_complex(PyObject *a1, int is_forward, double fct) { diff --git a/numpy/lib/_datasource.py b/numpy/lib/_datasource.py index aa793958e..139b8c0ca 100644 --- a/numpy/lib/_datasource.py +++ b/numpy/lib/_datasource.py @@ -70,70 +70,6 @@ def _check_mode(mode, encoding, newline): raise ValueError("Argument 'newline' not supported in binary mode") -def _python2_bz2open(fn, mode, encoding, newline): - """Wrapper to open bz2 in text mode. - - Parameters - ---------- - fn : str - File name - mode : {'r', 'w'} - File mode. Note that bz2 Text files are not supported. - encoding : str - Ignored, text bz2 files not supported in Python2. - newline : str - Ignored, text bz2 files not supported in Python2. - """ - import bz2 - - _check_mode(mode, encoding, newline) - - if "t" in mode: - # BZ2File is missing necessary functions for TextIOWrapper - warnings.warn("Assuming latin1 encoding for bz2 text file in Python2", - RuntimeWarning, stacklevel=5) - mode = mode.replace("t", "") - return bz2.BZ2File(fn, mode) - -def _python2_gzipopen(fn, mode, encoding, newline): - """ Wrapper to open gzip in text mode. - - Parameters - ---------- - fn : str, bytes, file - File path or opened file. - mode : str - File mode. The actual files are opened as binary, but will decoded - using the specified `encoding` and `newline`. - encoding : str - Encoding to be used when reading/writing as text. - newline : str - Newline to be used when reading/writing as text. - - """ - import gzip - # gzip is lacking read1 needed for TextIOWrapper - class GzipWrap(gzip.GzipFile): - def read1(self, n): - return self.read(n) - - _check_mode(mode, encoding, newline) - - gz_mode = mode.replace("t", "") - - if isinstance(fn, (str, bytes)): - binary_file = GzipWrap(fn, gz_mode) - elif hasattr(fn, "read") or hasattr(fn, "write"): - binary_file = GzipWrap(None, gz_mode, fileobj=fn) - else: - raise TypeError("filename must be a str or bytes object, or a file") - - if "t" in mode: - return io.TextIOWrapper(binary_file, encoding, newline=newline) - else: - return binary_file - - # Using a class instead of a module-level dictionary # to reduce the initial 'import numpy' overhead by # deferring the import of lzma, bz2 and gzip until needed @@ -174,19 +110,13 @@ class _FileOpeners: try: import bz2 - if sys.version_info[0] >= 3: - self._file_openers[".bz2"] = bz2.open - else: - self._file_openers[".bz2"] = _python2_bz2open + self._file_openers[".bz2"] = bz2.open except ImportError: pass try: import gzip - if sys.version_info[0] >= 3: - self._file_openers[".gz"] = gzip.open - else: - self._file_openers[".gz"] = _python2_gzipopen + self._file_openers[".gz"] = gzip.open except ImportError: pass @@ -547,14 +477,10 @@ class DataSource: if os.path.exists(path): return True - # We import this here because importing urllib2 is slow and + # We import this here because importing urllib is slow and # a significant fraction of numpy's total import time. - if sys.version_info[0] >= 3: - from urllib.request import urlopen - from urllib.error import URLError - else: - from urllib2 import urlopen - from urllib2 import URLError + from urllib.request import urlopen + from urllib.error import URLError # Test cached url upath = self.abspath(path) diff --git a/numpy/lib/format.py b/numpy/lib/format.py index 15a74518b..114bae287 100644 --- a/numpy/lib/format.py +++ b/numpy/lib/format.py @@ -162,7 +162,6 @@ evolved with time and this document is more current. """ import numpy -import sys import io import warnings from numpy.lib.utils import safe_eval @@ -213,10 +212,7 @@ def magic(major, minor): raise ValueError("major version must be 0 <= major < 256") if minor < 0 or minor > 255: raise ValueError("minor version must be 0 <= minor < 256") - if sys.version_info[0] < 3: - return MAGIC_PREFIX + chr(major) + chr(minor) - else: - return MAGIC_PREFIX + bytes([major, minor]) + return MAGIC_PREFIX + bytes([major, minor]) def read_magic(fp): """ Read the magic string to get the version of the file format. @@ -234,10 +230,7 @@ def read_magic(fp): if magic_str[:-2] != MAGIC_PREFIX: msg = "the magic string is not correct; expected %r, got %r" raise ValueError(msg % (MAGIC_PREFIX, magic_str[:-2])) - if sys.version_info[0] < 3: - major, minor = map(ord, magic_str[-2:]) - else: - major, minor = magic_str[-2:] + major, minor = magic_str[-2:] return major, minor def _has_metadata(dt): @@ -542,10 +535,7 @@ def _filter_header(s): """ import tokenize - if sys.version_info[0] >= 3: - from io import StringIO - else: - from StringIO import StringIO + from io import StringIO tokens = [] last_token_was_number = False @@ -738,12 +728,10 @@ def read_array(fp, allow_pickle=False, pickle_kwargs=None): try: array = pickle.load(fp, **pickle_kwargs) except UnicodeError as err: - if sys.version_info[0] >= 3: - # Friendlier error message - raise UnicodeError("Unpickling a python object failed: %r\n" - "You may need to pass the encoding= option " - "to numpy.load" % (err,)) - raise + # Friendlier error message + raise UnicodeError("Unpickling a python object failed: %r\n" + "You may need to pass the encoding= option " + "to numpy.load" % (err,)) else: if isfileobj(fp): # We can use the fast fromfile() function. diff --git a/numpy/lib/mixins.py b/numpy/lib/mixins.py index d4811b94d..50157069c 100644 --- a/numpy/lib/mixins.py +++ b/numpy/lib/mixins.py @@ -1,6 +1,4 @@ """Mixin classes for custom array types that don't inherit from ndarray.""" -import sys - from numpy.core import umath as um @@ -152,9 +150,7 @@ class NDArrayOperatorsMixin: __mul__, __rmul__, __imul__ = _numeric_methods(um.multiply, 'mul') __matmul__, __rmatmul__, __imatmul__ = _numeric_methods( um.matmul, 'matmul') - if sys.version_info.major < 3: - # Python 3 uses only __truediv__ and __floordiv__ - __div__, __rdiv__, __idiv__ = _numeric_methods(um.divide, 'div') + # Python 3 does not use __div__, __rdiv__, or __idiv__ __truediv__, __rtruediv__, __itruediv__ = _numeric_methods( um.true_divide, 'truediv') __floordiv__, __rfloordiv__, __ifloordiv__ = _numeric_methods( diff --git a/numpy/lib/npyio.py b/numpy/lib/npyio.py index c47e388c0..29af488d2 100644 --- a/numpy/lib/npyio.py +++ b/numpy/lib/npyio.py @@ -7,6 +7,7 @@ import warnings import weakref import contextlib from operator import itemgetter, index as opindex +from collections.abc import Mapping import numpy as np from . import format @@ -26,12 +27,6 @@ from numpy.compat import ( pickle, contextlib_nullcontext ) -if sys.version_info[0] >= 3: - from collections.abc import Mapping -else: - from future_builtins import map - from collections import Mapping - @set_module('numpy') def loads(*args, **kwargs): @@ -264,26 +259,25 @@ class NpzFile(Mapping): raise KeyError("%s is not a file in the archive" % key) - if sys.version_info.major == 3: - # deprecate the python 2 dict apis that we supported by accident in - # python 3. We forgot to implement itervalues() at all in earlier - # versions of numpy, so no need to deprecated it here. + # deprecate the python 2 dict apis that we supported by accident in + # python 3. We forgot to implement itervalues() at all in earlier + # versions of numpy, so no need to deprecated it here. - def iteritems(self): - # Numpy 1.15, 2018-02-20 - warnings.warn( - "NpzFile.iteritems is deprecated in python 3, to match the " - "removal of dict.itertems. Use .items() instead.", - DeprecationWarning, stacklevel=2) - return self.items() + def iteritems(self): + # Numpy 1.15, 2018-02-20 + warnings.warn( + "NpzFile.iteritems is deprecated in python 3, to match the " + "removal of dict.itertems. Use .items() instead.", + DeprecationWarning, stacklevel=2) + return self.items() - def iterkeys(self): - # Numpy 1.15, 2018-02-20 - warnings.warn( - "NpzFile.iterkeys is deprecated in python 3, to match the " - "removal of dict.iterkeys. Use .keys() instead.", - DeprecationWarning, stacklevel=2) - return self.keys() + def iterkeys(self): + # Numpy 1.15, 2018-02-20 + warnings.warn( + "NpzFile.iterkeys is deprecated in python 3, to match the " + "removal of dict.iterkeys. Use .keys() instead.", + DeprecationWarning, stacklevel=2) + return self.keys() @set_module('numpy') @@ -412,11 +406,7 @@ def load(file, mmap_mode=None, allow_pickle=False, fix_imports=True, # result can similarly silently corrupt numerical data. raise ValueError("encoding must be 'ASCII', 'latin1', or 'bytes'") - if sys.version_info[0] >= 3: - pickle_kwargs = dict(encoding=encoding, fix_imports=fix_imports) - else: - # Nothing to do on Python 2 - pickle_kwargs = {} + pickle_kwargs = dict(encoding=encoding, fix_imports=fix_imports) # TODO: Use contextlib.ExitStack once we drop Python 2 if hasattr(file, 'read'): @@ -539,16 +529,10 @@ def save(file, arr, allow_pickle=True, fix_imports=True): fid = open(file, "wb") own_fid = True - if sys.version_info[0] >= 3: - pickle_kwargs = dict(fix_imports=fix_imports) - else: - # Nothing to do on Python 2 - pickle_kwargs = None - try: arr = np.asanyarray(arr) format.write_array(fid, arr, allow_pickle=allow_pickle, - pickle_kwargs=pickle_kwargs) + pickle_kwargs=dict(fix_imports=fix_imports)) finally: if own_fid: fid.close() @@ -691,7 +675,7 @@ def savez_compressed(file, *args, **kwds): The ``.npz`` file format is a zipped archive of files named after the variables they contain. The archive is compressed with ``zipfile.ZIP_DEFLATED`` and each file in the archive contains one variable - in ``.npy`` format. For a description of the ``.npy`` format, see + in ``.npy`` format. For a description of the ``.npy`` format, see :py:mod:`numpy.lib.format`. @@ -1375,9 +1359,6 @@ def savetxt(fname, X, fmt='%.18e', delimiter=' ', newline='\n', header='', open(fname, 'wt').close() fh = np.lib._datasource.open(fname, 'wt', encoding=encoding) own_fh = True - # need to convert str to unicode for text io output - if sys.version_info[0] == 2: - fh = WriteWrap(fh, encoding or 'latin1') elif hasattr(fname, 'write'): # wrap to handle byte output streams fh = WriteWrap(fname, encoding or 'latin1') diff --git a/numpy/lib/recfunctions.py b/numpy/lib/recfunctions.py index 4e62169f4..af4cfa09d 100644 --- a/numpy/lib/recfunctions.py +++ b/numpy/lib/recfunctions.py @@ -5,7 +5,6 @@ Most of these functions were initially implemented by John Hunter for matplotlib. They have been rewritten and extended for convenience. """ -import sys import itertools import numpy as np import numpy.ma as ma @@ -17,9 +16,6 @@ from numpy.lib._iotools import _is_string_like from numpy.compat import basestring from numpy.testing import suppress_warnings -if sys.version_info[0] < 3: - from future_builtins import zip - _check_fill_value = np.ma.core._check_fill_value @@ -333,12 +329,7 @@ def _izip_records(seqarrays, fill_value=None, flatten=True): else: zipfunc = _izip_fields - if sys.version_info[0] >= 3: - zip_longest = itertools.zip_longest - else: - zip_longest = itertools.izip_longest - - for tup in zip_longest(*seqarrays, fillvalue=fill_value): + for tup in itertools.zip_longest(*seqarrays, fillvalue=fill_value): yield tuple(zipfunc(tup)) diff --git a/numpy/lib/tests/test_arraypad.py b/numpy/lib/tests/test_arraypad.py index cd75b4ac4..75db5928b 100644 --- a/numpy/lib/tests/test_arraypad.py +++ b/numpy/lib/tests/test_arraypad.py @@ -1260,24 +1260,29 @@ class TestPadWidth: with pytest.raises(ValueError, match=match): np.pad(arr, pad_width, mode) - @pytest.mark.parametrize("pad_width", [ - "3", - "word", - None, - object(), - 3.4, - ((2, 3, 4), (3, 2)), # dtype=object (tuple) - complex(1, -1), - ((-2.1, 3), (3, 2)), + @pytest.mark.parametrize("pad_width, dtype", [ + ("3", None), + ("word", None), + (None, None), + (object(), None), + (3.4, None), + (((2, 3, 4), (3, 2)), object), + (complex(1, -1), None), + (((-2.1, 3), (3, 2)), None), ]) @pytest.mark.parametrize("mode", _all_modes.keys()) - def test_bad_type(self, pad_width, mode): + def test_bad_type(self, pad_width, dtype, mode): arr = np.arange(30).reshape((6, 5)) match = "`pad_width` must be of integral type." - with pytest.raises(TypeError, match=match): - np.pad(arr, pad_width, mode) - with pytest.raises(TypeError, match=match): - np.pad(arr, np.array(pad_width), mode) + if dtype is not None: + # avoid DeprecationWarning when not specifying dtype + with pytest.raises(TypeError, match=match): + np.pad(arr, np.array(pad_width, dtype=dtype), mode) + else: + with pytest.raises(TypeError, match=match): + np.pad(arr, pad_width, mode) + with pytest.raises(TypeError, match=match): + np.pad(arr, np.array(pad_width), mode) def test_pad_width_as_ndarray(self): a = np.arange(12) diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py index 6460741b5..2d6f39e40 100644 --- a/numpy/lib/tests/test_io.py +++ b/numpy/lib/tests/test_io.py @@ -575,7 +575,7 @@ class TestSaveTxt: def test_large_zip(self): # The test takes at least 6GB of memory, writes a file larger than 4GB test_data = np.asarray([np.random.rand(np.random.randint(50,100),4) - for i in range(800000)]) + for i in range(800000)], dtype=object) with tempdir() as tmpdir: np.savez(os.path.join(tmpdir, 'test.npz'), test_data=test_data) diff --git a/numpy/lib/utils.py b/numpy/lib/utils.py index d41a6e541..152322115 100644 --- a/numpy/lib/utils.py +++ b/numpy/lib/utils.py @@ -871,11 +871,7 @@ def _lookfor_generate_cache(module, import_modules, regenerate): # Local import to speed up numpy's import time. import inspect - if sys.version_info[0] >= 3: - # In Python3 stderr, stdout are text files. - from io import StringIO - else: - from StringIO import StringIO + from io import StringIO if module is None: module = "numpy" diff --git a/numpy/linalg/lapack_lite/clapack_scrub.py b/numpy/linalg/lapack_lite/clapack_scrub.py index 2ddd083ea..531d861cf 100644 --- a/numpy/linalg/lapack_lite/clapack_scrub.py +++ b/numpy/linalg/lapack_lite/clapack_scrub.py @@ -1,15 +1,11 @@ #!/usr/bin/env python import sys, os import re +from io import StringIO + from plex import Scanner, Str, Lexicon, Opt, Bol, State, AnyChar, TEXT, IGNORE from plex.traditional import re as Re -PY2 = sys.version_info < (3, 0) - -if PY2: - from io import BytesIO as UStringIO -else: - from io import StringIO as UStringIO class MyScanner(Scanner): def __init__(self, info, name='<default>'): @@ -25,8 +21,8 @@ def sep_seq(sequence, sep): return pat def runScanner(data, scanner_class, lexicon=None): - info = UStringIO(data) - outfo = UStringIO() + info = StringIO(data) + outfo = StringIO() if lexicon is not None: scanner = scanner_class(lexicon, info) else: @@ -193,7 +189,7 @@ def cleanComments(source): return SourceLines state = SourceLines - for line in UStringIO(source): + for line in StringIO(source): state = state(line) comments.flushTo(lines) return lines.getValue() @@ -221,7 +217,7 @@ def removeHeader(source): return OutOfHeader state = LookingForHeader - for line in UStringIO(source): + for line in StringIO(source): state = state(line) return lines.getValue() @@ -230,7 +226,7 @@ def removeSubroutinePrototypes(source): r'/[*] Subroutine [*]/^\s*(?:(?:inline|static)\s+){0,2}(?!else|typedef|return)\w+\s+\*?\s*(\w+)\s*\([^0]+\)\s*;?' ) lines = LineQueue() - for line in UStringIO(source): + for line in StringIO(source): if not expression.match(line): lines.add(line) @@ -252,7 +248,7 @@ def removeBuiltinFunctions(source): return InBuiltInFunctions state = LookingForBuiltinFunctions - for line in UStringIO(source): + for line in StringIO(source): state = state(line) return lines.getValue() diff --git a/numpy/linalg/lapack_lite/f2c.h b/numpy/linalg/lapack_lite/f2c.h index 4462eaa74..d3fbfc177 100644 --- a/numpy/linalg/lapack_lite/f2c.h +++ b/numpy/linalg/lapack_lite/f2c.h @@ -11,6 +11,8 @@ #include "numpy/npy_common.h" #include "npy_cblas.h" +#include "lapack_lite_names.h" + typedef CBLAS_INT integer; typedef char *address; typedef short int shortint; @@ -383,6 +385,9 @@ extern void z_log(doublecomplex *, doublecomplex *); extern void z_sin(doublecomplex *, doublecomplex *); extern void z_sqrt(doublecomplex *, doublecomplex *); +extern double f__cabs(double, double); +extern double f__cabsf(float, float); + #ifdef __cplusplus } #endif diff --git a/numpy/linalg/lapack_lite/f2c_blas.c b/numpy/linalg/lapack_lite/f2c_blas.c index 44ad23bfe..65286892f 100644 --- a/numpy/linalg/lapack_lite/f2c_blas.c +++ b/numpy/linalg/lapack_lite/f2c_blas.c @@ -1,7 +1,7 @@ /* -NOTE: This is generated code. Look in Misc/lapack_lite for information on - remaking this file. -*/ + * NOTE: This is generated code. Look in numpy/linalg/lapack_lite for + * information on remaking this file. + */ #include "f2c.h" #ifdef HAVE_CONFIG @@ -4912,7 +4912,7 @@ L20: ( 1 + ( n - 1 )*abs( INCX ) ). Before entry, the incremented array X must contain the n element vector x. On exit, X is overwritten with the - transformed vector x. + tranformed vector x. INCX - INTEGER. On entry, INCX specifies the increment for the elements of @@ -9807,7 +9807,7 @@ L40: ( 1 + ( n - 1 )*abs( INCX ) ). Before entry, the incremented array X must contain the n element vector x. On exit, X is overwritten with the - transformed vector x. + tranformed vector x. INCX - INTEGER. On entry, INCX specifies the increment for the elements of @@ -14410,7 +14410,7 @@ L40: ( 1 + ( n - 1 )*abs( INCX ) ). Before entry, the incremented array X must contain the n element vector x. On exit, X is overwritten with the - transformed vector x. + tranformed vector x. INCX - INTEGER. On entry, INCX specifies the increment for the elements of @@ -19998,7 +19998,7 @@ L20: ( 1 + ( n - 1 )*abs( INCX ) ). Before entry, the incremented array X must contain the n element vector x. On exit, X is overwritten with the - transformed vector x. + tranformed vector x. INCX - INTEGER. On entry, INCX specifies the increment for the elements of diff --git a/numpy/linalg/lapack_lite/f2c_c_lapack.c b/numpy/linalg/lapack_lite/f2c_c_lapack.c index f52e1e157..c36c0e368 100644 --- a/numpy/linalg/lapack_lite/f2c_c_lapack.c +++ b/numpy/linalg/lapack_lite/f2c_c_lapack.c @@ -1,7 +1,7 @@ /* -NOTE: This is generated code. Look in Misc/lapack_lite for information on - remaking this file. -*/ + * NOTE: This is generated code. Look in numpy/linalg/lapack_lite for + * information on remaking this file. + */ #include "f2c.h" #ifdef HAVE_CONFIG diff --git a/numpy/linalg/lapack_lite/f2c_config.c b/numpy/linalg/lapack_lite/f2c_config.c index 2fe608227..3f59e0263 100644 --- a/numpy/linalg/lapack_lite/f2c_config.c +++ b/numpy/linalg/lapack_lite/f2c_config.c @@ -1,7 +1,7 @@ /* -NOTE: This is generated code. Look in Misc/lapack_lite for information on - remaking this file. -*/ + * NOTE: This is generated code. Look in numpy/linalg/lapack_lite for + * information on remaking this file. + */ #include "f2c.h" #ifdef HAVE_CONFIG diff --git a/numpy/linalg/lapack_lite/f2c_d_lapack.c b/numpy/linalg/lapack_lite/f2c_d_lapack.c index 1a6675ef1..233db74b9 100644 --- a/numpy/linalg/lapack_lite/f2c_d_lapack.c +++ b/numpy/linalg/lapack_lite/f2c_d_lapack.c @@ -1,7 +1,7 @@ /* -NOTE: This is generated code. Look in Misc/lapack_lite for information on - remaking this file. -*/ + * NOTE: This is generated code. Look in numpy/linalg/lapack_lite for + * information on remaking this file. + */ #include "f2c.h" #ifdef HAVE_CONFIG diff --git a/numpy/linalg/lapack_lite/f2c_lapack.c b/numpy/linalg/lapack_lite/f2c_lapack.c index d956ddbbb..752261044 100644 --- a/numpy/linalg/lapack_lite/f2c_lapack.c +++ b/numpy/linalg/lapack_lite/f2c_lapack.c @@ -1,7 +1,7 @@ /* -NOTE: This is generated code. Look in Misc/lapack_lite for information on - remaking this file. -*/ + * NOTE: This is generated code. Look in numpy/linalg/lapack_lite for + * information on remaking this file. + */ #include "f2c.h" #ifdef HAVE_CONFIG diff --git a/numpy/linalg/lapack_lite/f2c_s_lapack.c b/numpy/linalg/lapack_lite/f2c_s_lapack.c index fccb1f58b..2a32315c7 100644 --- a/numpy/linalg/lapack_lite/f2c_s_lapack.c +++ b/numpy/linalg/lapack_lite/f2c_s_lapack.c @@ -1,7 +1,7 @@ /* -NOTE: This is generated code. Look in Misc/lapack_lite for information on - remaking this file. -*/ + * NOTE: This is generated code. Look in numpy/linalg/lapack_lite for + * information on remaking this file. + */ #include "f2c.h" #ifdef HAVE_CONFIG diff --git a/numpy/linalg/lapack_lite/f2c_z_lapack.c b/numpy/linalg/lapack_lite/f2c_z_lapack.c index 0f11f2e72..8234eca41 100644 --- a/numpy/linalg/lapack_lite/f2c_z_lapack.c +++ b/numpy/linalg/lapack_lite/f2c_z_lapack.c @@ -1,7 +1,7 @@ /* -NOTE: This is generated code. Look in Misc/lapack_lite for information on - remaking this file. -*/ + * NOTE: This is generated code. Look in numpy/linalg/lapack_lite for + * information on remaking this file. + */ #include "f2c.h" #ifdef HAVE_CONFIG diff --git a/numpy/linalg/lapack_lite/lapack_lite_names.h b/numpy/linalg/lapack_lite/lapack_lite_names.h new file mode 100644 index 000000000..08fd7257d --- /dev/null +++ b/numpy/linalg/lapack_lite/lapack_lite_names.h @@ -0,0 +1,691 @@ +/* + * NOTE: This is generated code. Look in numpy/linalg/lapack_lite for + * information on remaking this file. + */ +/* + * This file renames all BLAS/LAPACK and f2c symbols to avoid + * dynamic symbol name conflicts, in cases where e.g. + * integer sizes do not match with 'standard' ABI. + */ +#define caxpy_ BLAS_FUNC(caxpy) +#define ccopy_ BLAS_FUNC(ccopy) +#define cdotc_ BLAS_FUNC(cdotc) +#define cdotu_ BLAS_FUNC(cdotu) +#define cgebak_ BLAS_FUNC(cgebak) +#define cgebal_ BLAS_FUNC(cgebal) +#define cgebd2_ BLAS_FUNC(cgebd2) +#define cgebrd_ BLAS_FUNC(cgebrd) +#define cgeev_ BLAS_FUNC(cgeev) +#define cgehd2_ BLAS_FUNC(cgehd2) +#define cgehrd_ BLAS_FUNC(cgehrd) +#define cgelq2_ BLAS_FUNC(cgelq2) +#define cgelqf_ BLAS_FUNC(cgelqf) +#define cgelsd_ BLAS_FUNC(cgelsd) +#define cgemm_ BLAS_FUNC(cgemm) +#define cgemv_ BLAS_FUNC(cgemv) +#define cgeqr2_ BLAS_FUNC(cgeqr2) +#define cgeqrf_ BLAS_FUNC(cgeqrf) +#define cgerc_ BLAS_FUNC(cgerc) +#define cgeru_ BLAS_FUNC(cgeru) +#define cgesdd_ BLAS_FUNC(cgesdd) +#define cgesv_ BLAS_FUNC(cgesv) +#define cgetf2_ BLAS_FUNC(cgetf2) +#define cgetrf_ BLAS_FUNC(cgetrf) +#define cgetrs_ BLAS_FUNC(cgetrs) +#define cheevd_ BLAS_FUNC(cheevd) +#define chemv_ BLAS_FUNC(chemv) +#define cher2_ BLAS_FUNC(cher2) +#define cher2k_ BLAS_FUNC(cher2k) +#define cherk_ BLAS_FUNC(cherk) +#define chetd2_ BLAS_FUNC(chetd2) +#define chetrd_ BLAS_FUNC(chetrd) +#define chseqr_ BLAS_FUNC(chseqr) +#define clabrd_ BLAS_FUNC(clabrd) +#define clacgv_ BLAS_FUNC(clacgv) +#define clacp2_ BLAS_FUNC(clacp2) +#define clacpy_ BLAS_FUNC(clacpy) +#define clacrm_ BLAS_FUNC(clacrm) +#define cladiv_ BLAS_FUNC(cladiv) +#define claed0_ BLAS_FUNC(claed0) +#define claed7_ BLAS_FUNC(claed7) +#define claed8_ BLAS_FUNC(claed8) +#define clahqr_ BLAS_FUNC(clahqr) +#define clahr2_ BLAS_FUNC(clahr2) +#define clals0_ BLAS_FUNC(clals0) +#define clalsa_ BLAS_FUNC(clalsa) +#define clalsd_ BLAS_FUNC(clalsd) +#define clange_ BLAS_FUNC(clange) +#define clanhe_ BLAS_FUNC(clanhe) +#define claqr0_ BLAS_FUNC(claqr0) +#define claqr1_ BLAS_FUNC(claqr1) +#define claqr2_ BLAS_FUNC(claqr2) +#define claqr3_ BLAS_FUNC(claqr3) +#define claqr4_ BLAS_FUNC(claqr4) +#define claqr5_ BLAS_FUNC(claqr5) +#define clarcm_ BLAS_FUNC(clarcm) +#define clarf_ BLAS_FUNC(clarf) +#define clarfb_ BLAS_FUNC(clarfb) +#define clarfg_ BLAS_FUNC(clarfg) +#define clarft_ BLAS_FUNC(clarft) +#define clartg_ BLAS_FUNC(clartg) +#define clascl_ BLAS_FUNC(clascl) +#define claset_ BLAS_FUNC(claset) +#define clasr_ BLAS_FUNC(clasr) +#define classq_ BLAS_FUNC(classq) +#define claswp_ BLAS_FUNC(claswp) +#define clatrd_ BLAS_FUNC(clatrd) +#define clatrs_ BLAS_FUNC(clatrs) +#define clauu2_ BLAS_FUNC(clauu2) +#define clauum_ BLAS_FUNC(clauum) +#define cpotf2_ BLAS_FUNC(cpotf2) +#define cpotrf_ BLAS_FUNC(cpotrf) +#define cpotri_ BLAS_FUNC(cpotri) +#define cpotrs_ BLAS_FUNC(cpotrs) +#define crot_ BLAS_FUNC(crot) +#define cscal_ BLAS_FUNC(cscal) +#define csrot_ BLAS_FUNC(csrot) +#define csscal_ BLAS_FUNC(csscal) +#define cstedc_ BLAS_FUNC(cstedc) +#define csteqr_ BLAS_FUNC(csteqr) +#define cswap_ BLAS_FUNC(cswap) +#define ctrevc_ BLAS_FUNC(ctrevc) +#define ctrexc_ BLAS_FUNC(ctrexc) +#define ctrmm_ BLAS_FUNC(ctrmm) +#define ctrmv_ BLAS_FUNC(ctrmv) +#define ctrsm_ BLAS_FUNC(ctrsm) +#define ctrsv_ BLAS_FUNC(ctrsv) +#define ctrti2_ BLAS_FUNC(ctrti2) +#define ctrtri_ BLAS_FUNC(ctrtri) +#define cung2r_ BLAS_FUNC(cung2r) +#define cungbr_ BLAS_FUNC(cungbr) +#define cunghr_ BLAS_FUNC(cunghr) +#define cungl2_ BLAS_FUNC(cungl2) +#define cunglq_ BLAS_FUNC(cunglq) +#define cungqr_ BLAS_FUNC(cungqr) +#define cunm2l_ BLAS_FUNC(cunm2l) +#define cunm2r_ BLAS_FUNC(cunm2r) +#define cunmbr_ BLAS_FUNC(cunmbr) +#define cunmhr_ BLAS_FUNC(cunmhr) +#define cunml2_ BLAS_FUNC(cunml2) +#define cunmlq_ BLAS_FUNC(cunmlq) +#define cunmql_ BLAS_FUNC(cunmql) +#define cunmqr_ BLAS_FUNC(cunmqr) +#define cunmtr_ BLAS_FUNC(cunmtr) +#define daxpy_ BLAS_FUNC(daxpy) +#define dbdsdc_ BLAS_FUNC(dbdsdc) +#define dbdsqr_ BLAS_FUNC(dbdsqr) +#define dcabs1_ BLAS_FUNC(dcabs1) +#define dcopy_ BLAS_FUNC(dcopy) +#define ddot_ BLAS_FUNC(ddot) +#define dgebak_ BLAS_FUNC(dgebak) +#define dgebal_ BLAS_FUNC(dgebal) +#define dgebd2_ BLAS_FUNC(dgebd2) +#define dgebrd_ BLAS_FUNC(dgebrd) +#define dgeev_ BLAS_FUNC(dgeev) +#define dgehd2_ BLAS_FUNC(dgehd2) +#define dgehrd_ BLAS_FUNC(dgehrd) +#define dgelq2_ BLAS_FUNC(dgelq2) +#define dgelqf_ BLAS_FUNC(dgelqf) +#define dgelsd_ BLAS_FUNC(dgelsd) +#define dgemm_ BLAS_FUNC(dgemm) +#define dgemv_ BLAS_FUNC(dgemv) +#define dgeqr2_ BLAS_FUNC(dgeqr2) +#define dgeqrf_ BLAS_FUNC(dgeqrf) +#define dger_ BLAS_FUNC(dger) +#define dgesdd_ BLAS_FUNC(dgesdd) +#define dgesv_ BLAS_FUNC(dgesv) +#define dgetf2_ BLAS_FUNC(dgetf2) +#define dgetrf_ BLAS_FUNC(dgetrf) +#define dgetrs_ BLAS_FUNC(dgetrs) +#define dhseqr_ BLAS_FUNC(dhseqr) +#define disnan_ BLAS_FUNC(disnan) +#define dlabad_ BLAS_FUNC(dlabad) +#define dlabrd_ BLAS_FUNC(dlabrd) +#define dlacpy_ BLAS_FUNC(dlacpy) +#define dladiv_ BLAS_FUNC(dladiv) +#define dlae2_ BLAS_FUNC(dlae2) +#define dlaed0_ BLAS_FUNC(dlaed0) +#define dlaed1_ BLAS_FUNC(dlaed1) +#define dlaed2_ BLAS_FUNC(dlaed2) +#define dlaed3_ BLAS_FUNC(dlaed3) +#define dlaed4_ BLAS_FUNC(dlaed4) +#define dlaed5_ BLAS_FUNC(dlaed5) +#define dlaed6_ BLAS_FUNC(dlaed6) +#define dlaed7_ BLAS_FUNC(dlaed7) +#define dlaed8_ BLAS_FUNC(dlaed8) +#define dlaed9_ BLAS_FUNC(dlaed9) +#define dlaeda_ BLAS_FUNC(dlaeda) +#define dlaev2_ BLAS_FUNC(dlaev2) +#define dlaexc_ BLAS_FUNC(dlaexc) +#define dlahqr_ BLAS_FUNC(dlahqr) +#define dlahr2_ BLAS_FUNC(dlahr2) +#define dlaisnan_ BLAS_FUNC(dlaisnan) +#define dlaln2_ BLAS_FUNC(dlaln2) +#define dlals0_ BLAS_FUNC(dlals0) +#define dlalsa_ BLAS_FUNC(dlalsa) +#define dlalsd_ BLAS_FUNC(dlalsd) +#define dlamc1_ BLAS_FUNC(dlamc1) +#define dlamc2_ BLAS_FUNC(dlamc2) +#define dlamc3_ BLAS_FUNC(dlamc3) +#define dlamc4_ BLAS_FUNC(dlamc4) +#define dlamc5_ BLAS_FUNC(dlamc5) +#define dlamch_ BLAS_FUNC(dlamch) +#define dlamrg_ BLAS_FUNC(dlamrg) +#define dlange_ BLAS_FUNC(dlange) +#define dlanst_ BLAS_FUNC(dlanst) +#define dlansy_ BLAS_FUNC(dlansy) +#define dlanv2_ BLAS_FUNC(dlanv2) +#define dlapy2_ BLAS_FUNC(dlapy2) +#define dlapy3_ BLAS_FUNC(dlapy3) +#define dlaqr0_ BLAS_FUNC(dlaqr0) +#define dlaqr1_ BLAS_FUNC(dlaqr1) +#define dlaqr2_ BLAS_FUNC(dlaqr2) +#define dlaqr3_ BLAS_FUNC(dlaqr3) +#define dlaqr4_ BLAS_FUNC(dlaqr4) +#define dlaqr5_ BLAS_FUNC(dlaqr5) +#define dlarf_ BLAS_FUNC(dlarf) +#define dlarfb_ BLAS_FUNC(dlarfb) +#define dlarfg_ BLAS_FUNC(dlarfg) +#define dlarft_ BLAS_FUNC(dlarft) +#define dlarfx_ BLAS_FUNC(dlarfx) +#define dlartg_ BLAS_FUNC(dlartg) +#define dlas2_ BLAS_FUNC(dlas2) +#define dlascl_ BLAS_FUNC(dlascl) +#define dlasd0_ BLAS_FUNC(dlasd0) +#define dlasd1_ BLAS_FUNC(dlasd1) +#define dlasd2_ BLAS_FUNC(dlasd2) +#define dlasd3_ BLAS_FUNC(dlasd3) +#define dlasd4_ BLAS_FUNC(dlasd4) +#define dlasd5_ BLAS_FUNC(dlasd5) +#define dlasd6_ BLAS_FUNC(dlasd6) +#define dlasd7_ BLAS_FUNC(dlasd7) +#define dlasd8_ BLAS_FUNC(dlasd8) +#define dlasda_ BLAS_FUNC(dlasda) +#define dlasdq_ BLAS_FUNC(dlasdq) +#define dlasdt_ BLAS_FUNC(dlasdt) +#define dlaset_ BLAS_FUNC(dlaset) +#define dlasq1_ BLAS_FUNC(dlasq1) +#define dlasq2_ BLAS_FUNC(dlasq2) +#define dlasq3_ BLAS_FUNC(dlasq3) +#define dlasq4_ BLAS_FUNC(dlasq4) +#define dlasq5_ BLAS_FUNC(dlasq5) +#define dlasq6_ BLAS_FUNC(dlasq6) +#define dlasr_ BLAS_FUNC(dlasr) +#define dlasrt_ BLAS_FUNC(dlasrt) +#define dlassq_ BLAS_FUNC(dlassq) +#define dlasv2_ BLAS_FUNC(dlasv2) +#define dlaswp_ BLAS_FUNC(dlaswp) +#define dlasy2_ BLAS_FUNC(dlasy2) +#define dlatrd_ BLAS_FUNC(dlatrd) +#define dlauu2_ BLAS_FUNC(dlauu2) +#define dlauum_ BLAS_FUNC(dlauum) +#define dnrm2_ BLAS_FUNC(dnrm2) +#define dorg2r_ BLAS_FUNC(dorg2r) +#define dorgbr_ BLAS_FUNC(dorgbr) +#define dorghr_ BLAS_FUNC(dorghr) +#define dorgl2_ BLAS_FUNC(dorgl2) +#define dorglq_ BLAS_FUNC(dorglq) +#define dorgqr_ BLAS_FUNC(dorgqr) +#define dorm2l_ BLAS_FUNC(dorm2l) +#define dorm2r_ BLAS_FUNC(dorm2r) +#define dormbr_ BLAS_FUNC(dormbr) +#define dormhr_ BLAS_FUNC(dormhr) +#define dorml2_ BLAS_FUNC(dorml2) +#define dormlq_ BLAS_FUNC(dormlq) +#define dormql_ BLAS_FUNC(dormql) +#define dormqr_ BLAS_FUNC(dormqr) +#define dormtr_ BLAS_FUNC(dormtr) +#define dpotf2_ BLAS_FUNC(dpotf2) +#define dpotrf_ BLAS_FUNC(dpotrf) +#define dpotri_ BLAS_FUNC(dpotri) +#define dpotrs_ BLAS_FUNC(dpotrs) +#define drot_ BLAS_FUNC(drot) +#define dscal_ BLAS_FUNC(dscal) +#define dstedc_ BLAS_FUNC(dstedc) +#define dsteqr_ BLAS_FUNC(dsteqr) +#define dsterf_ BLAS_FUNC(dsterf) +#define dswap_ BLAS_FUNC(dswap) +#define dsyevd_ BLAS_FUNC(dsyevd) +#define dsymv_ BLAS_FUNC(dsymv) +#define dsyr2_ BLAS_FUNC(dsyr2) +#define dsyr2k_ BLAS_FUNC(dsyr2k) +#define dsyrk_ BLAS_FUNC(dsyrk) +#define dsytd2_ BLAS_FUNC(dsytd2) +#define dsytrd_ BLAS_FUNC(dsytrd) +#define dtrevc_ BLAS_FUNC(dtrevc) +#define dtrexc_ BLAS_FUNC(dtrexc) +#define dtrmm_ BLAS_FUNC(dtrmm) +#define dtrmv_ BLAS_FUNC(dtrmv) +#define dtrsm_ BLAS_FUNC(dtrsm) +#define dtrti2_ BLAS_FUNC(dtrti2) +#define dtrtri_ BLAS_FUNC(dtrtri) +#define dzasum_ BLAS_FUNC(dzasum) +#define dznrm2_ BLAS_FUNC(dznrm2) +#define icamax_ BLAS_FUNC(icamax) +#define idamax_ BLAS_FUNC(idamax) +#define ieeeck_ BLAS_FUNC(ieeeck) +#define ilaclc_ BLAS_FUNC(ilaclc) +#define ilaclr_ BLAS_FUNC(ilaclr) +#define iladlc_ BLAS_FUNC(iladlc) +#define iladlr_ BLAS_FUNC(iladlr) +#define ilaenv_ BLAS_FUNC(ilaenv) +#define ilaslc_ BLAS_FUNC(ilaslc) +#define ilaslr_ BLAS_FUNC(ilaslr) +#define ilazlc_ BLAS_FUNC(ilazlc) +#define ilazlr_ BLAS_FUNC(ilazlr) +#define iparmq_ BLAS_FUNC(iparmq) +#define isamax_ BLAS_FUNC(isamax) +#define izamax_ BLAS_FUNC(izamax) +#define lsame_ BLAS_FUNC(lsame) +#define saxpy_ BLAS_FUNC(saxpy) +#define sbdsdc_ BLAS_FUNC(sbdsdc) +#define sbdsqr_ BLAS_FUNC(sbdsqr) +#define scabs1_ BLAS_FUNC(scabs1) +#define scasum_ BLAS_FUNC(scasum) +#define scnrm2_ BLAS_FUNC(scnrm2) +#define scopy_ BLAS_FUNC(scopy) +#define sdot_ BLAS_FUNC(sdot) +#define sgebak_ BLAS_FUNC(sgebak) +#define sgebal_ BLAS_FUNC(sgebal) +#define sgebd2_ BLAS_FUNC(sgebd2) +#define sgebrd_ BLAS_FUNC(sgebrd) +#define sgeev_ BLAS_FUNC(sgeev) +#define sgehd2_ BLAS_FUNC(sgehd2) +#define sgehrd_ BLAS_FUNC(sgehrd) +#define sgelq2_ BLAS_FUNC(sgelq2) +#define sgelqf_ BLAS_FUNC(sgelqf) +#define sgelsd_ BLAS_FUNC(sgelsd) +#define sgemm_ BLAS_FUNC(sgemm) +#define sgemv_ BLAS_FUNC(sgemv) +#define sgeqr2_ BLAS_FUNC(sgeqr2) +#define sgeqrf_ BLAS_FUNC(sgeqrf) +#define sger_ BLAS_FUNC(sger) +#define sgesdd_ BLAS_FUNC(sgesdd) +#define sgesv_ BLAS_FUNC(sgesv) +#define sgetf2_ BLAS_FUNC(sgetf2) +#define sgetrf_ BLAS_FUNC(sgetrf) +#define sgetrs_ BLAS_FUNC(sgetrs) +#define shseqr_ BLAS_FUNC(shseqr) +#define sisnan_ BLAS_FUNC(sisnan) +#define slabad_ BLAS_FUNC(slabad) +#define slabrd_ BLAS_FUNC(slabrd) +#define slacpy_ BLAS_FUNC(slacpy) +#define sladiv_ BLAS_FUNC(sladiv) +#define slae2_ BLAS_FUNC(slae2) +#define slaed0_ BLAS_FUNC(slaed0) +#define slaed1_ BLAS_FUNC(slaed1) +#define slaed2_ BLAS_FUNC(slaed2) +#define slaed3_ BLAS_FUNC(slaed3) +#define slaed4_ BLAS_FUNC(slaed4) +#define slaed5_ BLAS_FUNC(slaed5) +#define slaed6_ BLAS_FUNC(slaed6) +#define slaed7_ BLAS_FUNC(slaed7) +#define slaed8_ BLAS_FUNC(slaed8) +#define slaed9_ BLAS_FUNC(slaed9) +#define slaeda_ BLAS_FUNC(slaeda) +#define slaev2_ BLAS_FUNC(slaev2) +#define slaexc_ BLAS_FUNC(slaexc) +#define slahqr_ BLAS_FUNC(slahqr) +#define slahr2_ BLAS_FUNC(slahr2) +#define slaisnan_ BLAS_FUNC(slaisnan) +#define slaln2_ BLAS_FUNC(slaln2) +#define slals0_ BLAS_FUNC(slals0) +#define slalsa_ BLAS_FUNC(slalsa) +#define slalsd_ BLAS_FUNC(slalsd) +#define slamc1_ BLAS_FUNC(slamc1) +#define slamc2_ BLAS_FUNC(slamc2) +#define slamc3_ BLAS_FUNC(slamc3) +#define slamc4_ BLAS_FUNC(slamc4) +#define slamc5_ BLAS_FUNC(slamc5) +#define slamch_ BLAS_FUNC(slamch) +#define slamrg_ BLAS_FUNC(slamrg) +#define slange_ BLAS_FUNC(slange) +#define slanst_ BLAS_FUNC(slanst) +#define slansy_ BLAS_FUNC(slansy) +#define slanv2_ BLAS_FUNC(slanv2) +#define slapy2_ BLAS_FUNC(slapy2) +#define slapy3_ BLAS_FUNC(slapy3) +#define slaqr0_ BLAS_FUNC(slaqr0) +#define slaqr1_ BLAS_FUNC(slaqr1) +#define slaqr2_ BLAS_FUNC(slaqr2) +#define slaqr3_ BLAS_FUNC(slaqr3) +#define slaqr4_ BLAS_FUNC(slaqr4) +#define slaqr5_ BLAS_FUNC(slaqr5) +#define slarf_ BLAS_FUNC(slarf) +#define slarfb_ BLAS_FUNC(slarfb) +#define slarfg_ BLAS_FUNC(slarfg) +#define slarft_ BLAS_FUNC(slarft) +#define slarfx_ BLAS_FUNC(slarfx) +#define slartg_ BLAS_FUNC(slartg) +#define slas2_ BLAS_FUNC(slas2) +#define slascl_ BLAS_FUNC(slascl) +#define slasd0_ BLAS_FUNC(slasd0) +#define slasd1_ BLAS_FUNC(slasd1) +#define slasd2_ BLAS_FUNC(slasd2) +#define slasd3_ BLAS_FUNC(slasd3) +#define slasd4_ BLAS_FUNC(slasd4) +#define slasd5_ BLAS_FUNC(slasd5) +#define slasd6_ BLAS_FUNC(slasd6) +#define slasd7_ BLAS_FUNC(slasd7) +#define slasd8_ BLAS_FUNC(slasd8) +#define slasda_ BLAS_FUNC(slasda) +#define slasdq_ BLAS_FUNC(slasdq) +#define slasdt_ BLAS_FUNC(slasdt) +#define slaset_ BLAS_FUNC(slaset) +#define slasq1_ BLAS_FUNC(slasq1) +#define slasq2_ BLAS_FUNC(slasq2) +#define slasq3_ BLAS_FUNC(slasq3) +#define slasq4_ BLAS_FUNC(slasq4) +#define slasq5_ BLAS_FUNC(slasq5) +#define slasq6_ BLAS_FUNC(slasq6) +#define slasr_ BLAS_FUNC(slasr) +#define slasrt_ BLAS_FUNC(slasrt) +#define slassq_ BLAS_FUNC(slassq) +#define slasv2_ BLAS_FUNC(slasv2) +#define slaswp_ BLAS_FUNC(slaswp) +#define slasy2_ BLAS_FUNC(slasy2) +#define slatrd_ BLAS_FUNC(slatrd) +#define slauu2_ BLAS_FUNC(slauu2) +#define slauum_ BLAS_FUNC(slauum) +#define snrm2_ BLAS_FUNC(snrm2) +#define sorg2r_ BLAS_FUNC(sorg2r) +#define sorgbr_ BLAS_FUNC(sorgbr) +#define sorghr_ BLAS_FUNC(sorghr) +#define sorgl2_ BLAS_FUNC(sorgl2) +#define sorglq_ BLAS_FUNC(sorglq) +#define sorgqr_ BLAS_FUNC(sorgqr) +#define sorm2l_ BLAS_FUNC(sorm2l) +#define sorm2r_ BLAS_FUNC(sorm2r) +#define sormbr_ BLAS_FUNC(sormbr) +#define sormhr_ BLAS_FUNC(sormhr) +#define sorml2_ BLAS_FUNC(sorml2) +#define sormlq_ BLAS_FUNC(sormlq) +#define sormql_ BLAS_FUNC(sormql) +#define sormqr_ BLAS_FUNC(sormqr) +#define sormtr_ BLAS_FUNC(sormtr) +#define spotf2_ BLAS_FUNC(spotf2) +#define spotrf_ BLAS_FUNC(spotrf) +#define spotri_ BLAS_FUNC(spotri) +#define spotrs_ BLAS_FUNC(spotrs) +#define srot_ BLAS_FUNC(srot) +#define sscal_ BLAS_FUNC(sscal) +#define sstedc_ BLAS_FUNC(sstedc) +#define ssteqr_ BLAS_FUNC(ssteqr) +#define ssterf_ BLAS_FUNC(ssterf) +#define sswap_ BLAS_FUNC(sswap) +#define ssyevd_ BLAS_FUNC(ssyevd) +#define ssymv_ BLAS_FUNC(ssymv) +#define ssyr2_ BLAS_FUNC(ssyr2) +#define ssyr2k_ BLAS_FUNC(ssyr2k) +#define ssyrk_ BLAS_FUNC(ssyrk) +#define ssytd2_ BLAS_FUNC(ssytd2) +#define ssytrd_ BLAS_FUNC(ssytrd) +#define strevc_ BLAS_FUNC(strevc) +#define strexc_ BLAS_FUNC(strexc) +#define strmm_ BLAS_FUNC(strmm) +#define strmv_ BLAS_FUNC(strmv) +#define strsm_ BLAS_FUNC(strsm) +#define strti2_ BLAS_FUNC(strti2) +#define strtri_ BLAS_FUNC(strtri) +#define xerbla_ BLAS_FUNC(xerbla) +#define zaxpy_ BLAS_FUNC(zaxpy) +#define zcopy_ BLAS_FUNC(zcopy) +#define zdotc_ BLAS_FUNC(zdotc) +#define zdotu_ BLAS_FUNC(zdotu) +#define zdrot_ BLAS_FUNC(zdrot) +#define zdscal_ BLAS_FUNC(zdscal) +#define zgebak_ BLAS_FUNC(zgebak) +#define zgebal_ BLAS_FUNC(zgebal) +#define zgebd2_ BLAS_FUNC(zgebd2) +#define zgebrd_ BLAS_FUNC(zgebrd) +#define zgeev_ BLAS_FUNC(zgeev) +#define zgehd2_ BLAS_FUNC(zgehd2) +#define zgehrd_ BLAS_FUNC(zgehrd) +#define zgelq2_ BLAS_FUNC(zgelq2) +#define zgelqf_ BLAS_FUNC(zgelqf) +#define zgelsd_ BLAS_FUNC(zgelsd) +#define zgemm_ BLAS_FUNC(zgemm) +#define zgemv_ BLAS_FUNC(zgemv) +#define zgeqr2_ BLAS_FUNC(zgeqr2) +#define zgeqrf_ BLAS_FUNC(zgeqrf) +#define zgerc_ BLAS_FUNC(zgerc) +#define zgeru_ BLAS_FUNC(zgeru) +#define zgesdd_ BLAS_FUNC(zgesdd) +#define zgesv_ BLAS_FUNC(zgesv) +#define zgetf2_ BLAS_FUNC(zgetf2) +#define zgetrf_ BLAS_FUNC(zgetrf) +#define zgetrs_ BLAS_FUNC(zgetrs) +#define zheevd_ BLAS_FUNC(zheevd) +#define zhemv_ BLAS_FUNC(zhemv) +#define zher2_ BLAS_FUNC(zher2) +#define zher2k_ BLAS_FUNC(zher2k) +#define zherk_ BLAS_FUNC(zherk) +#define zhetd2_ BLAS_FUNC(zhetd2) +#define zhetrd_ BLAS_FUNC(zhetrd) +#define zhseqr_ BLAS_FUNC(zhseqr) +#define zlabrd_ BLAS_FUNC(zlabrd) +#define zlacgv_ BLAS_FUNC(zlacgv) +#define zlacp2_ BLAS_FUNC(zlacp2) +#define zlacpy_ BLAS_FUNC(zlacpy) +#define zlacrm_ BLAS_FUNC(zlacrm) +#define zladiv_ BLAS_FUNC(zladiv) +#define zlaed0_ BLAS_FUNC(zlaed0) +#define zlaed7_ BLAS_FUNC(zlaed7) +#define zlaed8_ BLAS_FUNC(zlaed8) +#define zlahqr_ BLAS_FUNC(zlahqr) +#define zlahr2_ BLAS_FUNC(zlahr2) +#define zlals0_ BLAS_FUNC(zlals0) +#define zlalsa_ BLAS_FUNC(zlalsa) +#define zlalsd_ BLAS_FUNC(zlalsd) +#define zlange_ BLAS_FUNC(zlange) +#define zlanhe_ BLAS_FUNC(zlanhe) +#define zlaqr0_ BLAS_FUNC(zlaqr0) +#define zlaqr1_ BLAS_FUNC(zlaqr1) +#define zlaqr2_ BLAS_FUNC(zlaqr2) +#define zlaqr3_ BLAS_FUNC(zlaqr3) +#define zlaqr4_ BLAS_FUNC(zlaqr4) +#define zlaqr5_ BLAS_FUNC(zlaqr5) +#define zlarcm_ BLAS_FUNC(zlarcm) +#define zlarf_ BLAS_FUNC(zlarf) +#define zlarfb_ BLAS_FUNC(zlarfb) +#define zlarfg_ BLAS_FUNC(zlarfg) +#define zlarft_ BLAS_FUNC(zlarft) +#define zlartg_ BLAS_FUNC(zlartg) +#define zlascl_ BLAS_FUNC(zlascl) +#define zlaset_ BLAS_FUNC(zlaset) +#define zlasr_ BLAS_FUNC(zlasr) +#define zlassq_ BLAS_FUNC(zlassq) +#define zlaswp_ BLAS_FUNC(zlaswp) +#define zlatrd_ BLAS_FUNC(zlatrd) +#define zlatrs_ BLAS_FUNC(zlatrs) +#define zlauu2_ BLAS_FUNC(zlauu2) +#define zlauum_ BLAS_FUNC(zlauum) +#define zpotf2_ BLAS_FUNC(zpotf2) +#define zpotrf_ BLAS_FUNC(zpotrf) +#define zpotri_ BLAS_FUNC(zpotri) +#define zpotrs_ BLAS_FUNC(zpotrs) +#define zrot_ BLAS_FUNC(zrot) +#define zscal_ BLAS_FUNC(zscal) +#define zstedc_ BLAS_FUNC(zstedc) +#define zsteqr_ BLAS_FUNC(zsteqr) +#define zswap_ BLAS_FUNC(zswap) +#define ztrevc_ BLAS_FUNC(ztrevc) +#define ztrexc_ BLAS_FUNC(ztrexc) +#define ztrmm_ BLAS_FUNC(ztrmm) +#define ztrmv_ BLAS_FUNC(ztrmv) +#define ztrsm_ BLAS_FUNC(ztrsm) +#define ztrsv_ BLAS_FUNC(ztrsv) +#define ztrti2_ BLAS_FUNC(ztrti2) +#define ztrtri_ BLAS_FUNC(ztrtri) +#define zung2r_ BLAS_FUNC(zung2r) +#define zungbr_ BLAS_FUNC(zungbr) +#define zunghr_ BLAS_FUNC(zunghr) +#define zungl2_ BLAS_FUNC(zungl2) +#define zunglq_ BLAS_FUNC(zunglq) +#define zungqr_ BLAS_FUNC(zungqr) +#define zunm2l_ BLAS_FUNC(zunm2l) +#define zunm2r_ BLAS_FUNC(zunm2r) +#define zunmbr_ BLAS_FUNC(zunmbr) +#define zunmhr_ BLAS_FUNC(zunmhr) +#define zunml2_ BLAS_FUNC(zunml2) +#define zunmlq_ BLAS_FUNC(zunmlq) +#define zunmql_ BLAS_FUNC(zunmql) +#define zunmqr_ BLAS_FUNC(zunmqr) +#define zunmtr_ BLAS_FUNC(zunmtr) + +/* Symbols exported by f2c.c */ +#define abort_ numpy_lapack_lite_abort_ +#define c_abs numpy_lapack_lite_c_abs +#define c_cos numpy_lapack_lite_c_cos +#define c_div numpy_lapack_lite_c_div +#define c_exp numpy_lapack_lite_c_exp +#define c_log numpy_lapack_lite_c_log +#define c_sin numpy_lapack_lite_c_sin +#define c_sqrt numpy_lapack_lite_c_sqrt +#define d_abs numpy_lapack_lite_d_abs +#define d_acos numpy_lapack_lite_d_acos +#define d_asin numpy_lapack_lite_d_asin +#define d_atan numpy_lapack_lite_d_atan +#define d_atn2 numpy_lapack_lite_d_atn2 +#define d_cnjg numpy_lapack_lite_d_cnjg +#define d_cos numpy_lapack_lite_d_cos +#define d_cosh numpy_lapack_lite_d_cosh +#define d_dim numpy_lapack_lite_d_dim +#define d_exp numpy_lapack_lite_d_exp +#define d_imag numpy_lapack_lite_d_imag +#define d_int numpy_lapack_lite_d_int +#define d_lg10 numpy_lapack_lite_d_lg10 +#define d_log numpy_lapack_lite_d_log +#define d_mod numpy_lapack_lite_d_mod +#define d_nint numpy_lapack_lite_d_nint +#define d_prod numpy_lapack_lite_d_prod +#define d_sign numpy_lapack_lite_d_sign +#define d_sin numpy_lapack_lite_d_sin +#define d_sinh numpy_lapack_lite_d_sinh +#define d_sqrt numpy_lapack_lite_d_sqrt +#define d_tan numpy_lapack_lite_d_tan +#define d_tanh numpy_lapack_lite_d_tanh +#define derf_ numpy_lapack_lite_derf_ +#define derfc_ numpy_lapack_lite_derfc_ +#define do_fio numpy_lapack_lite_do_fio +#define do_lio numpy_lapack_lite_do_lio +#define do_uio numpy_lapack_lite_do_uio +#define e_rdfe numpy_lapack_lite_e_rdfe +#define e_rdue numpy_lapack_lite_e_rdue +#define e_rsfe numpy_lapack_lite_e_rsfe +#define e_rsfi numpy_lapack_lite_e_rsfi +#define e_rsle numpy_lapack_lite_e_rsle +#define e_rsli numpy_lapack_lite_e_rsli +#define e_rsue numpy_lapack_lite_e_rsue +#define e_wdfe numpy_lapack_lite_e_wdfe +#define e_wdue numpy_lapack_lite_e_wdue +#define e_wsfe numpy_lapack_lite_e_wsfe +#define e_wsfi numpy_lapack_lite_e_wsfi +#define e_wsle numpy_lapack_lite_e_wsle +#define e_wsli numpy_lapack_lite_e_wsli +#define e_wsue numpy_lapack_lite_e_wsue +#define ef1asc_ numpy_lapack_lite_ef1asc_ +#define ef1cmc_ numpy_lapack_lite_ef1cmc_ +#define erf_ numpy_lapack_lite_erf_ +#define erfc_ numpy_lapack_lite_erfc_ +#define f__cabs numpy_lapack_lite_f__cabs +#define f__cabsf numpy_lapack_lite_f__cabsf +#define f_back numpy_lapack_lite_f_back +#define f_clos numpy_lapack_lite_f_clos +#define f_end numpy_lapack_lite_f_end +#define f_exit numpy_lapack_lite_f_exit +#define f_inqu numpy_lapack_lite_f_inqu +#define f_open numpy_lapack_lite_f_open +#define f_rew numpy_lapack_lite_f_rew +#define flush_ numpy_lapack_lite_flush_ +#define getarg_ numpy_lapack_lite_getarg_ +#define getenv_ numpy_lapack_lite_getenv_ +#define h_abs numpy_lapack_lite_h_abs +#define h_dim numpy_lapack_lite_h_dim +#define h_dnnt numpy_lapack_lite_h_dnnt +#define h_indx numpy_lapack_lite_h_indx +#define h_len numpy_lapack_lite_h_len +#define h_mod numpy_lapack_lite_h_mod +#define h_nint numpy_lapack_lite_h_nint +#define h_sign numpy_lapack_lite_h_sign +#define hl_ge numpy_lapack_lite_hl_ge +#define hl_gt numpy_lapack_lite_hl_gt +#define hl_le numpy_lapack_lite_hl_le +#define hl_lt numpy_lapack_lite_hl_lt +#define i_abs numpy_lapack_lite_i_abs +#define i_dim numpy_lapack_lite_i_dim +#define i_dnnt numpy_lapack_lite_i_dnnt +#define i_indx numpy_lapack_lite_i_indx +#define i_len numpy_lapack_lite_i_len +#define i_mod numpy_lapack_lite_i_mod +#define i_nint numpy_lapack_lite_i_nint +#define i_sign numpy_lapack_lite_i_sign +#define iargc_ numpy_lapack_lite_iargc_ +#define l_ge numpy_lapack_lite_l_ge +#define l_gt numpy_lapack_lite_l_gt +#define l_le numpy_lapack_lite_l_le +#define l_lt numpy_lapack_lite_l_lt +#define pow_ci numpy_lapack_lite_pow_ci +#define pow_dd numpy_lapack_lite_pow_dd +#define pow_di numpy_lapack_lite_pow_di +#define pow_hh numpy_lapack_lite_pow_hh +#define pow_ii numpy_lapack_lite_pow_ii +#define pow_ri numpy_lapack_lite_pow_ri +#define pow_zi numpy_lapack_lite_pow_zi +#define pow_zz numpy_lapack_lite_pow_zz +#define r_abs numpy_lapack_lite_r_abs +#define r_acos numpy_lapack_lite_r_acos +#define r_asin numpy_lapack_lite_r_asin +#define r_atan numpy_lapack_lite_r_atan +#define r_atn2 numpy_lapack_lite_r_atn2 +#define r_cnjg numpy_lapack_lite_r_cnjg +#define r_cos numpy_lapack_lite_r_cos +#define r_cosh numpy_lapack_lite_r_cosh +#define r_dim numpy_lapack_lite_r_dim +#define r_exp numpy_lapack_lite_r_exp +#define r_imag numpy_lapack_lite_r_imag +#define r_int numpy_lapack_lite_r_int +#define r_lg10 numpy_lapack_lite_r_lg10 +#define r_log numpy_lapack_lite_r_log +#define r_mod numpy_lapack_lite_r_mod +#define r_nint numpy_lapack_lite_r_nint +#define r_sign numpy_lapack_lite_r_sign +#define r_sin numpy_lapack_lite_r_sin +#define r_sinh numpy_lapack_lite_r_sinh +#define r_sqrt numpy_lapack_lite_r_sqrt +#define r_tan numpy_lapack_lite_r_tan +#define r_tanh numpy_lapack_lite_r_tanh +#define s_cat numpy_lapack_lite_s_cat +#define s_cmp numpy_lapack_lite_s_cmp +#define s_copy numpy_lapack_lite_s_copy +#define s_paus numpy_lapack_lite_s_paus +#define s_rdfe numpy_lapack_lite_s_rdfe +#define s_rdue numpy_lapack_lite_s_rdue +#define s_rnge numpy_lapack_lite_s_rnge +#define s_rsfe numpy_lapack_lite_s_rsfe +#define s_rsfi numpy_lapack_lite_s_rsfi +#define s_rsle numpy_lapack_lite_s_rsle +#define s_rsli numpy_lapack_lite_s_rsli +#define s_rsne numpy_lapack_lite_s_rsne +#define s_rsni numpy_lapack_lite_s_rsni +#define s_rsue numpy_lapack_lite_s_rsue +#define s_stop numpy_lapack_lite_s_stop +#define s_wdfe numpy_lapack_lite_s_wdfe +#define s_wdue numpy_lapack_lite_s_wdue +#define s_wsfe numpy_lapack_lite_s_wsfe +#define s_wsfi numpy_lapack_lite_s_wsfi +#define s_wsle numpy_lapack_lite_s_wsle +#define s_wsli numpy_lapack_lite_s_wsli +#define s_wsne numpy_lapack_lite_s_wsne +#define s_wsni numpy_lapack_lite_s_wsni +#define s_wsue numpy_lapack_lite_s_wsue +#define sig_die numpy_lapack_lite_sig_die +#define signal_ numpy_lapack_lite_signal_ +#define system_ numpy_lapack_lite_system_ +#define z_abs numpy_lapack_lite_z_abs +#define z_cos numpy_lapack_lite_z_cos +#define z_div numpy_lapack_lite_z_div +#define z_exp numpy_lapack_lite_z_exp +#define z_log numpy_lapack_lite_z_log +#define z_sin numpy_lapack_lite_z_sin +#define z_sqrt numpy_lapack_lite_z_sqrt diff --git a/numpy/linalg/lapack_lite/make_lite.py b/numpy/linalg/lapack_lite/make_lite.py index 4b1a0ad82..960f5e2d8 100755 --- a/numpy/linalg/lapack_lite/make_lite.py +++ b/numpy/linalg/lapack_lite/make_lite.py @@ -1,6 +1,6 @@ #!/usr/bin/env python """ -Usage: make_lite.py <wrapped_routines_file> <lapack_dir> <output_dir> +Usage: make_lite.py <wrapped_routines_file> <lapack_dir> Typical invocation: @@ -13,6 +13,7 @@ Requires the following to be on the path: """ import sys import os +import re import subprocess import shutil @@ -33,11 +34,14 @@ F2C_ARGS = ['-A', '-Nx800'] # The header to add to the top of the f2c_*.c file. Note that dlamch_() calls # will be replaced by the macros below by clapack_scrub.scrub_source() -HEADER = '''\ +HEADER_BLURB = '''\ /* -NOTE: This is generated code. Look in Misc/lapack_lite for information on - remaking this file. -*/ + * NOTE: This is generated code. Look in numpy/linalg/lapack_lite for + * information on remaking this file. + */ +''' + +HEADER = HEADER_BLURB + '''\ #include "f2c.h" #ifdef HAVE_CONFIG @@ -279,6 +283,52 @@ def ensure_executable(name): except: raise SystemExit(name + ' not found') +def create_name_header(output_dir): + routine_re = re.compile(r'^ (subroutine|.* function)\s+(\w+)\(.*$', + re.I) + extern_re = re.compile(r'^extern [a-z]+ ([a-z0-9_]+)\(.*$') + + # BLAS/LAPACK symbols + symbols = set(['xerbla']) + for fn in os.listdir(output_dir): + fn = os.path.join(output_dir, fn) + + if not fn.endswith('.f'): + continue + + with open(fn, 'r') as f: + for line in f: + m = routine_re.match(line) + if m: + symbols.add(m.group(2).lower()) + + # f2c symbols + f2c_symbols = set() + with open('f2c.h', 'r') as f: + for line in f: + m = extern_re.match(line) + if m: + f2c_symbols.add(m.group(1)) + + with open(os.path.join(output_dir, 'lapack_lite_names.h'), 'w') as f: + f.write(HEADER_BLURB) + f.write( + "/*\n" + " * This file renames all BLAS/LAPACK and f2c symbols to avoid\n" + " * dynamic symbol name conflicts, in cases where e.g.\n" + " * integer sizes do not match with 'standard' ABI.\n" + " */\n") + + # Rename BLAS/LAPACK symbols + for name in sorted(symbols): + f.write("#define %s_ BLAS_FUNC(%s)\n" % (name, name)) + + # Rename also symbols that f2c exports itself + f.write("\n" + "/* Symbols exported by f2c.c */\n") + for name in sorted(f2c_symbols): + f.write("#define %s numpy_lapack_lite_%s\n" % (name, name)) + def main(): if len(sys.argv) != 3: print(__doc__) @@ -328,12 +378,14 @@ def main(): print() + create_name_header(output_dir) + for fname in os.listdir(output_dir): - if fname.endswith('.c'): + if fname.endswith('.c') or fname == 'lapack_lite_names.h': print('Copying ' + fname) shutil.copy( os.path.join(output_dir, fname), - os.path.dirname(__file__), + os.path.abspath(os.path.dirname(__file__)), ) diff --git a/numpy/linalg/lapack_lite/python_xerbla.c b/numpy/linalg/lapack_lite/python_xerbla.c index 4dbb92e1f..fe2f718b2 100644 --- a/numpy/linalg/lapack_lite/python_xerbla.c +++ b/numpy/linalg/lapack_lite/python_xerbla.c @@ -2,9 +2,6 @@ #include "numpy/npy_common.h" #include "npy_cblas.h" -#undef c_abs -#include "f2c.h" - /* From the original manpage: -------------------------- diff --git a/numpy/linalg/setup.py b/numpy/linalg/setup.py index 66eed41b0..acfab0a68 100644 --- a/numpy/linalg/setup.py +++ b/numpy/linalg/setup.py @@ -38,8 +38,13 @@ def configuration(parent_package='', top_path=None): def calc_info(self): info = {'language': 'c'} if sys.maxsize > 2**32: - # Build lapack-lite in 64-bit integer mode - info['define_macros'] = [('HAVE_BLAS_ILP64', None)] + # Build lapack-lite in 64-bit integer mode. + # The suffix is arbitrary (lapack_lite symbols follow it), + # but use the "64_" convention here. + info['define_macros'] = [ + ('HAVE_BLAS_ILP64', None), + ('BLAS_SYMBOL_SUFFIX', '64_') + ] self.set_info(**info) lapack_info = numpy_linalg_lapack_lite().get_info(2) diff --git a/numpy/ma/core.py b/numpy/ma/core.py index 30aef7465..fcbd1d8d0 100644 --- a/numpy/ma/core.py +++ b/numpy/ma/core.py @@ -20,6 +20,7 @@ Released for unlimited redistribution. """ # pylint: disable-msg=E1002 +import builtins import sys import operator import warnings @@ -27,11 +28,6 @@ import textwrap import re from functools import reduce -if sys.version_info[0] >= 3: - import builtins -else: - import __builtin__ as builtins - import numpy as np import numpy.core.umath as umath import numpy.core.numerictypes as ntypes @@ -2826,8 +2822,8 @@ class MaskedArray(ndarray): elif isinstance(data, (tuple, list)): try: # If data is a sequence of masked array - mask = np.array([getmaskarray(m) for m in data], - dtype=mdtype) + mask = np.array([getmaskarray(np.asanyarray(m, dtype=mdtype)) + for m in data], dtype=mdtype) except ValueError: # If data is nested mask = nomask @@ -6394,10 +6390,6 @@ class MaskedConstant(MaskedArray): def __str__(self): return str(masked_print_option._display) - if sys.version_info.major < 3: - def __unicode__(self): - return unicode(masked_print_option._display) - def __repr__(self): if self is MaskedConstant.__singleton: return 'masked' diff --git a/numpy/ma/tests/test_core.py b/numpy/ma/tests/test_core.py index 7ef4b5030..bc8423188 100644 --- a/numpy/ma/tests/test_core.py +++ b/numpy/ma/tests/test_core.py @@ -934,7 +934,7 @@ class TestMaskedArray: def test_object_with_array(self): mx1 = masked_array([1.], mask=[True]) mx2 = masked_array([1., 2.]) - mx = masked_array([mx1, mx2], mask=[False, True]) + mx = masked_array([mx1, mx2], mask=[False, True], dtype=object) assert_(mx[0] is mx1) assert_(mx[1] is not mx2) assert_(np.all(mx[1].data == mx2.data)) diff --git a/numpy/random/_generator.pyx b/numpy/random/_generator.pyx index 5cd699ac7..32eda25f7 100644 --- a/numpy/random/_generator.pyx +++ b/numpy/random/_generator.pyx @@ -2919,7 +2919,6 @@ cdef class Generator: it = np.PyArray_MultiIterNew2(p_arr, n_arr) randoms = <np.ndarray>np.empty(it.shape, np.int64) - randoms_data = <np.int64_t *>np.PyArray_DATA(randoms) cnt = np.PyArray_SIZE(randoms) it = np.PyArray_MultiIterNew3(randoms, p_arr, n_arr) diff --git a/numpy/random/mtrand.pyx b/numpy/random/mtrand.pyx index a4d409f37..95921a8ea 100644 --- a/numpy/random/mtrand.pyx +++ b/numpy/random/mtrand.pyx @@ -3356,7 +3356,6 @@ cdef class RandomState: it = np.PyArray_MultiIterNew2(p_arr, n_arr) randoms = <np.ndarray>np.empty(it.shape, int) - randoms_data = <long *>np.PyArray_DATA(randoms) cnt = np.PyArray_SIZE(randoms) it = np.PyArray_MultiIterNew3(randoms, p_arr, n_arr) diff --git a/numpy/random/src/distributions/random_mvhg_count.c b/numpy/random/src/distributions/random_mvhg_count.c index 7cbed1f9e..1d4ed978e 100644 --- a/numpy/random/src/distributions/random_mvhg_count.c +++ b/numpy/random/src/distributions/random_mvhg_count.c @@ -1,8 +1,8 @@ +#include "numpy/random/distributions.h" #include <stdint.h> #include <stdlib.h> #include <stdbool.h> -#include "numpy/random/distributions.h" /* * random_multivariate_hypergeometric_count diff --git a/numpy/random/src/distributions/random_mvhg_marginals.c b/numpy/random/src/distributions/random_mvhg_marginals.c index 809d129de..689a85671 100644 --- a/numpy/random/src/distributions/random_mvhg_marginals.c +++ b/numpy/random/src/distributions/random_mvhg_marginals.c @@ -1,9 +1,9 @@ +#include "numpy/random/distributions.h" #include <stdint.h> #include <stddef.h> #include <stdbool.h> #include <math.h> -#include "numpy/random/distributions.h" #include "logfactorial.h" diff --git a/numpy/random/tests/test_generator_mt19937_regressions.py b/numpy/random/tests/test_generator_mt19937_regressions.py index fb0aac335..f0b502d06 100644 --- a/numpy/random/tests/test_generator_mt19937_regressions.py +++ b/numpy/random/tests/test_generator_mt19937_regressions.py @@ -55,9 +55,10 @@ class TestRegression: [1, (2, 2), (3, 3), None], [(1, 1), 2, 3, None]]: mt19937 = Generator(MT19937(12345)) - shuffled = list(t) + shuffled = np.array(t, dtype=object) mt19937.shuffle(shuffled) - assert_array_equal(shuffled, [t[2], t[0], t[3], t[1]]) + expected = np.array([t[2], t[0], t[3], t[1]], dtype=object) + assert_array_equal(np.array(shuffled, dtype=object), expected) def test_call_within_randomstate(self): # Check that custom BitGenerator does not call into global state @@ -117,7 +118,7 @@ class TestRegression: # a segfault on garbage collection. # See gh-7719 mt19937 = Generator(MT19937(1234)) - a = np.array([np.arange(1), np.arange(4)]) + a = np.array([np.arange(1), np.arange(4)], dtype=object) for _ in range(1000): mt19937.shuffle(a) diff --git a/numpy/random/tests/test_randomstate_regression.py b/numpy/random/tests/test_randomstate_regression.py index 3be9edf02..1d8a0ed5a 100644 --- a/numpy/random/tests/test_randomstate_regression.py +++ b/numpy/random/tests/test_randomstate_regression.py @@ -68,7 +68,8 @@ class TestRegression: random.seed(12345) shuffled = list(t) random.shuffle(shuffled) - assert_array_equal(shuffled, [t[0], t[3], t[1], t[2]]) + expected = np.array([t[0], t[3], t[1], t[2]], dtype=object) + assert_array_equal(np.array(shuffled, dtype=object), expected) def test_call_within_randomstate(self): # Check that custom RandomState does not call into global state @@ -128,7 +129,7 @@ class TestRegression: # a segfault on garbage collection. # See gh-7719 random.seed(1234) - a = np.array([np.arange(1), np.arange(4)]) + a = np.array([np.arange(1), np.arange(4)], dtype=object) for _ in range(1000): random.shuffle(a) diff --git a/numpy/random/tests/test_regression.py b/numpy/random/tests/test_regression.py index 7d77a31d8..cd8f3891c 100644 --- a/numpy/random/tests/test_regression.py +++ b/numpy/random/tests/test_regression.py @@ -64,7 +64,8 @@ class TestRegression: np.random.seed(12345) shuffled = list(t) random.shuffle(shuffled) - assert_array_equal(shuffled, [t[0], t[3], t[1], t[2]]) + expected = np.array([t[0], t[3], t[1], t[2]], dtype=object) + assert_array_equal(np.array(shuffled, dtype=object), expected) def test_call_within_randomstate(self): # Check that custom RandomState does not call into global state @@ -124,7 +125,7 @@ class TestRegression: # a segfault on garbage collection. # See gh-7719 np.random.seed(1234) - a = np.array([np.arange(1), np.arange(4)]) + a = np.array([np.arange(1), np.arange(4)], dtype=object) for _ in range(1000): np.random.shuffle(a) diff --git a/runtests.py b/runtests.py index d36e5bd39..07b3f6c87 100755 --- a/runtests.py +++ b/runtests.py @@ -53,6 +53,7 @@ else: import sys import os +import builtins # In case we are run from the source directory, we don't want to import the # project from there: @@ -181,7 +182,7 @@ def main(argv): sys.modules['__main__'] = types.ModuleType('__main__') ns = dict(__name__='__main__', __file__=extra_argv[0]) - exec_(script, ns) + exec(script, ns) sys.exit(0) else: import code @@ -479,25 +480,5 @@ def lcov_generate(): print("HTML output generated under build/lcov/") -# -# Python 3 support -# - -if sys.version_info[0] >= 3: - import builtins - exec_ = getattr(builtins, "exec") -else: - def exec_(code, globs=None, locs=None): - """Execute code in a namespace.""" - if globs is None: - frame = sys._getframe(1) - globs = frame.f_globals - if locs is None: - locs = frame.f_locals - del frame - elif locs is None: - locs = globs - exec("""exec code in globs, locs""") - if __name__ == "__main__": main(argv=sys.argv[1:]) diff --git a/shippable.yml b/shippable.yml index 4313a6de2..bea2a2206 100644 --- a/shippable.yml +++ b/shippable.yml @@ -21,9 +21,10 @@ runtime: build: ci: - # install dependencies + # install dependencies and newer toolchain for gfortran5 + - sudo add-apt-repository ppa:ubuntu-toolchain-r/test - sudo apt-get update - - sudo apt-get install gcc gfortran + - sudo apt-get install gcc gfortran libgfortran5 - target=$(python tools/openblas_support.py) - ls -lR "${target}" - sudo cp -r "${target}"/lib/* /usr/lib @@ -48,7 +49,7 @@ build: - extra_path=$(printf "%s:" "${extra_directories[@]}") - export PATH="${extra_path}${PATH}" # check OpenBLAS version - - python tools/openblas_support.py --check_version 0.3.7 + - python tools/openblas_support.py --check_version 0.3.8 # run the test suite - python runtests.py -n --debug-info --show-build-log -- -rsx --junit-xml=$SHIPPABLE_REPO_DIR/shippable/testresults/tests.xml -n 2 --durations=10 diff --git a/test_requirements.txt b/test_requirements.txt index 8ca2ce87b..3aea49079 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,5 +1,5 @@ cython==0.29.14 -pytest==5.3.2 +pytest==5.3.3 pytz==2019.3 pytest-cov==2.8.1 pickle5; python_version == '3.7' diff --git a/tools/changelog.py b/tools/changelog.py index 00ffdd9eb..2a0c4da43 100755 --- a/tools/changelog.py +++ b/tools/changelog.py @@ -36,13 +36,11 @@ From the bash command line with $GITHUB token:: import os import sys import re -import codecs from git import Repo from github import Github -if sys.version_info.major < 3: - UTF8Writer = codecs.getwriter('utf8') - sys.stdout = UTF8Writer(sys.stdout) +if sys.version_info[:2] < (3, 6): + raise RuntimeError("Python version must be >= 3.6") this_repo = Repo(os.path.join(os.path.dirname(__file__), "..")) diff --git a/tools/npy_tempita/__init__.py b/tools/npy_tempita/__init__.py index b84754988..ec4b0d5e7 100644 --- a/tools/npy_tempita/__init__.py +++ b/tools/npy_tempita/__init__.py @@ -317,7 +317,7 @@ class Template: arg0 = e_value.args[0] else: arg0 = coerce_text(e_value) - e_value.args = (self._add_line_info(arg0, pos),) + e_value.args = (self._add_line_info(arg0, pos),) if PY3: raise e_value else: @@ -461,14 +461,11 @@ def html_quote(value, force=True): return '' if not isinstance(value, basestring_): value = coerce_text(value) - if sys.version >= "3" and isinstance(value, bytes): + if isinstance(value, bytes): value = html_escape(value.decode('latin1'), 1) value = value.encode('latin1') else: value = html_escape(value, 1) - if sys.version < "3": - if is_unicode(value): - value = value.encode('ascii', 'xmlcharrefreplace') return value @@ -1286,7 +1283,7 @@ def fill_command(args=None): template_content = sys.stdin.read() template_name = '<stdin>' else: - with open(template_name, 'rb', encoding="latin-1") as f: + with open(template_name, 'rb', encoding="latin-1") as f: template_content = f.read() if options.use_html: TemplateClass = HTMLTemplate @@ -1295,7 +1292,7 @@ def fill_command(args=None): template = TemplateClass(template_content, name=template_name) result = template.substitute(vars) if options.output: - with open(options.output, 'wb') as f: + with open(options.output, 'wb') as f: f.write(result) else: sys.stdout.write(result) diff --git a/tools/npy_tempita/_looper.py b/tools/npy_tempita/_looper.py index 23121fe9e..8a1156678 100644 --- a/tools/npy_tempita/_looper.py +++ b/tools/npy_tempita/_looper.py @@ -61,9 +61,6 @@ class looper_iter: self.pos += 1 return result - if sys.version < "3": - next = __next__ - class loop_pos: @@ -94,9 +91,6 @@ class loop_pos: except IndexError: return None - if sys.version < "3": - next = __next__ - @property def previous(self): if self.pos == 0: diff --git a/tools/openblas_support.py b/tools/openblas_support.py index 2c1b70d6f..3708033d3 100644 --- a/tools/openblas_support.py +++ b/tools/openblas_support.py @@ -15,11 +15,11 @@ from tempfile import mkstemp, gettempdir import zipfile import tarfile -OPENBLAS_V = 'v0.3.7' -OPENBLAS_LONG = 'v0.3.7' +OPENBLAS_V = 'v0.3.8' +OPENBLAS_LONG = 'v0.3.5-605-gc815b8fb' # the 0.3.5 is misleading BASE_LOC = '' RACKSPACE = 'https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com' -ARCHITECTURES = ['', 'windows', 'darwin', 'arm', 'x86', 'ppc64'] +ARCHITECTURES = ['', 'windows', 'darwin', 'aarch64', 'x86', 'ppc64le', 's390x'] IS_32BIT = sys.maxsize < 2**32 def get_arch(): @@ -27,17 +27,12 @@ def get_arch(): ret = 'windows' elif platform.system() == 'Darwin': ret = 'darwin' - # Python3 returns a named tuple, but Python2 does not, so we are stuck - elif 'arm' in os.uname()[-1]: - ret = 'arm'; - elif 'aarch64' in os.uname()[-1]: - ret = 'arm'; - elif 'x86' in os.uname()[-1]: + elif 'x86' in platform.uname().machine: + # What do 32 bit machines report? + # If they are a docker, they report x86_64 ret = 'x86' - elif 'ppc64' in os.uname()[-1]: - ret = 'ppc64' else: - ret = '' + ret = platform.uname().machine assert ret in ARCHITECTURES return ret @@ -51,21 +46,10 @@ def get_ilp64(): def download_openblas(target, arch, ilp64): fnsuffix = {None: "", "64_": "64_"}[ilp64] filename = '' - if arch == 'arm': - # ARMv8 OpenBLAS built using script available here: - # https://github.com/tylerjereddy/openblas-static-gcc/tree/master/ARMv8 - # build done on GCC compile farm machine named gcc115 - # tarball uploaded manually to an unshared Dropbox location - filename = ('https://www.dropbox.com/s/vdeckao4omss187/' - 'openblas{}-{}-armv8.tar.gz?dl=1'.format(fnsuffix, OPENBLAS_V)) + if arch in ('aarch64', 'ppc64le', 's390x'): + filename = '{0}/openblas{1}-{2}-manylinux2014_{3}.tar.gz'.format( + RACKSPACE, fnsuffix, OPENBLAS_LONG, arch) typ = 'tar.gz' - elif arch == 'ppc64': - # build script for POWER8 OpenBLAS available here: - # https://github.com/tylerjereddy/openblas-static-gcc/blob/master/power8 - # built on GCC compile farm machine named gcc112 - # manually uploaded tarball to an unshared Dropbox location - filename = ('https://www.dropbox.com/s/yt0d2j86x1j8nh1/' - 'openblas{}-{}-ppc64le-power8.tar.gz?dl=1'.format(fnsuffix, OPENBLAS_V)) typ = 'tar.gz' elif arch == 'darwin': filename = '{0}/openblas{1}-{2}-macosx_10_9_x86_64-gf_1becaaa.tar.gz'.format( @@ -78,7 +62,7 @@ def download_openblas(target, arch, ilp64): suffix = 'win_amd64-gcc_7_1_0.zip' filename = '{0}/openblas{1}-{2}-{3}'.format(RACKSPACE, fnsuffix, OPENBLAS_LONG, suffix) typ = 'zip' - elif arch == 'x86': + elif 'x86' in arch: if IS_32BIT: suffix = 'manylinux1_i686.tar.gz' else: diff --git a/tools/refguide_check.py b/tools/refguide_check.py index 3b587b77b..ad3f93c42 100644 --- a/tools/refguide_check.py +++ b/tools/refguide_check.py @@ -195,7 +195,7 @@ def find_names(module, names_dict): names_dict : dict Dictionary which contains module name as key and a set of found function names and directives as value - + Returns ------- None @@ -229,12 +229,12 @@ def find_names(module, names_dict): def get_all_dict(module): """ Return a copy of the __all__ dict with irrelevant items removed. - + Parameters ---------- module : ModuleType The module whose __all__ dict has to be processed - + Returns ------- deprecated : list @@ -242,7 +242,7 @@ def get_all_dict(module): not_deprecated : list List of non callable or non deprecated sub modules others : list - List of remaining types of sub modules + List of remaining types of sub modules """ if hasattr(module, "__all__"): all_dict = copy.deepcopy(module.__all__) @@ -863,7 +863,7 @@ def check_doctests(module, verbose, ns=None, ns : dict Name space of module dots : bool - + doctest_warnings : bool Returns @@ -934,7 +934,7 @@ def check_doctests_testfile(fname, verbose, ns=None, ns : dict Name space - + dots : bool doctest_warnings : bool @@ -978,12 +978,8 @@ def check_doctests_testfile(fname, verbose, ns=None, return results full_name = fname - if sys.version_info.major <= 2: - with open(fname) as f: - text = f.read() - else: - with open(fname, encoding='utf-8') as f: - text = f.read() + with open(fname, encoding='utf-8') as f: + text = f.read() PSEUDOCODE = set(['some_function', 'some_module', 'import example', 'ctypes.CDLL', # likely need compiling, skip it @@ -1116,7 +1112,7 @@ def init_matplotlib(): def main(argv): """ Validates the docstrings of all the pre decided set of - modules for errors and docstring standards. + modules for errors and docstring standards. """ parser = ArgumentParser(usage=__doc__.lstrip()) parser.add_argument("module_names", metavar="SUBMODULES", default=[], diff --git a/tools/travis-before-install.sh b/tools/travis-before-install.sh index 8be8afb10..f53d18611 100755 --- a/tools/travis-before-install.sh +++ b/tools/travis-before-install.sh @@ -33,6 +33,7 @@ fi source venv/bin/activate python -V +gcc --version popd |
