summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--doc/release/1.16.0-notes.rst15
-rw-r--r--numpy/core/_dtype_ctypes.py15
-rw-r--r--numpy/core/tests/test_dtype.py30
-rw-r--r--numpy/distutils/ccompiler.py66
-rw-r--r--numpy/distutils/fcompiler/__init__.py21
-rw-r--r--numpy/lib/_datasource.py2
-rw-r--r--numpy/lib/tests/test__datasource.py15
-rw-r--r--numpy/lib/utils.py7
8 files changed, 136 insertions, 35 deletions
diff --git a/doc/release/1.16.0-notes.rst b/doc/release/1.16.0-notes.rst
index 412ed9d8f..3f7b7be32 100644
--- a/doc/release/1.16.0-notes.rst
+++ b/doc/release/1.16.0-notes.rst
@@ -314,6 +314,21 @@ This is possible thanks to the new
:c:function:`PyUFunc_FromFuncAndDataAndSignatureAndIdentity`, which allows
arbitrary values to be used as identities now.
+Improved conversion from ctypes objects
+---------------------------------------
+Numpy has always supported taking a value or type from ``ctypes`` and
+converting it into an array or dtype, but only behaved correctly for simpler
+types. As of this release, this caveat is lifted - now:
+
+* The ``_pack_`` attribute of ``ctypes.Structure``, used to emulate C's
+ ``__attribute__((packed))``, is respected.
+* Endianness of all ctypes objects is preserved
+* ``ctypes.Union`` is supported
+* Unrepresentable constructs raise exceptions, rather than producing
+ dangerously incorrect results:
+ * Bitfields are no longer interpreted as sub-arrays
+ * Pointers are no longer replaced with the type that they point to
+
Changes
=======
diff --git a/numpy/core/_dtype_ctypes.py b/numpy/core/_dtype_ctypes.py
index b6e4ddf9e..ca365d2cb 100644
--- a/numpy/core/_dtype_ctypes.py
+++ b/numpy/core/_dtype_ctypes.py
@@ -33,7 +33,6 @@ def _from_ctypes_array(t):
def _from_ctypes_structure(t):
- # TODO: gh-10533
for item in t._fields_:
if len(item) > 2:
raise TypeError(
@@ -67,6 +66,18 @@ def _from_ctypes_structure(t):
return np.dtype(fields, align=True)
+def dtype_from_ctypes_scalar(t):
+ """
+ Return the dtype type with endianness included if it's the case
+ """
+ if t.__ctype_be__ is t:
+ return np.dtype('>' + t._type_)
+ elif t.__ctype_le__ is t:
+ return np.dtype('<' + t._type_)
+ else:
+ return np.dtype(t._type_)
+
+
def dtype_from_ctypes_type(t):
"""
Construct a dtype object from a ctypes type
@@ -83,7 +94,7 @@ def dtype_from_ctypes_type(t):
"conversion from ctypes.Union types like {} to dtype"
.format(t.__name__))
elif isinstance(t._type_, str):
- return np.dtype(t._type_)
+ return dtype_from_ctypes_scalar(t)
else:
raise NotImplementedError(
"Unknown ctypes type {}".format(t.__name__))
diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py
index cfb67f592..f2e7f8f50 100644
--- a/numpy/core/tests/test_dtype.py
+++ b/numpy/core/tests/test_dtype.py
@@ -856,8 +856,26 @@ class TestFromCTypes(object):
itemsize=18))
self.check(PackedStructure, expected)
- @pytest.mark.xfail(sys.byteorder != 'little',
- reason="non-native endianness does not work - see gh-10533")
+ def test_big_endian_structure_packed(self):
+ class BigEndStruct(ctypes.BigEndianStructure):
+ _fields_ = [
+ ('one', ctypes.c_uint8),
+ ('two', ctypes.c_uint32)
+ ]
+ _pack_ = 1
+ expected = np.dtype([('one', 'u1'), ('two', '>u4')])
+ self.check(BigEndStruct, expected)
+
+ def test_little_endian_structure_packed(self):
+ class LittleEndStruct(ctypes.LittleEndianStructure):
+ _fields_ = [
+ ('one', ctypes.c_uint8),
+ ('two', ctypes.c_uint32)
+ ]
+ _pack_ = 1
+ expected = np.dtype([('one', 'u1'), ('two', '<u4')])
+ self.check(LittleEndStruct, expected)
+
def test_little_endian_structure(self):
class PaddedStruct(ctypes.LittleEndianStructure):
_fields_ = [
@@ -870,8 +888,6 @@ class TestFromCTypes(object):
], align=True)
self.check(PaddedStruct, expected)
- @pytest.mark.xfail(sys.byteorder != 'big',
- reason="non-native endianness does not work - see gh-10533")
def test_big_endian_structure(self):
class PaddedStruct(ctypes.BigEndianStructure):
_fields_ = [
@@ -883,3 +899,9 @@ class TestFromCTypes(object):
('b', '>H')
], align=True)
self.check(PaddedStruct, expected)
+
+ def test_simple_endian_types(self):
+ self.check(ctypes.c_uint16.__ctype_le__, np.dtype('<u2'))
+ self.check(ctypes.c_uint16.__ctype_be__, np.dtype('>u2'))
+ self.check(ctypes.c_uint8.__ctype_le__, np.dtype('u1'))
+ self.check(ctypes.c_uint8.__ctype_be__, np.dtype('u1'))
diff --git a/numpy/distutils/ccompiler.py b/numpy/distutils/ccompiler.py
index b03fb96b2..4b5b96aad 100644
--- a/numpy/distutils/ccompiler.py
+++ b/numpy/distutils/ccompiler.py
@@ -6,6 +6,7 @@ import sys
import types
import shlex
import time
+import subprocess
from copy import copy
from distutils import ccompiler
from distutils.ccompiler import *
@@ -16,7 +17,7 @@ from distutils.version import LooseVersion
from numpy.distutils import log
from numpy.distutils.compat import get_exception
-from numpy.distutils.exec_command import exec_command
+from numpy.distutils.exec_command import filepath_from_subprocess_output
from numpy.distutils.misc_util import cyg2win32, is_sequence, mingw32, \
quote_args, get_num_build_jobs, \
_commandline_dep_string
@@ -136,20 +137,39 @@ def CCompiler_spawn(self, cmd, display=None):
if is_sequence(display):
display = ' '.join(list(display))
log.info(display)
- s, o = exec_command(cmd)
- if s:
- if is_sequence(cmd):
- cmd = ' '.join(list(cmd))
- try:
- print(o)
- except UnicodeError:
- # When installing through pip, `o` can contain non-ascii chars
- pass
- if re.search('Too many open files', o):
- msg = '\nTry rerunning setup command until build succeeds.'
- else:
- msg = ''
- raise DistutilsExecError('Command "%s" failed with exit status %d%s' % (cmd, s, msg))
+ try:
+ subprocess.check_output(cmd)
+ except subprocess.CalledProcessError as exc:
+ o = exc.output
+ s = exc.returncode
+ except OSError:
+ # OSError doesn't have the same hooks for the exception
+ # output, but exec_command() historically would use an
+ # empty string for EnvironmentError (base class for
+ # OSError)
+ o = b''
+ # status previously used by exec_command() for parent
+ # of OSError
+ s = 127
+ else:
+ # use a convenience return here so that any kind of
+ # caught exception will execute the default code after the
+ # try / except block, which handles various exceptions
+ return None
+
+ if is_sequence(cmd):
+ cmd = ' '.join(list(cmd))
+ try:
+ print(o)
+ except UnicodeError:
+ # When installing through pip, `o` can contain non-ascii chars
+ pass
+ if re.search(b'Too many open files', o):
+ msg = '\nTry rerunning setup command until build succeeds.'
+ else:
+ msg = ''
+ raise DistutilsExecError('Command "%s" failed with exit status %d%s' %
+ (cmd, s, msg))
replace_method(CCompiler, 'spawn', CCompiler_spawn)
@@ -620,7 +640,21 @@ def CCompiler_get_version(self, force=False, ok_status=[0]):
version = m.group('version')
return version
- status, output = exec_command(version_cmd, use_tee=0)
+ try:
+ output = subprocess.check_output(version_cmd)
+ except subprocess.CalledProcessError as exc:
+ output = exc.output
+ status = exc.returncode
+ except OSError:
+ # match the historical returns for a parent
+ # exception class caught by exec_command()
+ status = 127
+ output = b''
+ else:
+ # output isn't actually a filepath but we do this
+ # for now to match previous distutils behavior
+ output = filepath_from_subprocess_output(output)
+ status = 0
version = None
if status in ok_status:
diff --git a/numpy/distutils/fcompiler/__init__.py b/numpy/distutils/fcompiler/__init__.py
index 3bd8057b4..12b32832e 100644
--- a/numpy/distutils/fcompiler/__init__.py
+++ b/numpy/distutils/fcompiler/__init__.py
@@ -22,6 +22,7 @@ import os
import sys
import re
import types
+import shlex
from numpy.compat import open_latin1
@@ -465,8 +466,10 @@ class FCompiler(CCompiler):
noarch = self.distutils_vars.get('noarch', noopt)
debug = self.distutils_vars.get('debug', False)
- f77 = self.command_vars.compiler_f77
- f90 = self.command_vars.compiler_f90
+ f77 = shlex.split(self.command_vars.compiler_f77,
+ posix=(os.name == 'posix'))
+ f90 = shlex.split(self.command_vars.compiler_f90,
+ posix=(os.name == 'posix'))
f77flags = []
f90flags = []
@@ -480,6 +483,14 @@ class FCompiler(CCompiler):
freeflags = self.flag_vars.free
# XXX Assuming that free format is default for f90 compiler.
fix = self.command_vars.compiler_fix
+ # NOTE: this and similar examples are probably just
+ # exluding --coverage flag when F90 = gfortran --coverage
+ # instead of putting that flag somewhere more appropriate
+ # this and similar examples where a Fortran compiler
+ # environment variable has been customized by CI or a user
+ # should perhaps eventually be more throughly tested and more
+ # robustly handled
+ fix = shlex.split(fix, posix=(os.name == 'posix'))
if fix:
fixflags = self.flag_vars.fix + f90flags
@@ -506,11 +517,11 @@ class FCompiler(CCompiler):
fflags = self.flag_vars.flags + dflags + oflags + aflags
if f77:
- self.set_commands(compiler_f77=[f77]+f77flags+fflags)
+ self.set_commands(compiler_f77=f77+f77flags+fflags)
if f90:
- self.set_commands(compiler_f90=[f90]+freeflags+f90flags+fflags)
+ self.set_commands(compiler_f90=f90+freeflags+f90flags+fflags)
if fix:
- self.set_commands(compiler_fix=[fix]+fixflags+fflags)
+ self.set_commands(compiler_fix=fix+fixflags+fflags)
#XXX: Do we need LDSHARED->SOSHARED, LDFLAGS->SOFLAGS
diff --git a/numpy/lib/_datasource.py b/numpy/lib/_datasource.py
index e34c376e5..30237b76f 100644
--- a/numpy/lib/_datasource.py
+++ b/numpy/lib/_datasource.py
@@ -328,7 +328,7 @@ class DataSource(object):
def __del__(self):
# Remove temp directories
- if self._istmpdest:
+ if hasattr(self, '_istmpdest') and self._istmpdest:
shutil.rmtree(self._destpath)
def _iszip(self, filename):
diff --git a/numpy/lib/tests/test__datasource.py b/numpy/lib/tests/test__datasource.py
index 1df8bebf6..8eac16b58 100644
--- a/numpy/lib/tests/test__datasource.py
+++ b/numpy/lib/tests/test__datasource.py
@@ -361,3 +361,18 @@ class TestOpenFunc(object):
fp = datasource.open(local_file)
assert_(fp)
fp.close()
+
+def test_del_attr_handling():
+ # DataSource __del__ can be called
+ # even if __init__ fails when the
+ # Exception object is caught by the
+ # caller as happens in refguide_check
+ # is_deprecated() function
+
+ ds = datasource.DataSource()
+ # simulate failed __init__ by removing key attribute
+ # produced within __init__ and expected by __del__
+ del ds._istmpdest
+ # should not raise an AttributeError if __del__
+ # gracefully handles failed __init__:
+ ds.__del__()
diff --git a/numpy/lib/utils.py b/numpy/lib/utils.py
index ce42c53b9..84edf4021 100644
--- a/numpy/lib/utils.py
+++ b/numpy/lib/utils.py
@@ -165,13 +165,6 @@ def deprecate(*args, **kwargs):
fn = args[0]
args = args[1:]
- # backward compatibility -- can be removed
- # after next release
- if 'newname' in kwargs:
- kwargs['new_name'] = kwargs.pop('newname')
- if 'oldname' in kwargs:
- kwargs['old_name'] = kwargs.pop('oldname')
-
return _Deprecate(*args, **kwargs)(fn)
else:
return _Deprecate(*args, **kwargs)
View Lorry Controller Status