summaryrefslogtreecommitdiff
path: root/numpy
diff options
context:
space:
mode:
Diffstat (limited to 'numpy')
-rw-r--r--numpy/core/code_generators/generate_umath.py57
-rw-r--r--numpy/core/setup.py1
-rw-r--r--numpy/core/src/umath/loops.h.src9
-rw-r--r--numpy/core/src/umath/loops_umath_fp.dispatch.c.src82
4 files changed, 108 insertions, 41 deletions
diff --git a/numpy/core/code_generators/generate_umath.py b/numpy/core/code_generators/generate_umath.py
index 89320128b..eb4c4155b 100644
--- a/numpy/core/code_generators/generate_umath.py
+++ b/numpy/core/code_generators/generate_umath.py
@@ -645,8 +645,7 @@ defdict = {
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.arccos'),
None,
- TD('e', f='acos', astype={'e': 'f'}),
- TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
+ TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(inexact, f='acos', astype={'e': 'f'}),
TD(P, f='arccos'),
),
@@ -654,8 +653,7 @@ defdict = {
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.arccosh'),
None,
- TD('e', f='acosh', astype={'e': 'f'}),
- TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
+ TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(inexact, f='acosh', astype={'e': 'f'}),
TD(P, f='arccosh'),
),
@@ -663,8 +661,7 @@ defdict = {
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.arcsin'),
None,
- TD('e', f='asin', astype={'e': 'f'}),
- TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
+ TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(inexact, f='asin', astype={'e': 'f'}),
TD(P, f='arcsin'),
),
@@ -672,8 +669,7 @@ defdict = {
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.arcsinh'),
None,
- TD('e', f='asinh', astype={'e': 'f'}),
- TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
+ TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(inexact, f='asinh', astype={'e': 'f'}),
TD(P, f='arcsinh'),
),
@@ -681,8 +677,7 @@ defdict = {
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.arctan'),
None,
- TD('e', f='atan', astype={'e': 'f'}),
- TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
+ TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(inexact, f='atan', astype={'e': 'f'}),
TD(P, f='arctan'),
),
@@ -690,8 +685,7 @@ defdict = {
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.arctanh'),
None,
- TD('e', f='atanh', astype={'e': 'f'}),
- TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
+ TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(inexact, f='atanh', astype={'e': 'f'}),
TD(P, f='arctanh'),
),
@@ -699,9 +693,8 @@ defdict = {
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.cos'),
None,
- TD('e', f='cos', astype={'e': 'f'}),
TD('f', dispatch=[('loops_trigonometric', 'f')]),
- TD('d', dispatch=[('loops_umath_fp', 'd')]),
+ TD('ed', dispatch=[('loops_umath_fp', 'ed')]),
TD('fdg' + cmplx, f='cos'),
TD(P, f='cos'),
),
@@ -709,9 +702,8 @@ defdict = {
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.sin'),
None,
- TD('e', f='sin', astype={'e': 'f'}),
TD('f', dispatch=[('loops_trigonometric', 'f')]),
- TD('d', dispatch=[('loops_umath_fp', 'd')]),
+ TD('ed', dispatch=[('loops_umath_fp', 'ed')]),
TD('fdg' + cmplx, f='sin'),
TD(P, f='sin'),
),
@@ -719,8 +711,7 @@ defdict = {
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.tan'),
None,
- TD('e', f='tan', astype={'e': 'f'}),
- TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
+ TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(inexact, f='tan', astype={'e': 'f'}),
TD(P, f='tan'),
),
@@ -728,8 +719,7 @@ defdict = {
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.cosh'),
None,
- TD('e', f='cosh', astype={'e': 'f'}),
- TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
+ TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(inexact, f='cosh', astype={'e': 'f'}),
TD(P, f='cosh'),
),
@@ -737,8 +727,7 @@ defdict = {
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.sinh'),
None,
- TD('e', f='sinh', astype={'e': 'f'}),
- TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
+ TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(inexact, f='sinh', astype={'e': 'f'}),
TD(P, f='sinh'),
),
@@ -746,7 +735,7 @@ defdict = {
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.tanh'),
None,
- TD('e', f='tanh', astype={'e': 'f'}),
+ TD('e', dispatch=[('loops_umath_fp', 'e')]),
TD('fd', dispatch=[('loops_hyperbolic', 'fd')]),
TD(inexact, f='tanh', astype={'e': 'f'}),
TD(P, f='tanh'),
@@ -755,7 +744,7 @@ defdict = {
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.exp'),
None,
- TD('e', f='exp', astype={'e': 'f'}),
+ TD('e', dispatch=[('loops_umath_fp', 'e')]),
TD('fd', dispatch=[('loops_exponent_log', 'fd')]),
TD('fdg' + cmplx, f='exp'),
TD(P, f='exp'),
@@ -764,8 +753,7 @@ defdict = {
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.exp2'),
None,
- TD('e', f='exp2', astype={'e': 'f'}),
- TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
+ TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(inexact, f='exp2', astype={'e': 'f'}),
TD(P, f='exp2'),
),
@@ -773,8 +761,7 @@ defdict = {
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.expm1'),
None,
- TD('e', f='expm1', astype={'e': 'f'}),
- TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
+ TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(inexact, f='expm1', astype={'e': 'f'}),
TD(P, f='expm1'),
),
@@ -782,7 +769,7 @@ defdict = {
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.log'),
None,
- TD('e', f='log', astype={'e': 'f'}),
+ TD('e', dispatch=[('loops_umath_fp', 'e')]),
TD('fd', dispatch=[('loops_exponent_log', 'fd')]),
TD('fdg' + cmplx, f='log'),
TD(P, f='log'),
@@ -791,8 +778,7 @@ defdict = {
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.log2'),
None,
- TD('e', f='log2', astype={'e': 'f'}),
- TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
+ TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(inexact, f='log2', astype={'e': 'f'}),
TD(P, f='log2'),
),
@@ -800,8 +786,7 @@ defdict = {
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.log10'),
None,
- TD('e', f='log10', astype={'e': 'f'}),
- TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
+ TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(inexact, f='log10', astype={'e': 'f'}),
TD(P, f='log10'),
),
@@ -809,8 +794,7 @@ defdict = {
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.log1p'),
None,
- TD('e', f='log1p', astype={'e': 'f'}),
- TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
+ TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(inexact, f='log1p', astype={'e': 'f'}),
TD(P, f='log1p'),
),
@@ -827,8 +811,7 @@ defdict = {
Ufunc(1, 1, None,
docstrings.get('numpy.core.umath.cbrt'),
None,
- TD('e', f='cbrt', astype={'e': 'f'}),
- TD('fd', dispatch=[('loops_umath_fp', 'fd')]),
+ TD('efd', dispatch=[('loops_umath_fp', 'efd')]),
TD(flts, f='cbrt', astype={'e': 'f'}),
TD(P, f='cbrt'),
),
diff --git a/numpy/core/setup.py b/numpy/core/setup.py
index fc09dba8b..3a239a70e 100644
--- a/numpy/core/setup.py
+++ b/numpy/core/setup.py
@@ -1077,7 +1077,6 @@ def configuration(parent_package='',top_path=None):
# actually the other way around, better performance and
# after all maintainable code.
svml_filter = (
- 'svml_z0_tanh_d_la.s', 'svml_z0_tanh_s_la.s'
)
if can_link_svml() and check_svml_submodule(svml_path):
svml_objs = glob.glob(svml_path + '/**/*.s', recursive=True)
diff --git a/numpy/core/src/umath/loops.h.src b/numpy/core/src/umath/loops.h.src
index 4b363d84b..424e204c1 100644
--- a/numpy/core/src/umath/loops.h.src
+++ b/numpy/core/src/umath/loops.h.src
@@ -274,6 +274,15 @@ NPY_CPU_DISPATCH_DECLARE(NPY_NO_EXPORT void @TYPE@_@func@,
/**end repeat**/
/**begin repeat
+ * #func = sin, cos, tan, exp, exp2, log, log2, log10, expm1, log1p, cbrt, arcsin, arccos, arctan, sinh, cosh, tanh, arcsinh, arccosh, arctanh#
+ */
+
+NPY_CPU_DISPATCH_DECLARE(NPY_NO_EXPORT void HALF_@func@,
+ (char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)))
+
+/**end repeat**/
+
+/**begin repeat
* #func = sin, cos#
*/
diff --git a/numpy/core/src/umath/loops_umath_fp.dispatch.c.src b/numpy/core/src/umath/loops_umath_fp.dispatch.c.src
index 538b5039f..46ce51824 100644
--- a/numpy/core/src/umath/loops_umath_fp.dispatch.c.src
+++ b/numpy/core/src/umath/loops_umath_fp.dispatch.c.src
@@ -82,7 +82,7 @@ simd_@func@_f64(const double *src, npy_intp ssrc,
/**begin repeat1
* #func = pow, atan2#
*/
-
+
static void
simd_@func@_@sfx@(const npyv_lanetype_@sfx@ *src1, npy_intp ssrc1,
const npyv_lanetype_@sfx@ *src2, npy_intp ssrc2,
@@ -96,14 +96,14 @@ simd_@func@_@sfx@(const npyv_lanetype_@sfx@ *src1, npy_intp ssrc1,
} else {
x1 = npyv_loadn_till_@sfx@(src1, ssrc1, len, 1);
}
-
+
npyv_@sfx@ x2;
if (ssrc2 == 1) {
x2 = npyv_load_till_@sfx@(src2, len, 1);
} else {
x2 = npyv_loadn_till_@sfx@(src2, ssrc2, len, 1);
}
-
+
npyv_@sfx@ out = __svml_@func@@func_suffix@(x1, x2);
if (sdst == 1) {
npyv_store_till_@sfx@(dst, len, out);
@@ -115,7 +115,83 @@ simd_@func@_@sfx@(const npyv_lanetype_@sfx@ *src1, npy_intp ssrc1,
/**end repeat1**/
/**end repeat**/
+typedef __m256i npyvh_f16;
+#define npyv_cvt_f16_f32 _mm512_cvtph_ps
+#define npyv_cvt_f32_f16 _mm512_cvtps_ph
+#define npyvh_load_f16(PTR) _mm256_loadu_si256((const __m256i*)(PTR))
+#define npyvh_store_f16(PTR, data) _mm256_storeu_si256((__m256i*)PTR, data)
+NPY_FINLINE npyvh_f16 npyvh_load_till_f16(const npy_half *ptr, npy_uintp nlane, npy_half fill)
+{
+ assert(nlane > 0);
+ const __m256i vfill = _mm256_set1_epi16(fill);
+ const __mmask16 mask = (0x0001 << nlane) - 0x0001;
+ return _mm256_mask_loadu_epi16(vfill, mask, ptr);
+}
+NPY_FINLINE void npyvh_store_till_f16(npy_half *ptr, npy_uintp nlane, npyvh_f16 data)
+{
+ assert(nlane > 0);
+ const __mmask16 mask = (0x0001 << nlane) - 0x0001;
+ _mm256_mask_storeu_epi16(ptr, mask, data);
+}
+
+/**begin repeat
+ * #func = sin, cos, tan, exp, exp2, expm1, log, log2, log10, log1p, cbrt, asin, acos, atan, sinh, cosh, tanh, asinh, acosh, atanh#
+ * #default_val = 0, 0, 0, 0, 0, 0x3c00, 0x3c00, 0x3c00, 0x3c00, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x3c00, 0#
+ */
+static void
+avx512_@func@_f16(const npy_half *src, npy_half *dst, npy_intp len)
+{
+ const int num_lanes = npyv_nlanes_f32;
+ npyvh_f16 x, out;
+ npyv_f32 x_ps, out_ps;
+ for (; len > 0; len -= num_lanes, src += num_lanes, dst += num_lanes) {
+ if (len >= num_lanes) {
+ x = npyvh_load_f16(src);
+ x_ps = npyv_cvt_f16_f32(x);
+ out_ps = __svml_@func@f16(x_ps);
+ out = npyv_cvt_f32_f16(out_ps, 0);
+ npyvh_store_f16(dst, out);
+ }
+ else {
+ x = npyvh_load_till_f16(src, len, @default_val@);
+ x_ps = npyv_cvt_f16_f32(x);
+ out_ps = __svml_@func@f16(x_ps);
+ out = npyv_cvt_f32_f16(out_ps, 0);
+ npyvh_store_till_f16(dst, len, out);
+ }
+ }
+ npyv_cleanup();
+}
+/**end repeat**/
+#endif
+
+/**begin repeat
+ * #func = sin, cos, tan, exp, exp2, expm1, log, log2, log10, log1p, cbrt, arcsin, arccos, arctan, sinh, cosh, tanh, arcsinh, arccosh, arctanh#
+ * #intrin = sin, cos, tan, exp, exp2, expm1, log, log2, log10, log1p, cbrt, asin, acos, atan, sinh, cosh, tanh, asinh, acosh, atanh#
+ */
+NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(HALF_@func@)
+(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(data))
+{
+#if NPY_SIMD && defined(NPY_HAVE_AVX512_SKX) && defined(NPY_CAN_LINK_SVML)
+ const npy_half *src = (npy_half*)args[0];
+ npy_half *dst = (npy_half*)args[1];
+ const int lsize = sizeof(src[0]);
+ const npy_intp ssrc = steps[0] / lsize;
+ const npy_intp sdst = steps[1] / lsize;
+ const npy_intp len = dimensions[0];
+ if (!is_mem_overlap(src, steps[0], dst, steps[1], len) &&
+ (ssrc == 1) &&
+ (sdst == 1)) {
+ avx512_@intrin@_f16(src, dst, len);
+ return;
+ }
#endif
+ UNARY_LOOP {
+ const npy_float in1 = npy_half_to_float(*(npy_half *)ip1);
+ *((npy_half *)op1) = npy_float_to_half(npy_@intrin@f(in1));
+ }
+}
+/**end repeat**/
/**begin repeat
* #TYPE = DOUBLE, FLOAT#
View Lorry Controller Status