1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
|
// Copyright Naoki Shibata and contributors 2010 - 2020.
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
#ifndef __VECTORTYPE_H__
#define __VECTORTYPE_H__
#include <math.h>
#include "sleef.h"
#ifdef ENABLE_SSE2
#include "helpersse2.h"
#endif
#ifdef ENABLE_AVX
#include "helperavx.h"
#endif
#ifdef ENABLE_AVX2
#include "helperavx2.h"
#endif
#ifdef ENABLE_AVX512F
#include "helperavx512f.h"
#endif
#ifdef ENABLE_NEON32
#include "helperneon32.h"
#endif
#ifdef ENABLE_ADVSIMD
#include "helperadvsimd.h"
#endif
#ifdef ENABLE_SVE
#include "helpersve.h"
#endif
#ifdef ENABLE_VSX
#include "helperpower_128.h"
#endif
#ifdef ENABLE_ZVECTOR
#include "helpers390x_128.h"
#endif
#ifdef ENABLE_ZVECTOR2
#include "helpers390x_128.h"
#endif
#ifdef ENABLE_VECEXT
#include "helpervecext.h"
#endif
#ifdef ENABLE_PUREC
#include "helperpurec.h"
#endif
#define IMPORT_IS_EXPORT
#include "sleefdft.h"
#if BASETYPEID == 1
#define LOG2VECWIDTH (LOG2VECTLENDP-1)
#define VECWIDTH (1 << LOG2VECWIDTH)
typedef double real;
typedef vdouble real2;
static int available(int name) { return vavailability_i(name); }
static INLINE real2 uminus(real2 d0) { return vneg_vd_vd(d0); }
static INLINE real2 uplusminus(real2 d0) { return vposneg_vd_vd(d0); }
static INLINE real2 uminusplus(real2 d0) { return vnegpos_vd_vd(d0); }
static INLINE real2 plus(real2 d0, real2 d1) { return vadd_vd_vd_vd(d0, d1); }
static INLINE real2 minus(real2 d0, real2 d1) { return vsub_vd_vd_vd(d0, d1); }
static INLINE real2 minusplus(real2 d0, real2 d1) { return vsubadd_vd_vd_vd(d0, d1); }
static INLINE real2 times(real2 d0, real2 d1) { return vmul_vd_vd_vd(d0, d1); }
static INLINE real2 timesminusplus(real2 d0, real2 d2, real2 d1) { return vmlsubadd_vd_vd_vd_vd(d0, d2, d1); }
static INLINE real2 ctimes(real2 d0, real d) { return vmul_vd_vd_vd(d0, vcast_vd_d(d)); }
static INLINE real2 ctimesminusplus(real2 d0, real c, real2 d1) { return vmlsubadd_vd_vd_vd_vd(d0, vcast_vd_d(c), d1); }
static INLINE real2 reverse(real2 d0) { return vrev21_vd_vd(d0); }
static INLINE real2 reverse2(real2 d0) { return vreva2_vd_vd(d0); }
static INLINE real2 loadc(real c) { return vcast_vd_d(c); }
static INLINE real2 load(const real *ptr, int offset) { return vload_vd_p(&ptr[2*offset]); }
static INLINE real2 loadu(const real *ptr, int offset) { return vloadu_vd_p(&ptr[2*offset]); }
static INLINE void store(real *ptr, int offset, real2 v) { vstore_v_p_vd(&ptr[2*offset], v); }
static INLINE void storeu(real *ptr, int offset, real2 v) { vstoreu_v_p_vd(&ptr[2*offset], v); }
static INLINE void stream(real *ptr, int offset, real2 v) { vstream_v_p_vd(&ptr[2*offset], v); }
static INLINE void scatter(real *ptr, int offset, int step, real2 v) { vscatter2_v_p_i_i_vd(ptr, offset, step, v); }
static INLINE void scstream(real *ptr, int offset, int step, real2 v) { vsscatter2_v_p_i_i_vd(ptr, offset, step, v); }
static INLINE void prefetch(real *ptr, int offset) { vprefetch_v_p(&ptr[2*offset]); }
#elif BASETYPEID == 2
#define LOG2VECWIDTH (LOG2VECTLENSP-1)
#define VECWIDTH (1 << LOG2VECWIDTH)
typedef float real;
typedef vfloat real2;
static int available(int name) { return vavailability_i(name); }
static INLINE real2 uminus(real2 d0) { return vneg_vf_vf(d0); }
static INLINE real2 uplusminus(real2 d0) { return vposneg_vf_vf(d0); }
static INLINE real2 uminusplus(real2 d0) { return vnegpos_vf_vf(d0); }
static INLINE real2 plus(real2 d0, real2 d1) { return vadd_vf_vf_vf(d0, d1); }
static INLINE real2 minus(real2 d0, real2 d1) { return vsub_vf_vf_vf(d0, d1); }
static INLINE real2 minusplus(real2 d0, real2 d1) { return vsubadd_vf_vf_vf(d0, d1); }
static INLINE real2 times(real2 d0, real2 d1) { return vmul_vf_vf_vf(d0, d1); }
static INLINE real2 ctimes(real2 d0, real d) { return vmul_vf_vf_vf(d0, vcast_vf_f(d)); }
static INLINE real2 timesminusplus(real2 d0, real2 d2, real2 d1) { return vmlsubadd_vf_vf_vf_vf(d0, d2, d1); }
static INLINE real2 ctimesminusplus(real2 d0, real c, real2 d1) { return vmlsubadd_vf_vf_vf_vf(d0, vcast_vf_f(c), d1); }
static INLINE real2 reverse(real2 d0) { return vrev21_vf_vf(d0); }
static INLINE real2 reverse2(real2 d0) { return vreva2_vf_vf(d0); }
static INLINE real2 loadc(real c) { return vcast_vf_f(c); }
static INLINE real2 load(const real *ptr, int offset) { return vload_vf_p(&ptr[2*offset]); }
static INLINE real2 loadu(const real *ptr, int offset) { return vloadu_vf_p(&ptr[2*offset]); }
static INLINE void store(real *ptr, int offset, real2 v) { vstore_v_p_vf(&ptr[2*offset], v); }
static INLINE void storeu(real *ptr, int offset, real2 v) { vstoreu_v_p_vf(&ptr[2*offset], v); }
static INLINE void stream(real *ptr, int offset, real2 v) { vstream_v_p_vf(&ptr[2*offset], v); }
static INLINE void scatter(real *ptr, int offset, int step, real2 v) { vscatter2_v_p_i_i_vf(ptr, offset, step, v); }
static INLINE void scstream(real *ptr, int offset, int step, real2 v) { vsscatter2_v_p_i_i_vf(ptr, offset, step, v); }
static INLINE void prefetch(real *ptr, int offset) { vprefetch_v_p(&ptr[2*offset]); }
#elif BASETYPEID == 3
#define LOG2VECWIDTH (LOG2VECTLENDP-1)
#define VECWIDTH (1 << LOG2VECWIDTH)
typedef long double real;
typedef vlongdouble real2;
static int available(int name) { return vavailability_i(name); }
static INLINE real2 uminus(real2 d0) { return vneg_vl_vl(d0); }
static INLINE real2 uplusminus(real2 d0) { return vposneg_vl_vl(d0); }
static INLINE real2 uminusplus(real2 d0) { return vnegpos_vl_vl(d0); }
static INLINE real2 plus(real2 d0, real2 d1) { return vadd_vl_vl_vl(d0, d1); }
static INLINE real2 minus(real2 d0, real2 d1) { return vsub_vl_vl_vl(d0, d1); }
static INLINE real2 minusplus(real2 d0, real2 d1) { return vsubadd_vl_vl_vl(d0, d1); }
static INLINE real2 times(real2 d0, real2 d1) { return vmul_vl_vl_vl(d0, d1); }
static INLINE real2 ctimes(real2 d0, real d) { return vmul_vl_vl_vl(d0, vcast_vl_l(d)); }
static INLINE real2 timesminusplus(real2 d0, real2 d2, real2 d1) { return vmlsubadd_vl_vl_vl_vl(d0, d2, d1); }
static INLINE real2 ctimesminusplus(real2 d0, real c, real2 d1) { return vmlsubadd_vl_vl_vl_vl(d0, vcast_vl_l(c), d1); }
static INLINE real2 reverse(real2 d0) { return vrev21_vl_vl(d0); }
static INLINE real2 reverse2(real2 d0) { return vreva2_vl_vl(d0); }
static INLINE real2 loadc(real c) { return vcast_vl_l(c); }
static INLINE real2 load(const real *ptr, int offset) { return vload_vl_p(&ptr[2*offset]); }
static INLINE real2 loadu(const real *ptr, int offset) { return vloadu_vl_p(&ptr[2*offset]); }
static INLINE void store(real *ptr, int offset, real2 v) { vstore_v_p_vl(&ptr[2*offset], v); }
static INLINE void storeu(real *ptr, int offset, real2 v) { vstoreu_v_p_vl(&ptr[2*offset], v); }
static INLINE void stream(real *ptr, int offset, real2 v) { vstream_v_p_vl(&ptr[2*offset], v); }
static INLINE void scatter(real *ptr, int offset, int step, real2 v) { vscatter2_v_p_i_i_vl(ptr, offset, step, v); }
static INLINE void scstream(real *ptr, int offset, int step, real2 v) { vsscatter2_v_p_i_i_vl(ptr, offset, step, v); }
static INLINE void prefetch(real *ptr, int offset) { vprefetch_v_p(&ptr[2*offset]); }
#elif BASETYPEID == 4
#define LOG2VECWIDTH (LOG2VECTLENDP-1)
#define VECWIDTH (1 << LOG2VECWIDTH)
typedef Sleef_quad real;
typedef vquad real2;
static int available(int name) { return vavailability_i(name); }
static INLINE real2 uminus(real2 d0) { return vneg_vq_vq(d0); }
static INLINE real2 uplusminus(real2 d0) { return vposneg_vq_vq(d0); }
static INLINE real2 uminusplus(real2 d0) { return vnegpos_vq_vq(d0); }
static INLINE real2 plus(real2 d0, real2 d1) { return vadd_vq_vq_vq(d0, d1); }
static INLINE real2 minus(real2 d0, real2 d1) { return vsub_vq_vq_vq(d0, d1); }
static INLINE real2 minusplus(real2 d0, real2 d1) { return vsubadd_vq_vq_vq(d0, d1); }
static INLINE real2 times(real2 d0, real2 d1) { return vmul_vq_vq_vq(d0, d1); }
static INLINE real2 ctimes(real2 d0, real d) { return vmul_vq_vq_vq(d0, vcast_vq_q(d)); }
static INLINE real2 timesminusplus(real2 d0, real2 d2, real2 d1) { return vmlsubadd_vq_vq_vq_vq(d0, d2, d1); }
static INLINE real2 ctimesminusplus(real2 d0, real c, real2 d1) { return vmlsubadd_vq_vq_vq_vq(d0, vcast_vq_q(c), d1); }
static INLINE real2 reverse(real2 d0) { return vrev21_vq_vq(d0); }
static INLINE real2 reverse2(real2 d0) { return vreva2_vq_vq(d0); }
static INLINE real2 loadc(real c) { return vcast_vq_q(c); }
static INLINE real2 load(const real *ptr, int offset) { return vload_vq_p(&ptr[2*offset]); }
static INLINE real2 loadu(const real *ptr, int offset) { return vloadu_vq_p(&ptr[2*offset]); }
static INLINE void store(real *ptr, int offset, real2 v) { vstore_v_p_vq(&ptr[2*offset], v); }
static INLINE void storeu(real *ptr, int offset, real2 v) { vstoreu_v_p_vq(&ptr[2*offset], v); }
static INLINE void stream(real *ptr, int offset, real2 v) { vstream_v_p_vq(&ptr[2*offset], v); }
static INLINE void scatter(real *ptr, int offset, int step, real2 v) { vscatter2_v_p_i_i_vq(ptr, offset, step, v); }
static INLINE void scstream(real *ptr, int offset, int step, real2 v) { vsscatter2_v_p_i_i_vq(ptr, offset, step, v); }
static INLINE void prefetch(real *ptr, int offset) { vprefetch_v_p(&ptr[2*offset]); }
#else
#error No BASETYPEID specified
#endif
#endif
|