12 #ifndef EIGEN_PACKET_MATH_NEON_H
13 #define EIGEN_PACKET_MATH_NEON_H
19 #ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
20 #define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
25 #ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
26 #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 8
29 typedef float32x4_t Packet4f;
30 typedef int32x4_t Packet4i;
31 typedef uint32x4_t Packet4ui;
33 #define _EIGEN_DECLARE_CONST_Packet4f(NAME,X) \
34 const Packet4f p4f_##NAME = pset1<Packet4f>(X)
36 #define _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \
37 const Packet4f p4f_##NAME = vreinterpretq_f32_u32(pset1<int>(X))
39 #define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \
40 const Packet4i p4i_##NAME = pset1<Packet4i>(X)
42 #if defined(__llvm__) && !defined(__clang__)
44 #define EIGEN_INIT_NEON_PACKET2(X, Y) {{X, Y}}
45 #define EIGEN_INIT_NEON_PACKET4(X, Y, Z, W) {{X, Y, Z, W}}
48 #define EIGEN_INIT_NEON_PACKET2(X, Y) {X, Y}
49 #define EIGEN_INIT_NEON_PACKET4(X, Y, Z, W) {X, Y, Z, W}
53 #define __pld(x) asm volatile ( " pld [%[addr]]\n" :: [addr] "r" (x) : "cc" );
56 template<>
struct packet_traits<float> : default_packet_traits
58 typedef Packet4f type;
73 template<>
struct packet_traits<int> : default_packet_traits
75 typedef Packet4i type;
84 #if EIGEN_GNUC_AT_MOST(4,4) && !defined(__llvm__)
86 EIGEN_STRONG_INLINE float32x4_t vld1q_f32(
const float* x) { return ::vld1q_f32((
const float32_t*)x); }
87 EIGEN_STRONG_INLINE float32x2_t vld1_f32 (
const float* x) { return ::vld1_f32 ((
const float32_t*)x); }
88 EIGEN_STRONG_INLINE
void vst1q_f32(
float* to, float32x4_t from) { ::vst1q_f32((float32_t*)to,from); }
89 EIGEN_STRONG_INLINE
void vst1_f32 (
float* to, float32x2_t from) { ::vst1_f32 ((float32_t*)to,from); }
92 template<>
struct unpacket_traits<Packet4f> {
typedef float type;
enum {size=4}; };
93 template<>
struct unpacket_traits<Packet4i> {
typedef int type;
enum {size=4}; };
95 template<> EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(
const float& from) {
return vdupq_n_f32(from); }
96 template<> EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(
const int& from) {
return vdupq_n_s32(from); }
98 template<> EIGEN_STRONG_INLINE Packet4f plset<float>(
const float& a)
100 Packet4f countdown = EIGEN_INIT_NEON_PACKET4(0, 1, 2, 3);
101 return vaddq_f32(pset1<Packet4f>(a), countdown);
103 template<> EIGEN_STRONG_INLINE Packet4i plset<int>(
const int& a)
105 Packet4i countdown = EIGEN_INIT_NEON_PACKET4(0, 1, 2, 3);
106 return vaddq_s32(pset1<Packet4i>(a), countdown);
109 template<> EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return vaddq_f32(a,b); }
110 template<> EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vaddq_s32(a,b); }
112 template<> EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return vsubq_f32(a,b); }
113 template<> EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vsubq_s32(a,b); }
115 template<> EIGEN_STRONG_INLINE Packet4f pnegate(
const Packet4f& a) {
return vnegq_f32(a); }
116 template<> EIGEN_STRONG_INLINE Packet4i pnegate(
const Packet4i& a) {
return vnegq_s32(a); }
118 template<> EIGEN_STRONG_INLINE Packet4f pconj(
const Packet4f& a) {
return a; }
119 template<> EIGEN_STRONG_INLINE Packet4i pconj(
const Packet4i& a) {
return a; }
121 template<> EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return vmulq_f32(a,b); }
122 template<> EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vmulq_s32(a,b); }
124 template<> EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(
const Packet4f& a,
const Packet4f& b)
126 Packet4f inv, restep, div;
133 inv = vrecpeq_f32(b);
137 restep = vrecpsq_f32(b, inv);
138 inv = vmulq_f32(restep, inv);
141 div = vmulq_f32(a, inv);
145 template<> EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(
const Packet4i& ,
const Packet4i& )
146 { eigen_assert(
false &&
"packet integer division are not supported by NEON");
147 return pset1<Packet4i>(0);
151 template<> EIGEN_STRONG_INLINE Packet4f pmadd(
const Packet4f& a,
const Packet4f& b,
const Packet4f& c) {
return vmlaq_f32(c,a,b); }
152 template<> EIGEN_STRONG_INLINE Packet4i pmadd(
const Packet4i& a,
const Packet4i& b,
const Packet4i& c) {
return vmlaq_s32(c,a,b); }
154 template<> EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return vminq_f32(a,b); }
155 template<> EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vminq_s32(a,b); }
157 template<> EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return vmaxq_f32(a,b); }
158 template<> EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vmaxq_s32(a,b); }
161 template<> EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(
const Packet4f& a,
const Packet4f& b)
163 return vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b)));
165 template<> EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vandq_s32(a,b); }
167 template<> EIGEN_STRONG_INLINE Packet4f por<Packet4f>(
const Packet4f& a,
const Packet4f& b)
169 return vreinterpretq_f32_u32(vorrq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b)));
171 template<> EIGEN_STRONG_INLINE Packet4i por<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vorrq_s32(a,b); }
173 template<> EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(
const Packet4f& a,
const Packet4f& b)
175 return vreinterpretq_f32_u32(veorq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b)));
177 template<> EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return veorq_s32(a,b); }
179 template<> EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(
const Packet4f& a,
const Packet4f& b)
181 return vreinterpretq_f32_u32(vbicq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b)));
183 template<> EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vbicq_s32(a,b); }
185 template<> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(
const float* from) { EIGEN_DEBUG_ALIGNED_LOAD
return vld1q_f32(from); }
186 template<> EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(
const int* from) { EIGEN_DEBUG_ALIGNED_LOAD
return vld1q_s32(from); }
188 template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(
const float* from) { EIGEN_DEBUG_UNALIGNED_LOAD
return vld1q_f32(from); }
189 template<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(
const int* from) { EIGEN_DEBUG_UNALIGNED_LOAD
return vld1q_s32(from); }
191 template<> EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(
const float* from)
194 lo = vld1_dup_f32(from);
195 hi = vld1_dup_f32(from+1);
196 return vcombine_f32(lo, hi);
198 template<> EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(
const int* from)
201 lo = vld1_dup_s32(from);
202 hi = vld1_dup_s32(from+1);
203 return vcombine_s32(lo, hi);
206 template<> EIGEN_STRONG_INLINE
void pstore<float>(
float* to,
const Packet4f& from) { EIGEN_DEBUG_ALIGNED_STORE vst1q_f32(to, from); }
207 template<> EIGEN_STRONG_INLINE
void pstore<int>(
int* to,
const Packet4i& from) { EIGEN_DEBUG_ALIGNED_STORE vst1q_s32(to, from); }
209 template<> EIGEN_STRONG_INLINE
void pstoreu<float>(
float* to,
const Packet4f& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1q_f32(to, from); }
210 template<> EIGEN_STRONG_INLINE
void pstoreu<int>(
int* to,
const Packet4i& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1q_s32(to, from); }
212 template<> EIGEN_STRONG_INLINE
void prefetch<float>(
const float* addr) { __pld(addr); }
213 template<> EIGEN_STRONG_INLINE
void prefetch<int>(
const int* addr) { __pld(addr); }
216 template<> EIGEN_STRONG_INLINE
float pfirst<Packet4f>(
const Packet4f& a) {
float EIGEN_ALIGN16 x[4]; vst1q_f32(x, a);
return x[0]; }
217 template<> EIGEN_STRONG_INLINE
int pfirst<Packet4i>(
const Packet4i& a) {
int EIGEN_ALIGN16 x[4]; vst1q_s32(x, a);
return x[0]; }
219 template<> EIGEN_STRONG_INLINE Packet4f preverse(
const Packet4f& a) {
220 float32x2_t a_lo, a_hi;
223 a_r64 = vrev64q_f32(a);
224 a_lo = vget_low_f32(a_r64);
225 a_hi = vget_high_f32(a_r64);
226 return vcombine_f32(a_hi, a_lo);
228 template<> EIGEN_STRONG_INLINE Packet4i preverse(
const Packet4i& a) {
229 int32x2_t a_lo, a_hi;
232 a_r64 = vrev64q_s32(a);
233 a_lo = vget_low_s32(a_r64);
234 a_hi = vget_high_s32(a_r64);
235 return vcombine_s32(a_hi, a_lo);
237 template<> EIGEN_STRONG_INLINE Packet4f pabs(
const Packet4f& a) {
return vabsq_f32(a); }
238 template<> EIGEN_STRONG_INLINE Packet4i pabs(
const Packet4i& a) {
return vabsq_s32(a); }
240 template<> EIGEN_STRONG_INLINE
float predux<Packet4f>(
const Packet4f& a)
242 float32x2_t a_lo, a_hi, sum;
244 a_lo = vget_low_f32(a);
245 a_hi = vget_high_f32(a);
246 sum = vpadd_f32(a_lo, a_hi);
247 sum = vpadd_f32(sum, sum);
248 return vget_lane_f32(sum, 0);
251 template<> EIGEN_STRONG_INLINE Packet4f preduxp<Packet4f>(
const Packet4f* vecs)
253 float32x4x2_t vtrn1, vtrn2, res1, res2;
254 Packet4f sum1, sum2, sum;
258 vtrn1 = vzipq_f32(vecs[0], vecs[2]);
259 vtrn2 = vzipq_f32(vecs[1], vecs[3]);
260 res1 = vzipq_f32(vtrn1.val[0], vtrn2.val[0]);
261 res2 = vzipq_f32(vtrn1.val[1], vtrn2.val[1]);
264 sum1 = vaddq_f32(res1.val[0], res1.val[1]);
265 sum2 = vaddq_f32(res2.val[0], res2.val[1]);
266 sum = vaddq_f32(sum1, sum2);
271 template<> EIGEN_STRONG_INLINE
int predux<Packet4i>(
const Packet4i& a)
273 int32x2_t a_lo, a_hi, sum;
275 a_lo = vget_low_s32(a);
276 a_hi = vget_high_s32(a);
277 sum = vpadd_s32(a_lo, a_hi);
278 sum = vpadd_s32(sum, sum);
279 return vget_lane_s32(sum, 0);
282 template<> EIGEN_STRONG_INLINE Packet4i preduxp<Packet4i>(
const Packet4i* vecs)
284 int32x4x2_t vtrn1, vtrn2, res1, res2;
285 Packet4i sum1, sum2, sum;
289 vtrn1 = vzipq_s32(vecs[0], vecs[2]);
290 vtrn2 = vzipq_s32(vecs[1], vecs[3]);
291 res1 = vzipq_s32(vtrn1.val[0], vtrn2.val[0]);
292 res2 = vzipq_s32(vtrn1.val[1], vtrn2.val[1]);
295 sum1 = vaddq_s32(res1.val[0], res1.val[1]);
296 sum2 = vaddq_s32(res2.val[0], res2.val[1]);
297 sum = vaddq_s32(sum1, sum2);
304 template<> EIGEN_STRONG_INLINE
float predux_mul<Packet4f>(
const Packet4f& a)
306 float32x2_t a_lo, a_hi, prod;
309 a_lo = vget_low_f32(a);
310 a_hi = vget_high_f32(a);
312 prod = vmul_f32(a_lo, a_hi);
314 prod = vmul_f32(prod, vrev64_f32(prod));
316 return vget_lane_f32(prod, 0);
318 template<> EIGEN_STRONG_INLINE
int predux_mul<Packet4i>(
const Packet4i& a)
320 int32x2_t a_lo, a_hi, prod;
323 a_lo = vget_low_s32(a);
324 a_hi = vget_high_s32(a);
326 prod = vmul_s32(a_lo, a_hi);
328 prod = vmul_s32(prod, vrev64_s32(prod));
330 return vget_lane_s32(prod, 0);
334 template<> EIGEN_STRONG_INLINE
float predux_min<Packet4f>(
const Packet4f& a)
336 float32x2_t a_lo, a_hi, min;
338 a_lo = vget_low_f32(a);
339 a_hi = vget_high_f32(a);
340 min = vpmin_f32(a_lo, a_hi);
341 min = vpmin_f32(min, min);
343 return vget_lane_f32(min, 0);
346 template<> EIGEN_STRONG_INLINE
int predux_min<Packet4i>(
const Packet4i& a)
348 int32x2_t a_lo, a_hi, min;
350 a_lo = vget_low_s32(a);
351 a_hi = vget_high_s32(a);
352 min = vpmin_s32(a_lo, a_hi);
353 min = vpmin_s32(min, min);
355 return vget_lane_s32(min, 0);
359 template<> EIGEN_STRONG_INLINE
float predux_max<Packet4f>(
const Packet4f& a)
361 float32x2_t a_lo, a_hi, max;
363 a_lo = vget_low_f32(a);
364 a_hi = vget_high_f32(a);
365 max = vpmax_f32(a_lo, a_hi);
366 max = vpmax_f32(max, max);
368 return vget_lane_f32(max, 0);
371 template<> EIGEN_STRONG_INLINE
int predux_max<Packet4i>(
const Packet4i& a)
373 int32x2_t a_lo, a_hi, max;
375 a_lo = vget_low_s32(a);
376 a_hi = vget_high_s32(a);
377 max = vpmax_s32(a_lo, a_hi);
379 return vget_lane_s32(max, 0);
384 #define PALIGN_NEON(Offset,Type,Command) \
386 struct palign_impl<Offset,Type>\
388 EIGEN_STRONG_INLINE static void run(Type& first, const Type& second)\
391 first = Command(first, second, Offset);\
395 PALIGN_NEON(0,Packet4f,vextq_f32)
396 PALIGN_NEON(1,Packet4f,vextq_f32)
397 PALIGN_NEON(2,Packet4f,vextq_f32)
398 PALIGN_NEON(3,Packet4f,vextq_f32)
399 PALIGN_NEON(0,Packet4i,vextq_s32)
400 PALIGN_NEON(1,Packet4i,vextq_s32)
401 PALIGN_NEON(2,Packet4i,vextq_s32)
402 PALIGN_NEON(3,Packet4i,vextq_s32)
410 #endif // EIGEN_PACKET_MATH_NEON_H
Definition: Eigen_Colamd.h:54