core/stdarch/crates/core_arch/src/aarch64/neon/
mod.rs

1//! ARMv8 ASIMD intrinsics
2
3#![allow(non_camel_case_types)]
4
5#[rustfmt::skip]
6mod generated;
7#[rustfmt::skip]
8#[stable(feature = "neon_intrinsics", since = "1.59.0")]
9pub use self::generated::*;
10
11// FIXME: replace neon with asimd
12
13use crate::{
14    core_arch::{arm_shared::*, simd::*},
15    hint::unreachable_unchecked,
16    intrinsics::simd::*,
17    mem::transmute,
18};
19#[cfg(test)]
20use stdarch_test::assert_instr;
21
22types! {
23    #![stable(feature = "neon_intrinsics", since = "1.59.0")]
24
25    /// ARM-specific 64-bit wide vector of one packed `f64`.
26    pub struct float64x1_t(1 x f64); // FIXME: check this!
27    /// ARM-specific 128-bit wide vector of two packed `f64`.
28    pub struct float64x2_t(2 x f64);
29}
30
31/// ARM-specific type containing two `float64x1_t` vectors.
32#[repr(C)]
33#[derive(Copy, Clone, Debug)]
34#[stable(feature = "neon_intrinsics", since = "1.59.0")]
35pub struct float64x1x2_t(pub float64x1_t, pub float64x1_t);
36/// ARM-specific type containing three `float64x1_t` vectors.
37#[repr(C)]
38#[derive(Copy, Clone, Debug)]
39#[stable(feature = "neon_intrinsics", since = "1.59.0")]
40pub struct float64x1x3_t(pub float64x1_t, pub float64x1_t, pub float64x1_t);
41/// ARM-specific type containing four `float64x1_t` vectors.
42#[repr(C)]
43#[derive(Copy, Clone, Debug)]
44#[stable(feature = "neon_intrinsics", since = "1.59.0")]
45pub struct float64x1x4_t(
46    pub float64x1_t,
47    pub float64x1_t,
48    pub float64x1_t,
49    pub float64x1_t,
50);
51
52/// ARM-specific type containing two `float64x2_t` vectors.
53#[repr(C)]
54#[derive(Copy, Clone, Debug)]
55#[stable(feature = "neon_intrinsics", since = "1.59.0")]
56pub struct float64x2x2_t(pub float64x2_t, pub float64x2_t);
57/// ARM-specific type containing three `float64x2_t` vectors.
58#[repr(C)]
59#[derive(Copy, Clone, Debug)]
60#[stable(feature = "neon_intrinsics", since = "1.59.0")]
61pub struct float64x2x3_t(pub float64x2_t, pub float64x2_t, pub float64x2_t);
62/// ARM-specific type containing four `float64x2_t` vectors.
63#[repr(C)]
64#[derive(Copy, Clone, Debug)]
65#[stable(feature = "neon_intrinsics", since = "1.59.0")]
66pub struct float64x2x4_t(
67    pub float64x2_t,
68    pub float64x2_t,
69    pub float64x2_t,
70    pub float64x2_t,
71);
72
73/// Duplicate vector element to vector or scalar
74#[inline]
75#[target_feature(enable = "neon")]
76#[cfg_attr(test, assert_instr(nop, N1 = 0, N2 = 0))]
77#[rustc_legacy_const_generics(1, 3)]
78#[stable(feature = "neon_intrinsics", since = "1.59.0")]
79pub fn vcopy_lane_s64<const N1: i32, const N2: i32>(_a: int64x1_t, b: int64x1_t) -> int64x1_t {
80    static_assert!(N1 == 0);
81    static_assert!(N2 == 0);
82    b
83}
84
85/// Duplicate vector element to vector or scalar
86#[inline]
87#[target_feature(enable = "neon")]
88#[cfg_attr(test, assert_instr(nop, N1 = 0, N2 = 0))]
89#[rustc_legacy_const_generics(1, 3)]
90#[stable(feature = "neon_intrinsics", since = "1.59.0")]
91pub fn vcopy_lane_u64<const N1: i32, const N2: i32>(_a: uint64x1_t, b: uint64x1_t) -> uint64x1_t {
92    static_assert!(N1 == 0);
93    static_assert!(N2 == 0);
94    b
95}
96
97/// Duplicate vector element to vector or scalar
98#[inline]
99#[target_feature(enable = "neon")]
100#[cfg_attr(test, assert_instr(nop, N1 = 0, N2 = 0))]
101#[rustc_legacy_const_generics(1, 3)]
102#[stable(feature = "neon_intrinsics", since = "1.59.0")]
103pub fn vcopy_lane_p64<const N1: i32, const N2: i32>(_a: poly64x1_t, b: poly64x1_t) -> poly64x1_t {
104    static_assert!(N1 == 0);
105    static_assert!(N2 == 0);
106    b
107}
108
109/// Duplicate vector element to vector or scalar
110#[inline]
111#[target_feature(enable = "neon")]
112#[cfg_attr(test, assert_instr(nop, N1 = 0, N2 = 0))]
113#[rustc_legacy_const_generics(1, 3)]
114#[stable(feature = "neon_intrinsics", since = "1.59.0")]
115pub fn vcopy_lane_f64<const N1: i32, const N2: i32>(
116    _a: float64x1_t,
117    b: float64x1_t,
118) -> float64x1_t {
119    static_assert!(N1 == 0);
120    static_assert!(N2 == 0);
121    b
122}
123
124/// Duplicate vector element to vector or scalar
125#[inline]
126#[target_feature(enable = "neon")]
127#[cfg_attr(test, assert_instr(nop, LANE1 = 0, LANE2 = 1))]
128#[rustc_legacy_const_generics(1, 3)]
129#[stable(feature = "neon_intrinsics", since = "1.59.0")]
130pub fn vcopy_laneq_s64<const LANE1: i32, const LANE2: i32>(
131    _a: int64x1_t,
132    b: int64x2_t,
133) -> int64x1_t {
134    static_assert!(LANE1 == 0);
135    static_assert_uimm_bits!(LANE2, 1);
136    unsafe { transmute::<i64, _>(simd_extract!(b, LANE2 as u32)) }
137}
138
139/// Duplicate vector element to vector or scalar
140#[inline]
141#[target_feature(enable = "neon")]
142#[cfg_attr(test, assert_instr(nop, LANE1 = 0, LANE2 = 1))]
143#[rustc_legacy_const_generics(1, 3)]
144#[stable(feature = "neon_intrinsics", since = "1.59.0")]
145pub fn vcopy_laneq_u64<const LANE1: i32, const LANE2: i32>(
146    _a: uint64x1_t,
147    b: uint64x2_t,
148) -> uint64x1_t {
149    static_assert!(LANE1 == 0);
150    static_assert_uimm_bits!(LANE2, 1);
151    unsafe { transmute::<u64, _>(simd_extract!(b, LANE2 as u32)) }
152}
153
154/// Duplicate vector element to vector or scalar
155#[inline]
156#[target_feature(enable = "neon")]
157#[cfg_attr(test, assert_instr(nop, LANE1 = 0, LANE2 = 1))]
158#[rustc_legacy_const_generics(1, 3)]
159#[stable(feature = "neon_intrinsics", since = "1.59.0")]
160pub fn vcopy_laneq_p64<const LANE1: i32, const LANE2: i32>(
161    _a: poly64x1_t,
162    b: poly64x2_t,
163) -> poly64x1_t {
164    static_assert!(LANE1 == 0);
165    static_assert_uimm_bits!(LANE2, 1);
166    unsafe { transmute::<u64, _>(simd_extract!(b, LANE2 as u32)) }
167}
168
169/// Duplicate vector element to vector or scalar
170#[inline]
171#[target_feature(enable = "neon")]
172#[cfg_attr(test, assert_instr(nop, LANE1 = 0, LANE2 = 1))]
173#[rustc_legacy_const_generics(1, 3)]
174#[stable(feature = "neon_intrinsics", since = "1.59.0")]
175pub fn vcopy_laneq_f64<const LANE1: i32, const LANE2: i32>(
176    _a: float64x1_t,
177    b: float64x2_t,
178) -> float64x1_t {
179    static_assert!(LANE1 == 0);
180    static_assert_uimm_bits!(LANE2, 1);
181    unsafe { transmute::<f64, _>(simd_extract!(b, LANE2 as u32)) }
182}
183
184/// Load multiple single-element structures to one, two, three, or four registers
185#[inline]
186#[target_feature(enable = "neon")]
187#[cfg_attr(test, assert_instr(ldr))]
188#[stable(feature = "neon_intrinsics", since = "1.59.0")]
189pub unsafe fn vld1_dup_f64(ptr: *const f64) -> float64x1_t {
190    vld1_f64(ptr)
191}
192
193/// Load multiple single-element structures to one, two, three, or four registers
194#[inline]
195#[target_feature(enable = "neon")]
196#[cfg_attr(test, assert_instr(ld1r))]
197#[stable(feature = "neon_intrinsics", since = "1.59.0")]
198pub unsafe fn vld1q_dup_f64(ptr: *const f64) -> float64x2_t {
199    let x = vld1q_lane_f64::<0>(ptr, transmute(f64x2::splat(0.)));
200    simd_shuffle!(x, x, [0, 0])
201}
202
203/// Load one single-element structure to one lane of one register.
204#[inline]
205#[target_feature(enable = "neon")]
206#[rustc_legacy_const_generics(2)]
207#[cfg_attr(test, assert_instr(ldr, LANE = 0))]
208#[stable(feature = "neon_intrinsics", since = "1.59.0")]
209pub unsafe fn vld1_lane_f64<const LANE: i32>(ptr: *const f64, src: float64x1_t) -> float64x1_t {
210    static_assert!(LANE == 0);
211    simd_insert!(src, LANE as u32, *ptr)
212}
213
214/// Load one single-element structure to one lane of one register.
215#[inline]
216#[target_feature(enable = "neon")]
217#[rustc_legacy_const_generics(2)]
218#[cfg_attr(test, assert_instr(ld1, LANE = 1))]
219#[stable(feature = "neon_intrinsics", since = "1.59.0")]
220pub unsafe fn vld1q_lane_f64<const LANE: i32>(ptr: *const f64, src: float64x2_t) -> float64x2_t {
221    static_assert_uimm_bits!(LANE, 1);
222    simd_insert!(src, LANE as u32, *ptr)
223}
224
225/// Bitwise Select instructions. This instruction sets each bit in the destination SIMD&FP register
226/// to the corresponding bit from the first source SIMD&FP register when the original
227/// destination bit was 1, otherwise from the second source SIMD&FP register.
228#[inline]
229#[target_feature(enable = "neon")]
230#[cfg_attr(test, assert_instr(bsl))]
231#[stable(feature = "neon_intrinsics", since = "1.59.0")]
232pub fn vbsl_f64(a: uint64x1_t, b: float64x1_t, c: float64x1_t) -> float64x1_t {
233    let not = int64x1_t::splat(-1);
234    unsafe {
235        transmute(simd_or(
236            simd_and(a, transmute(b)),
237            simd_and(simd_xor(a, transmute(not)), transmute(c)),
238        ))
239    }
240}
241/// Bitwise Select.
242#[inline]
243#[target_feature(enable = "neon")]
244#[cfg_attr(test, assert_instr(bsl))]
245#[stable(feature = "neon_intrinsics", since = "1.59.0")]
246pub fn vbsl_p64(a: poly64x1_t, b: poly64x1_t, c: poly64x1_t) -> poly64x1_t {
247    let not = int64x1_t::splat(-1);
248    unsafe { simd_or(simd_and(a, b), simd_and(simd_xor(a, transmute(not)), c)) }
249}
250/// Bitwise Select. (128-bit)
251#[inline]
252#[target_feature(enable = "neon")]
253#[cfg_attr(test, assert_instr(bsl))]
254#[stable(feature = "neon_intrinsics", since = "1.59.0")]
255pub fn vbslq_f64(a: uint64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t {
256    let not = int64x2_t::splat(-1);
257    unsafe {
258        transmute(simd_or(
259            simd_and(a, transmute(b)),
260            simd_and(simd_xor(a, transmute(not)), transmute(c)),
261        ))
262    }
263}
264/// Bitwise Select. (128-bit)
265#[inline]
266#[target_feature(enable = "neon")]
267#[cfg_attr(test, assert_instr(bsl))]
268#[stable(feature = "neon_intrinsics", since = "1.59.0")]
269pub fn vbslq_p64(a: poly64x2_t, b: poly64x2_t, c: poly64x2_t) -> poly64x2_t {
270    let not = int64x2_t::splat(-1);
271    unsafe { simd_or(simd_and(a, b), simd_and(simd_xor(a, transmute(not)), c)) }
272}
273
274/// Vector add.
275#[inline]
276#[target_feature(enable = "neon")]
277#[cfg_attr(test, assert_instr(fadd))]
278#[stable(feature = "neon_intrinsics", since = "1.59.0")]
279pub fn vadd_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t {
280    unsafe { simd_add(a, b) }
281}
282
283/// Vector add.
284#[inline]
285#[target_feature(enable = "neon")]
286#[cfg_attr(test, assert_instr(fadd))]
287#[stable(feature = "neon_intrinsics", since = "1.59.0")]
288pub fn vaddq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t {
289    unsafe { simd_add(a, b) }
290}
291
292/// Vector add.
293#[inline]
294#[target_feature(enable = "neon")]
295#[cfg_attr(test, assert_instr(add))]
296#[stable(feature = "neon_intrinsics", since = "1.59.0")]
297pub fn vadd_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t {
298    unsafe { simd_add(a, b) }
299}
300
301/// Vector add.
302#[inline]
303#[target_feature(enable = "neon")]
304#[cfg_attr(test, assert_instr(add))]
305#[stable(feature = "neon_intrinsics", since = "1.59.0")]
306pub fn vadd_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t {
307    unsafe { simd_add(a, b) }
308}
309
310/// Vector add.
311#[inline]
312#[target_feature(enable = "neon")]
313#[cfg_attr(test, assert_instr(add))]
314#[stable(feature = "neon_intrinsics", since = "1.59.0")]
315pub fn vaddd_s64(a: i64, b: i64) -> i64 {
316    a.wrapping_add(b)
317}
318
319/// Vector add.
320#[inline]
321#[target_feature(enable = "neon")]
322#[cfg_attr(test, assert_instr(add))]
323#[stable(feature = "neon_intrinsics", since = "1.59.0")]
324pub fn vaddd_u64(a: u64, b: u64) -> u64 {
325    a.wrapping_add(b)
326}
327
328/// Extract vector from pair of vectors
329#[inline]
330#[target_feature(enable = "neon")]
331#[cfg_attr(test, assert_instr(nop, N = 0))]
332#[rustc_legacy_const_generics(2)]
333#[stable(feature = "neon_intrinsics", since = "1.59.0")]
334pub fn vext_p64<const N: i32>(a: poly64x1_t, _b: poly64x1_t) -> poly64x1_t {
335    static_assert!(N == 0);
336    a
337}
338
339/// Extract vector from pair of vectors
340#[inline]
341#[target_feature(enable = "neon")]
342#[cfg_attr(test, assert_instr(nop, N = 0))]
343#[rustc_legacy_const_generics(2)]
344#[stable(feature = "neon_intrinsics", since = "1.59.0")]
345pub fn vext_f64<const N: i32>(a: float64x1_t, _b: float64x1_t) -> float64x1_t {
346    static_assert!(N == 0);
347    a
348}
349
350/// Duplicate vector element to vector or scalar
351#[inline]
352#[target_feature(enable = "neon")]
353#[cfg_attr(test, assert_instr(fmov))]
354#[stable(feature = "neon_intrinsics", since = "1.59.0")]
355pub fn vdup_n_p64(value: p64) -> poly64x1_t {
356    unsafe { transmute(u64x1::new(value)) }
357}
358
359/// Duplicate vector element to vector or scalar
360#[inline]
361#[target_feature(enable = "neon")]
362#[cfg_attr(test, assert_instr(nop))]
363#[stable(feature = "neon_intrinsics", since = "1.59.0")]
364pub fn vdup_n_f64(value: f64) -> float64x1_t {
365    float64x1_t::splat(value)
366}
367
368/// Duplicate vector element to vector or scalar
369#[inline]
370#[target_feature(enable = "neon")]
371#[cfg_attr(test, assert_instr(dup))]
372#[stable(feature = "neon_intrinsics", since = "1.59.0")]
373pub fn vdupq_n_p64(value: p64) -> poly64x2_t {
374    unsafe { transmute(u64x2::new(value, value)) }
375}
376
377/// Duplicate vector element to vector or scalar
378#[inline]
379#[target_feature(enable = "neon")]
380#[cfg_attr(test, assert_instr(dup))]
381#[stable(feature = "neon_intrinsics", since = "1.59.0")]
382pub fn vdupq_n_f64(value: f64) -> float64x2_t {
383    float64x2_t::splat(value)
384}
385
386/// Duplicate vector element to vector or scalar
387#[inline]
388#[target_feature(enable = "neon")]
389#[cfg_attr(test, assert_instr(fmov))]
390#[stable(feature = "neon_intrinsics", since = "1.59.0")]
391pub fn vmov_n_p64(value: p64) -> poly64x1_t {
392    vdup_n_p64(value)
393}
394
395/// Duplicate vector element to vector or scalar
396#[inline]
397#[target_feature(enable = "neon")]
398#[cfg_attr(test, assert_instr(nop))]
399#[stable(feature = "neon_intrinsics", since = "1.59.0")]
400pub fn vmov_n_f64(value: f64) -> float64x1_t {
401    vdup_n_f64(value)
402}
403
404/// Duplicate vector element to vector or scalar
405#[inline]
406#[target_feature(enable = "neon")]
407#[cfg_attr(test, assert_instr(dup))]
408#[stable(feature = "neon_intrinsics", since = "1.59.0")]
409pub fn vmovq_n_p64(value: p64) -> poly64x2_t {
410    vdupq_n_p64(value)
411}
412
413/// Duplicate vector element to vector or scalar
414#[inline]
415#[target_feature(enable = "neon")]
416#[cfg_attr(test, assert_instr(dup))]
417#[stable(feature = "neon_intrinsics", since = "1.59.0")]
418pub fn vmovq_n_f64(value: f64) -> float64x2_t {
419    vdupq_n_f64(value)
420}
421
422/// Duplicate vector element to vector or scalar
423#[inline]
424#[target_feature(enable = "neon")]
425#[cfg_attr(test, assert_instr(nop))]
426#[stable(feature = "neon_intrinsics", since = "1.59.0")]
427pub fn vget_high_f64(a: float64x2_t) -> float64x1_t {
428    unsafe { float64x1_t([simd_extract!(a, 1)]) }
429}
430
431/// Duplicate vector element to vector or scalar
432#[inline]
433#[target_feature(enable = "neon")]
434#[cfg_attr(test, assert_instr(ext))]
435#[stable(feature = "neon_intrinsics", since = "1.59.0")]
436pub fn vget_high_p64(a: poly64x2_t) -> poly64x1_t {
437    unsafe { transmute(u64x1::new(simd_extract!(a, 1))) }
438}
439
440/// Duplicate vector element to vector or scalar
441#[inline]
442#[target_feature(enable = "neon")]
443#[cfg_attr(test, assert_instr(nop))]
444#[stable(feature = "neon_intrinsics", since = "1.59.0")]
445pub fn vget_low_f64(a: float64x2_t) -> float64x1_t {
446    unsafe { float64x1_t([simd_extract!(a, 0)]) }
447}
448
449/// Duplicate vector element to vector or scalar
450#[inline]
451#[target_feature(enable = "neon")]
452#[cfg_attr(test, assert_instr(nop))]
453#[stable(feature = "neon_intrinsics", since = "1.59.0")]
454pub fn vget_low_p64(a: poly64x2_t) -> poly64x1_t {
455    unsafe { transmute(u64x1::new(simd_extract!(a, 0))) }
456}
457
458/// Duplicate vector element to vector or scalar
459#[inline]
460#[target_feature(enable = "neon")]
461#[rustc_legacy_const_generics(1)]
462#[stable(feature = "neon_intrinsics", since = "1.59.0")]
463#[cfg_attr(
464    all(test, any(target_arch = "aarch64", target_arch = "arm64ec")),
465    assert_instr(nop, IMM5 = 0)
466)]
467pub fn vget_lane_f64<const IMM5: i32>(v: float64x1_t) -> f64 {
468    static_assert!(IMM5 == 0);
469    unsafe { simd_extract!(v, IMM5 as u32) }
470}
471
472/// Duplicate vector element to vector or scalar
473#[inline]
474#[target_feature(enable = "neon")]
475#[rustc_legacy_const_generics(1)]
476#[stable(feature = "neon_intrinsics", since = "1.59.0")]
477#[cfg_attr(
478    all(test, any(target_arch = "aarch64", target_arch = "arm64ec")),
479    assert_instr(nop, IMM5 = 0)
480)]
481pub fn vgetq_lane_f64<const IMM5: i32>(v: float64x2_t) -> f64 {
482    static_assert_uimm_bits!(IMM5, 1);
483    unsafe { simd_extract!(v, IMM5 as u32) }
484}
485
486/// Vector combine
487#[inline]
488#[target_feature(enable = "neon")]
489#[cfg_attr(test, assert_instr(mov))]
490#[stable(feature = "neon_intrinsics", since = "1.59.0")]
491pub fn vcombine_f64(low: float64x1_t, high: float64x1_t) -> float64x2_t {
492    unsafe { simd_shuffle!(low, high, [0, 1]) }
493}
494
495/// Shift left
496#[inline]
497#[target_feature(enable = "neon")]
498#[cfg_attr(test, assert_instr(nop, N = 2))]
499#[rustc_legacy_const_generics(1)]
500#[stable(feature = "neon_intrinsics", since = "1.59.0")]
501pub fn vshld_n_s64<const N: i32>(a: i64) -> i64 {
502    static_assert_uimm_bits!(N, 6);
503    a << N
504}
505
506/// Shift left
507#[inline]
508#[target_feature(enable = "neon")]
509#[cfg_attr(test, assert_instr(nop, N = 2))]
510#[rustc_legacy_const_generics(1)]
511#[stable(feature = "neon_intrinsics", since = "1.59.0")]
512pub fn vshld_n_u64<const N: i32>(a: u64) -> u64 {
513    static_assert_uimm_bits!(N, 6);
514    a << N
515}
516
517/// Signed shift right
518#[inline]
519#[target_feature(enable = "neon")]
520#[cfg_attr(test, assert_instr(nop, N = 2))]
521#[rustc_legacy_const_generics(1)]
522#[stable(feature = "neon_intrinsics", since = "1.59.0")]
523pub fn vshrd_n_s64<const N: i32>(a: i64) -> i64 {
524    static_assert!(N >= 1 && N <= 64);
525    let n: i32 = if N == 64 { 63 } else { N };
526    a >> n
527}
528
529/// Unsigned shift right
530#[inline]
531#[target_feature(enable = "neon")]
532#[cfg_attr(test, assert_instr(nop, N = 2))]
533#[rustc_legacy_const_generics(1)]
534#[stable(feature = "neon_intrinsics", since = "1.59.0")]
535pub fn vshrd_n_u64<const N: i32>(a: u64) -> u64 {
536    static_assert!(N >= 1 && N <= 64);
537    let n: i32 = if N == 64 {
538        return 0;
539    } else {
540        N
541    };
542    a >> n
543}
544
545/// Signed shift right and accumulate
546#[inline]
547#[target_feature(enable = "neon")]
548#[cfg_attr(test, assert_instr(nop, N = 2))]
549#[rustc_legacy_const_generics(2)]
550#[stable(feature = "neon_intrinsics", since = "1.59.0")]
551pub fn vsrad_n_s64<const N: i32>(a: i64, b: i64) -> i64 {
552    static_assert!(N >= 1 && N <= 64);
553    a.wrapping_add(vshrd_n_s64::<N>(b))
554}
555
556/// Unsigned shift right and accumulate
557#[inline]
558#[target_feature(enable = "neon")]
559#[cfg_attr(test, assert_instr(nop, N = 2))]
560#[rustc_legacy_const_generics(2)]
561#[stable(feature = "neon_intrinsics", since = "1.59.0")]
562pub fn vsrad_n_u64<const N: i32>(a: u64, b: u64) -> u64 {
563    static_assert!(N >= 1 && N <= 64);
564    a.wrapping_add(vshrd_n_u64::<N>(b))
565}
566
567#[cfg(test)]
568mod tests {
569    use crate::core_arch::aarch64::test_support::*;
570    use crate::core_arch::arm_shared::test_support::*;
571    use crate::core_arch::{aarch64::neon::*, aarch64::*, simd::*};
572    use std::mem::transmute;
573    use stdarch_test::simd_test;
574
575    #[simd_test(enable = "neon")]
576    unsafe fn test_vadd_f64() {
577        let a = 1.;
578        let b = 8.;
579        let e = 9.;
580        let r: f64 = transmute(vadd_f64(transmute(a), transmute(b)));
581        assert_eq!(r, e);
582    }
583
584    #[simd_test(enable = "neon")]
585    unsafe fn test_vaddq_f64() {
586        let a = f64x2::new(1., 2.);
587        let b = f64x2::new(8., 7.);
588        let e = f64x2::new(9., 9.);
589        let r: f64x2 = transmute(vaddq_f64(transmute(a), transmute(b)));
590        assert_eq!(r, e);
591    }
592
593    #[simd_test(enable = "neon")]
594    unsafe fn test_vadd_s64() {
595        let a = 1_i64;
596        let b = 8_i64;
597        let e = 9_i64;
598        let r: i64 = transmute(vadd_s64(transmute(a), transmute(b)));
599        assert_eq!(r, e);
600    }
601
602    #[simd_test(enable = "neon")]
603    unsafe fn test_vadd_u64() {
604        let a = 1_u64;
605        let b = 8_u64;
606        let e = 9_u64;
607        let r: u64 = transmute(vadd_u64(transmute(a), transmute(b)));
608        assert_eq!(r, e);
609    }
610
611    #[simd_test(enable = "neon")]
612    unsafe fn test_vaddd_s64() {
613        let a = 1_i64;
614        let b = 8_i64;
615        let e = 9_i64;
616        let r: i64 = vaddd_s64(a, b);
617        assert_eq!(r, e);
618    }
619
620    #[simd_test(enable = "neon")]
621    unsafe fn test_vaddd_u64() {
622        let a = 1_u64;
623        let b = 8_u64;
624        let e = 9_u64;
625        let r: u64 = vaddd_u64(a, b);
626        assert_eq!(r, e);
627    }
628
629    #[simd_test(enable = "neon")]
630    unsafe fn test_vext_p64() {
631        let a: i64x1 = i64x1::new(0);
632        let b: i64x1 = i64x1::new(1);
633        let e: i64x1 = i64x1::new(0);
634        let r: i64x1 = transmute(vext_p64::<0>(transmute(a), transmute(b)));
635        assert_eq!(r, e);
636    }
637
638    #[simd_test(enable = "neon")]
639    unsafe fn test_vext_f64() {
640        let a: f64x1 = f64x1::new(0.);
641        let b: f64x1 = f64x1::new(1.);
642        let e: f64x1 = f64x1::new(0.);
643        let r: f64x1 = transmute(vext_f64::<0>(transmute(a), transmute(b)));
644        assert_eq!(r, e);
645    }
646
647    #[simd_test(enable = "neon")]
648    unsafe fn test_vshld_n_s64() {
649        let a: i64 = 1;
650        let e: i64 = 4;
651        let r: i64 = vshld_n_s64::<2>(a);
652        assert_eq!(r, e);
653    }
654
655    #[simd_test(enable = "neon")]
656    unsafe fn test_vshld_n_u64() {
657        let a: u64 = 1;
658        let e: u64 = 4;
659        let r: u64 = vshld_n_u64::<2>(a);
660        assert_eq!(r, e);
661    }
662
663    #[simd_test(enable = "neon")]
664    unsafe fn test_vshrd_n_s64() {
665        let a: i64 = 4;
666        let e: i64 = 1;
667        let r: i64 = vshrd_n_s64::<2>(a);
668        assert_eq!(r, e);
669    }
670
671    #[simd_test(enable = "neon")]
672    unsafe fn test_vshrd_n_u64() {
673        let a: u64 = 4;
674        let e: u64 = 1;
675        let r: u64 = vshrd_n_u64::<2>(a);
676        assert_eq!(r, e);
677    }
678
679    #[simd_test(enable = "neon")]
680    unsafe fn test_vsrad_n_s64() {
681        let a: i64 = 1;
682        let b: i64 = 4;
683        let e: i64 = 2;
684        let r: i64 = vsrad_n_s64::<2>(a, b);
685        assert_eq!(r, e);
686    }
687
688    #[simd_test(enable = "neon")]
689    unsafe fn test_vsrad_n_u64() {
690        let a: u64 = 1;
691        let b: u64 = 4;
692        let e: u64 = 2;
693        let r: u64 = vsrad_n_u64::<2>(a, b);
694        assert_eq!(r, e);
695    }
696
697    #[simd_test(enable = "neon")]
698    unsafe fn test_vdup_n_f64() {
699        let a: f64 = 3.3;
700        let e = f64x1::new(3.3);
701        let r: f64x1 = transmute(vdup_n_f64(a));
702        assert_eq!(r, e);
703    }
704
705    #[simd_test(enable = "neon")]
706    unsafe fn test_vdup_n_p64() {
707        let a: u64 = 3;
708        let e = u64x1::new(3);
709        let r: u64x1 = transmute(vdup_n_p64(a));
710        assert_eq!(r, e);
711    }
712
713    #[simd_test(enable = "neon")]
714    unsafe fn test_vdupq_n_f64() {
715        let a: f64 = 3.3;
716        let e = f64x2::new(3.3, 3.3);
717        let r: f64x2 = transmute(vdupq_n_f64(a));
718        assert_eq!(r, e);
719    }
720
721    #[simd_test(enable = "neon")]
722    unsafe fn test_vdupq_n_p64() {
723        let a: u64 = 3;
724        let e = u64x2::new(3, 3);
725        let r: u64x2 = transmute(vdupq_n_p64(a));
726        assert_eq!(r, e);
727    }
728
729    #[simd_test(enable = "neon")]
730    unsafe fn test_vmov_n_p64() {
731        let a: u64 = 3;
732        let e = u64x1::new(3);
733        let r: u64x1 = transmute(vmov_n_p64(a));
734        assert_eq!(r, e);
735    }
736
737    #[simd_test(enable = "neon")]
738    unsafe fn test_vmov_n_f64() {
739        let a: f64 = 3.3;
740        let e = f64x1::new(3.3);
741        let r: f64x1 = transmute(vmov_n_f64(a));
742        assert_eq!(r, e);
743    }
744
745    #[simd_test(enable = "neon")]
746    unsafe fn test_vmovq_n_p64() {
747        let a: u64 = 3;
748        let e = u64x2::new(3, 3);
749        let r: u64x2 = transmute(vmovq_n_p64(a));
750        assert_eq!(r, e);
751    }
752
753    #[simd_test(enable = "neon")]
754    unsafe fn test_vmovq_n_f64() {
755        let a: f64 = 3.3;
756        let e = f64x2::new(3.3, 3.3);
757        let r: f64x2 = transmute(vmovq_n_f64(a));
758        assert_eq!(r, e);
759    }
760
761    #[simd_test(enable = "neon")]
762    unsafe fn test_vget_high_f64() {
763        let a = f64x2::new(1.0, 2.0);
764        let e = f64x1::new(2.0);
765        let r: f64x1 = transmute(vget_high_f64(transmute(a)));
766        assert_eq!(r, e);
767    }
768
769    #[simd_test(enable = "neon")]
770    unsafe fn test_vget_high_p64() {
771        let a = u64x2::new(1, 2);
772        let e = u64x1::new(2);
773        let r: u64x1 = transmute(vget_high_p64(transmute(a)));
774        assert_eq!(r, e);
775    }
776
777    #[simd_test(enable = "neon")]
778    unsafe fn test_vget_low_f64() {
779        let a = f64x2::new(1.0, 2.0);
780        let e = f64x1::new(1.0);
781        let r: f64x1 = transmute(vget_low_f64(transmute(a)));
782        assert_eq!(r, e);
783    }
784
785    #[simd_test(enable = "neon")]
786    unsafe fn test_vget_low_p64() {
787        let a = u64x2::new(1, 2);
788        let e = u64x1::new(1);
789        let r: u64x1 = transmute(vget_low_p64(transmute(a)));
790        assert_eq!(r, e);
791    }
792
793    #[simd_test(enable = "neon")]
794    unsafe fn test_vget_lane_f64() {
795        let v = f64x1::new(1.0);
796        let r = vget_lane_f64::<0>(transmute(v));
797        assert_eq!(r, 1.0);
798    }
799
800    #[simd_test(enable = "neon")]
801    unsafe fn test_vgetq_lane_f64() {
802        let v = f64x2::new(0.0, 1.0);
803        let r = vgetq_lane_f64::<1>(transmute(v));
804        assert_eq!(r, 1.0);
805        let r = vgetq_lane_f64::<0>(transmute(v));
806        assert_eq!(r, 0.0);
807    }
808
809    #[simd_test(enable = "neon")]
810    unsafe fn test_vcopy_lane_s64() {
811        let a: i64x1 = i64x1::new(1);
812        let b: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF);
813        let e: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF);
814        let r: i64x1 = transmute(vcopy_lane_s64::<0, 0>(transmute(a), transmute(b)));
815        assert_eq!(r, e);
816    }
817
818    #[simd_test(enable = "neon")]
819    unsafe fn test_vcopy_lane_u64() {
820        let a: u64x1 = u64x1::new(1);
821        let b: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF);
822        let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF);
823        let r: u64x1 = transmute(vcopy_lane_u64::<0, 0>(transmute(a), transmute(b)));
824        assert_eq!(r, e);
825    }
826
827    #[simd_test(enable = "neon")]
828    unsafe fn test_vcopy_lane_p64() {
829        let a: i64x1 = i64x1::new(1);
830        let b: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF);
831        let e: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF);
832        let r: i64x1 = transmute(vcopy_lane_p64::<0, 0>(transmute(a), transmute(b)));
833        assert_eq!(r, e);
834    }
835
836    #[simd_test(enable = "neon")]
837    unsafe fn test_vcopy_lane_f64() {
838        let a: f64 = 1.;
839        let b: f64 = 0.;
840        let e: f64 = 0.;
841        let r: f64 = transmute(vcopy_lane_f64::<0, 0>(transmute(a), transmute(b)));
842        assert_eq!(r, e);
843    }
844
845    #[simd_test(enable = "neon")]
846    unsafe fn test_vcopy_laneq_s64() {
847        let a: i64x1 = i64x1::new(1);
848        let b: i64x2 = i64x2::new(0, 0x7F_FF_FF_FF_FF_FF_FF_FF);
849        let e: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF);
850        let r: i64x1 = transmute(vcopy_laneq_s64::<0, 1>(transmute(a), transmute(b)));
851        assert_eq!(r, e);
852    }
853
854    #[simd_test(enable = "neon")]
855    unsafe fn test_vcopy_laneq_u64() {
856        let a: u64x1 = u64x1::new(1);
857        let b: u64x2 = u64x2::new(0, 0xFF_FF_FF_FF_FF_FF_FF_FF);
858        let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF);
859        let r: u64x1 = transmute(vcopy_laneq_u64::<0, 1>(transmute(a), transmute(b)));
860        assert_eq!(r, e);
861    }
862
863    #[simd_test(enable = "neon")]
864    unsafe fn test_vcopy_laneq_p64() {
865        let a: i64x1 = i64x1::new(1);
866        let b: i64x2 = i64x2::new(0, 0x7F_FF_FF_FF_FF_FF_FF_FF);
867        let e: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF);
868        let r: i64x1 = transmute(vcopy_laneq_p64::<0, 1>(transmute(a), transmute(b)));
869        assert_eq!(r, e);
870    }
871
872    #[simd_test(enable = "neon")]
873    unsafe fn test_vcopy_laneq_f64() {
874        let a: f64 = 1.;
875        let b: f64x2 = f64x2::new(0., 0.5);
876        let e: f64 = 0.5;
877        let r: f64 = transmute(vcopy_laneq_f64::<0, 1>(transmute(a), transmute(b)));
878        assert_eq!(r, e);
879    }
880
881    #[simd_test(enable = "neon")]
882    unsafe fn test_vbsl_f64() {
883        let a = u64x1::new(0x8000000000000000);
884        let b = f64x1::new(-1.23f64);
885        let c = f64x1::new(2.34f64);
886        let e = f64x1::new(-2.34f64);
887        let r: f64x1 = transmute(vbsl_f64(transmute(a), transmute(b), transmute(c)));
888        assert_eq!(r, e);
889    }
890    #[simd_test(enable = "neon")]
891    unsafe fn test_vbsl_p64() {
892        let a = u64x1::new(1);
893        let b = u64x1::new(u64::MAX);
894        let c = u64x1::new(u64::MIN);
895        let e = u64x1::new(1);
896        let r: u64x1 = transmute(vbsl_p64(transmute(a), transmute(b), transmute(c)));
897        assert_eq!(r, e);
898    }
899    #[simd_test(enable = "neon")]
900    unsafe fn test_vbslq_f64() {
901        let a = u64x2::new(1, 0x8000000000000000);
902        let b = f64x2::new(f64::MAX, -1.23f64);
903        let c = f64x2::new(f64::MIN, 2.34f64);
904        let e = f64x2::new(f64::MIN, -2.34f64);
905        let r: f64x2 = transmute(vbslq_f64(transmute(a), transmute(b), transmute(c)));
906        assert_eq!(r, e);
907    }
908    #[simd_test(enable = "neon")]
909    unsafe fn test_vbslq_p64() {
910        let a = u64x2::new(u64::MAX, 1);
911        let b = u64x2::new(u64::MAX, u64::MAX);
912        let c = u64x2::new(u64::MIN, u64::MIN);
913        let e = u64x2::new(u64::MAX, 1);
914        let r: u64x2 = transmute(vbslq_p64(transmute(a), transmute(b), transmute(c)));
915        assert_eq!(r, e);
916    }
917
918    #[simd_test(enable = "neon")]
919    unsafe fn test_vld1_f64() {
920        let a: [f64; 2] = [0., 1.];
921        let e = f64x1::new(1.);
922        let r: f64x1 = transmute(vld1_f64(a[1..].as_ptr()));
923        assert_eq!(r, e)
924    }
925
926    #[simd_test(enable = "neon")]
927    unsafe fn test_vld1q_f64() {
928        let a: [f64; 3] = [0., 1., 2.];
929        let e = f64x2::new(1., 2.);
930        let r: f64x2 = transmute(vld1q_f64(a[1..].as_ptr()));
931        assert_eq!(r, e)
932    }
933
934    #[simd_test(enable = "neon")]
935    unsafe fn test_vld1_dup_f64() {
936        let a: [f64; 2] = [1., 42.];
937        let e = f64x1::new(42.);
938        let r: f64x1 = transmute(vld1_dup_f64(a[1..].as_ptr()));
939        assert_eq!(r, e)
940    }
941
942    #[simd_test(enable = "neon")]
943    unsafe fn test_vld1q_dup_f64() {
944        let elem: f64 = 42.;
945        let e = f64x2::new(42., 42.);
946        let r: f64x2 = transmute(vld1q_dup_f64(&elem));
947        assert_eq!(r, e)
948    }
949
950    #[simd_test(enable = "neon")]
951    unsafe fn test_vld1_lane_f64() {
952        let a = f64x1::new(0.);
953        let elem: f64 = 42.;
954        let e = f64x1::new(42.);
955        let r: f64x1 = transmute(vld1_lane_f64::<0>(&elem, transmute(a)));
956        assert_eq!(r, e)
957    }
958
959    #[simd_test(enable = "neon")]
960    unsafe fn test_vld1q_lane_f64() {
961        let a = f64x2::new(0., 1.);
962        let elem: f64 = 42.;
963        let e = f64x2::new(0., 42.);
964        let r: f64x2 = transmute(vld1q_lane_f64::<1>(&elem, transmute(a)));
965        assert_eq!(r, e)
966    }
967
968    #[simd_test(enable = "neon")]
969    unsafe fn test_vst1_f64() {
970        let mut vals = [0_f64; 2];
971        let a = f64x1::new(1.);
972
973        vst1_f64(vals[1..].as_mut_ptr(), transmute(a));
974
975        assert_eq!(vals[0], 0.);
976        assert_eq!(vals[1], 1.);
977    }
978
979    #[simd_test(enable = "neon")]
980    unsafe fn test_vst1q_f64() {
981        let mut vals = [0_f64; 3];
982        let a = f64x2::new(1., 2.);
983
984        vst1q_f64(vals[1..].as_mut_ptr(), transmute(a));
985
986        assert_eq!(vals[0], 0.);
987        assert_eq!(vals[1], 1.);
988        assert_eq!(vals[2], 2.);
989    }
990}
991
992#[cfg(test)]
993#[path = "../../arm_shared/neon/table_lookup_tests.rs"]
994mod table_lookup_tests;
995
996#[cfg(test)]
997#[path = "../../arm_shared/neon/shift_and_insert_tests.rs"]
998mod shift_and_insert_tests;
999
1000#[cfg(test)]
1001#[path = "../../arm_shared/neon/load_tests.rs"]
1002mod load_tests;
1003
1004#[cfg(test)]
1005#[path = "../../arm_shared/neon/store_tests.rs"]
1006mod store_tests;