core/stdarch/crates/core_arch/src/aarch64/neon/
mod.rs

1//! ARMv8 ASIMD intrinsics
2
3#![allow(non_camel_case_types)]
4
5#[rustfmt::skip]
6mod generated;
7#[rustfmt::skip]
8#[stable(feature = "neon_intrinsics", since = "1.59.0")]
9pub use self::generated::*;
10
11// FIXME: replace neon with asimd
12
13use crate::{
14    core_arch::{arm_shared::*, simd::*},
15    hint::unreachable_unchecked,
16    intrinsics::simd::*,
17    mem::transmute,
18};
19#[cfg(test)]
20use stdarch_test::assert_instr;
21
22types! {
23    #![stable(feature = "neon_intrinsics", since = "1.59.0")]
24
25    /// ARM-specific 64-bit wide vector of one packed `f64`.
26    pub struct float64x1_t(1 x f64); // FIXME: check this!
27    /// ARM-specific 128-bit wide vector of two packed `f64`.
28    pub struct float64x2_t(2 x f64);
29}
30
31/// ARM-specific type containing two `float64x1_t` vectors.
32#[repr(C)]
33#[derive(Copy, Clone, Debug)]
34#[stable(feature = "neon_intrinsics", since = "1.59.0")]
35pub struct float64x1x2_t(pub float64x1_t, pub float64x1_t);
36/// ARM-specific type containing three `float64x1_t` vectors.
37#[repr(C)]
38#[derive(Copy, Clone, Debug)]
39#[stable(feature = "neon_intrinsics", since = "1.59.0")]
40pub struct float64x1x3_t(pub float64x1_t, pub float64x1_t, pub float64x1_t);
41/// ARM-specific type containing four `float64x1_t` vectors.
42#[repr(C)]
43#[derive(Copy, Clone, Debug)]
44#[stable(feature = "neon_intrinsics", since = "1.59.0")]
45pub struct float64x1x4_t(
46    pub float64x1_t,
47    pub float64x1_t,
48    pub float64x1_t,
49    pub float64x1_t,
50);
51
52/// ARM-specific type containing two `float64x2_t` vectors.
53#[repr(C)]
54#[derive(Copy, Clone, Debug)]
55#[stable(feature = "neon_intrinsics", since = "1.59.0")]
56pub struct float64x2x2_t(pub float64x2_t, pub float64x2_t);
57/// ARM-specific type containing three `float64x2_t` vectors.
58#[repr(C)]
59#[derive(Copy, Clone, Debug)]
60#[stable(feature = "neon_intrinsics", since = "1.59.0")]
61pub struct float64x2x3_t(pub float64x2_t, pub float64x2_t, pub float64x2_t);
62/// ARM-specific type containing four `float64x2_t` vectors.
63#[repr(C)]
64#[derive(Copy, Clone, Debug)]
65#[stable(feature = "neon_intrinsics", since = "1.59.0")]
66pub struct float64x2x4_t(
67    pub float64x2_t,
68    pub float64x2_t,
69    pub float64x2_t,
70    pub float64x2_t,
71);
72
73/// Duplicate vector element to vector or scalar
74#[inline]
75#[target_feature(enable = "neon")]
76#[cfg_attr(test, assert_instr(nop, N1 = 0, N2 = 0))]
77#[rustc_legacy_const_generics(1, 3)]
78#[stable(feature = "neon_intrinsics", since = "1.59.0")]
79pub fn vcopy_lane_s64<const N1: i32, const N2: i32>(_a: int64x1_t, b: int64x1_t) -> int64x1_t {
80    static_assert!(N1 == 0);
81    static_assert!(N2 == 0);
82    b
83}
84
85/// Duplicate vector element to vector or scalar
86#[inline]
87#[target_feature(enable = "neon")]
88#[cfg_attr(test, assert_instr(nop, N1 = 0, N2 = 0))]
89#[rustc_legacy_const_generics(1, 3)]
90#[stable(feature = "neon_intrinsics", since = "1.59.0")]
91pub fn vcopy_lane_u64<const N1: i32, const N2: i32>(_a: uint64x1_t, b: uint64x1_t) -> uint64x1_t {
92    static_assert!(N1 == 0);
93    static_assert!(N2 == 0);
94    b
95}
96
97/// Duplicate vector element to vector or scalar
98#[inline]
99#[target_feature(enable = "neon")]
100#[cfg_attr(test, assert_instr(nop, N1 = 0, N2 = 0))]
101#[rustc_legacy_const_generics(1, 3)]
102#[stable(feature = "neon_intrinsics", since = "1.59.0")]
103pub fn vcopy_lane_p64<const N1: i32, const N2: i32>(_a: poly64x1_t, b: poly64x1_t) -> poly64x1_t {
104    static_assert!(N1 == 0);
105    static_assert!(N2 == 0);
106    b
107}
108
109/// Duplicate vector element to vector or scalar
110#[inline]
111#[target_feature(enable = "neon")]
112#[cfg_attr(test, assert_instr(nop, N1 = 0, N2 = 0))]
113#[rustc_legacy_const_generics(1, 3)]
114#[stable(feature = "neon_intrinsics", since = "1.59.0")]
115pub fn vcopy_lane_f64<const N1: i32, const N2: i32>(
116    _a: float64x1_t,
117    b: float64x1_t,
118) -> float64x1_t {
119    static_assert!(N1 == 0);
120    static_assert!(N2 == 0);
121    b
122}
123
124/// Duplicate vector element to vector or scalar
125#[inline]
126#[target_feature(enable = "neon")]
127#[cfg_attr(test, assert_instr(nop, LANE1 = 0, LANE2 = 1))]
128#[rustc_legacy_const_generics(1, 3)]
129#[stable(feature = "neon_intrinsics", since = "1.59.0")]
130pub fn vcopy_laneq_s64<const LANE1: i32, const LANE2: i32>(
131    _a: int64x1_t,
132    b: int64x2_t,
133) -> int64x1_t {
134    static_assert!(LANE1 == 0);
135    static_assert_uimm_bits!(LANE2, 1);
136    unsafe { transmute::<i64, _>(simd_extract!(b, LANE2 as u32)) }
137}
138
139/// Duplicate vector element to vector or scalar
140#[inline]
141#[target_feature(enable = "neon")]
142#[cfg_attr(test, assert_instr(nop, LANE1 = 0, LANE2 = 1))]
143#[rustc_legacy_const_generics(1, 3)]
144#[stable(feature = "neon_intrinsics", since = "1.59.0")]
145pub fn vcopy_laneq_u64<const LANE1: i32, const LANE2: i32>(
146    _a: uint64x1_t,
147    b: uint64x2_t,
148) -> uint64x1_t {
149    static_assert!(LANE1 == 0);
150    static_assert_uimm_bits!(LANE2, 1);
151    unsafe { transmute::<u64, _>(simd_extract!(b, LANE2 as u32)) }
152}
153
154/// Duplicate vector element to vector or scalar
155#[inline]
156#[target_feature(enable = "neon")]
157#[cfg_attr(test, assert_instr(nop, LANE1 = 0, LANE2 = 1))]
158#[rustc_legacy_const_generics(1, 3)]
159#[stable(feature = "neon_intrinsics", since = "1.59.0")]
160pub fn vcopy_laneq_p64<const LANE1: i32, const LANE2: i32>(
161    _a: poly64x1_t,
162    b: poly64x2_t,
163) -> poly64x1_t {
164    static_assert!(LANE1 == 0);
165    static_assert_uimm_bits!(LANE2, 1);
166    unsafe { transmute::<u64, _>(simd_extract!(b, LANE2 as u32)) }
167}
168
169/// Duplicate vector element to vector or scalar
170#[inline]
171#[target_feature(enable = "neon")]
172#[cfg_attr(test, assert_instr(nop, LANE1 = 0, LANE2 = 1))]
173#[rustc_legacy_const_generics(1, 3)]
174#[stable(feature = "neon_intrinsics", since = "1.59.0")]
175pub fn vcopy_laneq_f64<const LANE1: i32, const LANE2: i32>(
176    _a: float64x1_t,
177    b: float64x2_t,
178) -> float64x1_t {
179    static_assert!(LANE1 == 0);
180    static_assert_uimm_bits!(LANE2, 1);
181    unsafe { transmute::<f64, _>(simd_extract!(b, LANE2 as u32)) }
182}
183
184/// Load multiple single-element structures to one, two, three, or four registers
185#[inline]
186#[target_feature(enable = "neon")]
187#[cfg_attr(test, assert_instr(ldr))]
188#[stable(feature = "neon_intrinsics", since = "1.59.0")]
189pub unsafe fn vld1_dup_f64(ptr: *const f64) -> float64x1_t {
190    vld1_f64(ptr)
191}
192
193/// Load multiple single-element structures to one, two, three, or four registers
194#[inline]
195#[target_feature(enable = "neon")]
196#[cfg_attr(test, assert_instr(ld1r))]
197#[stable(feature = "neon_intrinsics", since = "1.59.0")]
198pub unsafe fn vld1q_dup_f64(ptr: *const f64) -> float64x2_t {
199    let x = vld1q_lane_f64::<0>(ptr, transmute(f64x2::splat(0.)));
200    simd_shuffle!(x, x, [0, 0])
201}
202
203/// Load one single-element structure to one lane of one register.
204#[inline]
205#[target_feature(enable = "neon")]
206#[rustc_legacy_const_generics(2)]
207#[cfg_attr(test, assert_instr(ldr, LANE = 0))]
208#[stable(feature = "neon_intrinsics", since = "1.59.0")]
209pub unsafe fn vld1_lane_f64<const LANE: i32>(ptr: *const f64, src: float64x1_t) -> float64x1_t {
210    static_assert!(LANE == 0);
211    simd_insert!(src, LANE as u32, *ptr)
212}
213
214/// Load one single-element structure to one lane of one register.
215#[inline]
216#[target_feature(enable = "neon")]
217#[rustc_legacy_const_generics(2)]
218#[cfg_attr(test, assert_instr(ld1, LANE = 1))]
219#[stable(feature = "neon_intrinsics", since = "1.59.0")]
220pub unsafe fn vld1q_lane_f64<const LANE: i32>(ptr: *const f64, src: float64x2_t) -> float64x2_t {
221    static_assert_uimm_bits!(LANE, 1);
222    simd_insert!(src, LANE as u32, *ptr)
223}
224
225/// Bitwise Select instructions. This instruction sets each bit in the destination SIMD&FP register
226/// to the corresponding bit from the first source SIMD&FP register when the original
227/// destination bit was 1, otherwise from the second source SIMD&FP register.
228#[inline]
229#[target_feature(enable = "neon")]
230#[cfg_attr(test, assert_instr(bsl))]
231#[stable(feature = "neon_intrinsics", since = "1.59.0")]
232pub fn vbsl_f64(a: uint64x1_t, b: float64x1_t, c: float64x1_t) -> float64x1_t {
233    let not = int64x1_t::splat(-1);
234    unsafe {
235        transmute(simd_or(
236            simd_and(a, transmute(b)),
237            simd_and(simd_xor(a, transmute(not)), transmute(c)),
238        ))
239    }
240}
241/// Bitwise Select.
242#[inline]
243#[target_feature(enable = "neon")]
244#[cfg_attr(test, assert_instr(bsl))]
245#[stable(feature = "neon_intrinsics", since = "1.59.0")]
246pub fn vbsl_p64(a: poly64x1_t, b: poly64x1_t, c: poly64x1_t) -> poly64x1_t {
247    let not = int64x1_t::splat(-1);
248    unsafe { simd_or(simd_and(a, b), simd_and(simd_xor(a, transmute(not)), c)) }
249}
250/// Bitwise Select. (128-bit)
251#[inline]
252#[target_feature(enable = "neon")]
253#[cfg_attr(test, assert_instr(bsl))]
254#[stable(feature = "neon_intrinsics", since = "1.59.0")]
255pub fn vbslq_f64(a: uint64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t {
256    let not = int64x2_t::splat(-1);
257    unsafe {
258        transmute(simd_or(
259            simd_and(a, transmute(b)),
260            simd_and(simd_xor(a, transmute(not)), transmute(c)),
261        ))
262    }
263}
264/// Bitwise Select. (128-bit)
265#[inline]
266#[target_feature(enable = "neon")]
267#[cfg_attr(test, assert_instr(bsl))]
268#[stable(feature = "neon_intrinsics", since = "1.59.0")]
269pub fn vbslq_p64(a: poly64x2_t, b: poly64x2_t, c: poly64x2_t) -> poly64x2_t {
270    let not = int64x2_t::splat(-1);
271    unsafe { simd_or(simd_and(a, b), simd_and(simd_xor(a, transmute(not)), c)) }
272}
273
274/// Vector add.
275#[inline]
276#[target_feature(enable = "neon")]
277#[cfg_attr(test, assert_instr(fadd))]
278#[stable(feature = "neon_intrinsics", since = "1.59.0")]
279pub fn vadd_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t {
280    unsafe { simd_add(a, b) }
281}
282
283/// Vector add.
284#[inline]
285#[target_feature(enable = "neon")]
286#[cfg_attr(test, assert_instr(fadd))]
287#[stable(feature = "neon_intrinsics", since = "1.59.0")]
288pub fn vaddq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t {
289    unsafe { simd_add(a, b) }
290}
291
292/// Vector add.
293#[inline]
294#[target_feature(enable = "neon")]
295#[cfg_attr(test, assert_instr(add))]
296#[stable(feature = "neon_intrinsics", since = "1.59.0")]
297pub fn vadd_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t {
298    unsafe { simd_add(a, b) }
299}
300
301/// Vector add.
302#[inline]
303#[target_feature(enable = "neon")]
304#[cfg_attr(test, assert_instr(add))]
305#[stable(feature = "neon_intrinsics", since = "1.59.0")]
306pub fn vadd_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t {
307    unsafe { simd_add(a, b) }
308}
309
310/// Vector add.
311#[inline]
312#[target_feature(enable = "neon")]
313#[cfg_attr(test, assert_instr(add))]
314#[stable(feature = "neon_intrinsics", since = "1.59.0")]
315pub fn vaddd_s64(a: i64, b: i64) -> i64 {
316    a.wrapping_add(b)
317}
318
319/// Vector add.
320#[inline]
321#[target_feature(enable = "neon")]
322#[cfg_attr(test, assert_instr(add))]
323#[stable(feature = "neon_intrinsics", since = "1.59.0")]
324pub fn vaddd_u64(a: u64, b: u64) -> u64 {
325    a.wrapping_add(b)
326}
327
328/// Extract vector from pair of vectors
329#[inline]
330#[target_feature(enable = "neon")]
331#[cfg_attr(test, assert_instr(nop, N = 0))]
332#[rustc_legacy_const_generics(2)]
333#[stable(feature = "neon_intrinsics", since = "1.59.0")]
334pub fn vext_p64<const N: i32>(a: poly64x1_t, _b: poly64x1_t) -> poly64x1_t {
335    static_assert!(N == 0);
336    a
337}
338
339/// Extract vector from pair of vectors
340#[inline]
341#[target_feature(enable = "neon")]
342#[cfg_attr(test, assert_instr(nop, N = 0))]
343#[rustc_legacy_const_generics(2)]
344#[stable(feature = "neon_intrinsics", since = "1.59.0")]
345pub fn vext_f64<const N: i32>(a: float64x1_t, _b: float64x1_t) -> float64x1_t {
346    static_assert!(N == 0);
347    a
348}
349
350/// Duplicate vector element to vector or scalar
351#[inline]
352#[target_feature(enable = "neon")]
353#[cfg_attr(test, assert_instr(fmov))]
354#[stable(feature = "neon_intrinsics", since = "1.59.0")]
355pub fn vdup_n_p64(value: p64) -> poly64x1_t {
356    unsafe { transmute(u64x1::new(value)) }
357}
358
359/// Duplicate vector element to vector or scalar
360#[inline]
361#[target_feature(enable = "neon")]
362#[cfg_attr(test, assert_instr(nop))]
363#[stable(feature = "neon_intrinsics", since = "1.59.0")]
364pub fn vdup_n_f64(value: f64) -> float64x1_t {
365    float64x1_t::splat(value)
366}
367
368/// Duplicate vector element to vector or scalar
369#[inline]
370#[target_feature(enable = "neon")]
371#[cfg_attr(test, assert_instr(dup))]
372#[stable(feature = "neon_intrinsics", since = "1.59.0")]
373pub fn vdupq_n_p64(value: p64) -> poly64x2_t {
374    unsafe { transmute(u64x2::new(value, value)) }
375}
376
377/// Duplicate vector element to vector or scalar
378#[inline]
379#[target_feature(enable = "neon")]
380#[cfg_attr(test, assert_instr(dup))]
381#[stable(feature = "neon_intrinsics", since = "1.59.0")]
382pub fn vdupq_n_f64(value: f64) -> float64x2_t {
383    float64x2_t::splat(value)
384}
385
386/// Duplicate vector element to vector or scalar
387#[inline]
388#[target_feature(enable = "neon")]
389#[cfg_attr(test, assert_instr(fmov))]
390#[stable(feature = "neon_intrinsics", since = "1.59.0")]
391pub fn vmov_n_p64(value: p64) -> poly64x1_t {
392    vdup_n_p64(value)
393}
394
395/// Duplicate vector element to vector or scalar
396#[inline]
397#[target_feature(enable = "neon")]
398#[cfg_attr(test, assert_instr(nop))]
399#[stable(feature = "neon_intrinsics", since = "1.59.0")]
400pub fn vmov_n_f64(value: f64) -> float64x1_t {
401    vdup_n_f64(value)
402}
403
404/// Duplicate vector element to vector or scalar
405#[inline]
406#[target_feature(enable = "neon")]
407#[cfg_attr(test, assert_instr(dup))]
408#[stable(feature = "neon_intrinsics", since = "1.59.0")]
409pub fn vmovq_n_p64(value: p64) -> poly64x2_t {
410    vdupq_n_p64(value)
411}
412
413/// Duplicate vector element to vector or scalar
414#[inline]
415#[target_feature(enable = "neon")]
416#[cfg_attr(test, assert_instr(dup))]
417#[stable(feature = "neon_intrinsics", since = "1.59.0")]
418pub fn vmovq_n_f64(value: f64) -> float64x2_t {
419    vdupq_n_f64(value)
420}
421
422/// Duplicate vector element to vector or scalar
423#[inline]
424#[target_feature(enable = "neon")]
425#[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(mov))]
426#[cfg_attr(all(test, target_env = "msvc"), assert_instr(dup))]
427#[stable(feature = "neon_intrinsics", since = "1.59.0")]
428pub fn vget_high_f64(a: float64x2_t) -> float64x1_t {
429    unsafe { float64x1_t([simd_extract!(a, 1)]) }
430}
431
432/// Duplicate vector element to vector or scalar
433#[inline]
434#[target_feature(enable = "neon")]
435#[cfg_attr(test, assert_instr(ext))]
436#[stable(feature = "neon_intrinsics", since = "1.59.0")]
437pub fn vget_high_p64(a: poly64x2_t) -> poly64x1_t {
438    unsafe { transmute(u64x1::new(simd_extract!(a, 1))) }
439}
440
441/// Duplicate vector element to vector or scalar
442#[inline]
443#[target_feature(enable = "neon")]
444#[cfg_attr(test, assert_instr(nop))]
445#[stable(feature = "neon_intrinsics", since = "1.59.0")]
446pub fn vget_low_f64(a: float64x2_t) -> float64x1_t {
447    unsafe { float64x1_t([simd_extract!(a, 0)]) }
448}
449
450/// Duplicate vector element to vector or scalar
451#[inline]
452#[target_feature(enable = "neon")]
453#[cfg_attr(test, assert_instr(nop))]
454#[stable(feature = "neon_intrinsics", since = "1.59.0")]
455pub fn vget_low_p64(a: poly64x2_t) -> poly64x1_t {
456    unsafe { transmute(u64x1::new(simd_extract!(a, 0))) }
457}
458
459/// Duplicate vector element to vector or scalar
460#[inline]
461#[target_feature(enable = "neon")]
462#[rustc_legacy_const_generics(1)]
463#[stable(feature = "neon_intrinsics", since = "1.59.0")]
464#[cfg_attr(
465    all(test, any(target_arch = "aarch64", target_arch = "arm64ec")),
466    assert_instr(nop, IMM5 = 0)
467)]
468pub fn vget_lane_f64<const IMM5: i32>(v: float64x1_t) -> f64 {
469    static_assert!(IMM5 == 0);
470    unsafe { simd_extract!(v, IMM5 as u32) }
471}
472
473/// Duplicate vector element to vector or scalar
474#[inline]
475#[target_feature(enable = "neon")]
476#[rustc_legacy_const_generics(1)]
477#[stable(feature = "neon_intrinsics", since = "1.59.0")]
478#[cfg_attr(
479    all(test, any(target_arch = "aarch64", target_arch = "arm64ec")),
480    assert_instr(nop, IMM5 = 0)
481)]
482pub fn vgetq_lane_f64<const IMM5: i32>(v: float64x2_t) -> f64 {
483    static_assert_uimm_bits!(IMM5, 1);
484    unsafe { simd_extract!(v, IMM5 as u32) }
485}
486
487/// Vector combine
488#[inline]
489#[target_feature(enable = "neon")]
490#[cfg_attr(test, assert_instr(mov))]
491#[stable(feature = "neon_intrinsics", since = "1.59.0")]
492pub fn vcombine_f64(low: float64x1_t, high: float64x1_t) -> float64x2_t {
493    unsafe { simd_shuffle!(low, high, [0, 1]) }
494}
495
496/// Shift left
497#[inline]
498#[target_feature(enable = "neon")]
499#[cfg_attr(test, assert_instr(nop, N = 2))]
500#[rustc_legacy_const_generics(1)]
501#[stable(feature = "neon_intrinsics", since = "1.59.0")]
502pub fn vshld_n_s64<const N: i32>(a: i64) -> i64 {
503    static_assert_uimm_bits!(N, 6);
504    a << N
505}
506
507/// Shift left
508#[inline]
509#[target_feature(enable = "neon")]
510#[cfg_attr(test, assert_instr(nop, N = 2))]
511#[rustc_legacy_const_generics(1)]
512#[stable(feature = "neon_intrinsics", since = "1.59.0")]
513pub fn vshld_n_u64<const N: i32>(a: u64) -> u64 {
514    static_assert_uimm_bits!(N, 6);
515    a << N
516}
517
518/// Signed shift right
519#[inline]
520#[target_feature(enable = "neon")]
521#[cfg_attr(test, assert_instr(nop, N = 2))]
522#[rustc_legacy_const_generics(1)]
523#[stable(feature = "neon_intrinsics", since = "1.59.0")]
524pub fn vshrd_n_s64<const N: i32>(a: i64) -> i64 {
525    static_assert!(N >= 1 && N <= 64);
526    let n: i32 = if N == 64 { 63 } else { N };
527    a >> n
528}
529
530/// Unsigned shift right
531#[inline]
532#[target_feature(enable = "neon")]
533#[cfg_attr(test, assert_instr(nop, N = 2))]
534#[rustc_legacy_const_generics(1)]
535#[stable(feature = "neon_intrinsics", since = "1.59.0")]
536pub fn vshrd_n_u64<const N: i32>(a: u64) -> u64 {
537    static_assert!(N >= 1 && N <= 64);
538    let n: i32 = if N == 64 {
539        return 0;
540    } else {
541        N
542    };
543    a >> n
544}
545
546/// Signed shift right and accumulate
547#[inline]
548#[target_feature(enable = "neon")]
549#[cfg_attr(test, assert_instr(nop, N = 2))]
550#[rustc_legacy_const_generics(2)]
551#[stable(feature = "neon_intrinsics", since = "1.59.0")]
552pub fn vsrad_n_s64<const N: i32>(a: i64, b: i64) -> i64 {
553    static_assert!(N >= 1 && N <= 64);
554    a.wrapping_add(vshrd_n_s64::<N>(b))
555}
556
557/// Unsigned shift right and accumulate
558#[inline]
559#[target_feature(enable = "neon")]
560#[cfg_attr(test, assert_instr(nop, N = 2))]
561#[rustc_legacy_const_generics(2)]
562#[stable(feature = "neon_intrinsics", since = "1.59.0")]
563pub fn vsrad_n_u64<const N: i32>(a: u64, b: u64) -> u64 {
564    static_assert!(N >= 1 && N <= 64);
565    a.wrapping_add(vshrd_n_u64::<N>(b))
566}
567
568#[cfg(test)]
569mod tests {
570    use crate::core_arch::aarch64::test_support::*;
571    use crate::core_arch::arm_shared::test_support::*;
572    use crate::core_arch::{aarch64::neon::*, aarch64::*, simd::*};
573    use std::mem::transmute;
574    use stdarch_test::simd_test;
575
576    #[simd_test(enable = "neon")]
577    unsafe fn test_vadd_f64() {
578        let a = 1.;
579        let b = 8.;
580        let e = 9.;
581        let r: f64 = transmute(vadd_f64(transmute(a), transmute(b)));
582        assert_eq!(r, e);
583    }
584
585    #[simd_test(enable = "neon")]
586    unsafe fn test_vaddq_f64() {
587        let a = f64x2::new(1., 2.);
588        let b = f64x2::new(8., 7.);
589        let e = f64x2::new(9., 9.);
590        let r: f64x2 = transmute(vaddq_f64(transmute(a), transmute(b)));
591        assert_eq!(r, e);
592    }
593
594    #[simd_test(enable = "neon")]
595    unsafe fn test_vadd_s64() {
596        let a = 1_i64;
597        let b = 8_i64;
598        let e = 9_i64;
599        let r: i64 = transmute(vadd_s64(transmute(a), transmute(b)));
600        assert_eq!(r, e);
601    }
602
603    #[simd_test(enable = "neon")]
604    unsafe fn test_vadd_u64() {
605        let a = 1_u64;
606        let b = 8_u64;
607        let e = 9_u64;
608        let r: u64 = transmute(vadd_u64(transmute(a), transmute(b)));
609        assert_eq!(r, e);
610    }
611
612    #[simd_test(enable = "neon")]
613    unsafe fn test_vaddd_s64() {
614        let a = 1_i64;
615        let b = 8_i64;
616        let e = 9_i64;
617        let r: i64 = vaddd_s64(a, b);
618        assert_eq!(r, e);
619    }
620
621    #[simd_test(enable = "neon")]
622    unsafe fn test_vaddd_u64() {
623        let a = 1_u64;
624        let b = 8_u64;
625        let e = 9_u64;
626        let r: u64 = vaddd_u64(a, b);
627        assert_eq!(r, e);
628    }
629
630    #[simd_test(enable = "neon")]
631    unsafe fn test_vext_p64() {
632        let a: i64x1 = i64x1::new(0);
633        let b: i64x1 = i64x1::new(1);
634        let e: i64x1 = i64x1::new(0);
635        let r: i64x1 = transmute(vext_p64::<0>(transmute(a), transmute(b)));
636        assert_eq!(r, e);
637    }
638
639    #[simd_test(enable = "neon")]
640    unsafe fn test_vext_f64() {
641        let a: f64x1 = f64x1::new(0.);
642        let b: f64x1 = f64x1::new(1.);
643        let e: f64x1 = f64x1::new(0.);
644        let r: f64x1 = transmute(vext_f64::<0>(transmute(a), transmute(b)));
645        assert_eq!(r, e);
646    }
647
648    #[simd_test(enable = "neon")]
649    unsafe fn test_vshld_n_s64() {
650        let a: i64 = 1;
651        let e: i64 = 4;
652        let r: i64 = vshld_n_s64::<2>(a);
653        assert_eq!(r, e);
654    }
655
656    #[simd_test(enable = "neon")]
657    unsafe fn test_vshld_n_u64() {
658        let a: u64 = 1;
659        let e: u64 = 4;
660        let r: u64 = vshld_n_u64::<2>(a);
661        assert_eq!(r, e);
662    }
663
664    #[simd_test(enable = "neon")]
665    unsafe fn test_vshrd_n_s64() {
666        let a: i64 = 4;
667        let e: i64 = 1;
668        let r: i64 = vshrd_n_s64::<2>(a);
669        assert_eq!(r, e);
670    }
671
672    #[simd_test(enable = "neon")]
673    unsafe fn test_vshrd_n_u64() {
674        let a: u64 = 4;
675        let e: u64 = 1;
676        let r: u64 = vshrd_n_u64::<2>(a);
677        assert_eq!(r, e);
678    }
679
680    #[simd_test(enable = "neon")]
681    unsafe fn test_vsrad_n_s64() {
682        let a: i64 = 1;
683        let b: i64 = 4;
684        let e: i64 = 2;
685        let r: i64 = vsrad_n_s64::<2>(a, b);
686        assert_eq!(r, e);
687    }
688
689    #[simd_test(enable = "neon")]
690    unsafe fn test_vsrad_n_u64() {
691        let a: u64 = 1;
692        let b: u64 = 4;
693        let e: u64 = 2;
694        let r: u64 = vsrad_n_u64::<2>(a, b);
695        assert_eq!(r, e);
696    }
697
698    #[simd_test(enable = "neon")]
699    unsafe fn test_vdup_n_f64() {
700        let a: f64 = 3.3;
701        let e = f64x1::new(3.3);
702        let r: f64x1 = transmute(vdup_n_f64(a));
703        assert_eq!(r, e);
704    }
705
706    #[simd_test(enable = "neon")]
707    unsafe fn test_vdup_n_p64() {
708        let a: u64 = 3;
709        let e = u64x1::new(3);
710        let r: u64x1 = transmute(vdup_n_p64(a));
711        assert_eq!(r, e);
712    }
713
714    #[simd_test(enable = "neon")]
715    unsafe fn test_vdupq_n_f64() {
716        let a: f64 = 3.3;
717        let e = f64x2::new(3.3, 3.3);
718        let r: f64x2 = transmute(vdupq_n_f64(a));
719        assert_eq!(r, e);
720    }
721
722    #[simd_test(enable = "neon")]
723    unsafe fn test_vdupq_n_p64() {
724        let a: u64 = 3;
725        let e = u64x2::new(3, 3);
726        let r: u64x2 = transmute(vdupq_n_p64(a));
727        assert_eq!(r, e);
728    }
729
730    #[simd_test(enable = "neon")]
731    unsafe fn test_vmov_n_p64() {
732        let a: u64 = 3;
733        let e = u64x1::new(3);
734        let r: u64x1 = transmute(vmov_n_p64(a));
735        assert_eq!(r, e);
736    }
737
738    #[simd_test(enable = "neon")]
739    unsafe fn test_vmov_n_f64() {
740        let a: f64 = 3.3;
741        let e = f64x1::new(3.3);
742        let r: f64x1 = transmute(vmov_n_f64(a));
743        assert_eq!(r, e);
744    }
745
746    #[simd_test(enable = "neon")]
747    unsafe fn test_vmovq_n_p64() {
748        let a: u64 = 3;
749        let e = u64x2::new(3, 3);
750        let r: u64x2 = transmute(vmovq_n_p64(a));
751        assert_eq!(r, e);
752    }
753
754    #[simd_test(enable = "neon")]
755    unsafe fn test_vmovq_n_f64() {
756        let a: f64 = 3.3;
757        let e = f64x2::new(3.3, 3.3);
758        let r: f64x2 = transmute(vmovq_n_f64(a));
759        assert_eq!(r, e);
760    }
761
762    #[simd_test(enable = "neon")]
763    unsafe fn test_vget_high_f64() {
764        let a = f64x2::new(1.0, 2.0);
765        let e = f64x1::new(2.0);
766        let r: f64x1 = transmute(vget_high_f64(transmute(a)));
767        assert_eq!(r, e);
768    }
769
770    #[simd_test(enable = "neon")]
771    unsafe fn test_vget_high_p64() {
772        let a = u64x2::new(1, 2);
773        let e = u64x1::new(2);
774        let r: u64x1 = transmute(vget_high_p64(transmute(a)));
775        assert_eq!(r, e);
776    }
777
778    #[simd_test(enable = "neon")]
779    unsafe fn test_vget_low_f64() {
780        let a = f64x2::new(1.0, 2.0);
781        let e = f64x1::new(1.0);
782        let r: f64x1 = transmute(vget_low_f64(transmute(a)));
783        assert_eq!(r, e);
784    }
785
786    #[simd_test(enable = "neon")]
787    unsafe fn test_vget_low_p64() {
788        let a = u64x2::new(1, 2);
789        let e = u64x1::new(1);
790        let r: u64x1 = transmute(vget_low_p64(transmute(a)));
791        assert_eq!(r, e);
792    }
793
794    #[simd_test(enable = "neon")]
795    unsafe fn test_vget_lane_f64() {
796        let v = f64x1::new(1.0);
797        let r = vget_lane_f64::<0>(transmute(v));
798        assert_eq!(r, 1.0);
799    }
800
801    #[simd_test(enable = "neon")]
802    unsafe fn test_vgetq_lane_f64() {
803        let v = f64x2::new(0.0, 1.0);
804        let r = vgetq_lane_f64::<1>(transmute(v));
805        assert_eq!(r, 1.0);
806        let r = vgetq_lane_f64::<0>(transmute(v));
807        assert_eq!(r, 0.0);
808    }
809
810    #[simd_test(enable = "neon")]
811    unsafe fn test_vcopy_lane_s64() {
812        let a: i64x1 = i64x1::new(1);
813        let b: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF);
814        let e: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF);
815        let r: i64x1 = transmute(vcopy_lane_s64::<0, 0>(transmute(a), transmute(b)));
816        assert_eq!(r, e);
817    }
818
819    #[simd_test(enable = "neon")]
820    unsafe fn test_vcopy_lane_u64() {
821        let a: u64x1 = u64x1::new(1);
822        let b: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF);
823        let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF);
824        let r: u64x1 = transmute(vcopy_lane_u64::<0, 0>(transmute(a), transmute(b)));
825        assert_eq!(r, e);
826    }
827
828    #[simd_test(enable = "neon")]
829    unsafe fn test_vcopy_lane_p64() {
830        let a: i64x1 = i64x1::new(1);
831        let b: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF);
832        let e: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF);
833        let r: i64x1 = transmute(vcopy_lane_p64::<0, 0>(transmute(a), transmute(b)));
834        assert_eq!(r, e);
835    }
836
837    #[simd_test(enable = "neon")]
838    unsafe fn test_vcopy_lane_f64() {
839        let a: f64 = 1.;
840        let b: f64 = 0.;
841        let e: f64 = 0.;
842        let r: f64 = transmute(vcopy_lane_f64::<0, 0>(transmute(a), transmute(b)));
843        assert_eq!(r, e);
844    }
845
846    #[simd_test(enable = "neon")]
847    unsafe fn test_vcopy_laneq_s64() {
848        let a: i64x1 = i64x1::new(1);
849        let b: i64x2 = i64x2::new(0, 0x7F_FF_FF_FF_FF_FF_FF_FF);
850        let e: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF);
851        let r: i64x1 = transmute(vcopy_laneq_s64::<0, 1>(transmute(a), transmute(b)));
852        assert_eq!(r, e);
853    }
854
855    #[simd_test(enable = "neon")]
856    unsafe fn test_vcopy_laneq_u64() {
857        let a: u64x1 = u64x1::new(1);
858        let b: u64x2 = u64x2::new(0, 0xFF_FF_FF_FF_FF_FF_FF_FF);
859        let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF);
860        let r: u64x1 = transmute(vcopy_laneq_u64::<0, 1>(transmute(a), transmute(b)));
861        assert_eq!(r, e);
862    }
863
864    #[simd_test(enable = "neon")]
865    unsafe fn test_vcopy_laneq_p64() {
866        let a: i64x1 = i64x1::new(1);
867        let b: i64x2 = i64x2::new(0, 0x7F_FF_FF_FF_FF_FF_FF_FF);
868        let e: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF);
869        let r: i64x1 = transmute(vcopy_laneq_p64::<0, 1>(transmute(a), transmute(b)));
870        assert_eq!(r, e);
871    }
872
873    #[simd_test(enable = "neon")]
874    unsafe fn test_vcopy_laneq_f64() {
875        let a: f64 = 1.;
876        let b: f64x2 = f64x2::new(0., 0.5);
877        let e: f64 = 0.5;
878        let r: f64 = transmute(vcopy_laneq_f64::<0, 1>(transmute(a), transmute(b)));
879        assert_eq!(r, e);
880    }
881
882    #[simd_test(enable = "neon")]
883    unsafe fn test_vbsl_f64() {
884        let a = u64x1::new(0x8000000000000000);
885        let b = f64x1::new(-1.23f64);
886        let c = f64x1::new(2.34f64);
887        let e = f64x1::new(-2.34f64);
888        let r: f64x1 = transmute(vbsl_f64(transmute(a), transmute(b), transmute(c)));
889        assert_eq!(r, e);
890    }
891    #[simd_test(enable = "neon")]
892    unsafe fn test_vbsl_p64() {
893        let a = u64x1::new(1);
894        let b = u64x1::new(u64::MAX);
895        let c = u64x1::new(u64::MIN);
896        let e = u64x1::new(1);
897        let r: u64x1 = transmute(vbsl_p64(transmute(a), transmute(b), transmute(c)));
898        assert_eq!(r, e);
899    }
900    #[simd_test(enable = "neon")]
901    unsafe fn test_vbslq_f64() {
902        let a = u64x2::new(1, 0x8000000000000000);
903        let b = f64x2::new(f64::MAX, -1.23f64);
904        let c = f64x2::new(f64::MIN, 2.34f64);
905        let e = f64x2::new(f64::MIN, -2.34f64);
906        let r: f64x2 = transmute(vbslq_f64(transmute(a), transmute(b), transmute(c)));
907        assert_eq!(r, e);
908    }
909    #[simd_test(enable = "neon")]
910    unsafe fn test_vbslq_p64() {
911        let a = u64x2::new(u64::MAX, 1);
912        let b = u64x2::new(u64::MAX, u64::MAX);
913        let c = u64x2::new(u64::MIN, u64::MIN);
914        let e = u64x2::new(u64::MAX, 1);
915        let r: u64x2 = transmute(vbslq_p64(transmute(a), transmute(b), transmute(c)));
916        assert_eq!(r, e);
917    }
918
919    #[simd_test(enable = "neon")]
920    unsafe fn test_vld1_f64() {
921        let a: [f64; 2] = [0., 1.];
922        let e = f64x1::new(1.);
923        let r: f64x1 = transmute(vld1_f64(a[1..].as_ptr()));
924        assert_eq!(r, e)
925    }
926
927    #[simd_test(enable = "neon")]
928    unsafe fn test_vld1q_f64() {
929        let a: [f64; 3] = [0., 1., 2.];
930        let e = f64x2::new(1., 2.);
931        let r: f64x2 = transmute(vld1q_f64(a[1..].as_ptr()));
932        assert_eq!(r, e)
933    }
934
935    #[simd_test(enable = "neon")]
936    unsafe fn test_vld1_dup_f64() {
937        let a: [f64; 2] = [1., 42.];
938        let e = f64x1::new(42.);
939        let r: f64x1 = transmute(vld1_dup_f64(a[1..].as_ptr()));
940        assert_eq!(r, e)
941    }
942
943    #[simd_test(enable = "neon")]
944    unsafe fn test_vld1q_dup_f64() {
945        let elem: f64 = 42.;
946        let e = f64x2::new(42., 42.);
947        let r: f64x2 = transmute(vld1q_dup_f64(&elem));
948        assert_eq!(r, e)
949    }
950
951    #[simd_test(enable = "neon")]
952    unsafe fn test_vld1_lane_f64() {
953        let a = f64x1::new(0.);
954        let elem: f64 = 42.;
955        let e = f64x1::new(42.);
956        let r: f64x1 = transmute(vld1_lane_f64::<0>(&elem, transmute(a)));
957        assert_eq!(r, e)
958    }
959
960    #[simd_test(enable = "neon")]
961    unsafe fn test_vld1q_lane_f64() {
962        let a = f64x2::new(0., 1.);
963        let elem: f64 = 42.;
964        let e = f64x2::new(0., 42.);
965        let r: f64x2 = transmute(vld1q_lane_f64::<1>(&elem, transmute(a)));
966        assert_eq!(r, e)
967    }
968
969    #[simd_test(enable = "neon")]
970    unsafe fn test_vst1_f64() {
971        let mut vals = [0_f64; 2];
972        let a = f64x1::new(1.);
973
974        vst1_f64(vals[1..].as_mut_ptr(), transmute(a));
975
976        assert_eq!(vals[0], 0.);
977        assert_eq!(vals[1], 1.);
978    }
979
980    #[simd_test(enable = "neon")]
981    unsafe fn test_vst1q_f64() {
982        let mut vals = [0_f64; 3];
983        let a = f64x2::new(1., 2.);
984
985        vst1q_f64(vals[1..].as_mut_ptr(), transmute(a));
986
987        assert_eq!(vals[0], 0.);
988        assert_eq!(vals[1], 1.);
989        assert_eq!(vals[2], 2.);
990    }
991}
992
993#[cfg(test)]
994#[path = "../../arm_shared/neon/table_lookup_tests.rs"]
995mod table_lookup_tests;
996
997#[cfg(test)]
998#[path = "../../arm_shared/neon/shift_and_insert_tests.rs"]
999mod shift_and_insert_tests;
1000
1001#[cfg(test)]
1002#[path = "../../arm_shared/neon/load_tests.rs"]
1003mod load_tests;
1004
1005#[cfg(test)]
1006#[path = "../../arm_shared/neon/store_tests.rs"]
1007mod store_tests;