core/stdarch/crates/core_arch/src/x86/
sse.rs

1//! Streaming SIMD Extensions (SSE)
2
3use crate::{
4    core_arch::{simd::*, x86::*},
5    intrinsics::simd::*,
6    intrinsics::sqrtf32,
7    mem, ptr,
8};
9
10#[cfg(test)]
11use stdarch_test::assert_instr;
12
13/// Adds the first component of `a` and `b`, the other components are copied
14/// from `a`.
15///
16/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_add_ss)
17#[inline]
18#[target_feature(enable = "sse")]
19#[cfg_attr(test, assert_instr(addss))]
20#[stable(feature = "simd_x86", since = "1.27.0")]
21pub fn _mm_add_ss(a: __m128, b: __m128) -> __m128 {
22    unsafe { simd_insert!(a, 0, _mm_cvtss_f32(a) + _mm_cvtss_f32(b)) }
23}
24
25/// Adds packed single-precision (32-bit) floating-point elements in `a` and
26/// `b`.
27///
28/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_add_ps)
29#[inline]
30#[target_feature(enable = "sse")]
31#[cfg_attr(test, assert_instr(addps))]
32#[stable(feature = "simd_x86", since = "1.27.0")]
33pub fn _mm_add_ps(a: __m128, b: __m128) -> __m128 {
34    unsafe { simd_add(a, b) }
35}
36
37/// Subtracts the first component of `b` from `a`, the other components are
38/// copied from `a`.
39///
40/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sub_ss)
41#[inline]
42#[target_feature(enable = "sse")]
43#[cfg_attr(test, assert_instr(subss))]
44#[stable(feature = "simd_x86", since = "1.27.0")]
45pub fn _mm_sub_ss(a: __m128, b: __m128) -> __m128 {
46    unsafe { simd_insert!(a, 0, _mm_cvtss_f32(a) - _mm_cvtss_f32(b)) }
47}
48
49/// Subtracts packed single-precision (32-bit) floating-point elements in `a` and
50/// `b`.
51///
52/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sub_ps)
53#[inline]
54#[target_feature(enable = "sse")]
55#[cfg_attr(test, assert_instr(subps))]
56#[stable(feature = "simd_x86", since = "1.27.0")]
57pub fn _mm_sub_ps(a: __m128, b: __m128) -> __m128 {
58    unsafe { simd_sub(a, b) }
59}
60
61/// Multiplies the first component of `a` and `b`, the other components are
62/// copied from `a`.
63///
64/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mul_ss)
65#[inline]
66#[target_feature(enable = "sse")]
67#[cfg_attr(test, assert_instr(mulss))]
68#[stable(feature = "simd_x86", since = "1.27.0")]
69pub fn _mm_mul_ss(a: __m128, b: __m128) -> __m128 {
70    unsafe { simd_insert!(a, 0, _mm_cvtss_f32(a) * _mm_cvtss_f32(b)) }
71}
72
73/// Multiplies packed single-precision (32-bit) floating-point elements in `a` and
74/// `b`.
75///
76/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mul_ps)
77#[inline]
78#[target_feature(enable = "sse")]
79#[cfg_attr(test, assert_instr(mulps))]
80#[stable(feature = "simd_x86", since = "1.27.0")]
81pub fn _mm_mul_ps(a: __m128, b: __m128) -> __m128 {
82    unsafe { simd_mul(a, b) }
83}
84
85/// Divides the first component of `b` by `a`, the other components are
86/// copied from `a`.
87///
88/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_div_ss)
89#[inline]
90#[target_feature(enable = "sse")]
91#[cfg_attr(test, assert_instr(divss))]
92#[stable(feature = "simd_x86", since = "1.27.0")]
93pub fn _mm_div_ss(a: __m128, b: __m128) -> __m128 {
94    unsafe { simd_insert!(a, 0, _mm_cvtss_f32(a) / _mm_cvtss_f32(b)) }
95}
96
97/// Divides packed single-precision (32-bit) floating-point elements in `a` and
98/// `b`.
99///
100/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_div_ps)
101#[inline]
102#[target_feature(enable = "sse")]
103#[cfg_attr(test, assert_instr(divps))]
104#[stable(feature = "simd_x86", since = "1.27.0")]
105pub fn _mm_div_ps(a: __m128, b: __m128) -> __m128 {
106    unsafe { simd_div(a, b) }
107}
108
109/// Returns the square root of the first single-precision (32-bit)
110/// floating-point element in `a`, the other elements are unchanged.
111///
112/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sqrt_ss)
113#[inline]
114#[target_feature(enable = "sse")]
115#[cfg_attr(test, assert_instr(sqrtss))]
116#[stable(feature = "simd_x86", since = "1.27.0")]
117pub fn _mm_sqrt_ss(a: __m128) -> __m128 {
118    unsafe { simd_insert!(a, 0, sqrtf32(_mm_cvtss_f32(a))) }
119}
120
121/// Returns the square root of packed single-precision (32-bit) floating-point
122/// elements in `a`.
123///
124/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sqrt_ps)
125#[inline]
126#[target_feature(enable = "sse")]
127#[cfg_attr(test, assert_instr(sqrtps))]
128#[stable(feature = "simd_x86", since = "1.27.0")]
129pub fn _mm_sqrt_ps(a: __m128) -> __m128 {
130    unsafe { simd_fsqrt(a) }
131}
132
133/// Returns the approximate reciprocal of the first single-precision
134/// (32-bit) floating-point element in `a`, the other elements are unchanged.
135///
136/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_rcp_ss)
137#[inline]
138#[target_feature(enable = "sse")]
139#[cfg_attr(test, assert_instr(rcpss))]
140#[stable(feature = "simd_x86", since = "1.27.0")]
141pub fn _mm_rcp_ss(a: __m128) -> __m128 {
142    unsafe { rcpss(a) }
143}
144
145/// Returns the approximate reciprocal of packed single-precision (32-bit)
146/// floating-point elements in `a`.
147///
148/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_rcp_ps)
149#[inline]
150#[target_feature(enable = "sse")]
151#[cfg_attr(test, assert_instr(rcpps))]
152#[stable(feature = "simd_x86", since = "1.27.0")]
153pub fn _mm_rcp_ps(a: __m128) -> __m128 {
154    unsafe { rcpps(a) }
155}
156
157/// Returns the approximate reciprocal square root of the first single-precision
158/// (32-bit) floating-point element in `a`, the other elements are unchanged.
159///
160/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_rsqrt_ss)
161#[inline]
162#[target_feature(enable = "sse")]
163#[cfg_attr(test, assert_instr(rsqrtss))]
164#[stable(feature = "simd_x86", since = "1.27.0")]
165pub fn _mm_rsqrt_ss(a: __m128) -> __m128 {
166    unsafe { rsqrtss(a) }
167}
168
169/// Returns the approximate reciprocal square root of packed single-precision
170/// (32-bit) floating-point elements in `a`.
171///
172/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_rsqrt_ps)
173#[inline]
174#[target_feature(enable = "sse")]
175#[cfg_attr(test, assert_instr(rsqrtps))]
176#[stable(feature = "simd_x86", since = "1.27.0")]
177pub fn _mm_rsqrt_ps(a: __m128) -> __m128 {
178    unsafe { rsqrtps(a) }
179}
180
181/// Compares the first single-precision (32-bit) floating-point element of `a`
182/// and `b`, and return the minimum value in the first element of the return
183/// value, the other elements are copied from `a`.
184///
185/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_min_ss)
186#[inline]
187#[target_feature(enable = "sse")]
188#[cfg_attr(test, assert_instr(minss))]
189#[stable(feature = "simd_x86", since = "1.27.0")]
190pub fn _mm_min_ss(a: __m128, b: __m128) -> __m128 {
191    unsafe { minss(a, b) }
192}
193
194/// Compares packed single-precision (32-bit) floating-point elements in `a` and
195/// `b`, and return the corresponding minimum values.
196///
197/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_min_ps)
198#[inline]
199#[target_feature(enable = "sse")]
200#[cfg_attr(test, assert_instr(minps))]
201#[stable(feature = "simd_x86", since = "1.27.0")]
202pub fn _mm_min_ps(a: __m128, b: __m128) -> __m128 {
203    // See the `test_mm_min_ps` test why this can't be implemented using `simd_fmin`.
204    unsafe { minps(a, b) }
205}
206
207/// Compares the first single-precision (32-bit) floating-point element of `a`
208/// and `b`, and return the maximum value in the first element of the return
209/// value, the other elements are copied from `a`.
210///
211/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_max_ss)
212#[inline]
213#[target_feature(enable = "sse")]
214#[cfg_attr(test, assert_instr(maxss))]
215#[stable(feature = "simd_x86", since = "1.27.0")]
216pub fn _mm_max_ss(a: __m128, b: __m128) -> __m128 {
217    unsafe { maxss(a, b) }
218}
219
220/// Compares packed single-precision (32-bit) floating-point elements in `a` and
221/// `b`, and return the corresponding maximum values.
222///
223/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_max_ps)
224#[inline]
225#[target_feature(enable = "sse")]
226#[cfg_attr(test, assert_instr(maxps))]
227#[stable(feature = "simd_x86", since = "1.27.0")]
228pub fn _mm_max_ps(a: __m128, b: __m128) -> __m128 {
229    // See the `test_mm_min_ps` test why this can't be implemented using `simd_fmax`.
230    unsafe { maxps(a, b) }
231}
232
233/// Bitwise AND of packed single-precision (32-bit) floating-point elements.
234///
235/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_and_ps)
236#[inline]
237#[target_feature(enable = "sse")]
238// i586 only seems to generate plain `and` instructions, so ignore it.
239#[cfg_attr(
240    all(test, any(target_arch = "x86_64", target_feature = "sse2")),
241    assert_instr(andps)
242)]
243#[stable(feature = "simd_x86", since = "1.27.0")]
244pub fn _mm_and_ps(a: __m128, b: __m128) -> __m128 {
245    unsafe {
246        let a: __m128i = mem::transmute(a);
247        let b: __m128i = mem::transmute(b);
248        mem::transmute(simd_and(a, b))
249    }
250}
251
252/// Bitwise AND-NOT of packed single-precision (32-bit) floating-point
253/// elements.
254///
255/// Computes `!a & b` for each bit in `a` and `b`.
256///
257/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_andnot_ps)
258#[inline]
259#[target_feature(enable = "sse")]
260// i586 only seems to generate plain `not` and `and` instructions, so ignore
261// it.
262#[cfg_attr(
263    all(test, any(target_arch = "x86_64", target_feature = "sse2")),
264    assert_instr(andnps)
265)]
266#[stable(feature = "simd_x86", since = "1.27.0")]
267pub fn _mm_andnot_ps(a: __m128, b: __m128) -> __m128 {
268    unsafe {
269        let a: __m128i = mem::transmute(a);
270        let b: __m128i = mem::transmute(b);
271        let mask: __m128i = mem::transmute(i32x4::splat(-1));
272        mem::transmute(simd_and(simd_xor(mask, a), b))
273    }
274}
275
276/// Bitwise OR of packed single-precision (32-bit) floating-point elements.
277///
278/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_or_ps)
279#[inline]
280#[target_feature(enable = "sse")]
281// i586 only seems to generate plain `or` instructions, so we ignore it.
282#[cfg_attr(
283    all(test, any(target_arch = "x86_64", target_feature = "sse2")),
284    assert_instr(orps)
285)]
286#[stable(feature = "simd_x86", since = "1.27.0")]
287pub fn _mm_or_ps(a: __m128, b: __m128) -> __m128 {
288    unsafe {
289        let a: __m128i = mem::transmute(a);
290        let b: __m128i = mem::transmute(b);
291        mem::transmute(simd_or(a, b))
292    }
293}
294
295/// Bitwise exclusive OR of packed single-precision (32-bit) floating-point
296/// elements.
297///
298/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_xor_ps)
299#[inline]
300#[target_feature(enable = "sse")]
301// i586 only seems to generate plain `xor` instructions, so we ignore it.
302#[cfg_attr(
303    all(test, any(target_arch = "x86_64", target_feature = "sse2")),
304    assert_instr(xorps)
305)]
306#[stable(feature = "simd_x86", since = "1.27.0")]
307pub fn _mm_xor_ps(a: __m128, b: __m128) -> __m128 {
308    unsafe {
309        let a: __m128i = mem::transmute(a);
310        let b: __m128i = mem::transmute(b);
311        mem::transmute(simd_xor(a, b))
312    }
313}
314
315/// Compares the lowest `f32` of both inputs for equality. The lowest 32 bits of
316/// the result will be `0xffffffff` if the two inputs are equal, or `0`
317/// otherwise. The upper 96 bits of the result are the upper 96 bits of `a`.
318///
319/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpeq_ss)
320#[inline]
321#[target_feature(enable = "sse")]
322#[cfg_attr(test, assert_instr(cmpeqss))]
323#[stable(feature = "simd_x86", since = "1.27.0")]
324pub fn _mm_cmpeq_ss(a: __m128, b: __m128) -> __m128 {
325    unsafe { cmpss(a, b, 0) }
326}
327
328/// Compares the lowest `f32` of both inputs for less than. The lowest 32 bits
329/// of the result will be `0xffffffff` if `a.extract(0)` is less than
330/// `b.extract(0)`, or `0` otherwise. The upper 96 bits of the result are the
331/// upper 96 bits of `a`.
332///
333/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmplt_ss)
334#[inline]
335#[target_feature(enable = "sse")]
336#[cfg_attr(test, assert_instr(cmpltss))]
337#[stable(feature = "simd_x86", since = "1.27.0")]
338pub fn _mm_cmplt_ss(a: __m128, b: __m128) -> __m128 {
339    unsafe { cmpss(a, b, 1) }
340}
341
342/// Compares the lowest `f32` of both inputs for less than or equal. The lowest
343/// 32 bits of the result will be `0xffffffff` if `a.extract(0)` is less than
344/// or equal `b.extract(0)`, or `0` otherwise. The upper 96 bits of the result
345/// are the upper 96 bits of `a`.
346///
347/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmple_ss)
348#[inline]
349#[target_feature(enable = "sse")]
350#[cfg_attr(test, assert_instr(cmpless))]
351#[stable(feature = "simd_x86", since = "1.27.0")]
352pub fn _mm_cmple_ss(a: __m128, b: __m128) -> __m128 {
353    unsafe { cmpss(a, b, 2) }
354}
355
356/// Compares the lowest `f32` of both inputs for greater than. The lowest 32
357/// bits of the result will be `0xffffffff` if `a.extract(0)` is greater
358/// than `b.extract(0)`, or `0` otherwise. The upper 96 bits of the result
359/// are the upper 96 bits of `a`.
360///
361/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpgt_ss)
362#[inline]
363#[target_feature(enable = "sse")]
364#[cfg_attr(test, assert_instr(cmpltss))]
365#[stable(feature = "simd_x86", since = "1.27.0")]
366pub fn _mm_cmpgt_ss(a: __m128, b: __m128) -> __m128 {
367    unsafe { simd_shuffle!(a, cmpss(b, a, 1), [4, 1, 2, 3]) }
368}
369
370/// Compares the lowest `f32` of both inputs for greater than or equal. The
371/// lowest 32 bits of the result will be `0xffffffff` if `a.extract(0)` is
372/// greater than or equal `b.extract(0)`, or `0` otherwise. The upper 96 bits
373/// of the result are the upper 96 bits of `a`.
374///
375/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpge_ss)
376#[inline]
377#[target_feature(enable = "sse")]
378#[cfg_attr(test, assert_instr(cmpless))]
379#[stable(feature = "simd_x86", since = "1.27.0")]
380pub fn _mm_cmpge_ss(a: __m128, b: __m128) -> __m128 {
381    unsafe { simd_shuffle!(a, cmpss(b, a, 2), [4, 1, 2, 3]) }
382}
383
384/// Compares the lowest `f32` of both inputs for inequality. The lowest 32 bits
385/// of the result will be `0xffffffff` if `a.extract(0)` is not equal to
386/// `b.extract(0)`, or `0` otherwise. The upper 96 bits of the result are the
387/// upper 96 bits of `a`.
388///
389/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpneq_ss)
390#[inline]
391#[target_feature(enable = "sse")]
392#[cfg_attr(test, assert_instr(cmpneqss))]
393#[stable(feature = "simd_x86", since = "1.27.0")]
394pub fn _mm_cmpneq_ss(a: __m128, b: __m128) -> __m128 {
395    unsafe { cmpss(a, b, 4) }
396}
397
398/// Compares the lowest `f32` of both inputs for not-less-than. The lowest 32
399/// bits of the result will be `0xffffffff` if `a.extract(0)` is not less than
400/// `b.extract(0)`, or `0` otherwise. The upper 96 bits of the result are the
401/// upper 96 bits of `a`.
402///
403/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpnlt_ss)
404#[inline]
405#[target_feature(enable = "sse")]
406#[cfg_attr(test, assert_instr(cmpnltss))]
407#[stable(feature = "simd_x86", since = "1.27.0")]
408pub fn _mm_cmpnlt_ss(a: __m128, b: __m128) -> __m128 {
409    unsafe { cmpss(a, b, 5) }
410}
411
412/// Compares the lowest `f32` of both inputs for not-less-than-or-equal. The
413/// lowest 32 bits of the result will be `0xffffffff` if `a.extract(0)` is not
414/// less than or equal to `b.extract(0)`, or `0` otherwise. The upper 96 bits
415/// of the result are the upper 96 bits of `a`.
416///
417/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpnle_ss)
418#[inline]
419#[target_feature(enable = "sse")]
420#[cfg_attr(test, assert_instr(cmpnless))]
421#[stable(feature = "simd_x86", since = "1.27.0")]
422pub fn _mm_cmpnle_ss(a: __m128, b: __m128) -> __m128 {
423    unsafe { cmpss(a, b, 6) }
424}
425
426/// Compares the lowest `f32` of both inputs for not-greater-than. The lowest 32
427/// bits of the result will be `0xffffffff` if `a.extract(0)` is not greater
428/// than `b.extract(0)`, or `0` otherwise. The upper 96 bits of the result are
429/// the upper 96 bits of `a`.
430///
431/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpngt_ss)
432#[inline]
433#[target_feature(enable = "sse")]
434#[cfg_attr(test, assert_instr(cmpnltss))]
435#[stable(feature = "simd_x86", since = "1.27.0")]
436pub fn _mm_cmpngt_ss(a: __m128, b: __m128) -> __m128 {
437    unsafe { simd_shuffle!(a, cmpss(b, a, 5), [4, 1, 2, 3]) }
438}
439
440/// Compares the lowest `f32` of both inputs for not-greater-than-or-equal. The
441/// lowest 32 bits of the result will be `0xffffffff` if `a.extract(0)` is not
442/// greater than or equal to `b.extract(0)`, or `0` otherwise. The upper 96
443/// bits of the result are the upper 96 bits of `a`.
444///
445/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpnge_ss)
446#[inline]
447#[target_feature(enable = "sse")]
448#[cfg_attr(test, assert_instr(cmpnless))]
449#[stable(feature = "simd_x86", since = "1.27.0")]
450pub fn _mm_cmpnge_ss(a: __m128, b: __m128) -> __m128 {
451    unsafe { simd_shuffle!(a, cmpss(b, a, 6), [4, 1, 2, 3]) }
452}
453
454/// Checks if the lowest `f32` of both inputs are ordered. The lowest 32 bits of
455/// the result will be `0xffffffff` if neither of `a.extract(0)` or
456/// `b.extract(0)` is a NaN, or `0` otherwise. The upper 96 bits of the result
457/// are the upper 96 bits of `a`.
458///
459/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpord_ss)
460#[inline]
461#[target_feature(enable = "sse")]
462#[cfg_attr(test, assert_instr(cmpordss))]
463#[stable(feature = "simd_x86", since = "1.27.0")]
464pub fn _mm_cmpord_ss(a: __m128, b: __m128) -> __m128 {
465    unsafe { cmpss(a, b, 7) }
466}
467
468/// Checks if the lowest `f32` of both inputs are unordered. The lowest 32 bits
469/// of the result will be `0xffffffff` if any of `a.extract(0)` or
470/// `b.extract(0)` is a NaN, or `0` otherwise. The upper 96 bits of the result
471/// are the upper 96 bits of `a`.
472///
473/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpunord_ss)
474#[inline]
475#[target_feature(enable = "sse")]
476#[cfg_attr(test, assert_instr(cmpunordss))]
477#[stable(feature = "simd_x86", since = "1.27.0")]
478pub fn _mm_cmpunord_ss(a: __m128, b: __m128) -> __m128 {
479    unsafe { cmpss(a, b, 3) }
480}
481
482/// Compares each of the four floats in `a` to the corresponding element in `b`.
483/// The result in the output vector will be `0xffffffff` if the input elements
484/// were equal, or `0` otherwise.
485///
486/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpeq_ps)
487#[inline]
488#[target_feature(enable = "sse")]
489#[cfg_attr(test, assert_instr(cmpeqps))]
490#[stable(feature = "simd_x86", since = "1.27.0")]
491pub fn _mm_cmpeq_ps(a: __m128, b: __m128) -> __m128 {
492    unsafe { cmpps(a, b, 0) }
493}
494
495/// Compares each of the four floats in `a` to the corresponding element in `b`.
496/// The result in the output vector will be `0xffffffff` if the input element
497/// in `a` is less than the corresponding element in `b`, or `0` otherwise.
498///
499/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmplt_ps)
500#[inline]
501#[target_feature(enable = "sse")]
502#[cfg_attr(test, assert_instr(cmpltps))]
503#[stable(feature = "simd_x86", since = "1.27.0")]
504pub fn _mm_cmplt_ps(a: __m128, b: __m128) -> __m128 {
505    unsafe { cmpps(a, b, 1) }
506}
507
508/// Compares each of the four floats in `a` to the corresponding element in `b`.
509/// The result in the output vector will be `0xffffffff` if the input element
510/// in `a` is less than or equal to the corresponding element in `b`, or `0`
511/// otherwise.
512///
513/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmple_ps)
514#[inline]
515#[target_feature(enable = "sse")]
516#[cfg_attr(test, assert_instr(cmpleps))]
517#[stable(feature = "simd_x86", since = "1.27.0")]
518pub fn _mm_cmple_ps(a: __m128, b: __m128) -> __m128 {
519    unsafe { cmpps(a, b, 2) }
520}
521
522/// Compares each of the four floats in `a` to the corresponding element in `b`.
523/// The result in the output vector will be `0xffffffff` if the input element
524/// in `a` is greater than the corresponding element in `b`, or `0` otherwise.
525///
526/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpgt_ps)
527#[inline]
528#[target_feature(enable = "sse")]
529#[cfg_attr(test, assert_instr(cmpltps))]
530#[stable(feature = "simd_x86", since = "1.27.0")]
531pub fn _mm_cmpgt_ps(a: __m128, b: __m128) -> __m128 {
532    unsafe { cmpps(b, a, 1) }
533}
534
535/// Compares each of the four floats in `a` to the corresponding element in `b`.
536/// The result in the output vector will be `0xffffffff` if the input element
537/// in `a` is greater than or equal to the corresponding element in `b`, or `0`
538/// otherwise.
539///
540/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpge_ps)
541#[inline]
542#[target_feature(enable = "sse")]
543#[cfg_attr(test, assert_instr(cmpleps))]
544#[stable(feature = "simd_x86", since = "1.27.0")]
545pub fn _mm_cmpge_ps(a: __m128, b: __m128) -> __m128 {
546    unsafe { cmpps(b, a, 2) }
547}
548
549/// Compares each of the four floats in `a` to the corresponding element in `b`.
550/// The result in the output vector will be `0xffffffff` if the input elements
551/// are **not** equal, or `0` otherwise.
552///
553/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpneq_ps)
554#[inline]
555#[target_feature(enable = "sse")]
556#[cfg_attr(test, assert_instr(cmpneqps))]
557#[stable(feature = "simd_x86", since = "1.27.0")]
558pub fn _mm_cmpneq_ps(a: __m128, b: __m128) -> __m128 {
559    unsafe { cmpps(a, b, 4) }
560}
561
562/// Compares each of the four floats in `a` to the corresponding element in `b`.
563/// The result in the output vector will be `0xffffffff` if the input element
564/// in `a` is **not** less than the corresponding element in `b`, or `0`
565/// otherwise.
566///
567/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpnlt_ps)
568#[inline]
569#[target_feature(enable = "sse")]
570#[cfg_attr(test, assert_instr(cmpnltps))]
571#[stable(feature = "simd_x86", since = "1.27.0")]
572pub fn _mm_cmpnlt_ps(a: __m128, b: __m128) -> __m128 {
573    unsafe { cmpps(a, b, 5) }
574}
575
576/// Compares each of the four floats in `a` to the corresponding element in `b`.
577/// The result in the output vector will be `0xffffffff` if the input element
578/// in `a` is **not** less than or equal to the corresponding element in `b`, or
579/// `0` otherwise.
580///
581/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpnle_ps)
582#[inline]
583#[target_feature(enable = "sse")]
584#[cfg_attr(test, assert_instr(cmpnleps))]
585#[stable(feature = "simd_x86", since = "1.27.0")]
586pub fn _mm_cmpnle_ps(a: __m128, b: __m128) -> __m128 {
587    unsafe { cmpps(a, b, 6) }
588}
589
590/// Compares each of the four floats in `a` to the corresponding element in `b`.
591/// The result in the output vector will be `0xffffffff` if the input element
592/// in `a` is **not** greater than the corresponding element in `b`, or `0`
593/// otherwise.
594///
595/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpngt_ps)
596#[inline]
597#[target_feature(enable = "sse")]
598#[cfg_attr(test, assert_instr(cmpnltps))]
599#[stable(feature = "simd_x86", since = "1.27.0")]
600pub fn _mm_cmpngt_ps(a: __m128, b: __m128) -> __m128 {
601    unsafe { cmpps(b, a, 5) }
602}
603
604/// Compares each of the four floats in `a` to the corresponding element in `b`.
605/// The result in the output vector will be `0xffffffff` if the input element
606/// in `a` is **not** greater than or equal to the corresponding element in `b`,
607/// or `0` otherwise.
608///
609/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpnge_ps)
610#[inline]
611#[target_feature(enable = "sse")]
612#[cfg_attr(test, assert_instr(cmpnleps))]
613#[stable(feature = "simd_x86", since = "1.27.0")]
614pub fn _mm_cmpnge_ps(a: __m128, b: __m128) -> __m128 {
615    unsafe { cmpps(b, a, 6) }
616}
617
618/// Compares each of the four floats in `a` to the corresponding element in `b`.
619/// Returns four floats that have one of two possible bit patterns. The element
620/// in the output vector will be `0xffffffff` if the input elements in `a` and
621/// `b` are ordered (i.e., neither of them is a NaN), or 0 otherwise.
622///
623/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpord_ps)
624#[inline]
625#[target_feature(enable = "sse")]
626#[cfg_attr(test, assert_instr(cmpordps))]
627#[stable(feature = "simd_x86", since = "1.27.0")]
628pub fn _mm_cmpord_ps(a: __m128, b: __m128) -> __m128 {
629    unsafe { cmpps(b, a, 7) }
630}
631
632/// Compares each of the four floats in `a` to the corresponding element in `b`.
633/// Returns four floats that have one of two possible bit patterns. The element
634/// in the output vector will be `0xffffffff` if the input elements in `a` and
635/// `b` are unordered (i.e., at least on of them is a NaN), or 0 otherwise.
636///
637/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpunord_ps)
638#[inline]
639#[target_feature(enable = "sse")]
640#[cfg_attr(test, assert_instr(cmpunordps))]
641#[stable(feature = "simd_x86", since = "1.27.0")]
642pub fn _mm_cmpunord_ps(a: __m128, b: __m128) -> __m128 {
643    unsafe { cmpps(b, a, 3) }
644}
645
646/// Compares two 32-bit floats from the low-order bits of `a` and `b`. Returns
647/// `1` if they are equal, or `0` otherwise.
648///
649/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_comieq_ss)
650#[inline]
651#[target_feature(enable = "sse")]
652#[cfg_attr(test, assert_instr(comiss))]
653#[stable(feature = "simd_x86", since = "1.27.0")]
654pub fn _mm_comieq_ss(a: __m128, b: __m128) -> i32 {
655    unsafe { comieq_ss(a, b) }
656}
657
658/// Compares two 32-bit floats from the low-order bits of `a` and `b`. Returns
659/// `1` if the value from `a` is less than the one from `b`, or `0` otherwise.
660///
661/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_comilt_ss)
662#[inline]
663#[target_feature(enable = "sse")]
664#[cfg_attr(test, assert_instr(comiss))]
665#[stable(feature = "simd_x86", since = "1.27.0")]
666pub fn _mm_comilt_ss(a: __m128, b: __m128) -> i32 {
667    unsafe { comilt_ss(a, b) }
668}
669
670/// Compares two 32-bit floats from the low-order bits of `a` and `b`. Returns
671/// `1` if the value from `a` is less than or equal to the one from `b`, or `0`
672/// otherwise.
673///
674/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_comile_ss)
675#[inline]
676#[target_feature(enable = "sse")]
677#[cfg_attr(test, assert_instr(comiss))]
678#[stable(feature = "simd_x86", since = "1.27.0")]
679pub fn _mm_comile_ss(a: __m128, b: __m128) -> i32 {
680    unsafe { comile_ss(a, b) }
681}
682
683/// Compares two 32-bit floats from the low-order bits of `a` and `b`. Returns
684/// `1` if the value from `a` is greater than the one from `b`, or `0`
685/// otherwise.
686///
687/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_comigt_ss)
688#[inline]
689#[target_feature(enable = "sse")]
690#[cfg_attr(test, assert_instr(comiss))]
691#[stable(feature = "simd_x86", since = "1.27.0")]
692pub fn _mm_comigt_ss(a: __m128, b: __m128) -> i32 {
693    unsafe { comigt_ss(a, b) }
694}
695
696/// Compares two 32-bit floats from the low-order bits of `a` and `b`. Returns
697/// `1` if the value from `a` is greater than or equal to the one from `b`, or
698/// `0` otherwise.
699///
700/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_comige_ss)
701#[inline]
702#[target_feature(enable = "sse")]
703#[cfg_attr(test, assert_instr(comiss))]
704#[stable(feature = "simd_x86", since = "1.27.0")]
705pub fn _mm_comige_ss(a: __m128, b: __m128) -> i32 {
706    unsafe { comige_ss(a, b) }
707}
708
709/// Compares two 32-bit floats from the low-order bits of `a` and `b`. Returns
710/// `1` if they are **not** equal, or `0` otherwise.
711///
712/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_comineq_ss)
713#[inline]
714#[target_feature(enable = "sse")]
715#[cfg_attr(test, assert_instr(comiss))]
716#[stable(feature = "simd_x86", since = "1.27.0")]
717pub fn _mm_comineq_ss(a: __m128, b: __m128) -> i32 {
718    unsafe { comineq_ss(a, b) }
719}
720
721/// Compares two 32-bit floats from the low-order bits of `a` and `b`. Returns
722/// `1` if they are equal, or `0` otherwise. This instruction will not signal
723/// an exception if either argument is a quiet NaN.
724///
725/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_ucomieq_ss)
726#[inline]
727#[target_feature(enable = "sse")]
728#[cfg_attr(test, assert_instr(ucomiss))]
729#[stable(feature = "simd_x86", since = "1.27.0")]
730pub fn _mm_ucomieq_ss(a: __m128, b: __m128) -> i32 {
731    unsafe { ucomieq_ss(a, b) }
732}
733
734/// Compares two 32-bit floats from the low-order bits of `a` and `b`. Returns
735/// `1` if the value from `a` is less than the one from `b`, or `0` otherwise.
736/// This instruction will not signal an exception if either argument is a quiet
737/// NaN.
738///
739/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_ucomilt_ss)
740#[inline]
741#[target_feature(enable = "sse")]
742#[cfg_attr(test, assert_instr(ucomiss))]
743#[stable(feature = "simd_x86", since = "1.27.0")]
744pub fn _mm_ucomilt_ss(a: __m128, b: __m128) -> i32 {
745    unsafe { ucomilt_ss(a, b) }
746}
747
748/// Compares two 32-bit floats from the low-order bits of `a` and `b`. Returns
749/// `1` if the value from `a` is less than or equal to the one from `b`, or `0`
750/// otherwise. This instruction will not signal an exception if either argument
751/// is a quiet NaN.
752///
753/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_ucomile_ss)
754#[inline]
755#[target_feature(enable = "sse")]
756#[cfg_attr(test, assert_instr(ucomiss))]
757#[stable(feature = "simd_x86", since = "1.27.0")]
758pub fn _mm_ucomile_ss(a: __m128, b: __m128) -> i32 {
759    unsafe { ucomile_ss(a, b) }
760}
761
762/// Compares two 32-bit floats from the low-order bits of `a` and `b`. Returns
763/// `1` if the value from `a` is greater than the one from `b`, or `0`
764/// otherwise. This instruction will not signal an exception if either argument
765/// is a quiet NaN.
766///
767/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_ucomigt_ss)
768#[inline]
769#[target_feature(enable = "sse")]
770#[cfg_attr(test, assert_instr(ucomiss))]
771#[stable(feature = "simd_x86", since = "1.27.0")]
772pub fn _mm_ucomigt_ss(a: __m128, b: __m128) -> i32 {
773    unsafe { ucomigt_ss(a, b) }
774}
775
776/// Compares two 32-bit floats from the low-order bits of `a` and `b`. Returns
777/// `1` if the value from `a` is greater than or equal to the one from `b`, or
778/// `0` otherwise. This instruction will not signal an exception if either
779/// argument is a quiet NaN.
780///
781/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_ucomige_ss)
782#[inline]
783#[target_feature(enable = "sse")]
784#[cfg_attr(test, assert_instr(ucomiss))]
785#[stable(feature = "simd_x86", since = "1.27.0")]
786pub fn _mm_ucomige_ss(a: __m128, b: __m128) -> i32 {
787    unsafe { ucomige_ss(a, b) }
788}
789
790/// Compares two 32-bit floats from the low-order bits of `a` and `b`. Returns
791/// `1` if they are **not** equal, or `0` otherwise. This instruction will not
792/// signal an exception if either argument is a quiet NaN.
793///
794/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_ucomineq_ss)
795#[inline]
796#[target_feature(enable = "sse")]
797#[cfg_attr(test, assert_instr(ucomiss))]
798#[stable(feature = "simd_x86", since = "1.27.0")]
799pub fn _mm_ucomineq_ss(a: __m128, b: __m128) -> i32 {
800    unsafe { ucomineq_ss(a, b) }
801}
802
803/// Converts the lowest 32 bit float in the input vector to a 32 bit integer.
804///
805/// The result is rounded according to the current rounding mode. If the result
806/// cannot be represented as a 32 bit integer the result will be `0x8000_0000`
807/// (`i32::MIN`).
808///
809/// This corresponds to the `CVTSS2SI` instruction (with 32 bit output).
810///
811/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtss_si32)
812#[inline]
813#[target_feature(enable = "sse")]
814#[cfg_attr(test, assert_instr(cvtss2si))]
815#[stable(feature = "simd_x86", since = "1.27.0")]
816pub fn _mm_cvtss_si32(a: __m128) -> i32 {
817    unsafe { cvtss2si(a) }
818}
819
820/// Alias for [`_mm_cvtss_si32`](fn._mm_cvtss_si32.html).
821///
822/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvt_ss2si)
823#[inline]
824#[target_feature(enable = "sse")]
825#[cfg_attr(test, assert_instr(cvtss2si))]
826#[stable(feature = "simd_x86", since = "1.27.0")]
827pub fn _mm_cvt_ss2si(a: __m128) -> i32 {
828    _mm_cvtss_si32(a)
829}
830
831/// Converts the lowest 32 bit float in the input vector to a 32 bit integer
832/// with
833/// truncation.
834///
835/// The result is rounded always using truncation (round towards zero). If the
836/// result cannot be represented as a 32 bit integer the result will be
837/// `0x8000_0000` (`i32::MIN`).
838///
839/// This corresponds to the `CVTTSS2SI` instruction (with 32 bit output).
840///
841/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvttss_si32)
842#[inline]
843#[target_feature(enable = "sse")]
844#[cfg_attr(test, assert_instr(cvttss2si))]
845#[stable(feature = "simd_x86", since = "1.27.0")]
846pub fn _mm_cvttss_si32(a: __m128) -> i32 {
847    unsafe { cvttss2si(a) }
848}
849
850/// Alias for [`_mm_cvttss_si32`](fn._mm_cvttss_si32.html).
851///
852/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtt_ss2si)
853#[inline]
854#[target_feature(enable = "sse")]
855#[cfg_attr(test, assert_instr(cvttss2si))]
856#[stable(feature = "simd_x86", since = "1.27.0")]
857pub fn _mm_cvtt_ss2si(a: __m128) -> i32 {
858    _mm_cvttss_si32(a)
859}
860
861/// Extracts the lowest 32 bit float from the input vector.
862///
863/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtss_f32)
864#[inline]
865#[target_feature(enable = "sse")]
866// No point in using assert_instrs. In Unix x86_64 calling convention this is a
867// no-op, and on msvc it's just a `mov`.
868#[stable(feature = "simd_x86", since = "1.27.0")]
869pub fn _mm_cvtss_f32(a: __m128) -> f32 {
870    unsafe { simd_extract!(a, 0) }
871}
872
873/// Converts a 32 bit integer to a 32 bit float. The result vector is the input
874/// vector `a` with the lowest 32 bit float replaced by the converted integer.
875///
876/// This intrinsic corresponds to the `CVTSI2SS` instruction (with 32 bit
877/// input).
878///
879/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtsi32_ss)
880#[inline]
881#[target_feature(enable = "sse")]
882#[cfg_attr(test, assert_instr(cvtsi2ss))]
883#[stable(feature = "simd_x86", since = "1.27.0")]
884pub fn _mm_cvtsi32_ss(a: __m128, b: i32) -> __m128 {
885    unsafe { cvtsi2ss(a, b) }
886}
887
888/// Alias for [`_mm_cvtsi32_ss`](fn._mm_cvtsi32_ss.html).
889///
890/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvt_si2ss)
891#[inline]
892#[target_feature(enable = "sse")]
893#[cfg_attr(test, assert_instr(cvtsi2ss))]
894#[stable(feature = "simd_x86", since = "1.27.0")]
895pub fn _mm_cvt_si2ss(a: __m128, b: i32) -> __m128 {
896    _mm_cvtsi32_ss(a, b)
897}
898
899/// Construct a `__m128` with the lowest element set to `a` and the rest set to
900/// zero.
901///
902/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_set_ss)
903#[inline]
904#[target_feature(enable = "sse")]
905#[cfg_attr(test, assert_instr(movss))]
906#[stable(feature = "simd_x86", since = "1.27.0")]
907pub fn _mm_set_ss(a: f32) -> __m128 {
908    __m128([a, 0.0, 0.0, 0.0])
909}
910
911/// Construct a `__m128` with all element set to `a`.
912///
913/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_set1_ps)
914#[inline]
915#[target_feature(enable = "sse")]
916#[cfg_attr(test, assert_instr(shufps))]
917#[stable(feature = "simd_x86", since = "1.27.0")]
918pub fn _mm_set1_ps(a: f32) -> __m128 {
919    __m128([a, a, a, a])
920}
921
922/// Alias for [`_mm_set1_ps`](fn._mm_set1_ps.html)
923///
924/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_set_ps1)
925#[inline]
926#[target_feature(enable = "sse")]
927#[cfg_attr(test, assert_instr(shufps))]
928#[stable(feature = "simd_x86", since = "1.27.0")]
929pub fn _mm_set_ps1(a: f32) -> __m128 {
930    _mm_set1_ps(a)
931}
932
933/// Construct a `__m128` from four floating point values highest to lowest.
934///
935/// Note that `a` will be the highest 32 bits of the result, and `d` the
936/// lowest. This matches the standard way of writing bit patterns on x86:
937///
938/// ```text
939///  bit    127 .. 96  95 .. 64  63 .. 32  31 .. 0
940///        +---------+---------+---------+---------+
941///        |    a    |    b    |    c    |    d    |   result
942///        +---------+---------+---------+---------+
943/// ```
944///
945/// Alternatively:
946///
947/// ```text
948/// let v = _mm_set_ps(d, c, b, a);
949/// ```
950///
951/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_set_ps)
952#[inline]
953#[target_feature(enable = "sse")]
954#[cfg_attr(test, assert_instr(unpcklps))]
955#[stable(feature = "simd_x86", since = "1.27.0")]
956pub fn _mm_set_ps(a: f32, b: f32, c: f32, d: f32) -> __m128 {
957    __m128([d, c, b, a])
958}
959
960/// Construct a `__m128` from four floating point values lowest to highest.
961///
962/// This matches the memory order of `__m128`, i.e., `a` will be the lowest 32
963/// bits of the result, and `d` the highest.
964///
965/// ```text
966/// assert_eq!(__m128::new(a, b, c, d), _mm_setr_ps(a, b, c, d));
967/// ```
968///
969/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_setr_ps)
970#[inline]
971#[target_feature(enable = "sse")]
972#[cfg_attr(
973    all(test, any(target_env = "msvc", target_arch = "x86_64")),
974    assert_instr(unpcklps)
975)]
976// On a 32-bit architecture on non-msvc it just copies the operands from the stack.
977#[cfg_attr(
978    all(test, all(not(target_env = "msvc"), target_arch = "x86")),
979    assert_instr(movaps)
980)]
981#[stable(feature = "simd_x86", since = "1.27.0")]
982pub fn _mm_setr_ps(a: f32, b: f32, c: f32, d: f32) -> __m128 {
983    __m128([a, b, c, d])
984}
985
986/// Construct a `__m128` with all elements initialized to zero.
987///
988/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_setzero_ps)
989#[inline]
990#[target_feature(enable = "sse")]
991#[cfg_attr(test, assert_instr(xorps))]
992#[stable(feature = "simd_x86", since = "1.27.0")]
993pub fn _mm_setzero_ps() -> __m128 {
994    const { unsafe { mem::zeroed() } }
995}
996
997/// A utility function for creating masks to use with Intel shuffle and
998/// permute intrinsics.
999#[inline]
1000#[allow(non_snake_case)]
1001#[unstable(feature = "stdarch_x86_mm_shuffle", issue = "111147")]
1002pub const fn _MM_SHUFFLE(z: u32, y: u32, x: u32, w: u32) -> i32 {
1003    ((z << 6) | (y << 4) | (x << 2) | w) as i32
1004}
1005
1006/// Shuffles packed single-precision (32-bit) floating-point elements in `a` and
1007/// `b` using `MASK`.
1008///
1009/// The lower half of result takes values from `a` and the higher half from
1010/// `b`. Mask is split to 2 control bits each to index the element from inputs.
1011///
1012/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_shuffle_ps)
1013///
1014/// Note that there appears to be a mistake within Intel's Intrinsics Guide.
1015/// `_mm_shuffle_ps` is supposed to take an `i32` instead of a `u32`
1016/// as is the case for [other shuffle intrinsics](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_shuffle_).
1017/// Performing an implicit type conversion between an unsigned integer and a signed integer
1018/// does not cause a problem in C, however Rust's commitment to strong typing does not allow this.
1019#[inline]
1020#[target_feature(enable = "sse")]
1021#[cfg_attr(test, assert_instr(shufps, MASK = 3))]
1022#[rustc_legacy_const_generics(2)]
1023#[stable(feature = "simd_x86", since = "1.27.0")]
1024pub fn _mm_shuffle_ps<const MASK: i32>(a: __m128, b: __m128) -> __m128 {
1025    static_assert_uimm_bits!(MASK, 8);
1026    unsafe {
1027        simd_shuffle!(
1028            a,
1029            b,
1030            [
1031                MASK as u32 & 0b11,
1032                (MASK as u32 >> 2) & 0b11,
1033                ((MASK as u32 >> 4) & 0b11) + 4,
1034                ((MASK as u32 >> 6) & 0b11) + 4,
1035            ],
1036        )
1037    }
1038}
1039
1040/// Unpacks and interleave single-precision (32-bit) floating-point elements
1041/// from the higher half of `a` and `b`.
1042///
1043/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_unpackhi_ps)
1044#[inline]
1045#[target_feature(enable = "sse")]
1046#[cfg_attr(test, assert_instr(unpckhps))]
1047#[stable(feature = "simd_x86", since = "1.27.0")]
1048pub fn _mm_unpackhi_ps(a: __m128, b: __m128) -> __m128 {
1049    unsafe { simd_shuffle!(a, b, [2, 6, 3, 7]) }
1050}
1051
1052/// Unpacks and interleave single-precision (32-bit) floating-point elements
1053/// from the lower half of `a` and `b`.
1054///
1055/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_unpacklo_ps)
1056#[inline]
1057#[target_feature(enable = "sse")]
1058#[cfg_attr(test, assert_instr(unpcklps))]
1059#[stable(feature = "simd_x86", since = "1.27.0")]
1060pub fn _mm_unpacklo_ps(a: __m128, b: __m128) -> __m128 {
1061    unsafe { simd_shuffle!(a, b, [0, 4, 1, 5]) }
1062}
1063
1064/// Combine higher half of `a` and `b`. The higher half of `b` occupies the
1065/// lower half of result.
1066///
1067/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_movehl_ps)
1068#[inline]
1069#[target_feature(enable = "sse")]
1070#[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(movhlps))]
1071#[stable(feature = "simd_x86", since = "1.27.0")]
1072pub fn _mm_movehl_ps(a: __m128, b: __m128) -> __m128 {
1073    // TODO; figure why this is a different instruction on msvc?
1074    unsafe { simd_shuffle!(a, b, [6, 7, 2, 3]) }
1075}
1076
1077/// Combine lower half of `a` and `b`. The lower half of `b` occupies the
1078/// higher half of result.
1079///
1080/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_movelh_ps)
1081#[inline]
1082#[target_feature(enable = "sse")]
1083#[cfg_attr(all(test, not(target_env = "msvc")), assert_instr(movlhps))]
1084#[stable(feature = "simd_x86", since = "1.27.0")]
1085pub fn _mm_movelh_ps(a: __m128, b: __m128) -> __m128 {
1086    unsafe { simd_shuffle!(a, b, [0, 1, 4, 5]) }
1087}
1088
1089/// Returns a mask of the most significant bit of each element in `a`.
1090///
1091/// The mask is stored in the 4 least significant bits of the return value.
1092/// All other bits are set to `0`.
1093///
1094/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_movemask_ps)
1095#[inline]
1096#[target_feature(enable = "sse")]
1097#[cfg_attr(test, assert_instr(movmskps))]
1098#[stable(feature = "simd_x86", since = "1.27.0")]
1099pub fn _mm_movemask_ps(a: __m128) -> i32 {
1100    // Propagate the highest bit to the rest, because simd_bitmask
1101    // requires all-1 or all-0.
1102    unsafe {
1103        let mask: i32x4 = simd_lt(transmute(a), i32x4::ZERO);
1104        simd_bitmask::<i32x4, u8>(mask).into()
1105    }
1106}
1107
1108/// Construct a `__m128` with the lowest element read from `p` and the other
1109/// elements set to zero.
1110///
1111/// This corresponds to instructions `VMOVSS` / `MOVSS`.
1112///
1113/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_load_ss)
1114#[inline]
1115#[target_feature(enable = "sse")]
1116#[cfg_attr(test, assert_instr(movss))]
1117#[stable(feature = "simd_x86", since = "1.27.0")]
1118pub unsafe fn _mm_load_ss(p: *const f32) -> __m128 {
1119    __m128([*p, 0.0, 0.0, 0.0])
1120}
1121
1122/// Construct a `__m128` by duplicating the value read from `p` into all
1123/// elements.
1124///
1125/// This corresponds to instructions `VMOVSS` / `MOVSS` followed by some
1126/// shuffling.
1127///
1128/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_load1_ps)
1129#[inline]
1130#[target_feature(enable = "sse")]
1131#[cfg_attr(test, assert_instr(movss))]
1132#[stable(feature = "simd_x86", since = "1.27.0")]
1133pub unsafe fn _mm_load1_ps(p: *const f32) -> __m128 {
1134    let a = *p;
1135    __m128([a, a, a, a])
1136}
1137
1138/// Alias for [`_mm_load1_ps`](fn._mm_load1_ps.html)
1139///
1140/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_load_ps1)
1141#[inline]
1142#[target_feature(enable = "sse")]
1143#[cfg_attr(test, assert_instr(movss))]
1144#[stable(feature = "simd_x86", since = "1.27.0")]
1145pub unsafe fn _mm_load_ps1(p: *const f32) -> __m128 {
1146    _mm_load1_ps(p)
1147}
1148
1149/// Loads four `f32` values from *aligned* memory into a `__m128`. If the
1150/// pointer is not aligned to a 128-bit boundary (16 bytes) a general
1151/// protection fault will be triggered (fatal program crash).
1152///
1153/// Use [`_mm_loadu_ps`](fn._mm_loadu_ps.html) for potentially unaligned
1154/// memory.
1155///
1156/// This corresponds to instructions `VMOVAPS` / `MOVAPS`.
1157///
1158/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_load_ps)
1159#[inline]
1160#[target_feature(enable = "sse")]
1161#[cfg_attr(test, assert_instr(movaps))]
1162#[stable(feature = "simd_x86", since = "1.27.0")]
1163#[allow(clippy::cast_ptr_alignment)]
1164pub unsafe fn _mm_load_ps(p: *const f32) -> __m128 {
1165    *(p as *const __m128)
1166}
1167
1168/// Loads four `f32` values from memory into a `__m128`. There are no
1169/// restrictions
1170/// on memory alignment. For aligned memory
1171/// [`_mm_load_ps`](fn._mm_load_ps.html)
1172/// may be faster.
1173///
1174/// This corresponds to instructions `VMOVUPS` / `MOVUPS`.
1175///
1176/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_loadu_ps)
1177#[inline]
1178#[target_feature(enable = "sse")]
1179#[cfg_attr(test, assert_instr(movups))]
1180#[stable(feature = "simd_x86", since = "1.27.0")]
1181pub unsafe fn _mm_loadu_ps(p: *const f32) -> __m128 {
1182    // Note: Using `*p` would require `f32` alignment, but `movups` has no
1183    // alignment restrictions.
1184    let mut dst = _mm_undefined_ps();
1185    ptr::copy_nonoverlapping(
1186        p as *const u8,
1187        ptr::addr_of_mut!(dst) as *mut u8,
1188        mem::size_of::<__m128>(),
1189    );
1190    dst
1191}
1192
1193/// Loads four `f32` values from aligned memory into a `__m128` in reverse
1194/// order.
1195///
1196/// If the pointer is not aligned to a 128-bit boundary (16 bytes) a general
1197/// protection fault will be triggered (fatal program crash).
1198///
1199/// Functionally equivalent to the following code sequence (assuming `p`
1200/// satisfies the alignment restrictions):
1201///
1202/// ```text
1203/// let a0 = *p;
1204/// let a1 = *p.add(1);
1205/// let a2 = *p.add(2);
1206/// let a3 = *p.add(3);
1207/// __m128::new(a3, a2, a1, a0)
1208/// ```
1209///
1210/// This corresponds to instructions `VMOVAPS` / `MOVAPS` followed by some
1211/// shuffling.
1212///
1213/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_loadr_ps)
1214#[inline]
1215#[target_feature(enable = "sse")]
1216#[cfg_attr(test, assert_instr(movaps))]
1217#[stable(feature = "simd_x86", since = "1.27.0")]
1218pub unsafe fn _mm_loadr_ps(p: *const f32) -> __m128 {
1219    let a = _mm_load_ps(p);
1220    simd_shuffle!(a, a, [3, 2, 1, 0])
1221}
1222
1223/// Stores the lowest 32 bit float of `a` into memory.
1224///
1225/// This intrinsic corresponds to the `MOVSS` instruction.
1226///
1227/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_store_ss)
1228#[inline]
1229#[target_feature(enable = "sse")]
1230#[cfg_attr(test, assert_instr(movss))]
1231#[stable(feature = "simd_x86", since = "1.27.0")]
1232pub unsafe fn _mm_store_ss(p: *mut f32, a: __m128) {
1233    *p = simd_extract!(a, 0);
1234}
1235
1236/// Stores the lowest 32 bit float of `a` repeated four times into *aligned*
1237/// memory.
1238///
1239/// If the pointer is not aligned to a 128-bit boundary (16 bytes) a general
1240/// protection fault will be triggered (fatal program crash).
1241///
1242/// Functionally equivalent to the following code sequence (assuming `p`
1243/// satisfies the alignment restrictions):
1244///
1245/// ```text
1246/// let x = a.extract(0);
1247/// *p = x;
1248/// *p.add(1) = x;
1249/// *p.add(2) = x;
1250/// *p.add(3) = x;
1251/// ```
1252///
1253/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_store1_ps)
1254#[inline]
1255#[target_feature(enable = "sse")]
1256#[cfg_attr(test, assert_instr(movaps))]
1257#[stable(feature = "simd_x86", since = "1.27.0")]
1258#[allow(clippy::cast_ptr_alignment)]
1259pub unsafe fn _mm_store1_ps(p: *mut f32, a: __m128) {
1260    let b: __m128 = simd_shuffle!(a, a, [0, 0, 0, 0]);
1261    *(p as *mut __m128) = b;
1262}
1263
1264/// Alias for [`_mm_store1_ps`](fn._mm_store1_ps.html)
1265///
1266/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_store_ps1)
1267#[inline]
1268#[target_feature(enable = "sse")]
1269#[cfg_attr(test, assert_instr(movaps))]
1270#[stable(feature = "simd_x86", since = "1.27.0")]
1271pub unsafe fn _mm_store_ps1(p: *mut f32, a: __m128) {
1272    _mm_store1_ps(p, a);
1273}
1274
1275/// Stores four 32-bit floats into *aligned* memory.
1276///
1277/// If the pointer is not aligned to a 128-bit boundary (16 bytes) a general
1278/// protection fault will be triggered (fatal program crash).
1279///
1280/// Use [`_mm_storeu_ps`](fn._mm_storeu_ps.html) for potentially unaligned
1281/// memory.
1282///
1283/// This corresponds to instructions `VMOVAPS` / `MOVAPS`.
1284///
1285/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_store_ps)
1286#[inline]
1287#[target_feature(enable = "sse")]
1288#[cfg_attr(test, assert_instr(movaps))]
1289#[stable(feature = "simd_x86", since = "1.27.0")]
1290#[allow(clippy::cast_ptr_alignment)]
1291pub unsafe fn _mm_store_ps(p: *mut f32, a: __m128) {
1292    *(p as *mut __m128) = a;
1293}
1294
1295/// Stores four 32-bit floats into memory. There are no restrictions on memory
1296/// alignment. For aligned memory [`_mm_store_ps`](fn._mm_store_ps.html) may be
1297/// faster.
1298///
1299/// This corresponds to instructions `VMOVUPS` / `MOVUPS`.
1300///
1301/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_storeu_ps)
1302#[inline]
1303#[target_feature(enable = "sse")]
1304#[cfg_attr(test, assert_instr(movups))]
1305#[stable(feature = "simd_x86", since = "1.27.0")]
1306pub unsafe fn _mm_storeu_ps(p: *mut f32, a: __m128) {
1307    ptr::copy_nonoverlapping(
1308        ptr::addr_of!(a) as *const u8,
1309        p as *mut u8,
1310        mem::size_of::<__m128>(),
1311    );
1312}
1313
1314/// Stores four 32-bit floats into *aligned* memory in reverse order.
1315///
1316/// If the pointer is not aligned to a 128-bit boundary (16 bytes) a general
1317/// protection fault will be triggered (fatal program crash).
1318///
1319/// Functionally equivalent to the following code sequence (assuming `p`
1320/// satisfies the alignment restrictions):
1321///
1322/// ```text
1323/// *p = a.extract(3);
1324/// *p.add(1) = a.extract(2);
1325/// *p.add(2) = a.extract(1);
1326/// *p.add(3) = a.extract(0);
1327/// ```
1328///
1329/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_storer_ps)
1330#[inline]
1331#[target_feature(enable = "sse")]
1332#[cfg_attr(test, assert_instr(movaps))]
1333#[stable(feature = "simd_x86", since = "1.27.0")]
1334#[allow(clippy::cast_ptr_alignment)]
1335pub unsafe fn _mm_storer_ps(p: *mut f32, a: __m128) {
1336    let b: __m128 = simd_shuffle!(a, a, [3, 2, 1, 0]);
1337    *(p as *mut __m128) = b;
1338}
1339
1340/// Returns a `__m128` with the first component from `b` and the remaining
1341/// components from `a`.
1342///
1343/// In other words for any `a` and `b`:
1344/// ```text
1345/// _mm_move_ss(a, b) == a.replace(0, b.extract(0))
1346/// ```
1347///
1348/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_move_ss)
1349#[inline]
1350#[target_feature(enable = "sse")]
1351#[cfg_attr(test, assert_instr(movss))]
1352#[stable(feature = "simd_x86", since = "1.27.0")]
1353pub fn _mm_move_ss(a: __m128, b: __m128) -> __m128 {
1354    unsafe { simd_shuffle!(a, b, [4, 1, 2, 3]) }
1355}
1356
1357/// Performs a serializing operation on all non-temporal ("streaming") store instructions that
1358/// were issued by the current thread prior to this instruction.
1359///
1360/// Guarantees that every non-temporal store instruction that precedes this fence, in program order, is
1361/// ordered before any load or store instruction which follows the fence in
1362/// synchronization order.
1363///
1364/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sfence)
1365/// (but note that Intel is only documenting the hardware-level concerns related to this
1366/// instruction; the Intel documentation does not take into account the extra concerns that arise
1367/// because the Rust memory model is different from the x86 memory model.)
1368///
1369/// # Safety of non-temporal stores
1370///
1371/// After using any non-temporal store intrinsic, but before any other access to the memory that the
1372/// intrinsic mutates, a call to `_mm_sfence` must be performed on the thread that used the
1373/// intrinsic.
1374///
1375/// Non-temporal stores behave very different from regular stores. For the purpose of the Rust
1376/// memory model, these stores are happening asynchronously in a background thread. This means a
1377/// non-temporal store can cause data races with other accesses, even other accesses on the same
1378/// thread. It also means that cross-thread synchronization does not work as expected: let's say the
1379/// intrinsic is called on thread T1, and T1 performs synchronization with some other thread T2. The
1380/// non-temporal store acts as if it happened not in T1 but in a different thread T3, and T2 has not
1381/// synchronized with T3! Calling `_mm_sfence` makes the current thread wait for and synchronize
1382/// with all the non-temporal stores previously started on this thread, which means in particular
1383/// that subsequent synchronization with other threads will then work as intended again.
1384///
1385/// The general pattern to use non-temporal stores correctly is to call `_mm_sfence` before your
1386/// code jumps back to code outside your library. This ensures all stores inside your function
1387/// are synchronized-before the return, and thus transitively synchronized-before everything
1388/// the caller does after your function returns.
1389//
1390// The following is not a doc comment since it's not clear whether we want to put this into the
1391// docs, but it should be written out somewhere.
1392//
1393// Formally, we consider non-temporal stores and sfences to be opaque blobs that the compiler cannot
1394// inspect, and that behave like the following functions. This explains where the docs above come
1395// from.
1396// ```
1397// #[thread_local]
1398// static mut PENDING_NONTEMP_WRITES = AtomicUsize::new(0);
1399//
1400// pub unsafe fn nontemporal_store<T>(ptr: *mut T, val: T) {
1401//     PENDING_NONTEMP_WRITES.fetch_add(1, Relaxed);
1402//     // Spawn a thread that will eventually do our write.
1403//     // We need to fetch a pointer to this thread's pending-write
1404//     // counter, so that we can access it from the background thread.
1405//     let pending_writes = addr_of!(PENDING_NONTEMP_WRITES);
1406//     // If this was actual Rust code we'd have to do some extra work
1407//     // because `ptr`, `val`, `pending_writes` are all `!Send`. We skip that here.
1408//     std::thread::spawn(move || {
1409//         // Do the write in the background thread.
1410//         ptr.write(val);
1411//         // Register the write as done. Crucially, this is `Release`, so it
1412//         // syncs-with the `Acquire in `sfence`.
1413//         (&*pending_writes).fetch_sub(1, Release);
1414//     });
1415// }
1416//
1417// pub fn sfence() {
1418//     unsafe {
1419//         // Wait until there are no more pending writes.
1420//         while PENDING_NONTEMP_WRITES.load(Acquire) > 0 {}
1421//     }
1422// }
1423// ```
1424#[inline]
1425#[target_feature(enable = "sse")]
1426#[cfg_attr(test, assert_instr(sfence))]
1427#[stable(feature = "simd_x86", since = "1.27.0")]
1428pub unsafe fn _mm_sfence() {
1429    sfence()
1430}
1431
1432/// Gets the unsigned 32-bit value of the MXCSR control and status register.
1433///
1434/// Note that Rust makes no guarantees whatsoever about the contents of this register: Rust
1435/// floating-point operations may or may not result in this register getting updated with exception
1436/// state, and the register can change between two invocations of this function even when no
1437/// floating-point operations appear in the source code (since floating-point operations appearing
1438/// earlier or later can be reordered).
1439///
1440/// If you need to perform some floating-point operations and check whether they raised an
1441/// exception, use an inline assembly block for the entire sequence of operations.
1442///
1443/// For more info see [`_mm_setcsr`](fn._mm_setcsr.html)
1444///
1445/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_getcsr)
1446#[inline]
1447#[target_feature(enable = "sse")]
1448#[cfg_attr(test, assert_instr(stmxcsr))]
1449#[stable(feature = "simd_x86", since = "1.27.0")]
1450#[deprecated(
1451    since = "1.75.0",
1452    note = "see `_mm_getcsr` documentation - use inline assembly instead"
1453)]
1454pub unsafe fn _mm_getcsr() -> u32 {
1455    unsafe {
1456        let mut result = 0_i32;
1457        stmxcsr(ptr::addr_of_mut!(result) as *mut i8);
1458        result as u32
1459    }
1460}
1461
1462/// Sets the MXCSR register with the 32-bit unsigned integer value.
1463///
1464/// This register controls how SIMD instructions handle floating point
1465/// operations. Modifying this register only affects the current thread.
1466///
1467/// It contains several groups of flags:
1468///
1469/// * *Exception flags* report which exceptions occurred since last they were reset.
1470///
1471/// * *Masking flags* can be used to mask (ignore) certain exceptions. By default
1472///   these flags are all set to 1, so all exceptions are masked. When
1473///   an exception is masked, the processor simply sets the exception flag and
1474///   continues the operation. If the exception is unmasked, the flag is also set
1475///   but additionally an exception handler is invoked.
1476///
1477/// * *Rounding mode flags* control the rounding mode of floating point
1478///   instructions.
1479///
1480/// * The *denormals-are-zero mode flag* turns all numbers which would be
1481///   denormalized (exponent bits are all zeros) into zeros.
1482///
1483/// Note that modifying the masking flags, rounding mode, or denormals-are-zero mode flags leads to
1484/// **immediate Undefined Behavior**: Rust assumes that these are always in their default state and
1485/// will optimize accordingly. This even applies when the register is altered and later reset to its
1486/// original value without any floating-point operations appearing in the source code between those
1487/// operations (since floating-point operations appearing earlier or later can be reordered).
1488///
1489/// If you need to perform some floating-point operations under a different masking flags, rounding
1490/// mode, or denormals-are-zero mode, use an inline assembly block and make sure to restore the
1491/// original MXCSR register state before the end of the block.
1492///
1493/// ## Exception Flags
1494///
1495/// * `_MM_EXCEPT_INVALID`: An invalid operation was performed (e.g., dividing
1496///   Infinity by Infinity).
1497///
1498/// * `_MM_EXCEPT_DENORM`: An operation attempted to operate on a denormalized
1499///   number. Mainly this can cause loss of precision.
1500///
1501/// * `_MM_EXCEPT_DIV_ZERO`: Division by zero occurred.
1502///
1503/// * `_MM_EXCEPT_OVERFLOW`: A numeric overflow exception occurred, i.e., a
1504///   result was too large to be represented (e.g., an `f32` with absolute
1505///   value greater than `2^128`).
1506///
1507/// * `_MM_EXCEPT_UNDERFLOW`: A numeric underflow exception occurred, i.e., a
1508///   result was too small to be represented in a normalized way (e.g., an
1509///   `f32` with absolute value smaller than `2^-126`.)
1510///
1511/// * `_MM_EXCEPT_INEXACT`: An inexact-result exception occurred (a.k.a.
1512///   precision exception). This means some precision was lost due to rounding.
1513///   For example, the fraction `1/3` cannot be represented accurately in a
1514///   32 or 64 bit float and computing it would cause this exception to be
1515///   raised. Precision exceptions are very common, so they are usually masked.
1516///
1517/// Exception flags can be read and set using the convenience functions
1518/// `_MM_GET_EXCEPTION_STATE` and `_MM_SET_EXCEPTION_STATE`. For example, to
1519/// check if an operation caused some overflow:
1520///
1521/// ```rust,ignore
1522/// _MM_SET_EXCEPTION_STATE(0); // clear all exception flags
1523///                             // perform calculations
1524/// if _MM_GET_EXCEPTION_STATE() & _MM_EXCEPT_OVERFLOW != 0 {
1525///     // handle overflow
1526/// }
1527/// ```
1528///
1529/// ## Masking Flags
1530///
1531/// There is one masking flag for each exception flag: `_MM_MASK_INVALID`,
1532/// `_MM_MASK_DENORM`, `_MM_MASK_DIV_ZERO`, `_MM_MASK_OVERFLOW`,
1533/// `_MM_MASK_UNDERFLOW`, `_MM_MASK_INEXACT`.
1534///
1535/// A single masking bit can be set via
1536///
1537/// ```rust,ignore
1538/// _MM_SET_EXCEPTION_MASK(_MM_MASK_UNDERFLOW);
1539/// ```
1540///
1541/// However, since mask bits are by default all set to 1, it is more common to
1542/// want to *disable* certain bits. For example, to unmask the underflow
1543/// exception, use:
1544///
1545/// ```rust,ignore
1546/// _mm_setcsr(_mm_getcsr() & !_MM_MASK_UNDERFLOW); // unmask underflow
1547/// exception
1548/// ```
1549///
1550/// Warning: an unmasked exception will cause an exception handler to be
1551/// called.
1552/// The standard handler will simply terminate the process. So, in this case
1553/// any underflow exception would terminate the current process with something
1554/// like `signal: 8, SIGFPE: erroneous arithmetic operation`.
1555///
1556/// ## Rounding Mode
1557///
1558/// The rounding mode is describe using two bits. It can be read and set using
1559/// the convenience wrappers `_MM_GET_ROUNDING_MODE()` and
1560/// `_MM_SET_ROUNDING_MODE(mode)`.
1561///
1562/// The rounding modes are:
1563///
1564/// * `_MM_ROUND_NEAREST`: (default) Round to closest to the infinite precision
1565///   value. If two values are equally close, round to even (i.e., least
1566///   significant bit will be zero).
1567///
1568/// * `_MM_ROUND_DOWN`: Round toward negative Infinity.
1569///
1570/// * `_MM_ROUND_UP`: Round toward positive Infinity.
1571///
1572/// * `_MM_ROUND_TOWARD_ZERO`: Round towards zero (truncate).
1573///
1574/// Example:
1575///
1576/// ```rust,ignore
1577/// _MM_SET_ROUNDING_MODE(_MM_ROUND_DOWN)
1578/// ```
1579///
1580/// ## Denormals-are-zero/Flush-to-zero Mode
1581///
1582/// If this bit is set, values that would be denormalized will be set to zero
1583/// instead. This is turned off by default.
1584///
1585/// You can read and enable/disable this mode via the helper functions
1586/// `_MM_GET_FLUSH_ZERO_MODE()` and `_MM_SET_FLUSH_ZERO_MODE()`:
1587///
1588/// ```rust,ignore
1589/// _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_OFF); // turn off (default)
1590/// _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON); // turn on
1591/// ```
1592///
1593///
1594/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_setcsr)
1595#[inline]
1596#[target_feature(enable = "sse")]
1597#[cfg_attr(test, assert_instr(ldmxcsr))]
1598#[stable(feature = "simd_x86", since = "1.27.0")]
1599#[deprecated(
1600    since = "1.75.0",
1601    note = "see `_mm_setcsr` documentation - use inline assembly instead"
1602)]
1603pub unsafe fn _mm_setcsr(val: u32) {
1604    ldmxcsr(ptr::addr_of!(val) as *const i8);
1605}
1606
1607/// See [`_mm_setcsr`](fn._mm_setcsr.html)
1608#[stable(feature = "simd_x86", since = "1.27.0")]
1609pub const _MM_EXCEPT_INVALID: u32 = 0x0001;
1610/// See [`_mm_setcsr`](fn._mm_setcsr.html)
1611#[stable(feature = "simd_x86", since = "1.27.0")]
1612pub const _MM_EXCEPT_DENORM: u32 = 0x0002;
1613/// See [`_mm_setcsr`](fn._mm_setcsr.html)
1614#[stable(feature = "simd_x86", since = "1.27.0")]
1615pub const _MM_EXCEPT_DIV_ZERO: u32 = 0x0004;
1616/// See [`_mm_setcsr`](fn._mm_setcsr.html)
1617#[stable(feature = "simd_x86", since = "1.27.0")]
1618pub const _MM_EXCEPT_OVERFLOW: u32 = 0x0008;
1619/// See [`_mm_setcsr`](fn._mm_setcsr.html)
1620#[stable(feature = "simd_x86", since = "1.27.0")]
1621pub const _MM_EXCEPT_UNDERFLOW: u32 = 0x0010;
1622/// See [`_mm_setcsr`](fn._mm_setcsr.html)
1623#[stable(feature = "simd_x86", since = "1.27.0")]
1624pub const _MM_EXCEPT_INEXACT: u32 = 0x0020;
1625/// See [`_MM_GET_EXCEPTION_STATE`](fn._MM_GET_EXCEPTION_STATE.html)
1626#[stable(feature = "simd_x86", since = "1.27.0")]
1627pub const _MM_EXCEPT_MASK: u32 = 0x003f;
1628
1629/// See [`_mm_setcsr`](fn._mm_setcsr.html)
1630#[stable(feature = "simd_x86", since = "1.27.0")]
1631pub const _MM_MASK_INVALID: u32 = 0x0080;
1632/// See [`_mm_setcsr`](fn._mm_setcsr.html)
1633#[stable(feature = "simd_x86", since = "1.27.0")]
1634pub const _MM_MASK_DENORM: u32 = 0x0100;
1635/// See [`_mm_setcsr`](fn._mm_setcsr.html)
1636#[stable(feature = "simd_x86", since = "1.27.0")]
1637pub const _MM_MASK_DIV_ZERO: u32 = 0x0200;
1638/// See [`_mm_setcsr`](fn._mm_setcsr.html)
1639#[stable(feature = "simd_x86", since = "1.27.0")]
1640pub const _MM_MASK_OVERFLOW: u32 = 0x0400;
1641/// See [`_mm_setcsr`](fn._mm_setcsr.html)
1642#[stable(feature = "simd_x86", since = "1.27.0")]
1643pub const _MM_MASK_UNDERFLOW: u32 = 0x0800;
1644/// See [`_mm_setcsr`](fn._mm_setcsr.html)
1645#[stable(feature = "simd_x86", since = "1.27.0")]
1646pub const _MM_MASK_INEXACT: u32 = 0x1000;
1647/// See [`_MM_GET_EXCEPTION_MASK`](fn._MM_GET_EXCEPTION_MASK.html)
1648#[stable(feature = "simd_x86", since = "1.27.0")]
1649pub const _MM_MASK_MASK: u32 = 0x1f80;
1650
1651/// See [`_mm_setcsr`](fn._mm_setcsr.html)
1652#[stable(feature = "simd_x86", since = "1.27.0")]
1653pub const _MM_ROUND_NEAREST: u32 = 0x0000;
1654/// See [`_mm_setcsr`](fn._mm_setcsr.html)
1655#[stable(feature = "simd_x86", since = "1.27.0")]
1656pub const _MM_ROUND_DOWN: u32 = 0x2000;
1657/// See [`_mm_setcsr`](fn._mm_setcsr.html)
1658#[stable(feature = "simd_x86", since = "1.27.0")]
1659pub const _MM_ROUND_UP: u32 = 0x4000;
1660/// See [`_mm_setcsr`](fn._mm_setcsr.html)
1661#[stable(feature = "simd_x86", since = "1.27.0")]
1662pub const _MM_ROUND_TOWARD_ZERO: u32 = 0x6000;
1663
1664/// See [`_MM_GET_ROUNDING_MODE`](fn._MM_GET_ROUNDING_MODE.html)
1665#[stable(feature = "simd_x86", since = "1.27.0")]
1666pub const _MM_ROUND_MASK: u32 = 0x6000;
1667
1668/// See [`_MM_GET_FLUSH_ZERO_MODE`](fn._MM_GET_FLUSH_ZERO_MODE.html)
1669#[stable(feature = "simd_x86", since = "1.27.0")]
1670pub const _MM_FLUSH_ZERO_MASK: u32 = 0x8000;
1671/// See [`_mm_setcsr`](fn._mm_setcsr.html)
1672#[stable(feature = "simd_x86", since = "1.27.0")]
1673pub const _MM_FLUSH_ZERO_ON: u32 = 0x8000;
1674/// See [`_mm_setcsr`](fn._mm_setcsr.html)
1675#[stable(feature = "simd_x86", since = "1.27.0")]
1676pub const _MM_FLUSH_ZERO_OFF: u32 = 0x0000;
1677
1678/// See [`_mm_setcsr`](fn._mm_setcsr.html)
1679///
1680/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_MM_GET_EXCEPTION_MASK)
1681#[inline]
1682#[allow(deprecated)] // Deprecated function implemented on top of deprecated function
1683#[allow(non_snake_case)]
1684#[target_feature(enable = "sse")]
1685#[stable(feature = "simd_x86", since = "1.27.0")]
1686#[deprecated(
1687    since = "1.75.0",
1688    note = "see `_mm_getcsr` documentation - use inline assembly instead"
1689)]
1690pub unsafe fn _MM_GET_EXCEPTION_MASK() -> u32 {
1691    _mm_getcsr() & _MM_MASK_MASK
1692}
1693
1694/// See [`_mm_setcsr`](fn._mm_setcsr.html)
1695///
1696/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_MM_GET_EXCEPTION_STATE)
1697#[inline]
1698#[allow(deprecated)] // Deprecated function implemented on top of deprecated function
1699#[allow(non_snake_case)]
1700#[target_feature(enable = "sse")]
1701#[stable(feature = "simd_x86", since = "1.27.0")]
1702#[deprecated(
1703    since = "1.75.0",
1704    note = "see `_mm_getcsr` documentation - use inline assembly instead"
1705)]
1706pub unsafe fn _MM_GET_EXCEPTION_STATE() -> u32 {
1707    _mm_getcsr() & _MM_EXCEPT_MASK
1708}
1709
1710/// See [`_mm_setcsr`](fn._mm_setcsr.html)
1711///
1712/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_MM_GET_FLUSH_ZERO_MODE)
1713#[inline]
1714#[allow(deprecated)] // Deprecated function implemented on top of deprecated function
1715#[allow(non_snake_case)]
1716#[target_feature(enable = "sse")]
1717#[stable(feature = "simd_x86", since = "1.27.0")]
1718#[deprecated(
1719    since = "1.75.0",
1720    note = "see `_mm_getcsr` documentation - use inline assembly instead"
1721)]
1722pub unsafe fn _MM_GET_FLUSH_ZERO_MODE() -> u32 {
1723    _mm_getcsr() & _MM_FLUSH_ZERO_MASK
1724}
1725
1726/// See [`_mm_setcsr`](fn._mm_setcsr.html)
1727///
1728/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_MM_GET_ROUNDING_MODE)
1729#[inline]
1730#[allow(deprecated)] // Deprecated function implemented on top of deprecated function
1731#[allow(non_snake_case)]
1732#[target_feature(enable = "sse")]
1733#[stable(feature = "simd_x86", since = "1.27.0")]
1734#[deprecated(
1735    since = "1.75.0",
1736    note = "see `_mm_getcsr` documentation - use inline assembly instead"
1737)]
1738pub unsafe fn _MM_GET_ROUNDING_MODE() -> u32 {
1739    _mm_getcsr() & _MM_ROUND_MASK
1740}
1741
1742/// See [`_mm_setcsr`](fn._mm_setcsr.html)
1743///
1744/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_MM_SET_EXCEPTION_MASK)
1745#[inline]
1746#[allow(deprecated)] // Deprecated function implemented on top of deprecated function
1747#[allow(non_snake_case)]
1748#[target_feature(enable = "sse")]
1749#[stable(feature = "simd_x86", since = "1.27.0")]
1750#[deprecated(
1751    since = "1.75.0",
1752    note = "see `_mm_setcsr` documentation - use inline assembly instead"
1753)]
1754pub unsafe fn _MM_SET_EXCEPTION_MASK(x: u32) {
1755    _mm_setcsr((_mm_getcsr() & !_MM_MASK_MASK) | (x & _MM_MASK_MASK))
1756}
1757
1758/// See [`_mm_setcsr`](fn._mm_setcsr.html)
1759///
1760/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_MM_SET_EXCEPTION_STATE)
1761#[inline]
1762#[allow(deprecated)] // Deprecated function implemented on top of deprecated function
1763#[allow(non_snake_case)]
1764#[target_feature(enable = "sse")]
1765#[stable(feature = "simd_x86", since = "1.27.0")]
1766#[deprecated(
1767    since = "1.75.0",
1768    note = "see `_mm_setcsr` documentation - use inline assembly instead"
1769)]
1770pub unsafe fn _MM_SET_EXCEPTION_STATE(x: u32) {
1771    _mm_setcsr((_mm_getcsr() & !_MM_EXCEPT_MASK) | (x & _MM_EXCEPT_MASK))
1772}
1773
1774/// See [`_mm_setcsr`](fn._mm_setcsr.html)
1775///
1776/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_MM_SET_FLUSH_ZERO_MODE)
1777#[inline]
1778#[allow(deprecated)] // Deprecated function implemented on top of deprecated function
1779#[allow(non_snake_case)]
1780#[target_feature(enable = "sse")]
1781#[stable(feature = "simd_x86", since = "1.27.0")]
1782#[deprecated(
1783    since = "1.75.0",
1784    note = "see `_mm_setcsr` documentation - use inline assembly instead"
1785)]
1786pub unsafe fn _MM_SET_FLUSH_ZERO_MODE(x: u32) {
1787    _mm_setcsr((_mm_getcsr() & !_MM_FLUSH_ZERO_MASK) | (x & _MM_FLUSH_ZERO_MASK))
1788}
1789
1790/// See [`_mm_setcsr`](fn._mm_setcsr.html)
1791///
1792/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_MM_SET_ROUNDING_MODE)
1793#[inline]
1794#[allow(deprecated)] // Deprecated function implemented on top of deprecated function
1795#[allow(non_snake_case)]
1796#[target_feature(enable = "sse")]
1797#[stable(feature = "simd_x86", since = "1.27.0")]
1798#[deprecated(
1799    since = "1.75.0",
1800    note = "see `_mm_setcsr` documentation - use inline assembly instead"
1801)]
1802pub unsafe fn _MM_SET_ROUNDING_MODE(x: u32) {
1803    _mm_setcsr((_mm_getcsr() & !_MM_ROUND_MASK) | (x & _MM_ROUND_MASK))
1804}
1805
1806/// See [`_mm_prefetch`](fn._mm_prefetch.html).
1807#[stable(feature = "simd_x86", since = "1.27.0")]
1808pub const _MM_HINT_T0: i32 = 3;
1809
1810/// See [`_mm_prefetch`](fn._mm_prefetch.html).
1811#[stable(feature = "simd_x86", since = "1.27.0")]
1812pub const _MM_HINT_T1: i32 = 2;
1813
1814/// See [`_mm_prefetch`](fn._mm_prefetch.html).
1815#[stable(feature = "simd_x86", since = "1.27.0")]
1816pub const _MM_HINT_T2: i32 = 1;
1817
1818/// See [`_mm_prefetch`](fn._mm_prefetch.html).
1819#[stable(feature = "simd_x86", since = "1.27.0")]
1820pub const _MM_HINT_NTA: i32 = 0;
1821
1822/// See [`_mm_prefetch`](fn._mm_prefetch.html).
1823#[stable(feature = "simd_x86", since = "1.27.0")]
1824pub const _MM_HINT_ET0: i32 = 7;
1825
1826/// See [`_mm_prefetch`](fn._mm_prefetch.html).
1827#[stable(feature = "simd_x86", since = "1.27.0")]
1828pub const _MM_HINT_ET1: i32 = 6;
1829
1830/// Fetch the cache line that contains address `p` using the given `STRATEGY`.
1831///
1832/// The `STRATEGY` must be one of:
1833///
1834/// * [`_MM_HINT_T0`](constant._MM_HINT_T0.html): Fetch into all levels of the
1835///   cache hierarchy.
1836///
1837/// * [`_MM_HINT_T1`](constant._MM_HINT_T1.html): Fetch into L2 and higher.
1838///
1839/// * [`_MM_HINT_T2`](constant._MM_HINT_T2.html): Fetch into L3 and higher or
1840///   an implementation-specific choice (e.g., L2 if there is no L3).
1841///
1842/// * [`_MM_HINT_NTA`](constant._MM_HINT_NTA.html): Fetch data using the
1843///   non-temporal access (NTA) hint. It may be a place closer than main memory
1844///   but outside of the cache hierarchy. This is used to reduce access latency
1845///   without polluting the cache.
1846///
1847/// * [`_MM_HINT_ET0`](constant._MM_HINT_ET0.html) and
1848///   [`_MM_HINT_ET1`](constant._MM_HINT_ET1.html) are similar to `_MM_HINT_T0`
1849///   and `_MM_HINT_T1` but indicate an anticipation to write to the address.
1850///
1851/// The actual implementation depends on the particular CPU. This instruction
1852/// is considered a hint, so the CPU is also free to simply ignore the request.
1853///
1854/// The amount of prefetched data depends on the cache line size of the
1855/// specific CPU, but it will be at least 32 bytes.
1856///
1857/// Common caveats:
1858///
1859/// * Most modern CPUs already automatically prefetch data based on predicted
1860///   access patterns.
1861///
1862/// * Data is usually not fetched if this would cause a TLB miss or a page
1863///   fault.
1864///
1865/// * Too much prefetching can cause unnecessary cache evictions.
1866///
1867/// * Prefetching may also fail if there are not enough memory-subsystem
1868///   resources (e.g., request buffers).
1869///
1870///
1871/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_prefetch)
1872#[inline]
1873#[target_feature(enable = "sse")]
1874#[cfg_attr(test, assert_instr(prefetcht0, STRATEGY = _MM_HINT_T0))]
1875#[cfg_attr(test, assert_instr(prefetcht1, STRATEGY = _MM_HINT_T1))]
1876#[cfg_attr(test, assert_instr(prefetcht2, STRATEGY = _MM_HINT_T2))]
1877#[cfg_attr(test, assert_instr(prefetchnta, STRATEGY = _MM_HINT_NTA))]
1878#[rustc_legacy_const_generics(1)]
1879#[stable(feature = "simd_x86", since = "1.27.0")]
1880pub unsafe fn _mm_prefetch<const STRATEGY: i32>(p: *const i8) {
1881    static_assert_uimm_bits!(STRATEGY, 3);
1882    // We use the `llvm.prefetch` intrinsic with `cache type` = 1 (data cache).
1883    // `locality` and `rw` are based on our `STRATEGY`.
1884    prefetch(p, (STRATEGY >> 2) & 1, STRATEGY & 3, 1);
1885}
1886
1887/// Returns vector of type __m128 with indeterminate elements.with indetermination elements.
1888/// Despite using the word "undefined" (following Intel's naming scheme), this non-deterministically
1889/// picks some valid value and is not equivalent to [`mem::MaybeUninit`].
1890/// In practice, this is typically equivalent to [`mem::zeroed`].
1891///
1892/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_undefined_ps)
1893#[inline]
1894#[target_feature(enable = "sse")]
1895#[stable(feature = "simd_x86", since = "1.27.0")]
1896pub fn _mm_undefined_ps() -> __m128 {
1897    const { unsafe { mem::zeroed() } }
1898}
1899
1900/// Transpose the 4x4 matrix formed by 4 rows of __m128 in place.
1901///
1902/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_MM_TRANSPOSE4_PS)
1903#[inline]
1904#[allow(non_snake_case)]
1905#[target_feature(enable = "sse")]
1906#[stable(feature = "simd_x86", since = "1.27.0")]
1907pub fn _MM_TRANSPOSE4_PS(
1908    row0: &mut __m128,
1909    row1: &mut __m128,
1910    row2: &mut __m128,
1911    row3: &mut __m128,
1912) {
1913    let tmp0 = _mm_unpacklo_ps(*row0, *row1);
1914    let tmp2 = _mm_unpacklo_ps(*row2, *row3);
1915    let tmp1 = _mm_unpackhi_ps(*row0, *row1);
1916    let tmp3 = _mm_unpackhi_ps(*row2, *row3);
1917
1918    *row0 = _mm_movelh_ps(tmp0, tmp2);
1919    *row1 = _mm_movehl_ps(tmp2, tmp0);
1920    *row2 = _mm_movelh_ps(tmp1, tmp3);
1921    *row3 = _mm_movehl_ps(tmp3, tmp1);
1922}
1923
1924#[allow(improper_ctypes)]
1925unsafe extern "C" {
1926    #[link_name = "llvm.x86.sse.rcp.ss"]
1927    fn rcpss(a: __m128) -> __m128;
1928    #[link_name = "llvm.x86.sse.rcp.ps"]
1929    fn rcpps(a: __m128) -> __m128;
1930    #[link_name = "llvm.x86.sse.rsqrt.ss"]
1931    fn rsqrtss(a: __m128) -> __m128;
1932    #[link_name = "llvm.x86.sse.rsqrt.ps"]
1933    fn rsqrtps(a: __m128) -> __m128;
1934    #[link_name = "llvm.x86.sse.min.ss"]
1935    fn minss(a: __m128, b: __m128) -> __m128;
1936    #[link_name = "llvm.x86.sse.min.ps"]
1937    fn minps(a: __m128, b: __m128) -> __m128;
1938    #[link_name = "llvm.x86.sse.max.ss"]
1939    fn maxss(a: __m128, b: __m128) -> __m128;
1940    #[link_name = "llvm.x86.sse.max.ps"]
1941    fn maxps(a: __m128, b: __m128) -> __m128;
1942    #[link_name = "llvm.x86.sse.cmp.ps"]
1943    fn cmpps(a: __m128, b: __m128, imm8: i8) -> __m128;
1944    #[link_name = "llvm.x86.sse.comieq.ss"]
1945    fn comieq_ss(a: __m128, b: __m128) -> i32;
1946    #[link_name = "llvm.x86.sse.comilt.ss"]
1947    fn comilt_ss(a: __m128, b: __m128) -> i32;
1948    #[link_name = "llvm.x86.sse.comile.ss"]
1949    fn comile_ss(a: __m128, b: __m128) -> i32;
1950    #[link_name = "llvm.x86.sse.comigt.ss"]
1951    fn comigt_ss(a: __m128, b: __m128) -> i32;
1952    #[link_name = "llvm.x86.sse.comige.ss"]
1953    fn comige_ss(a: __m128, b: __m128) -> i32;
1954    #[link_name = "llvm.x86.sse.comineq.ss"]
1955    fn comineq_ss(a: __m128, b: __m128) -> i32;
1956    #[link_name = "llvm.x86.sse.ucomieq.ss"]
1957    fn ucomieq_ss(a: __m128, b: __m128) -> i32;
1958    #[link_name = "llvm.x86.sse.ucomilt.ss"]
1959    fn ucomilt_ss(a: __m128, b: __m128) -> i32;
1960    #[link_name = "llvm.x86.sse.ucomile.ss"]
1961    fn ucomile_ss(a: __m128, b: __m128) -> i32;
1962    #[link_name = "llvm.x86.sse.ucomigt.ss"]
1963    fn ucomigt_ss(a: __m128, b: __m128) -> i32;
1964    #[link_name = "llvm.x86.sse.ucomige.ss"]
1965    fn ucomige_ss(a: __m128, b: __m128) -> i32;
1966    #[link_name = "llvm.x86.sse.ucomineq.ss"]
1967    fn ucomineq_ss(a: __m128, b: __m128) -> i32;
1968    #[link_name = "llvm.x86.sse.cvtss2si"]
1969    fn cvtss2si(a: __m128) -> i32;
1970    #[link_name = "llvm.x86.sse.cvttss2si"]
1971    fn cvttss2si(a: __m128) -> i32;
1972    #[link_name = "llvm.x86.sse.cvtsi2ss"]
1973    fn cvtsi2ss(a: __m128, b: i32) -> __m128;
1974    #[link_name = "llvm.x86.sse.sfence"]
1975    fn sfence();
1976    #[link_name = "llvm.x86.sse.stmxcsr"]
1977    fn stmxcsr(p: *mut i8);
1978    #[link_name = "llvm.x86.sse.ldmxcsr"]
1979    fn ldmxcsr(p: *const i8);
1980    #[link_name = "llvm.prefetch"]
1981    fn prefetch(p: *const i8, rw: i32, loc: i32, ty: i32);
1982    #[link_name = "llvm.x86.sse.cmp.ss"]
1983    fn cmpss(a: __m128, b: __m128, imm8: i8) -> __m128;
1984}
1985
1986/// Stores `a` into the memory at `mem_addr` using a non-temporal memory hint.
1987///
1988/// `mem_addr` must be aligned on a 16-byte boundary or a general-protection
1989/// exception _may_ be generated.
1990///
1991/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_stream_ps)
1992///
1993/// # Safety of non-temporal stores
1994///
1995/// After using this intrinsic, but before any other access to the memory that this intrinsic
1996/// mutates, a call to [`_mm_sfence`] must be performed by the thread that used the intrinsic. In
1997/// particular, functions that call this intrinsic should generally call `_mm_sfence` before they
1998/// return.
1999///
2000/// See [`_mm_sfence`] for details.
2001#[inline]
2002#[target_feature(enable = "sse")]
2003#[cfg_attr(test, assert_instr(movntps))]
2004#[stable(feature = "simd_x86", since = "1.27.0")]
2005#[allow(clippy::cast_ptr_alignment)]
2006pub unsafe fn _mm_stream_ps(mem_addr: *mut f32, a: __m128) {
2007    crate::arch::asm!(
2008        vps!("movntps", ",{a}"),
2009        p = in(reg) mem_addr,
2010        a = in(xmm_reg) a,
2011        options(nostack, preserves_flags),
2012    );
2013}
2014
2015#[cfg(test)]
2016mod tests {
2017    use crate::{hint::black_box, mem::transmute, ptr};
2018    use std::boxed;
2019    use stdarch_test::simd_test;
2020
2021    use crate::core_arch::{simd::*, x86::*};
2022
2023    const NAN: f32 = f32::NAN;
2024
2025    #[simd_test(enable = "sse")]
2026    unsafe fn test_mm_add_ps() {
2027        let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0);
2028        let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0);
2029        let r = _mm_add_ps(a, b);
2030        assert_eq_m128(r, _mm_setr_ps(-101.0, 25.0, 0.0, -15.0));
2031    }
2032
2033    #[simd_test(enable = "sse")]
2034    unsafe fn test_mm_add_ss() {
2035        let a = _mm_set_ps(-1.0, 5.0, 0.0, -10.0);
2036        let b = _mm_set_ps(-100.0, 20.0, 0.0, -5.0);
2037        let r = _mm_add_ss(a, b);
2038        assert_eq_m128(r, _mm_set_ps(-1.0, 5.0, 0.0, -15.0));
2039    }
2040
2041    #[simd_test(enable = "sse")]
2042    unsafe fn test_mm_sub_ps() {
2043        let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0);
2044        let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0);
2045        let r = _mm_sub_ps(a, b);
2046        assert_eq_m128(r, _mm_setr_ps(99.0, -15.0, 0.0, -5.0));
2047    }
2048
2049    #[simd_test(enable = "sse")]
2050    unsafe fn test_mm_sub_ss() {
2051        let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0);
2052        let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0);
2053        let r = _mm_sub_ss(a, b);
2054        assert_eq_m128(r, _mm_setr_ps(99.0, 5.0, 0.0, -10.0));
2055    }
2056
2057    #[simd_test(enable = "sse")]
2058    unsafe fn test_mm_mul_ps() {
2059        let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0);
2060        let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0);
2061        let r = _mm_mul_ps(a, b);
2062        assert_eq_m128(r, _mm_setr_ps(100.0, 100.0, 0.0, 50.0));
2063    }
2064
2065    #[simd_test(enable = "sse")]
2066    unsafe fn test_mm_mul_ss() {
2067        let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0);
2068        let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0);
2069        let r = _mm_mul_ss(a, b);
2070        assert_eq_m128(r, _mm_setr_ps(100.0, 5.0, 0.0, -10.0));
2071    }
2072
2073    #[simd_test(enable = "sse")]
2074    unsafe fn test_mm_div_ps() {
2075        let a = _mm_setr_ps(-1.0, 5.0, 2.0, -10.0);
2076        let b = _mm_setr_ps(-100.0, 20.0, 0.2, -5.0);
2077        let r = _mm_div_ps(a, b);
2078        assert_eq_m128(r, _mm_setr_ps(0.01, 0.25, 10.0, 2.0));
2079    }
2080
2081    #[simd_test(enable = "sse")]
2082    unsafe fn test_mm_div_ss() {
2083        let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0);
2084        let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0);
2085        let r = _mm_div_ss(a, b);
2086        assert_eq_m128(r, _mm_setr_ps(0.01, 5.0, 0.0, -10.0));
2087    }
2088
2089    #[simd_test(enable = "sse")]
2090    unsafe fn test_mm_sqrt_ss() {
2091        let a = _mm_setr_ps(4.0, 13.0, 16.0, 100.0);
2092        let r = _mm_sqrt_ss(a);
2093        let e = _mm_setr_ps(2.0, 13.0, 16.0, 100.0);
2094        assert_eq_m128(r, e);
2095    }
2096
2097    #[simd_test(enable = "sse")]
2098    unsafe fn test_mm_sqrt_ps() {
2099        let a = _mm_setr_ps(4.0, 13.0, 16.0, 100.0);
2100        let r = _mm_sqrt_ps(a);
2101        let e = _mm_setr_ps(2.0, 3.6055512, 4.0, 10.0);
2102        assert_eq_m128(r, e);
2103    }
2104
2105    #[simd_test(enable = "sse")]
2106    unsafe fn test_mm_rcp_ss() {
2107        let a = _mm_setr_ps(4.0, 13.0, 16.0, 100.0);
2108        let r = _mm_rcp_ss(a);
2109        let e = _mm_setr_ps(0.24993896, 13.0, 16.0, 100.0);
2110        let rel_err = 0.00048828125;
2111        assert_approx_eq!(get_m128(r, 0), get_m128(e, 0), 2. * rel_err);
2112        for i in 1..4 {
2113            assert_eq!(get_m128(r, i), get_m128(e, i));
2114        }
2115    }
2116
2117    #[simd_test(enable = "sse")]
2118    unsafe fn test_mm_rcp_ps() {
2119        let a = _mm_setr_ps(4.0, 13.0, 16.0, 100.0);
2120        let r = _mm_rcp_ps(a);
2121        let e = _mm_setr_ps(0.24993896, 0.0769043, 0.06248474, 0.0099983215);
2122        let rel_err = 0.00048828125;
2123        for i in 0..4 {
2124            assert_approx_eq!(get_m128(r, i), get_m128(e, i), 2. * rel_err);
2125        }
2126    }
2127
2128    #[simd_test(enable = "sse")]
2129    unsafe fn test_mm_rsqrt_ss() {
2130        let a = _mm_setr_ps(4.0, 13.0, 16.0, 100.0);
2131        let r = _mm_rsqrt_ss(a);
2132        let e = _mm_setr_ps(0.49987793, 13.0, 16.0, 100.0);
2133        let rel_err = 0.00048828125;
2134        for i in 0..4 {
2135            assert_approx_eq!(get_m128(r, i), get_m128(e, i), 2. * rel_err);
2136        }
2137    }
2138
2139    #[simd_test(enable = "sse")]
2140    unsafe fn test_mm_rsqrt_ps() {
2141        let a = _mm_setr_ps(4.0, 13.0, 16.0, 100.0);
2142        let r = _mm_rsqrt_ps(a);
2143        let e = _mm_setr_ps(0.49987793, 0.2772827, 0.24993896, 0.099990845);
2144        let rel_err = 0.00048828125;
2145        for i in 0..4 {
2146            assert_approx_eq!(get_m128(r, i), get_m128(e, i), 2. * rel_err);
2147        }
2148    }
2149
2150    #[simd_test(enable = "sse")]
2151    unsafe fn test_mm_min_ss() {
2152        let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0);
2153        let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0);
2154        let r = _mm_min_ss(a, b);
2155        assert_eq_m128(r, _mm_setr_ps(-100.0, 5.0, 0.0, -10.0));
2156    }
2157
2158    #[simd_test(enable = "sse")]
2159    unsafe fn test_mm_min_ps() {
2160        let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0);
2161        let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0);
2162        let r = _mm_min_ps(a, b);
2163        assert_eq_m128(r, _mm_setr_ps(-100.0, 5.0, 0.0, -10.0));
2164
2165        // `_mm_min_ps` can **not** be implemented using the `simd_min` rust intrinsic. `simd_min`
2166        // is lowered by the llvm codegen backend to `llvm.minnum.v*` llvm intrinsic. This intrinsic
2167        // doesn't specify how -0.0 is handled. Unfortunately it happens to behave different from
2168        // the `minps` x86 instruction on x86. The `llvm.minnum.v*` llvm intrinsic equals
2169        // `r1` to `a` and `r2` to `b`.
2170        let a = _mm_setr_ps(-0.0, 0.0, 0.0, 0.0);
2171        let b = _mm_setr_ps(0.0, 0.0, 0.0, 0.0);
2172        let r1: [u8; 16] = transmute(_mm_min_ps(a, b));
2173        let r2: [u8; 16] = transmute(_mm_min_ps(b, a));
2174        let a: [u8; 16] = transmute(a);
2175        let b: [u8; 16] = transmute(b);
2176        assert_eq!(r1, b);
2177        assert_eq!(r2, a);
2178        assert_ne!(a, b); // sanity check that -0.0 is actually present
2179    }
2180
2181    #[simd_test(enable = "sse")]
2182    unsafe fn test_mm_max_ss() {
2183        let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0);
2184        let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0);
2185        let r = _mm_max_ss(a, b);
2186        assert_eq_m128(r, _mm_setr_ps(-1.0, 5.0, 0.0, -10.0));
2187    }
2188
2189    #[simd_test(enable = "sse")]
2190    unsafe fn test_mm_max_ps() {
2191        let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0);
2192        let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0);
2193        let r = _mm_max_ps(a, b);
2194        assert_eq_m128(r, _mm_setr_ps(-1.0, 20.0, 0.0, -5.0));
2195
2196        // Check SSE-specific semantics for -0.0 handling.
2197        let a = _mm_setr_ps(-0.0, 0.0, 0.0, 0.0);
2198        let b = _mm_setr_ps(0.0, 0.0, 0.0, 0.0);
2199        let r1: [u8; 16] = transmute(_mm_max_ps(a, b));
2200        let r2: [u8; 16] = transmute(_mm_max_ps(b, a));
2201        let a: [u8; 16] = transmute(a);
2202        let b: [u8; 16] = transmute(b);
2203        assert_eq!(r1, b);
2204        assert_eq!(r2, a);
2205        assert_ne!(a, b); // sanity check that -0.0 is actually present
2206    }
2207
2208    #[simd_test(enable = "sse")]
2209    unsafe fn test_mm_and_ps() {
2210        let a = transmute(u32x4::splat(0b0011));
2211        let b = transmute(u32x4::splat(0b0101));
2212        let r = _mm_and_ps(*black_box(&a), *black_box(&b));
2213        let e = transmute(u32x4::splat(0b0001));
2214        assert_eq_m128(r, e);
2215    }
2216
2217    #[simd_test(enable = "sse")]
2218    unsafe fn test_mm_andnot_ps() {
2219        let a = transmute(u32x4::splat(0b0011));
2220        let b = transmute(u32x4::splat(0b0101));
2221        let r = _mm_andnot_ps(*black_box(&a), *black_box(&b));
2222        let e = transmute(u32x4::splat(0b0100));
2223        assert_eq_m128(r, e);
2224    }
2225
2226    #[simd_test(enable = "sse")]
2227    unsafe fn test_mm_or_ps() {
2228        let a = transmute(u32x4::splat(0b0011));
2229        let b = transmute(u32x4::splat(0b0101));
2230        let r = _mm_or_ps(*black_box(&a), *black_box(&b));
2231        let e = transmute(u32x4::splat(0b0111));
2232        assert_eq_m128(r, e);
2233    }
2234
2235    #[simd_test(enable = "sse")]
2236    unsafe fn test_mm_xor_ps() {
2237        let a = transmute(u32x4::splat(0b0011));
2238        let b = transmute(u32x4::splat(0b0101));
2239        let r = _mm_xor_ps(*black_box(&a), *black_box(&b));
2240        let e = transmute(u32x4::splat(0b0110));
2241        assert_eq_m128(r, e);
2242    }
2243
2244    #[simd_test(enable = "sse")]
2245    unsafe fn test_mm_cmpeq_ss() {
2246        let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0);
2247        let b = _mm_setr_ps(-1.0, 5.0, 6.0, 7.0);
2248        let r: u32x4 = transmute(_mm_cmpeq_ss(a, b));
2249        let e: u32x4 = transmute(_mm_setr_ps(f32::from_bits(0), 2.0, 3.0, 4.0));
2250        assert_eq!(r, e);
2251
2252        let b2 = _mm_setr_ps(1.0, 5.0, 6.0, 7.0);
2253        let r2: u32x4 = transmute(_mm_cmpeq_ss(a, b2));
2254        let e2: u32x4 = transmute(_mm_setr_ps(f32::from_bits(0xffffffff), 2.0, 3.0, 4.0));
2255        assert_eq!(r2, e2);
2256    }
2257
2258    #[simd_test(enable = "sse")]
2259    unsafe fn test_mm_cmplt_ss() {
2260        let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0);
2261        let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0);
2262        let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0);
2263        let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0);
2264
2265        let b1 = 0u32; // a.extract(0) < b.extract(0)
2266        let c1 = 0u32; // a.extract(0) < c.extract(0)
2267        let d1 = !0u32; // a.extract(0) < d.extract(0)
2268
2269        let rb: u32x4 = transmute(_mm_cmplt_ss(a, b));
2270        let eb: u32x4 = transmute(_mm_setr_ps(f32::from_bits(b1), 2.0, 3.0, 4.0));
2271        assert_eq!(rb, eb);
2272
2273        let rc: u32x4 = transmute(_mm_cmplt_ss(a, c));
2274        let ec: u32x4 = transmute(_mm_setr_ps(f32::from_bits(c1), 2.0, 3.0, 4.0));
2275        assert_eq!(rc, ec);
2276
2277        let rd: u32x4 = transmute(_mm_cmplt_ss(a, d));
2278        let ed: u32x4 = transmute(_mm_setr_ps(f32::from_bits(d1), 2.0, 3.0, 4.0));
2279        assert_eq!(rd, ed);
2280    }
2281
2282    #[simd_test(enable = "sse")]
2283    unsafe fn test_mm_cmple_ss() {
2284        let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0);
2285        let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0);
2286        let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0);
2287        let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0);
2288
2289        let b1 = 0u32; // a.extract(0) <= b.extract(0)
2290        let c1 = !0u32; // a.extract(0) <= c.extract(0)
2291        let d1 = !0u32; // a.extract(0) <= d.extract(0)
2292
2293        let rb: u32x4 = transmute(_mm_cmple_ss(a, b));
2294        let eb: u32x4 = transmute(_mm_setr_ps(f32::from_bits(b1), 2.0, 3.0, 4.0));
2295        assert_eq!(rb, eb);
2296
2297        let rc: u32x4 = transmute(_mm_cmple_ss(a, c));
2298        let ec: u32x4 = transmute(_mm_setr_ps(f32::from_bits(c1), 2.0, 3.0, 4.0));
2299        assert_eq!(rc, ec);
2300
2301        let rd: u32x4 = transmute(_mm_cmple_ss(a, d));
2302        let ed: u32x4 = transmute(_mm_setr_ps(f32::from_bits(d1), 2.0, 3.0, 4.0));
2303        assert_eq!(rd, ed);
2304    }
2305
2306    #[simd_test(enable = "sse")]
2307    unsafe fn test_mm_cmpgt_ss() {
2308        let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0);
2309        let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0);
2310        let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0);
2311        let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0);
2312
2313        let b1 = !0u32; // a.extract(0) > b.extract(0)
2314        let c1 = 0u32; // a.extract(0) > c.extract(0)
2315        let d1 = 0u32; // a.extract(0) > d.extract(0)
2316
2317        let rb: u32x4 = transmute(_mm_cmpgt_ss(a, b));
2318        let eb: u32x4 = transmute(_mm_setr_ps(f32::from_bits(b1), 2.0, 3.0, 4.0));
2319        assert_eq!(rb, eb);
2320
2321        let rc: u32x4 = transmute(_mm_cmpgt_ss(a, c));
2322        let ec: u32x4 = transmute(_mm_setr_ps(f32::from_bits(c1), 2.0, 3.0, 4.0));
2323        assert_eq!(rc, ec);
2324
2325        let rd: u32x4 = transmute(_mm_cmpgt_ss(a, d));
2326        let ed: u32x4 = transmute(_mm_setr_ps(f32::from_bits(d1), 2.0, 3.0, 4.0));
2327        assert_eq!(rd, ed);
2328    }
2329
2330    #[simd_test(enable = "sse")]
2331    unsafe fn test_mm_cmpge_ss() {
2332        let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0);
2333        let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0);
2334        let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0);
2335        let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0);
2336
2337        let b1 = !0u32; // a.extract(0) >= b.extract(0)
2338        let c1 = !0u32; // a.extract(0) >= c.extract(0)
2339        let d1 = 0u32; // a.extract(0) >= d.extract(0)
2340
2341        let rb: u32x4 = transmute(_mm_cmpge_ss(a, b));
2342        let eb: u32x4 = transmute(_mm_setr_ps(f32::from_bits(b1), 2.0, 3.0, 4.0));
2343        assert_eq!(rb, eb);
2344
2345        let rc: u32x4 = transmute(_mm_cmpge_ss(a, c));
2346        let ec: u32x4 = transmute(_mm_setr_ps(f32::from_bits(c1), 2.0, 3.0, 4.0));
2347        assert_eq!(rc, ec);
2348
2349        let rd: u32x4 = transmute(_mm_cmpge_ss(a, d));
2350        let ed: u32x4 = transmute(_mm_setr_ps(f32::from_bits(d1), 2.0, 3.0, 4.0));
2351        assert_eq!(rd, ed);
2352    }
2353
2354    #[simd_test(enable = "sse")]
2355    unsafe fn test_mm_cmpneq_ss() {
2356        let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0);
2357        let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0);
2358        let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0);
2359        let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0);
2360
2361        let b1 = !0u32; // a.extract(0) != b.extract(0)
2362        let c1 = 0u32; // a.extract(0) != c.extract(0)
2363        let d1 = !0u32; // a.extract(0) != d.extract(0)
2364
2365        let rb: u32x4 = transmute(_mm_cmpneq_ss(a, b));
2366        let eb: u32x4 = transmute(_mm_setr_ps(f32::from_bits(b1), 2.0, 3.0, 4.0));
2367        assert_eq!(rb, eb);
2368
2369        let rc: u32x4 = transmute(_mm_cmpneq_ss(a, c));
2370        let ec: u32x4 = transmute(_mm_setr_ps(f32::from_bits(c1), 2.0, 3.0, 4.0));
2371        assert_eq!(rc, ec);
2372
2373        let rd: u32x4 = transmute(_mm_cmpneq_ss(a, d));
2374        let ed: u32x4 = transmute(_mm_setr_ps(f32::from_bits(d1), 2.0, 3.0, 4.0));
2375        assert_eq!(rd, ed);
2376    }
2377
2378    #[simd_test(enable = "sse")]
2379    unsafe fn test_mm_cmpnlt_ss() {
2380        // TODO: this test is exactly the same as for `_mm_cmpge_ss`, but there
2381        // must be a difference. It may have to do with behavior in the
2382        // presence of NaNs (signaling or quiet). If so, we should add tests
2383        // for those.
2384
2385        let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0);
2386        let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0);
2387        let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0);
2388        let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0);
2389
2390        let b1 = !0u32; // a.extract(0) >= b.extract(0)
2391        let c1 = !0u32; // a.extract(0) >= c.extract(0)
2392        let d1 = 0u32; // a.extract(0) >= d.extract(0)
2393
2394        let rb: u32x4 = transmute(_mm_cmpnlt_ss(a, b));
2395        let eb: u32x4 = transmute(_mm_setr_ps(f32::from_bits(b1), 2.0, 3.0, 4.0));
2396        assert_eq!(rb, eb);
2397
2398        let rc: u32x4 = transmute(_mm_cmpnlt_ss(a, c));
2399        let ec: u32x4 = transmute(_mm_setr_ps(f32::from_bits(c1), 2.0, 3.0, 4.0));
2400        assert_eq!(rc, ec);
2401
2402        let rd: u32x4 = transmute(_mm_cmpnlt_ss(a, d));
2403        let ed: u32x4 = transmute(_mm_setr_ps(f32::from_bits(d1), 2.0, 3.0, 4.0));
2404        assert_eq!(rd, ed);
2405    }
2406
2407    #[simd_test(enable = "sse")]
2408    unsafe fn test_mm_cmpnle_ss() {
2409        // TODO: this test is exactly the same as for `_mm_cmpgt_ss`, but there
2410        // must be a difference. It may have to do with behavior in the
2411        // presence
2412        // of NaNs (signaling or quiet). If so, we should add tests for those.
2413
2414        let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0);
2415        let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0);
2416        let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0);
2417        let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0);
2418
2419        let b1 = !0u32; // a.extract(0) > b.extract(0)
2420        let c1 = 0u32; // a.extract(0) > c.extract(0)
2421        let d1 = 0u32; // a.extract(0) > d.extract(0)
2422
2423        let rb: u32x4 = transmute(_mm_cmpnle_ss(a, b));
2424        let eb: u32x4 = transmute(_mm_setr_ps(f32::from_bits(b1), 2.0, 3.0, 4.0));
2425        assert_eq!(rb, eb);
2426
2427        let rc: u32x4 = transmute(_mm_cmpnle_ss(a, c));
2428        let ec: u32x4 = transmute(_mm_setr_ps(f32::from_bits(c1), 2.0, 3.0, 4.0));
2429        assert_eq!(rc, ec);
2430
2431        let rd: u32x4 = transmute(_mm_cmpnle_ss(a, d));
2432        let ed: u32x4 = transmute(_mm_setr_ps(f32::from_bits(d1), 2.0, 3.0, 4.0));
2433        assert_eq!(rd, ed);
2434    }
2435
2436    #[simd_test(enable = "sse")]
2437    unsafe fn test_mm_cmpngt_ss() {
2438        // TODO: this test is exactly the same as for `_mm_cmple_ss`, but there
2439        // must be a difference. It may have to do with behavior in the
2440        // presence of NaNs (signaling or quiet). If so, we should add tests
2441        // for those.
2442
2443        let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0);
2444        let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0);
2445        let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0);
2446        let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0);
2447
2448        let b1 = 0u32; // a.extract(0) <= b.extract(0)
2449        let c1 = !0u32; // a.extract(0) <= c.extract(0)
2450        let d1 = !0u32; // a.extract(0) <= d.extract(0)
2451
2452        let rb: u32x4 = transmute(_mm_cmpngt_ss(a, b));
2453        let eb: u32x4 = transmute(_mm_setr_ps(f32::from_bits(b1), 2.0, 3.0, 4.0));
2454        assert_eq!(rb, eb);
2455
2456        let rc: u32x4 = transmute(_mm_cmpngt_ss(a, c));
2457        let ec: u32x4 = transmute(_mm_setr_ps(f32::from_bits(c1), 2.0, 3.0, 4.0));
2458        assert_eq!(rc, ec);
2459
2460        let rd: u32x4 = transmute(_mm_cmpngt_ss(a, d));
2461        let ed: u32x4 = transmute(_mm_setr_ps(f32::from_bits(d1), 2.0, 3.0, 4.0));
2462        assert_eq!(rd, ed);
2463    }
2464
2465    #[simd_test(enable = "sse")]
2466    unsafe fn test_mm_cmpnge_ss() {
2467        // TODO: this test is exactly the same as for `_mm_cmplt_ss`, but there
2468        // must be a difference. It may have to do with behavior in the
2469        // presence of NaNs (signaling or quiet). If so, we should add tests
2470        // for those.
2471
2472        let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0);
2473        let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0);
2474        let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0);
2475        let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0);
2476
2477        let b1 = 0u32; // a.extract(0) < b.extract(0)
2478        let c1 = 0u32; // a.extract(0) < c.extract(0)
2479        let d1 = !0u32; // a.extract(0) < d.extract(0)
2480
2481        let rb: u32x4 = transmute(_mm_cmpnge_ss(a, b));
2482        let eb: u32x4 = transmute(_mm_setr_ps(f32::from_bits(b1), 2.0, 3.0, 4.0));
2483        assert_eq!(rb, eb);
2484
2485        let rc: u32x4 = transmute(_mm_cmpnge_ss(a, c));
2486        let ec: u32x4 = transmute(_mm_setr_ps(f32::from_bits(c1), 2.0, 3.0, 4.0));
2487        assert_eq!(rc, ec);
2488
2489        let rd: u32x4 = transmute(_mm_cmpnge_ss(a, d));
2490        let ed: u32x4 = transmute(_mm_setr_ps(f32::from_bits(d1), 2.0, 3.0, 4.0));
2491        assert_eq!(rd, ed);
2492    }
2493
2494    #[simd_test(enable = "sse")]
2495    unsafe fn test_mm_cmpord_ss() {
2496        let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0);
2497        let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0);
2498        let c = _mm_setr_ps(NAN, 5.0, 6.0, 7.0);
2499        let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0);
2500
2501        let b1 = !0u32; // a.extract(0) ord b.extract(0)
2502        let c1 = 0u32; // a.extract(0) ord c.extract(0)
2503        let d1 = !0u32; // a.extract(0) ord d.extract(0)
2504
2505        let rb: u32x4 = transmute(_mm_cmpord_ss(a, b));
2506        let eb: u32x4 = transmute(_mm_setr_ps(f32::from_bits(b1), 2.0, 3.0, 4.0));
2507        assert_eq!(rb, eb);
2508
2509        let rc: u32x4 = transmute(_mm_cmpord_ss(a, c));
2510        let ec: u32x4 = transmute(_mm_setr_ps(f32::from_bits(c1), 2.0, 3.0, 4.0));
2511        assert_eq!(rc, ec);
2512
2513        let rd: u32x4 = transmute(_mm_cmpord_ss(a, d));
2514        let ed: u32x4 = transmute(_mm_setr_ps(f32::from_bits(d1), 2.0, 3.0, 4.0));
2515        assert_eq!(rd, ed);
2516    }
2517
2518    #[simd_test(enable = "sse")]
2519    unsafe fn test_mm_cmpunord_ss() {
2520        let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0);
2521        let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0);
2522        let c = _mm_setr_ps(NAN, 5.0, 6.0, 7.0);
2523        let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0);
2524
2525        let b1 = 0u32; // a.extract(0) unord b.extract(0)
2526        let c1 = !0u32; // a.extract(0) unord c.extract(0)
2527        let d1 = 0u32; // a.extract(0) unord d.extract(0)
2528
2529        let rb: u32x4 = transmute(_mm_cmpunord_ss(a, b));
2530        let eb: u32x4 = transmute(_mm_setr_ps(f32::from_bits(b1), 2.0, 3.0, 4.0));
2531        assert_eq!(rb, eb);
2532
2533        let rc: u32x4 = transmute(_mm_cmpunord_ss(a, c));
2534        let ec: u32x4 = transmute(_mm_setr_ps(f32::from_bits(c1), 2.0, 3.0, 4.0));
2535        assert_eq!(rc, ec);
2536
2537        let rd: u32x4 = transmute(_mm_cmpunord_ss(a, d));
2538        let ed: u32x4 = transmute(_mm_setr_ps(f32::from_bits(d1), 2.0, 3.0, 4.0));
2539        assert_eq!(rd, ed);
2540    }
2541
2542    #[simd_test(enable = "sse")]
2543    unsafe fn test_mm_cmpeq_ps() {
2544        let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN);
2545        let b = _mm_setr_ps(15.0, 20.0, 1.0, NAN);
2546        let tru = !0u32;
2547        let fls = 0u32;
2548
2549        let e = u32x4::new(fls, fls, tru, fls);
2550        let r: u32x4 = transmute(_mm_cmpeq_ps(a, b));
2551        assert_eq!(r, e);
2552    }
2553
2554    #[simd_test(enable = "sse")]
2555    unsafe fn test_mm_cmplt_ps() {
2556        let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN);
2557        let b = _mm_setr_ps(15.0, 20.0, 1.0, NAN);
2558        let tru = !0u32;
2559        let fls = 0u32;
2560
2561        let e = u32x4::new(tru, fls, fls, fls);
2562        let r: u32x4 = transmute(_mm_cmplt_ps(a, b));
2563        assert_eq!(r, e);
2564    }
2565
2566    #[simd_test(enable = "sse")]
2567    unsafe fn test_mm_cmple_ps() {
2568        let a = _mm_setr_ps(10.0, 50.0, 1.0, 4.0);
2569        let b = _mm_setr_ps(15.0, 20.0, 1.0, NAN);
2570        let tru = !0u32;
2571        let fls = 0u32;
2572
2573        let e = u32x4::new(tru, fls, tru, fls);
2574        let r: u32x4 = transmute(_mm_cmple_ps(a, b));
2575        assert_eq!(r, e);
2576    }
2577
2578    #[simd_test(enable = "sse")]
2579    unsafe fn test_mm_cmpgt_ps() {
2580        let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN);
2581        let b = _mm_setr_ps(15.0, 20.0, 1.0, 42.0);
2582        let tru = !0u32;
2583        let fls = 0u32;
2584
2585        let e = u32x4::new(fls, tru, fls, fls);
2586        let r: u32x4 = transmute(_mm_cmpgt_ps(a, b));
2587        assert_eq!(r, e);
2588    }
2589
2590    #[simd_test(enable = "sse")]
2591    unsafe fn test_mm_cmpge_ps() {
2592        let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN);
2593        let b = _mm_setr_ps(15.0, 20.0, 1.0, 42.0);
2594        let tru = !0u32;
2595        let fls = 0u32;
2596
2597        let e = u32x4::new(fls, tru, tru, fls);
2598        let r: u32x4 = transmute(_mm_cmpge_ps(a, b));
2599        assert_eq!(r, e);
2600    }
2601
2602    #[simd_test(enable = "sse")]
2603    unsafe fn test_mm_cmpneq_ps() {
2604        let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN);
2605        let b = _mm_setr_ps(15.0, 20.0, 1.0, NAN);
2606        let tru = !0u32;
2607        let fls = 0u32;
2608
2609        let e = u32x4::new(tru, tru, fls, tru);
2610        let r: u32x4 = transmute(_mm_cmpneq_ps(a, b));
2611        assert_eq!(r, e);
2612    }
2613
2614    #[simd_test(enable = "sse")]
2615    unsafe fn test_mm_cmpnlt_ps() {
2616        let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN);
2617        let b = _mm_setr_ps(15.0, 20.0, 1.0, 5.0);
2618        let tru = !0u32;
2619        let fls = 0u32;
2620
2621        let e = u32x4::new(fls, tru, tru, tru);
2622        let r: u32x4 = transmute(_mm_cmpnlt_ps(a, b));
2623        assert_eq!(r, e);
2624    }
2625
2626    #[simd_test(enable = "sse")]
2627    unsafe fn test_mm_cmpnle_ps() {
2628        let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN);
2629        let b = _mm_setr_ps(15.0, 20.0, 1.0, 5.0);
2630        let tru = !0u32;
2631        let fls = 0u32;
2632
2633        let e = u32x4::new(fls, tru, fls, tru);
2634        let r: u32x4 = transmute(_mm_cmpnle_ps(a, b));
2635        assert_eq!(r, e);
2636    }
2637
2638    #[simd_test(enable = "sse")]
2639    unsafe fn test_mm_cmpngt_ps() {
2640        let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN);
2641        let b = _mm_setr_ps(15.0, 20.0, 1.0, 5.0);
2642        let tru = !0u32;
2643        let fls = 0u32;
2644
2645        let e = u32x4::new(tru, fls, tru, tru);
2646        let r: u32x4 = transmute(_mm_cmpngt_ps(a, b));
2647        assert_eq!(r, e);
2648    }
2649
2650    #[simd_test(enable = "sse")]
2651    unsafe fn test_mm_cmpnge_ps() {
2652        let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN);
2653        let b = _mm_setr_ps(15.0, 20.0, 1.0, 5.0);
2654        let tru = !0u32;
2655        let fls = 0u32;
2656
2657        let e = u32x4::new(tru, fls, fls, tru);
2658        let r: u32x4 = transmute(_mm_cmpnge_ps(a, b));
2659        assert_eq!(r, e);
2660    }
2661
2662    #[simd_test(enable = "sse")]
2663    unsafe fn test_mm_cmpord_ps() {
2664        let a = _mm_setr_ps(10.0, 50.0, NAN, NAN);
2665        let b = _mm_setr_ps(15.0, NAN, 1.0, NAN);
2666        let tru = !0u32;
2667        let fls = 0u32;
2668
2669        let e = u32x4::new(tru, fls, fls, fls);
2670        let r: u32x4 = transmute(_mm_cmpord_ps(a, b));
2671        assert_eq!(r, e);
2672    }
2673
2674    #[simd_test(enable = "sse")]
2675    unsafe fn test_mm_cmpunord_ps() {
2676        let a = _mm_setr_ps(10.0, 50.0, NAN, NAN);
2677        let b = _mm_setr_ps(15.0, NAN, 1.0, NAN);
2678        let tru = !0u32;
2679        let fls = 0u32;
2680
2681        let e = u32x4::new(fls, tru, tru, tru);
2682        let r: u32x4 = transmute(_mm_cmpunord_ps(a, b));
2683        assert_eq!(r, e);
2684    }
2685
2686    #[simd_test(enable = "sse")]
2687    unsafe fn test_mm_comieq_ss() {
2688        let aa = &[3.0f32, 12.0, 23.0, NAN];
2689        let bb = &[3.0f32, 47.5, 1.5, NAN];
2690
2691        let ee = &[1i32, 0, 0, 0];
2692
2693        for i in 0..4 {
2694            let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0);
2695            let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0);
2696
2697            let r = _mm_comieq_ss(a, b);
2698
2699            assert_eq!(
2700                ee[i], r,
2701                "_mm_comieq_ss({:?}, {:?}) = {}, expected: {} (i={})",
2702                a, b, r, ee[i], i
2703            );
2704        }
2705    }
2706
2707    #[simd_test(enable = "sse")]
2708    unsafe fn test_mm_comilt_ss() {
2709        let aa = &[3.0f32, 12.0, 23.0, NAN];
2710        let bb = &[3.0f32, 47.5, 1.5, NAN];
2711
2712        let ee = &[0i32, 1, 0, 0];
2713
2714        for i in 0..4 {
2715            let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0);
2716            let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0);
2717
2718            let r = _mm_comilt_ss(a, b);
2719
2720            assert_eq!(
2721                ee[i], r,
2722                "_mm_comilt_ss({:?}, {:?}) = {}, expected: {} (i={})",
2723                a, b, r, ee[i], i
2724            );
2725        }
2726    }
2727
2728    #[simd_test(enable = "sse")]
2729    unsafe fn test_mm_comile_ss() {
2730        let aa = &[3.0f32, 12.0, 23.0, NAN];
2731        let bb = &[3.0f32, 47.5, 1.5, NAN];
2732
2733        let ee = &[1i32, 1, 0, 0];
2734
2735        for i in 0..4 {
2736            let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0);
2737            let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0);
2738
2739            let r = _mm_comile_ss(a, b);
2740
2741            assert_eq!(
2742                ee[i], r,
2743                "_mm_comile_ss({:?}, {:?}) = {}, expected: {} (i={})",
2744                a, b, r, ee[i], i
2745            );
2746        }
2747    }
2748
2749    #[simd_test(enable = "sse")]
2750    unsafe fn test_mm_comigt_ss() {
2751        let aa = &[3.0f32, 12.0, 23.0, NAN];
2752        let bb = &[3.0f32, 47.5, 1.5, NAN];
2753
2754        let ee = &[1i32, 0, 1, 0];
2755
2756        for i in 0..4 {
2757            let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0);
2758            let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0);
2759
2760            let r = _mm_comige_ss(a, b);
2761
2762            assert_eq!(
2763                ee[i], r,
2764                "_mm_comige_ss({:?}, {:?}) = {}, expected: {} (i={})",
2765                a, b, r, ee[i], i
2766            );
2767        }
2768    }
2769
2770    #[simd_test(enable = "sse")]
2771    unsafe fn test_mm_comineq_ss() {
2772        let aa = &[3.0f32, 12.0, 23.0, NAN];
2773        let bb = &[3.0f32, 47.5, 1.5, NAN];
2774
2775        let ee = &[0i32, 1, 1, 1];
2776
2777        for i in 0..4 {
2778            let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0);
2779            let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0);
2780
2781            let r = _mm_comineq_ss(a, b);
2782
2783            assert_eq!(
2784                ee[i], r,
2785                "_mm_comineq_ss({:?}, {:?}) = {}, expected: {} (i={})",
2786                a, b, r, ee[i], i
2787            );
2788        }
2789    }
2790
2791    #[simd_test(enable = "sse")]
2792    unsafe fn test_mm_ucomieq_ss() {
2793        let aa = &[3.0f32, 12.0, 23.0, NAN];
2794        let bb = &[3.0f32, 47.5, 1.5, NAN];
2795
2796        let ee = &[1i32, 0, 0, 0];
2797
2798        for i in 0..4 {
2799            let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0);
2800            let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0);
2801
2802            let r = _mm_ucomieq_ss(a, b);
2803
2804            assert_eq!(
2805                ee[i], r,
2806                "_mm_ucomieq_ss({:?}, {:?}) = {}, expected: {} (i={})",
2807                a, b, r, ee[i], i
2808            );
2809        }
2810    }
2811
2812    #[simd_test(enable = "sse")]
2813    unsafe fn test_mm_ucomilt_ss() {
2814        let aa = &[3.0f32, 12.0, 23.0, NAN];
2815        let bb = &[3.0f32, 47.5, 1.5, NAN];
2816
2817        let ee = &[0i32, 1, 0, 0];
2818
2819        for i in 0..4 {
2820            let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0);
2821            let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0);
2822
2823            let r = _mm_ucomilt_ss(a, b);
2824
2825            assert_eq!(
2826                ee[i], r,
2827                "_mm_ucomilt_ss({:?}, {:?}) = {}, expected: {} (i={})",
2828                a, b, r, ee[i], i
2829            );
2830        }
2831    }
2832
2833    #[simd_test(enable = "sse")]
2834    unsafe fn test_mm_ucomile_ss() {
2835        let aa = &[3.0f32, 12.0, 23.0, NAN];
2836        let bb = &[3.0f32, 47.5, 1.5, NAN];
2837
2838        let ee = &[1i32, 1, 0, 0];
2839
2840        for i in 0..4 {
2841            let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0);
2842            let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0);
2843
2844            let r = _mm_ucomile_ss(a, b);
2845
2846            assert_eq!(
2847                ee[i], r,
2848                "_mm_ucomile_ss({:?}, {:?}) = {}, expected: {} (i={})",
2849                a, b, r, ee[i], i
2850            );
2851        }
2852    }
2853
2854    #[simd_test(enable = "sse")]
2855    unsafe fn test_mm_ucomigt_ss() {
2856        let aa = &[3.0f32, 12.0, 23.0, NAN];
2857        let bb = &[3.0f32, 47.5, 1.5, NAN];
2858
2859        let ee = &[0i32, 0, 1, 0];
2860
2861        for i in 0..4 {
2862            let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0);
2863            let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0);
2864
2865            let r = _mm_ucomigt_ss(a, b);
2866
2867            assert_eq!(
2868                ee[i], r,
2869                "_mm_ucomigt_ss({:?}, {:?}) = {}, expected: {} (i={})",
2870                a, b, r, ee[i], i
2871            );
2872        }
2873    }
2874
2875    #[simd_test(enable = "sse")]
2876    unsafe fn test_mm_ucomige_ss() {
2877        let aa = &[3.0f32, 12.0, 23.0, NAN];
2878        let bb = &[3.0f32, 47.5, 1.5, NAN];
2879
2880        let ee = &[1i32, 0, 1, 0];
2881
2882        for i in 0..4 {
2883            let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0);
2884            let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0);
2885
2886            let r = _mm_ucomige_ss(a, b);
2887
2888            assert_eq!(
2889                ee[i], r,
2890                "_mm_ucomige_ss({:?}, {:?}) = {}, expected: {} (i={})",
2891                a, b, r, ee[i], i
2892            );
2893        }
2894    }
2895
2896    #[simd_test(enable = "sse")]
2897    unsafe fn test_mm_ucomineq_ss() {
2898        let aa = &[3.0f32, 12.0, 23.0, NAN];
2899        let bb = &[3.0f32, 47.5, 1.5, NAN];
2900
2901        let ee = &[0i32, 1, 1, 1];
2902
2903        for i in 0..4 {
2904            let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0);
2905            let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0);
2906
2907            let r = _mm_ucomineq_ss(a, b);
2908
2909            assert_eq!(
2910                ee[i], r,
2911                "_mm_ucomineq_ss({:?}, {:?}) = {}, expected: {} (i={})",
2912                a, b, r, ee[i], i
2913            );
2914        }
2915    }
2916
2917    #[simd_test(enable = "sse")]
2918    unsafe fn test_mm_cvtss_si32() {
2919        let inputs = &[42.0f32, -3.1, 4.0e10, 4.0e-20, NAN, 2147483500.1];
2920        let result = &[42i32, -3, i32::MIN, 0, i32::MIN, 2147483520];
2921        for i in 0..inputs.len() {
2922            let x = _mm_setr_ps(inputs[i], 1.0, 3.0, 4.0);
2923            let e = result[i];
2924            let r = _mm_cvtss_si32(x);
2925            assert_eq!(
2926                e, r,
2927                "TestCase #{} _mm_cvtss_si32({:?}) = {}, expected: {}",
2928                i, x, r, e
2929            );
2930        }
2931    }
2932
2933    #[simd_test(enable = "sse")]
2934    unsafe fn test_mm_cvttss_si32() {
2935        let inputs = &[
2936            (42.0f32, 42i32),
2937            (-31.4, -31),
2938            (-33.5, -33),
2939            (-34.5, -34),
2940            (10.999, 10),
2941            (-5.99, -5),
2942            (4.0e10, i32::MIN),
2943            (4.0e-10, 0),
2944            (NAN, i32::MIN),
2945            (2147483500.1, 2147483520),
2946        ];
2947        for (i, &(xi, e)) in inputs.iter().enumerate() {
2948            let x = _mm_setr_ps(xi, 1.0, 3.0, 4.0);
2949            let r = _mm_cvttss_si32(x);
2950            assert_eq!(
2951                e, r,
2952                "TestCase #{} _mm_cvttss_si32({:?}) = {}, expected: {}",
2953                i, x, r, e
2954            );
2955        }
2956    }
2957
2958    #[simd_test(enable = "sse")]
2959    unsafe fn test_mm_cvtsi32_ss() {
2960        let inputs = &[
2961            (4555i32, 4555.0f32),
2962            (322223333, 322223330.0),
2963            (-432, -432.0),
2964            (-322223333, -322223330.0),
2965        ];
2966
2967        for &(x, f) in inputs.iter() {
2968            let a = _mm_setr_ps(5.0, 6.0, 7.0, 8.0);
2969            let r = _mm_cvtsi32_ss(a, x);
2970            let e = _mm_setr_ps(f, 6.0, 7.0, 8.0);
2971            assert_eq_m128(e, r);
2972        }
2973    }
2974
2975    #[simd_test(enable = "sse")]
2976    unsafe fn test_mm_cvtss_f32() {
2977        let a = _mm_setr_ps(312.0134, 5.0, 6.0, 7.0);
2978        assert_eq!(_mm_cvtss_f32(a), 312.0134);
2979    }
2980
2981    #[simd_test(enable = "sse")]
2982    unsafe fn test_mm_set_ss() {
2983        let r = _mm_set_ss(black_box(4.25));
2984        assert_eq_m128(r, _mm_setr_ps(4.25, 0.0, 0.0, 0.0));
2985    }
2986
2987    #[simd_test(enable = "sse")]
2988    unsafe fn test_mm_set1_ps() {
2989        let r1 = _mm_set1_ps(black_box(4.25));
2990        let r2 = _mm_set_ps1(black_box(4.25));
2991        assert_eq!(get_m128(r1, 0), 4.25);
2992        assert_eq!(get_m128(r1, 1), 4.25);
2993        assert_eq!(get_m128(r1, 2), 4.25);
2994        assert_eq!(get_m128(r1, 3), 4.25);
2995        assert_eq!(get_m128(r2, 0), 4.25);
2996        assert_eq!(get_m128(r2, 1), 4.25);
2997        assert_eq!(get_m128(r2, 2), 4.25);
2998        assert_eq!(get_m128(r2, 3), 4.25);
2999    }
3000
3001    #[simd_test(enable = "sse")]
3002    unsafe fn test_mm_set_ps() {
3003        let r = _mm_set_ps(
3004            black_box(1.0),
3005            black_box(2.0),
3006            black_box(3.0),
3007            black_box(4.0),
3008        );
3009        assert_eq!(get_m128(r, 0), 4.0);
3010        assert_eq!(get_m128(r, 1), 3.0);
3011        assert_eq!(get_m128(r, 2), 2.0);
3012        assert_eq!(get_m128(r, 3), 1.0);
3013    }
3014
3015    #[simd_test(enable = "sse")]
3016    unsafe fn test_mm_setr_ps() {
3017        let r = _mm_setr_ps(
3018            black_box(1.0),
3019            black_box(2.0),
3020            black_box(3.0),
3021            black_box(4.0),
3022        );
3023        assert_eq_m128(r, _mm_setr_ps(1.0, 2.0, 3.0, 4.0));
3024    }
3025
3026    #[simd_test(enable = "sse")]
3027    unsafe fn test_mm_setzero_ps() {
3028        let r = *black_box(&_mm_setzero_ps());
3029        assert_eq_m128(r, _mm_set1_ps(0.0));
3030    }
3031
3032    #[simd_test(enable = "sse")]
3033    unsafe fn test_mm_shuffle() {
3034        assert_eq!(_MM_SHUFFLE(0, 1, 1, 3), 0b00_01_01_11);
3035        assert_eq!(_MM_SHUFFLE(3, 1, 1, 0), 0b11_01_01_00);
3036        assert_eq!(_MM_SHUFFLE(1, 2, 2, 1), 0b01_10_10_01);
3037    }
3038
3039    #[simd_test(enable = "sse")]
3040    unsafe fn test_mm_shuffle_ps() {
3041        let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0);
3042        let b = _mm_setr_ps(5.0, 6.0, 7.0, 8.0);
3043        let r = _mm_shuffle_ps::<0b00_01_01_11>(a, b);
3044        assert_eq_m128(r, _mm_setr_ps(4.0, 2.0, 6.0, 5.0));
3045    }
3046
3047    #[simd_test(enable = "sse")]
3048    unsafe fn test_mm_unpackhi_ps() {
3049        let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0);
3050        let b = _mm_setr_ps(5.0, 6.0, 7.0, 8.0);
3051        let r = _mm_unpackhi_ps(a, b);
3052        assert_eq_m128(r, _mm_setr_ps(3.0, 7.0, 4.0, 8.0));
3053    }
3054
3055    #[simd_test(enable = "sse")]
3056    unsafe fn test_mm_unpacklo_ps() {
3057        let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0);
3058        let b = _mm_setr_ps(5.0, 6.0, 7.0, 8.0);
3059        let r = _mm_unpacklo_ps(a, b);
3060        assert_eq_m128(r, _mm_setr_ps(1.0, 5.0, 2.0, 6.0));
3061    }
3062
3063    #[simd_test(enable = "sse")]
3064    unsafe fn test_mm_movehl_ps() {
3065        let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0);
3066        let b = _mm_setr_ps(5.0, 6.0, 7.0, 8.0);
3067        let r = _mm_movehl_ps(a, b);
3068        assert_eq_m128(r, _mm_setr_ps(7.0, 8.0, 3.0, 4.0));
3069    }
3070
3071    #[simd_test(enable = "sse")]
3072    unsafe fn test_mm_movelh_ps() {
3073        let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0);
3074        let b = _mm_setr_ps(5.0, 6.0, 7.0, 8.0);
3075        let r = _mm_movelh_ps(a, b);
3076        assert_eq_m128(r, _mm_setr_ps(1.0, 2.0, 5.0, 6.0));
3077    }
3078
3079    #[simd_test(enable = "sse")]
3080    unsafe fn test_mm_load_ss() {
3081        let a = 42.0f32;
3082        let r = _mm_load_ss(ptr::addr_of!(a));
3083        assert_eq_m128(r, _mm_setr_ps(42.0, 0.0, 0.0, 0.0));
3084    }
3085
3086    #[simd_test(enable = "sse")]
3087    unsafe fn test_mm_load1_ps() {
3088        let a = 42.0f32;
3089        let r = _mm_load1_ps(ptr::addr_of!(a));
3090        assert_eq_m128(r, _mm_setr_ps(42.0, 42.0, 42.0, 42.0));
3091    }
3092
3093    #[simd_test(enable = "sse")]
3094    unsafe fn test_mm_load_ps() {
3095        let vals = &[1.0f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0];
3096
3097        let mut p = vals.as_ptr();
3098        let mut fixup = 0.0f32;
3099
3100        // Make sure p is aligned, otherwise we might get a
3101        // (signal: 11, SIGSEGV: invalid memory reference)
3102
3103        let unalignment = (p as usize) & 0xf;
3104        if unalignment != 0 {
3105            let delta = (16 - unalignment) >> 2;
3106            fixup = delta as f32;
3107            p = p.add(delta);
3108        }
3109
3110        let r = _mm_load_ps(p);
3111        let e = _mm_add_ps(_mm_setr_ps(1.0, 2.0, 3.0, 4.0), _mm_set1_ps(fixup));
3112        assert_eq_m128(r, e);
3113    }
3114
3115    #[simd_test(enable = "sse")]
3116    unsafe fn test_mm_loadu_ps() {
3117        let vals = &[1.0f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0];
3118        let p = vals.as_ptr().add(3);
3119        let r = _mm_loadu_ps(black_box(p));
3120        assert_eq_m128(r, _mm_setr_ps(4.0, 5.0, 6.0, 7.0));
3121    }
3122
3123    #[simd_test(enable = "sse")]
3124    unsafe fn test_mm_loadr_ps() {
3125        let vals = &[1.0f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0];
3126
3127        let mut p = vals.as_ptr();
3128        let mut fixup = 0.0f32;
3129
3130        // Make sure p is aligned, otherwise we might get a
3131        // (signal: 11, SIGSEGV: invalid memory reference)
3132
3133        let unalignment = (p as usize) & 0xf;
3134        if unalignment != 0 {
3135            let delta = (16 - unalignment) >> 2;
3136            fixup = delta as f32;
3137            p = p.add(delta);
3138        }
3139
3140        let r = _mm_loadr_ps(p);
3141        let e = _mm_add_ps(_mm_setr_ps(4.0, 3.0, 2.0, 1.0), _mm_set1_ps(fixup));
3142        assert_eq_m128(r, e);
3143    }
3144
3145    #[simd_test(enable = "sse")]
3146    unsafe fn test_mm_store_ss() {
3147        let mut vals = [0.0f32; 8];
3148        let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0);
3149        _mm_store_ss(vals.as_mut_ptr().add(1), a);
3150
3151        assert_eq!(vals[0], 0.0);
3152        assert_eq!(vals[1], 1.0);
3153        assert_eq!(vals[2], 0.0);
3154    }
3155
3156    #[simd_test(enable = "sse")]
3157    unsafe fn test_mm_store1_ps() {
3158        let mut vals = [0.0f32; 8];
3159        let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0);
3160
3161        let mut ofs = 0;
3162        let mut p = vals.as_mut_ptr();
3163
3164        if (p as usize) & 0xf != 0 {
3165            ofs = (16 - ((p as usize) & 0xf)) >> 2;
3166            p = p.add(ofs);
3167        }
3168
3169        _mm_store1_ps(p, *black_box(&a));
3170
3171        if ofs > 0 {
3172            assert_eq!(vals[ofs - 1], 0.0);
3173        }
3174        assert_eq!(vals[ofs + 0], 1.0);
3175        assert_eq!(vals[ofs + 1], 1.0);
3176        assert_eq!(vals[ofs + 2], 1.0);
3177        assert_eq!(vals[ofs + 3], 1.0);
3178        assert_eq!(vals[ofs + 4], 0.0);
3179    }
3180
3181    #[simd_test(enable = "sse")]
3182    unsafe fn test_mm_store_ps() {
3183        let mut vals = [0.0f32; 8];
3184        let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0);
3185
3186        let mut ofs = 0;
3187        let mut p = vals.as_mut_ptr();
3188
3189        // Align p to 16-byte boundary
3190        if (p as usize) & 0xf != 0 {
3191            ofs = (16 - ((p as usize) & 0xf)) >> 2;
3192            p = p.add(ofs);
3193        }
3194
3195        _mm_store_ps(p, *black_box(&a));
3196
3197        if ofs > 0 {
3198            assert_eq!(vals[ofs - 1], 0.0);
3199        }
3200        assert_eq!(vals[ofs + 0], 1.0);
3201        assert_eq!(vals[ofs + 1], 2.0);
3202        assert_eq!(vals[ofs + 2], 3.0);
3203        assert_eq!(vals[ofs + 3], 4.0);
3204        assert_eq!(vals[ofs + 4], 0.0);
3205    }
3206
3207    #[simd_test(enable = "sse")]
3208    unsafe fn test_mm_storer_ps() {
3209        let mut vals = [0.0f32; 8];
3210        let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0);
3211
3212        let mut ofs = 0;
3213        let mut p = vals.as_mut_ptr();
3214
3215        // Align p to 16-byte boundary
3216        if (p as usize) & 0xf != 0 {
3217            ofs = (16 - ((p as usize) & 0xf)) >> 2;
3218            p = p.add(ofs);
3219        }
3220
3221        _mm_storer_ps(p, *black_box(&a));
3222
3223        if ofs > 0 {
3224            assert_eq!(vals[ofs - 1], 0.0);
3225        }
3226        assert_eq!(vals[ofs + 0], 4.0);
3227        assert_eq!(vals[ofs + 1], 3.0);
3228        assert_eq!(vals[ofs + 2], 2.0);
3229        assert_eq!(vals[ofs + 3], 1.0);
3230        assert_eq!(vals[ofs + 4], 0.0);
3231    }
3232
3233    #[simd_test(enable = "sse")]
3234    unsafe fn test_mm_storeu_ps() {
3235        let mut vals = [0.0f32; 8];
3236        let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0);
3237
3238        let mut ofs = 0;
3239        let mut p = vals.as_mut_ptr();
3240
3241        // Make sure p is **not** aligned to 16-byte boundary
3242        if (p as usize) & 0xf == 0 {
3243            ofs = 1;
3244            p = p.add(1);
3245        }
3246
3247        _mm_storeu_ps(p, *black_box(&a));
3248
3249        if ofs > 0 {
3250            assert_eq!(vals[ofs - 1], 0.0);
3251        }
3252        assert_eq!(vals[ofs + 0], 1.0);
3253        assert_eq!(vals[ofs + 1], 2.0);
3254        assert_eq!(vals[ofs + 2], 3.0);
3255        assert_eq!(vals[ofs + 3], 4.0);
3256        assert_eq!(vals[ofs + 4], 0.0);
3257    }
3258
3259    #[simd_test(enable = "sse")]
3260    unsafe fn test_mm_move_ss() {
3261        let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0);
3262        let b = _mm_setr_ps(5.0, 6.0, 7.0, 8.0);
3263
3264        let r = _mm_move_ss(a, b);
3265        let e = _mm_setr_ps(5.0, 2.0, 3.0, 4.0);
3266        assert_eq_m128(e, r);
3267    }
3268
3269    #[simd_test(enable = "sse")]
3270    unsafe fn test_mm_movemask_ps() {
3271        let r = _mm_movemask_ps(_mm_setr_ps(-1.0, 5.0, -5.0, 0.0));
3272        assert_eq!(r, 0b0101);
3273
3274        let r = _mm_movemask_ps(_mm_setr_ps(-1.0, -5.0, -5.0, 0.0));
3275        assert_eq!(r, 0b0111);
3276    }
3277
3278    #[simd_test(enable = "sse")]
3279    // Miri cannot support this until it is clear how it fits in the Rust memory model
3280    #[cfg_attr(miri, ignore)]
3281    unsafe fn test_mm_sfence() {
3282        _mm_sfence();
3283    }
3284
3285    #[simd_test(enable = "sse")]
3286    unsafe fn test_MM_TRANSPOSE4_PS() {
3287        let mut a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0);
3288        let mut b = _mm_setr_ps(5.0, 6.0, 7.0, 8.0);
3289        let mut c = _mm_setr_ps(9.0, 10.0, 11.0, 12.0);
3290        let mut d = _mm_setr_ps(13.0, 14.0, 15.0, 16.0);
3291
3292        _MM_TRANSPOSE4_PS(&mut a, &mut b, &mut c, &mut d);
3293
3294        assert_eq_m128(a, _mm_setr_ps(1.0, 5.0, 9.0, 13.0));
3295        assert_eq_m128(b, _mm_setr_ps(2.0, 6.0, 10.0, 14.0));
3296        assert_eq_m128(c, _mm_setr_ps(3.0, 7.0, 11.0, 15.0));
3297        assert_eq_m128(d, _mm_setr_ps(4.0, 8.0, 12.0, 16.0));
3298    }
3299
3300    #[repr(align(16))]
3301    struct Memory {
3302        pub data: [f32; 4],
3303    }
3304
3305    #[simd_test(enable = "sse")]
3306    // Miri cannot support this until it is clear how it fits in the Rust memory model
3307    // (non-temporal store)
3308    #[cfg_attr(miri, ignore)]
3309    unsafe fn test_mm_stream_ps() {
3310        let a = _mm_set1_ps(7.0);
3311        let mut mem = Memory { data: [-1.0; 4] };
3312
3313        _mm_stream_ps(ptr::addr_of_mut!(mem.data[0]), a);
3314        for i in 0..4 {
3315            assert_eq!(mem.data[i], get_m128(a, i));
3316        }
3317    }
3318}