1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
use rustc_apfloat::{ieee::Double, ieee::Single};
use rustc_middle::mir;
use rustc_middle::ty::layout::LayoutOf as _;
use rustc_middle::ty::Ty;
use rustc_span::Symbol;
use rustc_target::spec::abi::Abi;

use super::{
    bin_op_simd_float_all, conditional_dot_product, convert_float_to_int, horizontal_bin_op,
    mask_load, mask_store, round_all, test_bits_masked, test_high_bits_masked, unary_op_ps,
    FloatBinOp, FloatUnaryOp,
};
use crate::*;

impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {}
pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
    fn emulate_x86_avx_intrinsic(
        &mut self,
        link_name: Symbol,
        abi: Abi,
        args: &[OpTy<'tcx>],
        dest: &MPlaceTy<'tcx>,
    ) -> InterpResult<'tcx, EmulateItemResult> {
        let this = self.eval_context_mut();
        this.expect_target_feature_for_intrinsic(link_name, "avx")?;
        // Prefix should have already been checked.
        let unprefixed_name = link_name.as_str().strip_prefix("llvm.x86.avx.").unwrap();

        match unprefixed_name {
            // Used to implement _mm256_min_ps and _mm256_max_ps functions.
            // Note that the semantics are a bit different from Rust simd_min
            // and simd_max intrinsics regarding handling of NaN and -0.0: Rust
            // matches the IEEE min/max operations, while x86 has different
            // semantics.
            "min.ps.256" | "max.ps.256" => {
                let [left, right] =
                    this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;

                let which = match unprefixed_name {
                    "min.ps.256" => FloatBinOp::Min,
                    "max.ps.256" => FloatBinOp::Max,
                    _ => unreachable!(),
                };

                bin_op_simd_float_all::<Single>(this, which, left, right, dest)?;
            }
            // Used to implement _mm256_min_pd and _mm256_max_pd functions.
            "min.pd.256" | "max.pd.256" => {
                let [left, right] =
                    this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;

                let which = match unprefixed_name {
                    "min.pd.256" => FloatBinOp::Min,
                    "max.pd.256" => FloatBinOp::Max,
                    _ => unreachable!(),
                };

                bin_op_simd_float_all::<Double>(this, which, left, right, dest)?;
            }
            // Used to implement the _mm256_round_ps function.
            // Rounds the elements of `op` according to `rounding`.
            "round.ps.256" => {
                let [op, rounding] =
                    this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;

                round_all::<rustc_apfloat::ieee::Single>(this, op, rounding, dest)?;
            }
            // Used to implement the _mm256_round_pd function.
            // Rounds the elements of `op` according to `rounding`.
            "round.pd.256" => {
                let [op, rounding] =
                    this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;

                round_all::<rustc_apfloat::ieee::Double>(this, op, rounding, dest)?;
            }
            // Used to implement _mm256_{rcp,rsqrt}_ps functions.
            // Performs the operations on all components of `op`.
            "rcp.ps.256" | "rsqrt.ps.256" => {
                let [op] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;

                let which = match unprefixed_name {
                    "rcp.ps.256" => FloatUnaryOp::Rcp,
                    "rsqrt.ps.256" => FloatUnaryOp::Rsqrt,
                    _ => unreachable!(),
                };

                unary_op_ps(this, which, op, dest)?;
            }
            // Used to implement the _mm256_dp_ps function.
            "dp.ps.256" => {
                let [left, right, imm] =
                    this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;

                conditional_dot_product(this, left, right, imm, dest)?;
            }
            // Used to implement the _mm256_h{add,sub}_p{s,d} functions.
            // Horizontally add/subtract adjacent floating point values
            // in `left` and `right`.
            "hadd.ps.256" | "hadd.pd.256" | "hsub.ps.256" | "hsub.pd.256" => {
                let [left, right] =
                    this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;

                let which = match unprefixed_name {
                    "hadd.ps.256" | "hadd.pd.256" => mir::BinOp::Add,
                    "hsub.ps.256" | "hsub.pd.256" => mir::BinOp::Sub,
                    _ => unreachable!(),
                };

                horizontal_bin_op(this, which, /*saturating*/ false, left, right, dest)?;
            }
            // Used to implement the _mm256_cmp_ps function.
            // Performs a comparison operation on each component of `left`
            // and `right`. For each component, returns 0 if false or u32::MAX
            // if true.
            "cmp.ps.256" => {
                let [left, right, imm] =
                    this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;

                let which =
                    FloatBinOp::cmp_from_imm(this, this.read_scalar(imm)?.to_i8()?, link_name)?;

                bin_op_simd_float_all::<Single>(this, which, left, right, dest)?;
            }
            // Used to implement the _mm256_cmp_pd function.
            // Performs a comparison operation on each component of `left`
            // and `right`. For each component, returns 0 if false or u64::MAX
            // if true.
            "cmp.pd.256" => {
                let [left, right, imm] =
                    this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;

                let which =
                    FloatBinOp::cmp_from_imm(this, this.read_scalar(imm)?.to_i8()?, link_name)?;

                bin_op_simd_float_all::<Double>(this, which, left, right, dest)?;
            }
            // Used to implement the _mm256_cvtps_epi32, _mm256_cvttps_epi32, _mm256_cvtpd_epi32
            // and _mm256_cvttpd_epi32 functions.
            // Converts packed f32/f64 to packed i32.
            "cvt.ps2dq.256" | "cvtt.ps2dq.256" | "cvt.pd2dq.256" | "cvtt.pd2dq.256" => {
                let [op] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;

                let rnd = match unprefixed_name {
                    // "current SSE rounding mode", assume nearest
                    "cvt.ps2dq.256" | "cvt.pd2dq.256" => rustc_apfloat::Round::NearestTiesToEven,
                    // always truncate
                    "cvtt.ps2dq.256" | "cvtt.pd2dq.256" => rustc_apfloat::Round::TowardZero,
                    _ => unreachable!(),
                };

                convert_float_to_int(this, op, rnd, dest)?;
            }
            // Used to implement the _mm_permutevar_ps and _mm256_permutevar_ps functions.
            // Shuffles 32-bit floats from `data` using `control` as control. Each 128-bit
            // chunk is shuffled independently: this means that we view the vector as a
            // sequence of 4-element arrays, and we shuffle each of these arrays, where
            // `control` determines which element of the current `data` array is written.
            "vpermilvar.ps" | "vpermilvar.ps.256" => {
                let [data, control] =
                    this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;

                let (data, data_len) = this.operand_to_simd(data)?;
                let (control, control_len) = this.operand_to_simd(control)?;
                let (dest, dest_len) = this.mplace_to_simd(dest)?;

                assert_eq!(dest_len, data_len);
                assert_eq!(dest_len, control_len);

                for i in 0..dest_len {
                    let control = this.project_index(&control, i)?;

                    // Each 128-bit chunk is shuffled independently. Since each chunk contains
                    // four 32-bit elements, only two bits from `control` are used. To read the
                    // value from the current chunk, add the destination index truncated to a multiple
                    // of 4.
                    let chunk_base = i & !0b11;
                    let src_i = u64::from(this.read_scalar(&control)?.to_u32()? & 0b11)
                        .strict_add(chunk_base);

                    this.copy_op(
                        &this.project_index(&data, src_i)?,
                        &this.project_index(&dest, i)?,
                    )?;
                }
            }
            // Used to implement the _mm_permutevar_pd and _mm256_permutevar_pd functions.
            // Shuffles 64-bit floats from `left` using `right` as control. Each 128-bit
            // chunk is shuffled independently: this means that we view the vector as
            // a sequence of 2-element arrays, and we shuffle each of these arrays,
            // where `right` determines which element of the current `left` array is
            // written.
            "vpermilvar.pd" | "vpermilvar.pd.256" => {
                let [data, control] =
                    this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;

                let (data, data_len) = this.operand_to_simd(data)?;
                let (control, control_len) = this.operand_to_simd(control)?;
                let (dest, dest_len) = this.mplace_to_simd(dest)?;

                assert_eq!(dest_len, data_len);
                assert_eq!(dest_len, control_len);

                for i in 0..dest_len {
                    let control = this.project_index(&control, i)?;

                    // Each 128-bit chunk is shuffled independently. Since each chunk contains
                    // two 64-bit elements, only the second bit from `control` is used (yes, the
                    // second instead of the first, ask Intel). To read the value from the current
                    // chunk, add the destination index truncated to a multiple of 2.
                    let chunk_base = i & !1;
                    let src_i =
                        ((this.read_scalar(&control)?.to_u64()? >> 1) & 1).strict_add(chunk_base);

                    this.copy_op(
                        &this.project_index(&data, src_i)?,
                        &this.project_index(&dest, i)?,
                    )?;
                }
            }
            // Used to implement the _mm256_permute2f128_ps, _mm256_permute2f128_pd and
            // _mm256_permute2f128_si256 functions. Regardless of the suffix in the name
            // thay all can be considered to operate on vectors of 128-bit elements.
            // For each 128-bit element of `dest`, copies one from `left`, `right` or
            // zero, according to `imm`.
            "vperm2f128.ps.256" | "vperm2f128.pd.256" | "vperm2f128.si.256" => {
                let [left, right, imm] =
                    this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;

                assert_eq!(dest.layout, left.layout);
                assert_eq!(dest.layout, right.layout);
                assert_eq!(dest.layout.size.bits(), 256);

                // Transmute to `[u128; 2]` to process each 128-bit chunk independently.
                let u128x2_layout =
                    this.layout_of(Ty::new_array(this.tcx.tcx, this.tcx.types.u128, 2))?;
                let left = left.transmute(u128x2_layout, this)?;
                let right = right.transmute(u128x2_layout, this)?;
                let dest = dest.transmute(u128x2_layout, this)?;

                let imm = this.read_scalar(imm)?.to_u8()?;

                for i in 0..2 {
                    let dest = this.project_index(&dest, i)?;

                    let imm = match i {
                        0 => imm & 0xF,
                        1 => imm >> 4,
                        _ => unreachable!(),
                    };
                    if imm & 0b100 != 0 {
                        this.write_scalar(Scalar::from_u128(0), &dest)?;
                    } else {
                        let src = match imm {
                            0b00 => this.project_index(&left, 0)?,
                            0b01 => this.project_index(&left, 1)?,
                            0b10 => this.project_index(&right, 0)?,
                            0b11 => this.project_index(&right, 1)?,
                            _ => unreachable!(),
                        };
                        this.copy_op(&src, &dest)?;
                    }
                }
            }
            // Used to implement the _mm_maskload_ps, _mm_maskload_pd, _mm256_maskload_ps
            // and _mm256_maskload_pd functions.
            // For the element `i`, if the high bit of the `i`-th element of `mask`
            // is one, it is loaded from `ptr.wrapping_add(i)`, otherwise zero is
            // loaded.
            "maskload.ps" | "maskload.pd" | "maskload.ps.256" | "maskload.pd.256" => {
                let [ptr, mask] =
                    this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;

                mask_load(this, ptr, mask, dest)?;
            }
            // Used to implement the _mm_maskstore_ps, _mm_maskstore_pd, _mm256_maskstore_ps
            // and _mm256_maskstore_pd functions.
            // For the element `i`, if the high bit of the element `i`-th of `mask`
            // is one, it is stored into `ptr.wapping_add(i)`.
            // Unlike SSE2's _mm_maskmoveu_si128, these are not non-temporal stores.
            "maskstore.ps" | "maskstore.pd" | "maskstore.ps.256" | "maskstore.pd.256" => {
                let [ptr, mask, value] =
                    this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;

                mask_store(this, ptr, mask, value)?;
            }
            // Used to implement the _mm256_lddqu_si256 function.
            // Reads a 256-bit vector from an unaligned pointer. This intrinsic
            // is expected to perform better than a regular unaligned read when
            // the data crosses a cache line, but for Miri this is just a regular
            // unaligned read.
            "ldu.dq.256" => {
                let [src_ptr] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
                let src_ptr = this.read_pointer(src_ptr)?;
                let dest = dest.force_mplace(this)?;

                // Unaligned copy, which is what we want.
                this.mem_copy(src_ptr, dest.ptr(), dest.layout.size, /*nonoverlapping*/ true)?;
            }
            // Used to implement the _mm256_testz_si256, _mm256_testc_si256 and
            // _mm256_testnzc_si256 functions.
            // Tests `op & mask == 0`, `op & mask == mask` or
            // `op & mask != 0 && op & mask != mask`
            "ptestz.256" | "ptestc.256" | "ptestnzc.256" => {
                let [op, mask] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;

                let (all_zero, masked_set) = test_bits_masked(this, op, mask)?;
                let res = match unprefixed_name {
                    "ptestz.256" => all_zero,
                    "ptestc.256" => masked_set,
                    "ptestnzc.256" => !all_zero && !masked_set,
                    _ => unreachable!(),
                };

                this.write_scalar(Scalar::from_i32(res.into()), dest)?;
            }
            // Used to implement the _mm256_testz_pd, _mm256_testc_pd, _mm256_testnzc_pd
            // _mm_testz_pd, _mm_testc_pd, _mm_testnzc_pd, _mm256_testz_ps,
            // _mm256_testc_ps, _mm256_testnzc_ps, _mm_testz_ps, _mm_testc_ps and
            // _mm_testnzc_ps functions.
            // Calculates two booleans:
            // `direct`, which is true when the highest bit of each element of `op & mask` is zero.
            // `negated`, which is true when the highest bit of each element of `!op & mask` is zero.
            // Return `direct` (testz), `negated` (testc) or `!direct & !negated` (testnzc)
            "vtestz.pd.256" | "vtestc.pd.256" | "vtestnzc.pd.256" | "vtestz.pd" | "vtestc.pd"
            | "vtestnzc.pd" | "vtestz.ps.256" | "vtestc.ps.256" | "vtestnzc.ps.256"
            | "vtestz.ps" | "vtestc.ps" | "vtestnzc.ps" => {
                let [op, mask] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;

                let (direct, negated) = test_high_bits_masked(this, op, mask)?;
                let res = match unprefixed_name {
                    "vtestz.pd.256" | "vtestz.pd" | "vtestz.ps.256" | "vtestz.ps" => direct,
                    "vtestc.pd.256" | "vtestc.pd" | "vtestc.ps.256" | "vtestc.ps" => negated,
                    "vtestnzc.pd.256" | "vtestnzc.pd" | "vtestnzc.ps.256" | "vtestnzc.ps" =>
                        !direct && !negated,
                    _ => unreachable!(),
                };

                this.write_scalar(Scalar::from_i32(res.into()), dest)?;
            }
            // Used to implement the `_mm256_zeroupper` and `_mm256_zeroall` functions.
            // These function clear out the upper 128 bits of all avx registers or
            // zero out all avx registers respectively.
            "vzeroupper" | "vzeroall" => {
                // These functions are purely a performance hint for the CPU.
                // Any registers currently in use will be saved beforehand by the
                // compiler, making these functions no-ops.

                // The only thing that needs to be ensured is the correct calling convention.
                let [] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
            }
            _ => return Ok(EmulateItemResult::NotSupported),
        }
        Ok(EmulateItemResult::NeedsReturn)
    }
}