1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355
use rustc_apfloat::{ieee::Double, ieee::Single};
use rustc_middle::mir;
use rustc_middle::ty::layout::LayoutOf as _;
use rustc_middle::ty::Ty;
use rustc_span::Symbol;
use rustc_target::spec::abi::Abi;
use super::{
bin_op_simd_float_all, conditional_dot_product, convert_float_to_int, horizontal_bin_op,
mask_load, mask_store, round_all, test_bits_masked, test_high_bits_masked, unary_op_ps,
FloatBinOp, FloatUnaryOp,
};
use crate::*;
impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {}
pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
fn emulate_x86_avx_intrinsic(
&mut self,
link_name: Symbol,
abi: Abi,
args: &[OpTy<'tcx>],
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, EmulateItemResult> {
let this = self.eval_context_mut();
this.expect_target_feature_for_intrinsic(link_name, "avx")?;
// Prefix should have already been checked.
let unprefixed_name = link_name.as_str().strip_prefix("llvm.x86.avx.").unwrap();
match unprefixed_name {
// Used to implement _mm256_min_ps and _mm256_max_ps functions.
// Note that the semantics are a bit different from Rust simd_min
// and simd_max intrinsics regarding handling of NaN and -0.0: Rust
// matches the IEEE min/max operations, while x86 has different
// semantics.
"min.ps.256" | "max.ps.256" => {
let [left, right] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let which = match unprefixed_name {
"min.ps.256" => FloatBinOp::Min,
"max.ps.256" => FloatBinOp::Max,
_ => unreachable!(),
};
bin_op_simd_float_all::<Single>(this, which, left, right, dest)?;
}
// Used to implement _mm256_min_pd and _mm256_max_pd functions.
"min.pd.256" | "max.pd.256" => {
let [left, right] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let which = match unprefixed_name {
"min.pd.256" => FloatBinOp::Min,
"max.pd.256" => FloatBinOp::Max,
_ => unreachable!(),
};
bin_op_simd_float_all::<Double>(this, which, left, right, dest)?;
}
// Used to implement the _mm256_round_ps function.
// Rounds the elements of `op` according to `rounding`.
"round.ps.256" => {
let [op, rounding] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
round_all::<rustc_apfloat::ieee::Single>(this, op, rounding, dest)?;
}
// Used to implement the _mm256_round_pd function.
// Rounds the elements of `op` according to `rounding`.
"round.pd.256" => {
let [op, rounding] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
round_all::<rustc_apfloat::ieee::Double>(this, op, rounding, dest)?;
}
// Used to implement _mm256_{rcp,rsqrt}_ps functions.
// Performs the operations on all components of `op`.
"rcp.ps.256" | "rsqrt.ps.256" => {
let [op] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let which = match unprefixed_name {
"rcp.ps.256" => FloatUnaryOp::Rcp,
"rsqrt.ps.256" => FloatUnaryOp::Rsqrt,
_ => unreachable!(),
};
unary_op_ps(this, which, op, dest)?;
}
// Used to implement the _mm256_dp_ps function.
"dp.ps.256" => {
let [left, right, imm] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
conditional_dot_product(this, left, right, imm, dest)?;
}
// Used to implement the _mm256_h{add,sub}_p{s,d} functions.
// Horizontally add/subtract adjacent floating point values
// in `left` and `right`.
"hadd.ps.256" | "hadd.pd.256" | "hsub.ps.256" | "hsub.pd.256" => {
let [left, right] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let which = match unprefixed_name {
"hadd.ps.256" | "hadd.pd.256" => mir::BinOp::Add,
"hsub.ps.256" | "hsub.pd.256" => mir::BinOp::Sub,
_ => unreachable!(),
};
horizontal_bin_op(this, which, /*saturating*/ false, left, right, dest)?;
}
// Used to implement the _mm256_cmp_ps function.
// Performs a comparison operation on each component of `left`
// and `right`. For each component, returns 0 if false or u32::MAX
// if true.
"cmp.ps.256" => {
let [left, right, imm] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let which =
FloatBinOp::cmp_from_imm(this, this.read_scalar(imm)?.to_i8()?, link_name)?;
bin_op_simd_float_all::<Single>(this, which, left, right, dest)?;
}
// Used to implement the _mm256_cmp_pd function.
// Performs a comparison operation on each component of `left`
// and `right`. For each component, returns 0 if false or u64::MAX
// if true.
"cmp.pd.256" => {
let [left, right, imm] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let which =
FloatBinOp::cmp_from_imm(this, this.read_scalar(imm)?.to_i8()?, link_name)?;
bin_op_simd_float_all::<Double>(this, which, left, right, dest)?;
}
// Used to implement the _mm256_cvtps_epi32, _mm256_cvttps_epi32, _mm256_cvtpd_epi32
// and _mm256_cvttpd_epi32 functions.
// Converts packed f32/f64 to packed i32.
"cvt.ps2dq.256" | "cvtt.ps2dq.256" | "cvt.pd2dq.256" | "cvtt.pd2dq.256" => {
let [op] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let rnd = match unprefixed_name {
// "current SSE rounding mode", assume nearest
"cvt.ps2dq.256" | "cvt.pd2dq.256" => rustc_apfloat::Round::NearestTiesToEven,
// always truncate
"cvtt.ps2dq.256" | "cvtt.pd2dq.256" => rustc_apfloat::Round::TowardZero,
_ => unreachable!(),
};
convert_float_to_int(this, op, rnd, dest)?;
}
// Used to implement the _mm_permutevar_ps and _mm256_permutevar_ps functions.
// Shuffles 32-bit floats from `data` using `control` as control. Each 128-bit
// chunk is shuffled independently: this means that we view the vector as a
// sequence of 4-element arrays, and we shuffle each of these arrays, where
// `control` determines which element of the current `data` array is written.
"vpermilvar.ps" | "vpermilvar.ps.256" => {
let [data, control] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let (data, data_len) = this.operand_to_simd(data)?;
let (control, control_len) = this.operand_to_simd(control)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
assert_eq!(dest_len, data_len);
assert_eq!(dest_len, control_len);
for i in 0..dest_len {
let control = this.project_index(&control, i)?;
// Each 128-bit chunk is shuffled independently. Since each chunk contains
// four 32-bit elements, only two bits from `control` are used. To read the
// value from the current chunk, add the destination index truncated to a multiple
// of 4.
let chunk_base = i & !0b11;
let src_i = u64::from(this.read_scalar(&control)?.to_u32()? & 0b11)
.strict_add(chunk_base);
this.copy_op(
&this.project_index(&data, src_i)?,
&this.project_index(&dest, i)?,
)?;
}
}
// Used to implement the _mm_permutevar_pd and _mm256_permutevar_pd functions.
// Shuffles 64-bit floats from `left` using `right` as control. Each 128-bit
// chunk is shuffled independently: this means that we view the vector as
// a sequence of 2-element arrays, and we shuffle each of these arrays,
// where `right` determines which element of the current `left` array is
// written.
"vpermilvar.pd" | "vpermilvar.pd.256" => {
let [data, control] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let (data, data_len) = this.operand_to_simd(data)?;
let (control, control_len) = this.operand_to_simd(control)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
assert_eq!(dest_len, data_len);
assert_eq!(dest_len, control_len);
for i in 0..dest_len {
let control = this.project_index(&control, i)?;
// Each 128-bit chunk is shuffled independently. Since each chunk contains
// two 64-bit elements, only the second bit from `control` is used (yes, the
// second instead of the first, ask Intel). To read the value from the current
// chunk, add the destination index truncated to a multiple of 2.
let chunk_base = i & !1;
let src_i =
((this.read_scalar(&control)?.to_u64()? >> 1) & 1).strict_add(chunk_base);
this.copy_op(
&this.project_index(&data, src_i)?,
&this.project_index(&dest, i)?,
)?;
}
}
// Used to implement the _mm256_permute2f128_ps, _mm256_permute2f128_pd and
// _mm256_permute2f128_si256 functions. Regardless of the suffix in the name
// thay all can be considered to operate on vectors of 128-bit elements.
// For each 128-bit element of `dest`, copies one from `left`, `right` or
// zero, according to `imm`.
"vperm2f128.ps.256" | "vperm2f128.pd.256" | "vperm2f128.si.256" => {
let [left, right, imm] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
assert_eq!(dest.layout, left.layout);
assert_eq!(dest.layout, right.layout);
assert_eq!(dest.layout.size.bits(), 256);
// Transmute to `[u128; 2]` to process each 128-bit chunk independently.
let u128x2_layout =
this.layout_of(Ty::new_array(this.tcx.tcx, this.tcx.types.u128, 2))?;
let left = left.transmute(u128x2_layout, this)?;
let right = right.transmute(u128x2_layout, this)?;
let dest = dest.transmute(u128x2_layout, this)?;
let imm = this.read_scalar(imm)?.to_u8()?;
for i in 0..2 {
let dest = this.project_index(&dest, i)?;
let imm = match i {
0 => imm & 0xF,
1 => imm >> 4,
_ => unreachable!(),
};
if imm & 0b100 != 0 {
this.write_scalar(Scalar::from_u128(0), &dest)?;
} else {
let src = match imm {
0b00 => this.project_index(&left, 0)?,
0b01 => this.project_index(&left, 1)?,
0b10 => this.project_index(&right, 0)?,
0b11 => this.project_index(&right, 1)?,
_ => unreachable!(),
};
this.copy_op(&src, &dest)?;
}
}
}
// Used to implement the _mm_maskload_ps, _mm_maskload_pd, _mm256_maskload_ps
// and _mm256_maskload_pd functions.
// For the element `i`, if the high bit of the `i`-th element of `mask`
// is one, it is loaded from `ptr.wrapping_add(i)`, otherwise zero is
// loaded.
"maskload.ps" | "maskload.pd" | "maskload.ps.256" | "maskload.pd.256" => {
let [ptr, mask] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
mask_load(this, ptr, mask, dest)?;
}
// Used to implement the _mm_maskstore_ps, _mm_maskstore_pd, _mm256_maskstore_ps
// and _mm256_maskstore_pd functions.
// For the element `i`, if the high bit of the element `i`-th of `mask`
// is one, it is stored into `ptr.wapping_add(i)`.
// Unlike SSE2's _mm_maskmoveu_si128, these are not non-temporal stores.
"maskstore.ps" | "maskstore.pd" | "maskstore.ps.256" | "maskstore.pd.256" => {
let [ptr, mask, value] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
mask_store(this, ptr, mask, value)?;
}
// Used to implement the _mm256_lddqu_si256 function.
// Reads a 256-bit vector from an unaligned pointer. This intrinsic
// is expected to perform better than a regular unaligned read when
// the data crosses a cache line, but for Miri this is just a regular
// unaligned read.
"ldu.dq.256" => {
let [src_ptr] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let src_ptr = this.read_pointer(src_ptr)?;
let dest = dest.force_mplace(this)?;
// Unaligned copy, which is what we want.
this.mem_copy(src_ptr, dest.ptr(), dest.layout.size, /*nonoverlapping*/ true)?;
}
// Used to implement the _mm256_testz_si256, _mm256_testc_si256 and
// _mm256_testnzc_si256 functions.
// Tests `op & mask == 0`, `op & mask == mask` or
// `op & mask != 0 && op & mask != mask`
"ptestz.256" | "ptestc.256" | "ptestnzc.256" => {
let [op, mask] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let (all_zero, masked_set) = test_bits_masked(this, op, mask)?;
let res = match unprefixed_name {
"ptestz.256" => all_zero,
"ptestc.256" => masked_set,
"ptestnzc.256" => !all_zero && !masked_set,
_ => unreachable!(),
};
this.write_scalar(Scalar::from_i32(res.into()), dest)?;
}
// Used to implement the _mm256_testz_pd, _mm256_testc_pd, _mm256_testnzc_pd
// _mm_testz_pd, _mm_testc_pd, _mm_testnzc_pd, _mm256_testz_ps,
// _mm256_testc_ps, _mm256_testnzc_ps, _mm_testz_ps, _mm_testc_ps and
// _mm_testnzc_ps functions.
// Calculates two booleans:
// `direct`, which is true when the highest bit of each element of `op & mask` is zero.
// `negated`, which is true when the highest bit of each element of `!op & mask` is zero.
// Return `direct` (testz), `negated` (testc) or `!direct & !negated` (testnzc)
"vtestz.pd.256" | "vtestc.pd.256" | "vtestnzc.pd.256" | "vtestz.pd" | "vtestc.pd"
| "vtestnzc.pd" | "vtestz.ps.256" | "vtestc.ps.256" | "vtestnzc.ps.256"
| "vtestz.ps" | "vtestc.ps" | "vtestnzc.ps" => {
let [op, mask] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let (direct, negated) = test_high_bits_masked(this, op, mask)?;
let res = match unprefixed_name {
"vtestz.pd.256" | "vtestz.pd" | "vtestz.ps.256" | "vtestz.ps" => direct,
"vtestc.pd.256" | "vtestc.pd" | "vtestc.ps.256" | "vtestc.ps" => negated,
"vtestnzc.pd.256" | "vtestnzc.pd" | "vtestnzc.ps.256" | "vtestnzc.ps" =>
!direct && !negated,
_ => unreachable!(),
};
this.write_scalar(Scalar::from_i32(res.into()), dest)?;
}
// Used to implement the `_mm256_zeroupper` and `_mm256_zeroall` functions.
// These function clear out the upper 128 bits of all avx registers or
// zero out all avx registers respectively.
"vzeroupper" | "vzeroall" => {
// These functions are purely a performance hint for the CPU.
// Any registers currently in use will be saved beforehand by the
// compiler, making these functions no-ops.
// The only thing that needs to be ensured is the correct calling convention.
let [] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
}
_ => return Ok(EmulateItemResult::NotSupported),
}
Ok(EmulateItemResult::NeedsReturn)
}
}