use rand::Rng as _;
use rustc_apfloat::{ieee::Single, Float};
use rustc_middle::ty::layout::LayoutOf as _;
use rustc_middle::ty::Ty;
use rustc_middle::{mir, ty};
use rustc_span::Symbol;
use rustc_target::abi::Size;
use rustc_target::spec::abi::Abi;
use crate::*;
use helpers::bool_to_simd_element;
mod aesni;
mod avx;
mod avx2;
mod bmi;
mod sha;
mod sse;
mod sse2;
mod sse3;
mod sse41;
mod sse42;
mod ssse3;
impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {}
pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
fn emulate_x86_intrinsic(
&mut self,
link_name: Symbol,
abi: Abi,
args: &[OpTy<'tcx>],
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, EmulateItemResult> {
let this = self.eval_context_mut();
let unprefixed_name = link_name.as_str().strip_prefix("llvm.x86.").unwrap();
match unprefixed_name {
"addcarry.32" | "addcarry.64" | "subborrow.32" | "subborrow.64" => {
if unprefixed_name.ends_with("64") && this.tcx.sess.target.arch != "x86_64" {
return Ok(EmulateItemResult::NotSupported);
}
let [cb_in, a, b] = this.check_shim(abi, Abi::Unadjusted, link_name, args)?;
let op = if unprefixed_name.starts_with("add") {
mir::BinOp::AddWithOverflow
} else {
mir::BinOp::SubWithOverflow
};
let (sum, cb_out) = carrying_add(this, cb_in, a, b, op)?;
this.write_scalar(cb_out, &this.project_field(dest, 0)?)?;
this.write_immediate(*sum, &this.project_field(dest, 1)?)?;
}
"addcarryx.u32" | "addcarryx.u64" => {
this.expect_target_feature_for_intrinsic(link_name, "adx")?;
let is_u64 = unprefixed_name.ends_with("64");
if is_u64 && this.tcx.sess.target.arch != "x86_64" {
return Ok(EmulateItemResult::NotSupported);
}
let [c_in, a, b, out] = this.check_shim(abi, Abi::Unadjusted, link_name, args)?;
let out = this.deref_pointer_as(
out,
if is_u64 { this.machine.layouts.u64 } else { this.machine.layouts.u32 },
)?;
let (sum, c_out) = carrying_add(this, c_in, a, b, mir::BinOp::AddWithOverflow)?;
this.write_scalar(c_out, dest)?;
this.write_immediate(*sum, &out)?;
}
"sse2.pause" => {
let [] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
if this.tcx.sess.unstable_target_features.contains(&Symbol::intern("sse2")) {
this.yield_active_thread();
}
}
"pclmulqdq" => {
let [left, right, imm] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
pclmulqdq(this, left, right, imm, dest)?;
}
name if name.starts_with("bmi.") => {
return bmi::EvalContextExt::emulate_x86_bmi_intrinsic(
this, link_name, abi, args, dest,
);
}
name if name.starts_with("sha") => {
return sha::EvalContextExt::emulate_x86_sha_intrinsic(
this, link_name, abi, args, dest,
);
}
name if name.starts_with("sse.") => {
return sse::EvalContextExt::emulate_x86_sse_intrinsic(
this, link_name, abi, args, dest,
);
}
name if name.starts_with("sse2.") => {
return sse2::EvalContextExt::emulate_x86_sse2_intrinsic(
this, link_name, abi, args, dest,
);
}
name if name.starts_with("sse3.") => {
return sse3::EvalContextExt::emulate_x86_sse3_intrinsic(
this, link_name, abi, args, dest,
);
}
name if name.starts_with("ssse3.") => {
return ssse3::EvalContextExt::emulate_x86_ssse3_intrinsic(
this, link_name, abi, args, dest,
);
}
name if name.starts_with("sse41.") => {
return sse41::EvalContextExt::emulate_x86_sse41_intrinsic(
this, link_name, abi, args, dest,
);
}
name if name.starts_with("sse42.") => {
return sse42::EvalContextExt::emulate_x86_sse42_intrinsic(
this, link_name, abi, args, dest,
);
}
name if name.starts_with("aesni.") => {
return aesni::EvalContextExt::emulate_x86_aesni_intrinsic(
this, link_name, abi, args, dest,
);
}
name if name.starts_with("avx.") => {
return avx::EvalContextExt::emulate_x86_avx_intrinsic(
this, link_name, abi, args, dest,
);
}
name if name.starts_with("avx2.") => {
return avx2::EvalContextExt::emulate_x86_avx2_intrinsic(
this, link_name, abi, args, dest,
);
}
_ => return Ok(EmulateItemResult::NotSupported),
}
Ok(EmulateItemResult::NeedsReturn)
}
}
#[derive(Copy, Clone)]
enum FloatBinOp {
Cmp {
gt: bool,
lt: bool,
eq: bool,
unord: bool,
},
Min,
Max,
}
impl FloatBinOp {
fn cmp_from_imm<'tcx>(
this: &crate::MiriInterpCx<'tcx>,
imm: i8,
intrinsic: Symbol,
) -> InterpResult<'tcx, Self> {
if imm & !0b1_1111 != 0 {
panic!("invalid `imm` parameter of {intrinsic}: 0x{imm:x}");
}
let (gt, lt, eq, mut unord) = match imm & 0b111 {
0x0 => (false, false, true, false),
0x1 => (false, true, false, false),
0x2 => (false, true, true, false),
0x3 => (false, false, false, true),
0x4 => (true, true, false, true),
0x5 => (true, false, true, true),
0x6 => (true, false, false, true),
0x7 => (true, true, true, false),
_ => unreachable!(),
};
if imm & 0b1000 != 0 {
this.expect_target_feature_for_intrinsic(intrinsic, "avx")?;
unord = !unord;
}
Ok(Self::Cmp { gt, lt, eq, unord })
}
}
fn bin_op_float<'tcx, F: rustc_apfloat::Float>(
which: FloatBinOp,
left: &ImmTy<'tcx>,
right: &ImmTy<'tcx>,
) -> InterpResult<'tcx, Scalar> {
match which {
FloatBinOp::Cmp { gt, lt, eq, unord } => {
let left = left.to_scalar().to_float::<F>()?;
let right = right.to_scalar().to_float::<F>()?;
let res = match left.partial_cmp(&right) {
None => unord,
Some(std::cmp::Ordering::Less) => lt,
Some(std::cmp::Ordering::Equal) => eq,
Some(std::cmp::Ordering::Greater) => gt,
};
Ok(bool_to_simd_element(res, Size::from_bits(F::BITS)))
}
FloatBinOp::Min => {
let left_scalar = left.to_scalar();
let left = left_scalar.to_float::<F>()?;
let right_scalar = right.to_scalar();
let right = right_scalar.to_float::<F>()?;
if (left == F::ZERO && right == F::ZERO)
|| left.is_nan()
|| right.is_nan()
|| left >= right
{
Ok(right_scalar)
} else {
Ok(left_scalar)
}
}
FloatBinOp::Max => {
let left_scalar = left.to_scalar();
let left = left_scalar.to_float::<F>()?;
let right_scalar = right.to_scalar();
let right = right_scalar.to_float::<F>()?;
if (left == F::ZERO && right == F::ZERO)
|| left.is_nan()
|| right.is_nan()
|| left <= right
{
Ok(right_scalar)
} else {
Ok(left_scalar)
}
}
}
}
fn bin_op_simd_float_first<'tcx, F: rustc_apfloat::Float>(
this: &mut crate::MiriInterpCx<'tcx>,
which: FloatBinOp,
left: &OpTy<'tcx>,
right: &OpTy<'tcx>,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
let (left, left_len) = this.operand_to_simd(left)?;
let (right, right_len) = this.operand_to_simd(right)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
assert_eq!(dest_len, left_len);
assert_eq!(dest_len, right_len);
let res0 = bin_op_float::<F>(
which,
&this.read_immediate(&this.project_index(&left, 0)?)?,
&this.read_immediate(&this.project_index(&right, 0)?)?,
)?;
this.write_scalar(res0, &this.project_index(&dest, 0)?)?;
for i in 1..dest_len {
this.copy_op(&this.project_index(&left, i)?, &this.project_index(&dest, i)?)?;
}
Ok(())
}
fn bin_op_simd_float_all<'tcx, F: rustc_apfloat::Float>(
this: &mut crate::MiriInterpCx<'tcx>,
which: FloatBinOp,
left: &OpTy<'tcx>,
right: &OpTy<'tcx>,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
let (left, left_len) = this.operand_to_simd(left)?;
let (right, right_len) = this.operand_to_simd(right)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
assert_eq!(dest_len, left_len);
assert_eq!(dest_len, right_len);
for i in 0..dest_len {
let left = this.read_immediate(&this.project_index(&left, i)?)?;
let right = this.read_immediate(&this.project_index(&right, i)?)?;
let dest = this.project_index(&dest, i)?;
let res = bin_op_float::<F>(which, &left, &right)?;
this.write_scalar(res, &dest)?;
}
Ok(())
}
#[derive(Copy, Clone)]
enum FloatUnaryOp {
Rcp,
Rsqrt,
}
#[allow(clippy::arithmetic_side_effects)] fn unary_op_f32<'tcx>(
this: &mut crate::MiriInterpCx<'tcx>,
which: FloatUnaryOp,
op: &ImmTy<'tcx>,
) -> InterpResult<'tcx, Scalar> {
match which {
FloatUnaryOp::Rcp => {
let op = op.to_scalar().to_f32()?;
let div = (Single::from_u128(1).value / op).value;
let res = apply_random_float_error(this, div, -12);
Ok(Scalar::from_f32(res))
}
FloatUnaryOp::Rsqrt => {
let op = op.to_scalar().to_u32()?;
let sqrt = Single::from_bits(f32::from_bits(op).sqrt().to_bits().into());
let rsqrt = (Single::from_u128(1).value / sqrt).value;
let res = apply_random_float_error(this, rsqrt, -12);
Ok(Scalar::from_f32(res))
}
}
}
#[allow(clippy::arithmetic_side_effects)] fn apply_random_float_error<F: rustc_apfloat::Float>(
this: &mut crate::MiriInterpCx<'_>,
val: F,
err_scale: i32,
) -> F {
let rng = this.machine.rng.get_mut();
let err = F::from_u128(rng.gen::<u64>().into()).value.scalbn(err_scale.strict_sub(64));
let err = if rng.gen::<bool>() { -err } else { err };
(val * (F::from_u128(1).value + err).value).value
}
fn unary_op_ss<'tcx>(
this: &mut crate::MiriInterpCx<'tcx>,
which: FloatUnaryOp,
op: &OpTy<'tcx>,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
let (op, op_len) = this.operand_to_simd(op)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
assert_eq!(dest_len, op_len);
let res0 = unary_op_f32(this, which, &this.read_immediate(&this.project_index(&op, 0)?)?)?;
this.write_scalar(res0, &this.project_index(&dest, 0)?)?;
for i in 1..dest_len {
this.copy_op(&this.project_index(&op, i)?, &this.project_index(&dest, i)?)?;
}
Ok(())
}
fn unary_op_ps<'tcx>(
this: &mut crate::MiriInterpCx<'tcx>,
which: FloatUnaryOp,
op: &OpTy<'tcx>,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
let (op, op_len) = this.operand_to_simd(op)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
assert_eq!(dest_len, op_len);
for i in 0..dest_len {
let op = this.read_immediate(&this.project_index(&op, i)?)?;
let dest = this.project_index(&dest, i)?;
let res = unary_op_f32(this, which, &op)?;
this.write_scalar(res, &dest)?;
}
Ok(())
}
enum ShiftOp {
Left,
RightLogic,
RightArith,
}
fn shift_simd_by_scalar<'tcx>(
this: &mut crate::MiriInterpCx<'tcx>,
left: &OpTy<'tcx>,
right: &OpTy<'tcx>,
which: ShiftOp,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
let (left, left_len) = this.operand_to_simd(left)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
assert_eq!(dest_len, left_len);
let shift = u32::try_from(extract_first_u64(this, right)?).unwrap_or(u32::MAX);
for i in 0..dest_len {
let left = this.read_scalar(&this.project_index(&left, i)?)?;
let dest = this.project_index(&dest, i)?;
let res = match which {
ShiftOp::Left => {
let left = left.to_uint(dest.layout.size)?;
let res = left.checked_shl(shift).unwrap_or(0);
Scalar::from_uint(dest.layout.size.truncate(res), dest.layout.size)
}
ShiftOp::RightLogic => {
let left = left.to_uint(dest.layout.size)?;
let res = left.checked_shr(shift).unwrap_or(0);
Scalar::from_uint(res, dest.layout.size)
}
ShiftOp::RightArith => {
let left = left.to_int(dest.layout.size)?;
let res = left.checked_shr(shift).unwrap_or(left >> 127);
Scalar::from_int(res, dest.layout.size)
}
};
this.write_scalar(res, &dest)?;
}
Ok(())
}
fn shift_simd_by_simd<'tcx>(
this: &mut crate::MiriInterpCx<'tcx>,
left: &OpTy<'tcx>,
right: &OpTy<'tcx>,
which: ShiftOp,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
let (left, left_len) = this.operand_to_simd(left)?;
let (right, right_len) = this.operand_to_simd(right)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
assert_eq!(dest_len, left_len);
assert_eq!(dest_len, right_len);
for i in 0..dest_len {
let left = this.read_scalar(&this.project_index(&left, i)?)?;
let right = this.read_scalar(&this.project_index(&right, i)?)?;
let dest = this.project_index(&dest, i)?;
let shift = u32::try_from(right.to_uint(dest.layout.size)?).unwrap_or(u32::MAX);
let res = match which {
ShiftOp::Left => {
let left = left.to_uint(dest.layout.size)?;
let res = left.checked_shl(shift).unwrap_or(0);
Scalar::from_uint(dest.layout.size.truncate(res), dest.layout.size)
}
ShiftOp::RightLogic => {
let left = left.to_uint(dest.layout.size)?;
let res = left.checked_shr(shift).unwrap_or(0);
Scalar::from_uint(res, dest.layout.size)
}
ShiftOp::RightArith => {
let left = left.to_int(dest.layout.size)?;
let res = left.checked_shr(shift).unwrap_or(left >> 127);
Scalar::from_int(res, dest.layout.size)
}
};
this.write_scalar(res, &dest)?;
}
Ok(())
}
fn extract_first_u64<'tcx>(
this: &crate::MiriInterpCx<'tcx>,
op: &OpTy<'tcx>,
) -> InterpResult<'tcx, u64> {
let array_layout = this.layout_of(Ty::new_array(this.tcx.tcx, this.tcx.types.u64, 2))?;
let op = op.transmute(array_layout, this)?;
this.read_scalar(&this.project_index(&op, 0)?)?.to_u64()
}
fn round_first<'tcx, F: rustc_apfloat::Float>(
this: &mut crate::MiriInterpCx<'tcx>,
left: &OpTy<'tcx>,
right: &OpTy<'tcx>,
rounding: &OpTy<'tcx>,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
let (left, left_len) = this.operand_to_simd(left)?;
let (right, right_len) = this.operand_to_simd(right)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
assert_eq!(dest_len, left_len);
assert_eq!(dest_len, right_len);
let rounding = rounding_from_imm(this.read_scalar(rounding)?.to_i32()?)?;
let op0: F = this.read_scalar(&this.project_index(&right, 0)?)?.to_float()?;
let res = op0.round_to_integral(rounding).value;
this.write_scalar(
Scalar::from_uint(res.to_bits(), Size::from_bits(F::BITS)),
&this.project_index(&dest, 0)?,
)?;
for i in 1..dest_len {
this.copy_op(&this.project_index(&left, i)?, &this.project_index(&dest, i)?)?;
}
Ok(())
}
fn round_all<'tcx, F: rustc_apfloat::Float>(
this: &mut crate::MiriInterpCx<'tcx>,
op: &OpTy<'tcx>,
rounding: &OpTy<'tcx>,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
let (op, op_len) = this.operand_to_simd(op)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
assert_eq!(dest_len, op_len);
let rounding = rounding_from_imm(this.read_scalar(rounding)?.to_i32()?)?;
for i in 0..dest_len {
let op: F = this.read_scalar(&this.project_index(&op, i)?)?.to_float()?;
let res = op.round_to_integral(rounding).value;
this.write_scalar(
Scalar::from_uint(res.to_bits(), Size::from_bits(F::BITS)),
&this.project_index(&dest, i)?,
)?;
}
Ok(())
}
fn rounding_from_imm<'tcx>(rounding: i32) -> InterpResult<'tcx, rustc_apfloat::Round> {
match rounding & !0b1000 {
0b000 => Ok(rustc_apfloat::Round::NearestTiesToEven),
0b001 => Ok(rustc_apfloat::Round::TowardNegative),
0b010 => Ok(rustc_apfloat::Round::TowardPositive),
0b011 => Ok(rustc_apfloat::Round::TowardZero),
0b100..=0b111 => Ok(rustc_apfloat::Round::NearestTiesToEven),
rounding => panic!("invalid rounding mode 0x{rounding:02x}"),
}
}
fn convert_float_to_int<'tcx>(
this: &mut crate::MiriInterpCx<'tcx>,
op: &OpTy<'tcx>,
rnd: rustc_apfloat::Round,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
let (op, op_len) = this.operand_to_simd(op)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
assert!(matches!(dest.layout.field(this, 0).ty.kind(), ty::Int(_)));
for i in 0..op_len.min(dest_len) {
let op = this.read_immediate(&this.project_index(&op, i)?)?;
let dest = this.project_index(&dest, i)?;
let res = this.float_to_int_checked(&op, dest.layout, rnd)?.unwrap_or_else(|| {
ImmTy::from_int(dest.layout.size.signed_int_min(), dest.layout)
});
this.write_immediate(*res, &dest)?;
}
for i in op_len..dest_len {
let dest = this.project_index(&dest, i)?;
this.write_scalar(Scalar::from_int(0, dest.layout.size), &dest)?;
}
Ok(())
}
fn int_abs<'tcx>(
this: &mut crate::MiriInterpCx<'tcx>,
op: &OpTy<'tcx>,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
let (op, op_len) = this.operand_to_simd(op)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
assert_eq!(op_len, dest_len);
let zero = ImmTy::from_int(0, op.layout.field(this, 0));
for i in 0..dest_len {
let op = this.read_immediate(&this.project_index(&op, i)?)?;
let dest = this.project_index(&dest, i)?;
let lt_zero = this.binary_op(mir::BinOp::Lt, &op, &zero)?;
let res =
if lt_zero.to_scalar().to_bool()? { this.unary_op(mir::UnOp::Neg, &op)? } else { op };
this.write_immediate(*res, &dest)?;
}
Ok(())
}
fn split_simd_to_128bit_chunks<'tcx, P: Projectable<'tcx, Provenance>>(
this: &mut crate::MiriInterpCx<'tcx>,
op: &P,
) -> InterpResult<'tcx, (u64, u64, P)> {
let simd_layout = op.layout();
let (simd_len, element_ty) = simd_layout.ty.simd_size_and_type(this.tcx.tcx);
assert_eq!(simd_layout.size.bits() % 128, 0);
let num_chunks = simd_layout.size.bits() / 128;
let items_per_chunk = simd_len.strict_div(num_chunks);
let chunked_layout = this
.layout_of(Ty::new_array(
this.tcx.tcx,
Ty::new_array(this.tcx.tcx, element_ty, items_per_chunk),
num_chunks,
))
.unwrap();
let chunked_op = op.transmute(chunked_layout, this)?;
Ok((num_chunks, items_per_chunk, chunked_op))
}
fn horizontal_bin_op<'tcx>(
this: &mut crate::MiriInterpCx<'tcx>,
which: mir::BinOp,
saturating: bool,
left: &OpTy<'tcx>,
right: &OpTy<'tcx>,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
assert_eq!(left.layout, dest.layout);
assert_eq!(right.layout, dest.layout);
let (num_chunks, items_per_chunk, left) = split_simd_to_128bit_chunks(this, left)?;
let (_, _, right) = split_simd_to_128bit_chunks(this, right)?;
let (_, _, dest) = split_simd_to_128bit_chunks(this, dest)?;
let middle = items_per_chunk / 2;
for i in 0..num_chunks {
let left = this.project_index(&left, i)?;
let right = this.project_index(&right, i)?;
let dest = this.project_index(&dest, i)?;
for j in 0..items_per_chunk {
let (k, src) = if j < middle { (j, &left) } else { (j.strict_sub(middle), &right) };
let base_i = k.strict_mul(2);
let lhs = this.read_immediate(&this.project_index(src, base_i)?)?;
let rhs = this.read_immediate(&this.project_index(src, base_i.strict_add(1))?)?;
let res = if saturating {
Immediate::from(this.saturating_arith(which, &lhs, &rhs)?)
} else {
*this.binary_op(which, &lhs, &rhs)?
};
this.write_immediate(res, &this.project_index(&dest, j)?)?;
}
}
Ok(())
}
fn conditional_dot_product<'tcx>(
this: &mut crate::MiriInterpCx<'tcx>,
left: &OpTy<'tcx>,
right: &OpTy<'tcx>,
imm: &OpTy<'tcx>,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
assert_eq!(left.layout, dest.layout);
assert_eq!(right.layout, dest.layout);
let (num_chunks, items_per_chunk, left) = split_simd_to_128bit_chunks(this, left)?;
let (_, _, right) = split_simd_to_128bit_chunks(this, right)?;
let (_, _, dest) = split_simd_to_128bit_chunks(this, dest)?;
let element_layout = left.layout.field(this, 0).field(this, 0);
assert!(items_per_chunk <= 4);
let imm = this.read_scalar(imm)?.to_uint(imm.layout.size)?;
for i in 0..num_chunks {
let left = this.project_index(&left, i)?;
let right = this.project_index(&right, i)?;
let dest = this.project_index(&dest, i)?;
let mut sum = ImmTy::from_int(0u8, element_layout);
for j in 0..items_per_chunk {
if imm & (1 << j.strict_add(4)) != 0 {
let left = this.read_immediate(&this.project_index(&left, j)?)?;
let right = this.read_immediate(&this.project_index(&right, j)?)?;
let mul = this.binary_op(mir::BinOp::Mul, &left, &right)?;
sum = this.binary_op(mir::BinOp::Add, &sum, &mul)?;
}
}
for j in 0..items_per_chunk {
let dest = this.project_index(&dest, j)?;
if imm & (1 << j) != 0 {
this.write_immediate(*sum, &dest)?;
} else {
this.write_scalar(Scalar::from_int(0u8, element_layout.size), &dest)?;
}
}
}
Ok(())
}
fn test_bits_masked<'tcx>(
this: &crate::MiriInterpCx<'tcx>,
op: &OpTy<'tcx>,
mask: &OpTy<'tcx>,
) -> InterpResult<'tcx, (bool, bool)> {
assert_eq!(op.layout, mask.layout);
let (op, op_len) = this.operand_to_simd(op)?;
let (mask, mask_len) = this.operand_to_simd(mask)?;
assert_eq!(op_len, mask_len);
let mut all_zero = true;
let mut masked_set = true;
for i in 0..op_len {
let op = this.project_index(&op, i)?;
let mask = this.project_index(&mask, i)?;
let op = this.read_scalar(&op)?.to_uint(op.layout.size)?;
let mask = this.read_scalar(&mask)?.to_uint(mask.layout.size)?;
all_zero &= (op & mask) == 0;
masked_set &= (op & mask) == mask;
}
Ok((all_zero, masked_set))
}
fn test_high_bits_masked<'tcx>(
this: &crate::MiriInterpCx<'tcx>,
op: &OpTy<'tcx>,
mask: &OpTy<'tcx>,
) -> InterpResult<'tcx, (bool, bool)> {
assert_eq!(op.layout, mask.layout);
let (op, op_len) = this.operand_to_simd(op)?;
let (mask, mask_len) = this.operand_to_simd(mask)?;
assert_eq!(op_len, mask_len);
let high_bit_offset = op.layout.field(this, 0).size.bits().strict_sub(1);
let mut direct = true;
let mut negated = true;
for i in 0..op_len {
let op = this.project_index(&op, i)?;
let mask = this.project_index(&mask, i)?;
let op = this.read_scalar(&op)?.to_uint(op.layout.size)?;
let mask = this.read_scalar(&mask)?.to_uint(mask.layout.size)?;
direct &= (op & mask) >> high_bit_offset == 0;
negated &= (!op & mask) >> high_bit_offset == 0;
}
Ok((direct, negated))
}
fn mask_load<'tcx>(
this: &mut crate::MiriInterpCx<'tcx>,
ptr: &OpTy<'tcx>,
mask: &OpTy<'tcx>,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
let (mask, mask_len) = this.operand_to_simd(mask)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
assert_eq!(dest_len, mask_len);
let mask_item_size = mask.layout.field(this, 0).size;
let high_bit_offset = mask_item_size.bits().strict_sub(1);
let ptr = this.read_pointer(ptr)?;
for i in 0..dest_len {
let mask = this.project_index(&mask, i)?;
let dest = this.project_index(&dest, i)?;
if this.read_scalar(&mask)?.to_uint(mask_item_size)? >> high_bit_offset != 0 {
let ptr = ptr.wrapping_offset(dest.layout.size * i, &this.tcx);
this.mem_copy(ptr, dest.ptr(), dest.layout.size, true)?;
} else {
this.write_scalar(Scalar::from_int(0, dest.layout.size), &dest)?;
}
}
Ok(())
}
fn mask_store<'tcx>(
this: &mut crate::MiriInterpCx<'tcx>,
ptr: &OpTy<'tcx>,
mask: &OpTy<'tcx>,
value: &OpTy<'tcx>,
) -> InterpResult<'tcx, ()> {
let (mask, mask_len) = this.operand_to_simd(mask)?;
let (value, value_len) = this.operand_to_simd(value)?;
assert_eq!(value_len, mask_len);
let mask_item_size = mask.layout.field(this, 0).size;
let high_bit_offset = mask_item_size.bits().strict_sub(1);
let ptr = this.read_pointer(ptr)?;
for i in 0..value_len {
let mask = this.project_index(&mask, i)?;
let value = this.project_index(&value, i)?;
if this.read_scalar(&mask)?.to_uint(mask_item_size)? >> high_bit_offset != 0 {
let ptr = ptr.wrapping_offset(value.layout.size * i, &this.tcx);
this.mem_copy(value.ptr(), ptr, value.layout.size, true)?;
}
}
Ok(())
}
fn mpsadbw<'tcx>(
this: &mut crate::MiriInterpCx<'tcx>,
left: &OpTy<'tcx>,
right: &OpTy<'tcx>,
imm: &OpTy<'tcx>,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
assert_eq!(left.layout, right.layout);
assert_eq!(left.layout.size, dest.layout.size);
let (num_chunks, op_items_per_chunk, left) = split_simd_to_128bit_chunks(this, left)?;
let (_, _, right) = split_simd_to_128bit_chunks(this, right)?;
let (_, dest_items_per_chunk, dest) = split_simd_to_128bit_chunks(this, dest)?;
assert_eq!(op_items_per_chunk, dest_items_per_chunk.strict_mul(2));
let imm = this.read_scalar(imm)?.to_uint(imm.layout.size)?;
let left_offset = u64::try_from((imm >> 2) & 1).unwrap().strict_mul(4);
let right_offset = u64::try_from(imm & 0b11).unwrap().strict_mul(4);
for i in 0..num_chunks {
let left = this.project_index(&left, i)?;
let right = this.project_index(&right, i)?;
let dest = this.project_index(&dest, i)?;
for j in 0..dest_items_per_chunk {
let left_offset = left_offset.strict_add(j);
let mut res: u16 = 0;
for k in 0..4 {
let left = this
.read_scalar(&this.project_index(&left, left_offset.strict_add(k))?)?
.to_u8()?;
let right = this
.read_scalar(&this.project_index(&right, right_offset.strict_add(k))?)?
.to_u8()?;
res = res.strict_add(left.abs_diff(right).into());
}
this.write_scalar(Scalar::from_u16(res), &this.project_index(&dest, j)?)?;
}
}
Ok(())
}
fn pmulhrsw<'tcx>(
this: &mut crate::MiriInterpCx<'tcx>,
left: &OpTy<'tcx>,
right: &OpTy<'tcx>,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
let (left, left_len) = this.operand_to_simd(left)?;
let (right, right_len) = this.operand_to_simd(right)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
assert_eq!(dest_len, left_len);
assert_eq!(dest_len, right_len);
for i in 0..dest_len {
let left = this.read_scalar(&this.project_index(&left, i)?)?.to_i16()?;
let right = this.read_scalar(&this.project_index(&right, i)?)?.to_i16()?;
let dest = this.project_index(&dest, i)?;
let res = (i32::from(left).strict_mul(right.into()) >> 14).strict_add(1) >> 1;
#[allow(clippy::cast_possible_truncation)]
let res = res as i16;
this.write_scalar(Scalar::from_i16(res), &dest)?;
}
Ok(())
}
fn pclmulqdq<'tcx>(
this: &mut MiriInterpCx<'tcx>,
left: &OpTy<'tcx>,
right: &OpTy<'tcx>,
imm8: &OpTy<'tcx>,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
assert_eq!(left.layout, right.layout);
assert_eq!(left.layout.size, dest.layout.size);
let array_layout = this.layout_of(Ty::new_array(this.tcx.tcx, this.tcx.types.u64, 2))?;
let left = left.transmute(array_layout, this)?;
let right = right.transmute(array_layout, this)?;
let dest = dest.transmute(array_layout, this)?;
let imm8 = this.read_scalar(imm8)?.to_u8()?;
let index = if (imm8 & 0x01) == 0 { 0 } else { 1 };
let left = this.read_scalar(&this.project_index(&left, index)?)?.to_u64()?;
let index = if (imm8 & 0x10) == 0 { 0 } else { 1 };
let right = this.read_scalar(&this.project_index(&right, index)?)?.to_u64()?;
let mut result: u128 = 0;
for i in 0..64 {
if (right & (1 << i)) != 0 {
result ^= u128::from(left) << i;
}
}
let result_low = (result & 0xFFFF_FFFF_FFFF_FFFF) as u64;
let result_high = (result >> 64) as u64;
let dest_low = this.project_index(&dest, 0)?;
this.write_scalar(Scalar::from_u64(result_low), &dest_low)?;
let dest_high = this.project_index(&dest, 1)?;
this.write_scalar(Scalar::from_u64(result_high), &dest_high)?;
Ok(())
}
fn pack_generic<'tcx>(
this: &mut crate::MiriInterpCx<'tcx>,
left: &OpTy<'tcx>,
right: &OpTy<'tcx>,
dest: &MPlaceTy<'tcx>,
f: impl Fn(Scalar) -> InterpResult<'tcx, Scalar>,
) -> InterpResult<'tcx, ()> {
assert_eq!(left.layout, right.layout);
assert_eq!(left.layout.size, dest.layout.size);
let (num_chunks, op_items_per_chunk, left) = split_simd_to_128bit_chunks(this, left)?;
let (_, _, right) = split_simd_to_128bit_chunks(this, right)?;
let (_, dest_items_per_chunk, dest) = split_simd_to_128bit_chunks(this, dest)?;
assert_eq!(dest_items_per_chunk, op_items_per_chunk.strict_mul(2));
for i in 0..num_chunks {
let left = this.project_index(&left, i)?;
let right = this.project_index(&right, i)?;
let dest = this.project_index(&dest, i)?;
for j in 0..op_items_per_chunk {
let left = this.read_scalar(&this.project_index(&left, j)?)?;
let right = this.read_scalar(&this.project_index(&right, j)?)?;
let left_dest = this.project_index(&dest, j)?;
let right_dest = this.project_index(&dest, j.strict_add(op_items_per_chunk))?;
let left_res = f(left)?;
let right_res = f(right)?;
this.write_scalar(left_res, &left_dest)?;
this.write_scalar(right_res, &right_dest)?;
}
}
Ok(())
}
fn packsswb<'tcx>(
this: &mut crate::MiriInterpCx<'tcx>,
left: &OpTy<'tcx>,
right: &OpTy<'tcx>,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
pack_generic(this, left, right, dest, |op| {
let op = op.to_i16()?;
let res = i8::try_from(op).unwrap_or(if op < 0 { i8::MIN } else { i8::MAX });
Ok(Scalar::from_i8(res))
})
}
fn packuswb<'tcx>(
this: &mut crate::MiriInterpCx<'tcx>,
left: &OpTy<'tcx>,
right: &OpTy<'tcx>,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
pack_generic(this, left, right, dest, |op| {
let op = op.to_i16()?;
let res = u8::try_from(op).unwrap_or(if op < 0 { 0 } else { u8::MAX });
Ok(Scalar::from_u8(res))
})
}
fn packssdw<'tcx>(
this: &mut crate::MiriInterpCx<'tcx>,
left: &OpTy<'tcx>,
right: &OpTy<'tcx>,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
pack_generic(this, left, right, dest, |op| {
let op = op.to_i32()?;
let res = i16::try_from(op).unwrap_or(if op < 0 { i16::MIN } else { i16::MAX });
Ok(Scalar::from_i16(res))
})
}
fn packusdw<'tcx>(
this: &mut crate::MiriInterpCx<'tcx>,
left: &OpTy<'tcx>,
right: &OpTy<'tcx>,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
pack_generic(this, left, right, dest, |op| {
let op = op.to_i32()?;
let res = u16::try_from(op).unwrap_or(if op < 0 { 0 } else { u16::MAX });
Ok(Scalar::from_u16(res))
})
}
fn psign<'tcx>(
this: &mut crate::MiriInterpCx<'tcx>,
left: &OpTy<'tcx>,
right: &OpTy<'tcx>,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
let (left, left_len) = this.operand_to_simd(left)?;
let (right, right_len) = this.operand_to_simd(right)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
assert_eq!(dest_len, left_len);
assert_eq!(dest_len, right_len);
for i in 0..dest_len {
let dest = this.project_index(&dest, i)?;
let left = this.read_immediate(&this.project_index(&left, i)?)?;
let right = this.read_scalar(&this.project_index(&right, i)?)?.to_int(dest.layout.size)?;
let res =
this.binary_op(mir::BinOp::Mul, &left, &ImmTy::from_int(right.signum(), dest.layout))?;
this.write_immediate(*res, &dest)?;
}
Ok(())
}
fn carrying_add<'tcx>(
this: &mut crate::MiriInterpCx<'tcx>,
cb_in: &OpTy<'tcx>,
a: &OpTy<'tcx>,
b: &OpTy<'tcx>,
op: mir::BinOp,
) -> InterpResult<'tcx, (ImmTy<'tcx>, Scalar)> {
assert!(op == mir::BinOp::AddWithOverflow || op == mir::BinOp::SubWithOverflow);
let cb_in = this.read_scalar(cb_in)?.to_u8()? != 0;
let a = this.read_immediate(a)?;
let b = this.read_immediate(b)?;
let (sum, overflow1) = this.binary_op(op, &a, &b)?.to_pair(this);
let (sum, overflow2) =
this.binary_op(op, &sum, &ImmTy::from_uint(cb_in, a.layout))?.to_pair(this);
let cb_out = overflow1.to_scalar().to_bool()? | overflow2.to_scalar().to_bool()?;
Ok((sum, Scalar::from_u8(cb_out.into())))
}