#![warn(clippy::arithmetic_side_effects)]
mod atomic;
mod simd;
use std::iter;
use rand::Rng;
use rustc_apfloat::{Float, Round};
use rustc_middle::ty::layout::LayoutOf;
use rustc_middle::{
mir,
ty::{self, FloatTy},
};
use rustc_span::{sym, Symbol};
use rustc_target::abi::Size;
use crate::*;
use atomic::EvalContextExt as _;
use helpers::{check_arg_count, ToHost, ToSoft};
use simd::EvalContextExt as _;
impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {}
pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
fn call_intrinsic(
&mut self,
instance: ty::Instance<'tcx>,
args: &[OpTy<'tcx>],
dest: &MPlaceTy<'tcx>,
ret: Option<mir::BasicBlock>,
unwind: mir::UnwindAction,
) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>> {
let this = self.eval_context_mut();
if this.eval_intrinsic(instance, args, dest, ret)? {
return Ok(None);
}
let intrinsic_name = this.tcx.item_name(instance.def_id());
let intrinsic_name = intrinsic_name.as_str();
match this.emulate_intrinsic_by_name(intrinsic_name, instance.args, args, dest, ret)? {
EmulateItemResult::NotSupported => {
if this.tcx.intrinsic(instance.def_id()).unwrap().must_be_overridden {
throw_unsup_format!("unimplemented intrinsic: `{intrinsic_name}`")
}
let intrinsic_fallback_is_spec = Symbol::intern("intrinsic_fallback_is_spec");
if this
.tcx
.get_attrs_by_path(instance.def_id(), &[sym::miri, intrinsic_fallback_is_spec])
.next()
.is_none()
{
throw_unsup_format!(
"Miri can only use intrinsic fallback bodies that exactly reflect the specification: they fully check for UB and are as non-deterministic as possible. After verifying that `{intrinsic_name}` does so, add the `#[miri::intrinsic_fallback_is_spec]` attribute to it; also ping @rust-lang/miri when you do that"
);
}
Ok(Some(ty::Instance {
def: ty::InstanceKind::Item(instance.def_id()),
args: instance.args,
}))
}
EmulateItemResult::NeedsReturn => {
trace!("{:?}", this.dump_place(&dest.clone().into()));
this.return_to_block(ret)?;
Ok(None)
}
EmulateItemResult::NeedsUnwind => {
this.unwind_to_block(unwind)?;
Ok(None)
}
EmulateItemResult::AlreadyJumped => Ok(None),
}
}
fn emulate_intrinsic_by_name(
&mut self,
intrinsic_name: &str,
generic_args: ty::GenericArgsRef<'tcx>,
args: &[OpTy<'tcx>],
dest: &MPlaceTy<'tcx>,
ret: Option<mir::BasicBlock>,
) -> InterpResult<'tcx, EmulateItemResult> {
let this = self.eval_context_mut();
if let Some(name) = intrinsic_name.strip_prefix("atomic_") {
return this.emulate_atomic_intrinsic(name, args, dest);
}
if let Some(name) = intrinsic_name.strip_prefix("simd_") {
return this.emulate_simd_intrinsic(name, generic_args, args, dest);
}
match intrinsic_name {
"abort" => {
throw_machine_stop!(TerminationInfo::Abort(
"the program aborted execution".to_owned()
));
}
"catch_unwind" => {
this.handle_catch_unwind(args, dest, ret)?;
return Ok(EmulateItemResult::AlreadyJumped);
}
"volatile_load" => {
let [place] = check_arg_count(args)?;
let place = this.deref_pointer(place)?;
this.copy_op(&place, dest)?;
}
"volatile_store" => {
let [place, dest] = check_arg_count(args)?;
let place = this.deref_pointer(place)?;
this.copy_op(dest, &place)?;
}
"write_bytes" | "volatile_set_memory" => {
let [ptr, val_byte, count] = check_arg_count(args)?;
let ty = ptr.layout.ty.builtin_deref(true).unwrap();
let ty_layout = this.layout_of(ty)?;
let val_byte = this.read_scalar(val_byte)?.to_u8()?;
let ptr = this.read_pointer(ptr)?;
let count = this.read_target_usize(count)?;
let byte_count = ty_layout.size.checked_mul(count, this).ok_or_else(|| {
err_ub_format!("overflow computing total size of `{intrinsic_name}`")
})?;
this.write_bytes_ptr(ptr, iter::repeat(val_byte).take(byte_count.bytes_usize()))?;
}
"ptr_mask" => {
let [ptr, mask] = check_arg_count(args)?;
let ptr = this.read_pointer(ptr)?;
let mask = this.read_target_usize(mask)?;
let masked_addr = Size::from_bytes(ptr.addr().bytes() & mask);
this.write_pointer(Pointer::new(ptr.provenance, masked_addr), dest)?;
}
"is_val_statically_known" => {
let [arg] = check_arg_count(args)?;
this.validate_operand(arg, false)?;
let branch: bool = this.machine.rng.get_mut().gen();
this.write_scalar(Scalar::from_bool(branch), dest)?;
}
"fabsf32" => {
let [f] = check_arg_count(args)?;
let f = this.read_scalar(f)?.to_f32()?;
this.write_scalar(Scalar::from_f32(f.abs()), dest)?;
}
"fabsf64" => {
let [f] = check_arg_count(args)?;
let f = this.read_scalar(f)?.to_f64()?;
this.write_scalar(Scalar::from_f64(f.abs()), dest)?;
}
"floorf32" | "ceilf32" | "truncf32" | "roundf32" | "rintf32" => {
let [f] = check_arg_count(args)?;
let f = this.read_scalar(f)?.to_f32()?;
let mode = match intrinsic_name {
"floorf32" => Round::TowardNegative,
"ceilf32" => Round::TowardPositive,
"truncf32" => Round::TowardZero,
"roundf32" => Round::NearestTiesToAway,
"rintf32" => Round::NearestTiesToEven,
_ => bug!(),
};
let res = f.round_to_integral(mode).value;
let res = this.adjust_nan(res, &[f]);
this.write_scalar(res, dest)?;
}
"floorf64" | "ceilf64" | "truncf64" | "roundf64" | "rintf64" => {
let [f] = check_arg_count(args)?;
let f = this.read_scalar(f)?.to_f64()?;
let mode = match intrinsic_name {
"floorf64" => Round::TowardNegative,
"ceilf64" => Round::TowardPositive,
"truncf64" => Round::TowardZero,
"roundf64" => Round::NearestTiesToAway,
"rintf64" => Round::NearestTiesToEven,
_ => bug!(),
};
let res = f.round_to_integral(mode).value;
let res = this.adjust_nan(res, &[f]);
this.write_scalar(res, dest)?;
}
#[rustfmt::skip]
| "sinf32"
| "cosf32"
| "sqrtf32"
| "expf32"
| "exp2f32"
| "logf32"
| "log10f32"
| "log2f32"
=> {
let [f] = check_arg_count(args)?;
let f = this.read_scalar(f)?.to_f32()?;
let f_host = f.to_host();
let res = match intrinsic_name {
"sinf32" => f_host.sin(),
"cosf32" => f_host.cos(),
"sqrtf32" => f_host.sqrt(), "expf32" => f_host.exp(),
"exp2f32" => f_host.exp2(),
"logf32" => f_host.ln(),
"log10f32" => f_host.log10(),
"log2f32" => f_host.log2(),
_ => bug!(),
};
let res = res.to_soft();
let res = this.adjust_nan(res, &[f]);
this.write_scalar(res, dest)?;
}
#[rustfmt::skip]
| "sinf64"
| "cosf64"
| "sqrtf64"
| "expf64"
| "exp2f64"
| "logf64"
| "log10f64"
| "log2f64"
=> {
let [f] = check_arg_count(args)?;
let f = this.read_scalar(f)?.to_f64()?;
let f_host = f.to_host();
let res = match intrinsic_name {
"sinf64" => f_host.sin(),
"cosf64" => f_host.cos(),
"sqrtf64" => f_host.sqrt(), "expf64" => f_host.exp(),
"exp2f64" => f_host.exp2(),
"logf64" => f_host.ln(),
"log10f64" => f_host.log10(),
"log2f64" => f_host.log2(),
_ => bug!(),
};
let res = res.to_soft();
let res = this.adjust_nan(res, &[f]);
this.write_scalar(res, dest)?;
}
"minnumf32" | "maxnumf32" | "copysignf32" => {
let [a, b] = check_arg_count(args)?;
let a = this.read_scalar(a)?.to_f32()?;
let b = this.read_scalar(b)?.to_f32()?;
let res = match intrinsic_name {
"minnumf32" => this.adjust_nan(a.min(b), &[a, b]),
"maxnumf32" => this.adjust_nan(a.max(b), &[a, b]),
"copysignf32" => a.copy_sign(b), _ => bug!(),
};
this.write_scalar(Scalar::from_f32(res), dest)?;
}
"minnumf64" | "maxnumf64" | "copysignf64" => {
let [a, b] = check_arg_count(args)?;
let a = this.read_scalar(a)?.to_f64()?;
let b = this.read_scalar(b)?.to_f64()?;
let res = match intrinsic_name {
"minnumf64" => this.adjust_nan(a.min(b), &[a, b]),
"maxnumf64" => this.adjust_nan(a.max(b), &[a, b]),
"copysignf64" => a.copy_sign(b), _ => bug!(),
};
this.write_scalar(Scalar::from_f64(res), dest)?;
}
"fmaf32" => {
let [a, b, c] = check_arg_count(args)?;
let a = this.read_scalar(a)?.to_f32()?;
let b = this.read_scalar(b)?.to_f32()?;
let c = this.read_scalar(c)?.to_f32()?;
let res = a.to_host().mul_add(b.to_host(), c.to_host()).to_soft();
let res = this.adjust_nan(res, &[a, b, c]);
this.write_scalar(res, dest)?;
}
"fmaf64" => {
let [a, b, c] = check_arg_count(args)?;
let a = this.read_scalar(a)?.to_f64()?;
let b = this.read_scalar(b)?.to_f64()?;
let c = this.read_scalar(c)?.to_f64()?;
let res = a.to_host().mul_add(b.to_host(), c.to_host()).to_soft();
let res = this.adjust_nan(res, &[a, b, c]);
this.write_scalar(res, dest)?;
}
"powf32" => {
let [f1, f2] = check_arg_count(args)?;
let f1 = this.read_scalar(f1)?.to_f32()?;
let f2 = this.read_scalar(f2)?.to_f32()?;
let res = f1.to_host().powf(f2.to_host()).to_soft();
let res = this.adjust_nan(res, &[f1, f2]);
this.write_scalar(res, dest)?;
}
"powf64" => {
let [f1, f2] = check_arg_count(args)?;
let f1 = this.read_scalar(f1)?.to_f64()?;
let f2 = this.read_scalar(f2)?.to_f64()?;
let res = f1.to_host().powf(f2.to_host()).to_soft();
let res = this.adjust_nan(res, &[f1, f2]);
this.write_scalar(res, dest)?;
}
"powif32" => {
let [f, i] = check_arg_count(args)?;
let f = this.read_scalar(f)?.to_f32()?;
let i = this.read_scalar(i)?.to_i32()?;
let res = f.to_host().powi(i).to_soft();
let res = this.adjust_nan(res, &[f]);
this.write_scalar(res, dest)?;
}
"powif64" => {
let [f, i] = check_arg_count(args)?;
let f = this.read_scalar(f)?.to_f64()?;
let i = this.read_scalar(i)?.to_i32()?;
let res = f.to_host().powi(i).to_soft();
let res = this.adjust_nan(res, &[f]);
this.write_scalar(res, dest)?;
}
#[rustfmt::skip]
| "fadd_algebraic"
| "fsub_algebraic"
| "fmul_algebraic"
| "fdiv_algebraic"
| "frem_algebraic"
=> {
let [a, b] = check_arg_count(args)?;
let a = this.read_immediate(a)?;
let b = this.read_immediate(b)?;
let op = match intrinsic_name {
"fadd_algebraic" => mir::BinOp::Add,
"fsub_algebraic" => mir::BinOp::Sub,
"fmul_algebraic" => mir::BinOp::Mul,
"fdiv_algebraic" => mir::BinOp::Div,
"frem_algebraic" => mir::BinOp::Rem,
_ => bug!(),
};
let res = this.binary_op(op, &a, &b)?;
this.write_immediate(*res, dest)?;
}
#[rustfmt::skip]
| "fadd_fast"
| "fsub_fast"
| "fmul_fast"
| "fdiv_fast"
| "frem_fast"
=> {
let [a, b] = check_arg_count(args)?;
let a = this.read_immediate(a)?;
let b = this.read_immediate(b)?;
let op = match intrinsic_name {
"fadd_fast" => mir::BinOp::Add,
"fsub_fast" => mir::BinOp::Sub,
"fmul_fast" => mir::BinOp::Mul,
"fdiv_fast" => mir::BinOp::Div,
"frem_fast" => mir::BinOp::Rem,
_ => bug!(),
};
let float_finite = |x: &ImmTy<'tcx>| -> InterpResult<'tcx, bool> {
let ty::Float(fty) = x.layout.ty.kind() else {
bug!("float_finite: non-float input type {}", x.layout.ty)
};
Ok(match fty {
FloatTy::F16 => x.to_scalar().to_f16()?.is_finite(),
FloatTy::F32 => x.to_scalar().to_f32()?.is_finite(),
FloatTy::F64 => x.to_scalar().to_f64()?.is_finite(),
FloatTy::F128 => x.to_scalar().to_f128()?.is_finite(),
})
};
match (float_finite(&a)?, float_finite(&b)?) {
(false, false) => throw_ub_format!(
"`{intrinsic_name}` intrinsic called with non-finite value as both parameters",
),
(false, _) => throw_ub_format!(
"`{intrinsic_name}` intrinsic called with non-finite value as first parameter",
),
(_, false) => throw_ub_format!(
"`{intrinsic_name}` intrinsic called with non-finite value as second parameter",
),
_ => {}
}
let res = this.binary_op(op, &a, &b)?;
if !float_finite(&res)? {
throw_ub_format!("`{intrinsic_name}` intrinsic produced non-finite value as result");
}
this.write_immediate(*res, dest)?;
}
"float_to_int_unchecked" => {
let [val] = check_arg_count(args)?;
let val = this.read_immediate(val)?;
let res = this
.float_to_int_checked(&val, dest.layout, Round::TowardZero)?
.ok_or_else(|| {
err_ub_format!(
"`float_to_int_unchecked` intrinsic called on {val} which cannot be represented in target type `{:?}`",
dest.layout.ty
)
})?;
this.write_immediate(*res, dest)?;
}
"breakpoint" => {
let [] = check_arg_count(args)?;
throw_machine_stop!(TerminationInfo::Abort(format!("trace/breakpoint trap")))
}
_ => return Ok(EmulateItemResult::NotSupported),
}
Ok(EmulateItemResult::NeedsReturn)
}
}