miri/intrinsics/
mod.rs

1#![warn(clippy::arithmetic_side_effects)]
2
3mod aarch64;
4mod atomic;
5mod loongarch;
6mod math;
7mod simd;
8mod x86;
9
10pub use self::atomic::AtomicRmwOp;
11
12#[rustfmt::skip] // prevent `use` reordering
13use rand::RngExt;
14use rustc_abi::{Endian, Size};
15use rustc_middle::{mir, ty};
16use rustc_span::{Symbol, sym};
17use rustc_target::spec::Arch;
18
19use self::atomic::EvalContextExt as _;
20use self::math::EvalContextExt as _;
21use self::simd::EvalContextExt as _;
22use crate::*;
23
24/// Check that the number of args is what we expect.
25fn check_intrinsic_arg_count<'a, 'tcx, const N: usize>(
26    args: &'a [OpTy<'tcx>],
27) -> InterpResult<'tcx, &'a [OpTy<'tcx>; N]>
28where
29    &'a [OpTy<'tcx>; N]: TryFrom<&'a [OpTy<'tcx>]>,
30{
31    if let Ok(ops) = args.try_into() {
32        return interp_ok(ops);
33    }
34    throw_ub_format!(
35        "incorrect number of arguments for intrinsic: got {}, expected {}",
36        args.len(),
37        N
38    )
39}
40
41impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {}
42pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
43    fn call_intrinsic(
44        &mut self,
45        instance: ty::Instance<'tcx>,
46        args: &[OpTy<'tcx>],
47        dest: &PlaceTy<'tcx>,
48        ret: Option<mir::BasicBlock>,
49        unwind: mir::UnwindAction,
50    ) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>> {
51        let this = self.eval_context_mut();
52
53        // See if the core engine can handle this intrinsic.
54        if this.eval_intrinsic(instance, args, dest, ret)? {
55            return interp_ok(None);
56        }
57        let intrinsic_name = this.tcx.item_name(instance.def_id());
58        let intrinsic_name = intrinsic_name.as_str();
59
60        // FIXME: avoid allocating memory
61        let dest = this.force_allocation(dest)?;
62
63        let res =
64            this.emulate_intrinsic_by_name(intrinsic_name, instance.args, args, &dest, ret)?;
65        res.jump_to_next_block(this, &dest, ret, Some(unwind), |this| {
66            // We haven't handled the intrinsic, let's see if we can use a fallback body.
67            if this.tcx.intrinsic(instance.def_id()).unwrap().must_be_overridden {
68                throw_unsup_format!("unimplemented intrinsic: `{intrinsic_name}`")
69            }
70            let intrinsic_fallback_is_spec = Symbol::intern("intrinsic_fallback_is_spec");
71            if this
72                .tcx
73                .get_attrs_by_path(instance.def_id(), &[sym::miri, intrinsic_fallback_is_spec])
74                .next()
75                .is_none()
76            {
77                throw_unsup_format!(
78                    "Miri can only use intrinsic fallback bodies that exactly reflect the specification: they fully check for UB and are as non-deterministic as possible. After verifying that `{intrinsic_name}` does so, add the `#[miri::intrinsic_fallback_is_spec]` attribute to it; also ping @rust-lang/miri when you do that"
79                );
80            }
81            interp_ok(Some(ty::Instance {
82                def: ty::InstanceKind::Item(instance.def_id()),
83                args: instance.args,
84            }))
85        })
86    }
87
88    /// Emulates a Miri-supported intrinsic (not supported by the core engine).
89    /// Returns `Ok(true)` if the intrinsic was handled.
90    fn emulate_intrinsic_by_name(
91        &mut self,
92        intrinsic_name: &str,
93        generic_args: ty::GenericArgsRef<'tcx>,
94        args: &[OpTy<'tcx>],
95        dest: &MPlaceTy<'tcx>,
96        ret: Option<mir::BasicBlock>,
97    ) -> InterpResult<'tcx, EmulateItemResult> {
98        let this = self.eval_context_mut();
99
100        if let Some(name) = intrinsic_name.strip_prefix("atomic_") {
101            return this.emulate_atomic_intrinsic(name, generic_args, args, dest);
102        }
103        if let Some(name) = intrinsic_name.strip_prefix("simd_") {
104            return this.emulate_simd_intrinsic(name, args, dest);
105        }
106
107        match intrinsic_name {
108            // Basic control flow
109            "abort" => {
110                throw_machine_stop!(TerminationInfo::Abort(
111                    "the program aborted execution".to_owned()
112                ));
113            }
114            "catch_unwind" => {
115                let [try_fn, data, catch_fn] = check_intrinsic_arg_count(args)?;
116                this.handle_catch_unwind(try_fn, data, catch_fn, dest, ret)?;
117                // This pushed a stack frame, don't jump to `ret`.
118                return interp_ok(EmulateItemResult::AlreadyJumped);
119            }
120
121            // Memory model / provenance manipulation
122            "ptr_mask" => {
123                let [ptr, mask] = check_intrinsic_arg_count(args)?;
124
125                let ptr = this.read_pointer(ptr)?;
126                let mask = this.read_target_usize(mask)?;
127
128                let masked_addr = Size::from_bytes(ptr.addr().bytes() & mask);
129
130                this.write_pointer(Pointer::new(ptr.provenance, masked_addr), dest)?;
131            }
132
133            // We want to return either `true` or `false` at random, or else something like
134            // ```
135            // if !is_val_statically_known(0) { unreachable_unchecked(); }
136            // ```
137            // Would not be considered UB, or the other way around (`is_val_statically_known(0)`).
138            "is_val_statically_known" => {
139                let [_arg] = check_intrinsic_arg_count(args)?;
140                // FIXME: should we check for validity here? It's tricky because we do not have a
141                // place. Codegen does not seem to set any attributes like `noundef` for intrinsic
142                // calls, so we don't *have* to do anything.
143                let branch: bool = this.machine.rng.get_mut().random();
144                this.write_scalar(Scalar::from_bool(branch), dest)?;
145            }
146
147            // Other
148            "breakpoint" => {
149                let [] = check_intrinsic_arg_count(args)?;
150                // normally this would raise a SIGTRAP, which aborts if no debugger is connected
151                throw_machine_stop!(TerminationInfo::Abort(format!("trace/breakpoint trap")))
152            }
153
154            "assert_inhabited" | "assert_zero_valid" | "assert_mem_uninitialized_valid" => {
155                // Make these a NOP, so we get the better Miri-native error messages.
156            }
157
158            _ => return this.emulate_math_intrinsic(intrinsic_name, generic_args, args, dest),
159        }
160
161        interp_ok(EmulateItemResult::NeedsReturn)
162    }
163
164    fn call_llvm_intrinsic(
165        &mut self,
166        instance: ty::Instance<'tcx>,
167        args: &[OpTy<'tcx>],
168        dest: &PlaceTy<'tcx>,
169        ret: Option<mir::BasicBlock>,
170    ) -> InterpResult<'tcx> {
171        let this = self.eval_context_mut();
172
173        let link_name = this.tcx.codegen_fn_attrs(instance.def_id()).symbol_name.unwrap();
174
175        // FIXME: avoid allocating memory
176        let dest = this.force_allocation(dest)?;
177
178        let res = 'handled: {
179            match link_name.as_str() {
180                // LLVM intrinsics
181                "llvm.prefetch.p0" => {
182                    let [p, rw, loc, ty] = this.check_shim_sig_unadjusted(link_name, args)?;
183
184                    let _ = this.read_pointer(p)?;
185                    let rw = this.read_scalar(rw)?.to_i32()?;
186                    let loc = this.read_scalar(loc)?.to_i32()?;
187                    let ty = this.read_scalar(ty)?.to_i32()?;
188
189                    if ty == 1 {
190                        // Data cache prefetch.
191                        // Notably, we do not have to check the pointer, this operation is never UB!
192
193                        if !matches!(rw, 0 | 1) {
194                            throw_unsup_format!(
195                                "invalid `rw` value passed to `llvm.prefetch`: {rw}"
196                            );
197                        }
198                        if !matches!(loc, 0..=3) {
199                            throw_unsup_format!(
200                                "invalid `loc` value passed to `llvm.prefetch`: {loc}"
201                            );
202                        }
203                    } else {
204                        throw_unsup_format!("unsupported `llvm.prefetch` type argument: {ty}");
205                    }
206                }
207                // Used to implement the x86 `_mm{,256,512}_popcnt_epi{8,16,32,64}` and wasm
208                // `{i,u}8x16_popcnt` functions.
209                name if name.starts_with("llvm.ctpop.v")
210                    && this.tcx.sess.target.endian == Endian::Little =>
211                {
212                    let [op] = this.check_shim_sig_unadjusted(link_name, args)?;
213
214                    let (op, op_len) = this.project_to_simd(op)?;
215                    let (dest, dest_len) = this.project_to_simd(&dest)?;
216
217                    assert_eq!(dest_len, op_len);
218
219                    for i in 0..dest_len {
220                        let op = this.read_immediate(&this.project_index(&op, i)?)?;
221                        // Use `to_uint` to get a zero-extended `u128`. Those
222                        // extra zeros will not affect `count_ones`.
223                        let res = op.to_scalar().to_uint(op.layout.size)?.count_ones();
224
225                        this.write_scalar(
226                            Scalar::from_uint(res, op.layout.size),
227                            &this.project_index(&dest, i)?,
228                        )?;
229                    }
230                }
231
232                // Target-specific shims
233                name if name.starts_with("llvm.x86.")
234                    && matches!(this.tcx.sess.target.arch, Arch::X86 | Arch::X86_64)
235                    && this.tcx.sess.target.endian == Endian::Little =>
236                    break 'handled x86::EvalContextExt::emulate_x86_intrinsic(
237                        this, link_name, args, &dest,
238                    )?,
239                name if name.starts_with("llvm.aarch64.")
240                    && this.tcx.sess.target.arch == Arch::AArch64
241                    && this.tcx.sess.target.endian == Endian::Little =>
242                    break 'handled aarch64::EvalContextExt::emulate_aarch64_intrinsic(
243                        this, link_name, args, &dest,
244                    )?,
245                name if name.starts_with("llvm.loongarch.")
246                    && matches!(
247                        this.tcx.sess.target.arch,
248                        Arch::LoongArch32 | Arch::LoongArch64
249                    )
250                    && this.tcx.sess.target.endian == Endian::Little =>
251                    break 'handled loongarch::EvalContextExt::emulate_loongarch_intrinsic(
252                        this, link_name, args, &dest,
253                    )?,
254                _ => break 'handled EmulateItemResult::NotSupported,
255            }
256            EmulateItemResult::NeedsReturn
257        };
258
259        // The rest either implements the logic, or falls back to `lookup_exported_symbol`.
260        res.jump_to_next_block(this, &dest, ret, None, |this| {
261            throw_machine_stop!(TerminationInfo::UnsupportedForeignItem(format!(
262                "can't call LLVM intrinsic `{link_name}` on architecture `{arch}`",
263                arch = this.tcx.sess.target.arch,
264            )));
265        })
266    }
267}
miri/intrinsics/mod.rs

miri/intrinsics/
mod.rs
