miri/intrinsics/
mod.rs

1#![warn(clippy::arithmetic_side_effects)]
2
3mod aarch64;
4mod loongarch;
5mod math;
6mod simd;
7mod x86;
8
9#[rustfmt::skip] // prevent `use` reordering
10use rand::RngExt;
11use rustc_abi::{Endian, Size};
12use rustc_middle::{mir, ty};
13use rustc_span::sym;
14use rustc_target::spec::Arch;
15
16use self::math::EvalContextExt as _;
17use self::simd::EvalContextExt as _;
18use crate::*;
19
20/// Check that the number of args is what we expect.
21fn check_intrinsic_arg_count<'a, 'tcx, const N: usize>(
22    args: &'a [OpTy<'tcx>],
23) -> InterpResult<'tcx, &'a [OpTy<'tcx>; N]>
24where
25    &'a [OpTy<'tcx>; N]: TryFrom<&'a [OpTy<'tcx>]>,
26{
27    if let Ok(ops) = args.try_into() {
28        return interp_ok(ops);
29    }
30    throw_ub_format!(
31        "incorrect number of arguments for intrinsic: got {}, expected {}",
32        args.len(),
33        N
34    )
35}
36
37impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {}
38pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
39    fn call_intrinsic(
40        &mut self,
41        instance: ty::Instance<'tcx>,
42        args: &[OpTy<'tcx>],
43        dest: &PlaceTy<'tcx>,
44        ret: Option<mir::BasicBlock>,
45        unwind: mir::UnwindAction,
46    ) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>> {
47        let ecx = self.eval_context_mut();
48
49        // See if the core engine can handle this intrinsic.
50        if ecx.eval_intrinsic(instance, args, dest, ret)? {
51            return interp_ok(None);
52        }
53        let intrinsic_name = ecx.tcx.item_name(instance.def_id());
54        let intrinsic_name = intrinsic_name.as_str();
55
56        let res = ecx.emulate_intrinsic_by_name(intrinsic_name, instance.args, args, dest, ret)?;
57        res.jump_to_next_block(ecx, dest, ret, Some(unwind), |ecx| {
58            // We haven't handled the intrinsic, let's see if we can use a fallback body.
59            if ecx.tcx.intrinsic(instance.def_id()).unwrap().must_be_overridden {
60                throw_unsup_format!("unimplemented intrinsic: `{intrinsic_name}`")
61            }
62            let intrinsic_fallback_is_spec = ecx
63                .tcx
64                .tool_attrs(
65                    instance.def_id(),
66                    &[sym::miri, sym::intrinsic_fallback_is_spec],
67                )
68                .next()
69                .is_some();
70            if !intrinsic_fallback_is_spec {
71                throw_unsup_format!(
72                    "Miri can only use intrinsic fallback bodies that exactly reflect the specification: they fully check for UB and are as non-deterministic as possible. After verifying that `{intrinsic_name}` does so, add the `#[miri::intrinsic_fallback_is_spec]` attribute to it; also ping @rust-lang/miri when you do that"
73                );
74            }
75            interp_ok(Some(ty::Instance {
76                def: ty::InstanceKind::Item(instance.def_id()),
77                args: instance.args,
78            }))
79        })
80    }
81
82    /// Emulates a Miri-supported intrinsic (not supported by the core engine).
83    /// Returns `Ok(true)` if the intrinsic was handled.
84    fn emulate_intrinsic_by_name(
85        &mut self,
86        intrinsic_name: &str,
87        generic_args: ty::GenericArgs<'tcx>,
88        args: &[OpTy<'tcx>],
89        dest: &PlaceTy<'tcx>,
90        ret: Option<mir::BasicBlock>,
91    ) -> InterpResult<'tcx, EmulateItemResult> {
92        let ecx = self.eval_context_mut();
93
94        if let Some(name) = intrinsic_name.strip_prefix("simd_") {
95            return ecx.emulate_simd_intrinsic(name, args, dest);
96        }
97
98        match intrinsic_name {
99            // Basic control flow
100            "abort_immediate" => {
101                // Note that `abort_immediate` is also hooked separately in CTFE.
102                throw_machine_stop!(TerminationInfo::Abort(
103                    "the program aborted execution".to_owned()
104                ));
105            }
106            "catch_unwind" => {
107                let [try_fn, data, catch_fn] = check_intrinsic_arg_count(args)?;
108                ecx.handle_catch_unwind(try_fn, data, catch_fn, dest, ret)?;
109                // This pushed a stack frame, don't jump to `ret`.
110                return interp_ok(EmulateItemResult::AlreadyJumped);
111            }
112
113            // Memory model / provenance manipulation
114            "ptr_mask" => {
115                let [ptr, mask] = check_intrinsic_arg_count(args)?;
116
117                let ptr = ecx.read_pointer(ptr)?;
118                let mask = ecx.read_target_usize(mask)?;
119
120                let masked_addr = Size::from_bytes(ptr.addr().bytes() & mask);
121
122                ecx.write_pointer(Pointer::new(ptr.provenance, masked_addr), dest)?;
123            }
124
125            // We want to return either `true` or `false` at random, or else something like
126            // ```
127            // if !is_val_statically_known(0) { unreachable_unchecked(); }
128            // ```
129            // Would not be considered UB, or the other way around (`is_val_statically_known(0)`).
130            "is_val_statically_known" => {
131                let [_arg] = check_intrinsic_arg_count(args)?;
132                // FIXME: should we check for validity here? It's tricky because we do not have a
133                // place. Codegen does not seem to set any attributes like `noundef` for intrinsic
134                // calls, so we don't *have* to do anything.
135                let branch: bool = ecx.machine.rng.get_mut().random();
136                ecx.write_scalar(Scalar::from_bool(branch), dest)?;
137            }
138
139            // Other
140            "breakpoint" => {
141                let [] = check_intrinsic_arg_count(args)?;
142                // normally this would raise a SIGTRAP, which aborts if no debugger is connected
143                throw_machine_stop!(TerminationInfo::Abort(format!("trace/breakpoint trap")))
144            }
145
146            "assert_inhabited" | "assert_zero_valid" | "assert_mem_uninitialized_valid" => {
147                // Make these a NOP, so we get the better Miri-native error messages.
148            }
149
150            _ => return ecx.emulate_math_intrinsic(intrinsic_name, generic_args, args, dest),
151        }
152
153        interp_ok(EmulateItemResult::NeedsReturn)
154    }
155
156    fn call_llvm_intrinsic(
157        &mut self,
158        instance: ty::Instance<'tcx>,
159        args: &[OpTy<'tcx>],
160        dest: &PlaceTy<'tcx>,
161        ret: Option<mir::BasicBlock>,
162    ) -> InterpResult<'tcx> {
163        let ecx = self.eval_context_mut();
164
165        let link_name = ecx.tcx.codegen_fn_attrs(instance.def_id()).symbol_name.unwrap();
166
167        // These are anyway mostly vector intrinsics and vectors live in memory.
168        let dest = ecx.force_allocation(dest)?;
169
170        let res = 'handled: {
171            match link_name.as_str() {
172                // LLVM intrinsics
173                "llvm.prefetch.p0" => {
174                    let [p, rw, loc, ty] = ecx.check_shim_sig_llvm_intrinsic(link_name, args)?;
175
176                    let _ = ecx.read_pointer(p)?;
177                    let rw = ecx.read_scalar(rw)?.to_i32()?;
178                    let loc = ecx.read_scalar(loc)?.to_i32()?;
179                    let ty = ecx.read_scalar(ty)?.to_i32()?;
180
181                    if ty == 1 {
182                        // Data cache prefetch.
183                        // Notably, we do not have to check the pointer, this operation is never UB!
184
185                        if !matches!(rw, 0 | 1) {
186                            throw_unsup_format!(
187                                "invalid `rw` value passed to `llvm.prefetch`: {rw}"
188                            );
189                        }
190                        if !matches!(loc, 0..=3) {
191                            throw_unsup_format!(
192                                "invalid `loc` value passed to `llvm.prefetch`: {loc}"
193                            );
194                        }
195                    } else {
196                        throw_unsup_format!("unsupported `llvm.prefetch` type argument: {ty}");
197                    }
198                }
199                // Used to implement the x86 `_mm{,256,512}_popcnt_epi{8,16,32,64}` and wasm
200                // `{i,u}8x16_popcnt` functions.
201                name if name.starts_with("llvm.ctpop.v")
202                    && ecx.tcx.sess.target.endian == Endian::Little =>
203                {
204                    let [op] = ecx.check_shim_sig_llvm_intrinsic(link_name, args)?;
205
206                    let (op, op_len) = ecx.project_to_simd(op)?;
207                    let (dest, dest_len) = ecx.project_to_simd(&dest)?;
208
209                    assert_eq!(dest_len, op_len);
210
211                    for i in 0..dest_len {
212                        let op = ecx.read_immediate(&ecx.project_index(&op, i)?)?;
213                        // Use `to_uint` to get a zero-extended `u128`. Those
214                        // extra zeros will not affect `count_ones`.
215                        let res = op.to_scalar().to_uint(op.layout.size)?.count_ones();
216
217                        ecx.write_scalar(
218                            Scalar::from_uint(res, op.layout.size),
219                            &ecx.project_index(&dest, i)?,
220                        )?;
221                    }
222                }
223
224                // Target-specific shims
225                name if name.starts_with("llvm.x86.")
226                    && matches!(ecx.tcx.sess.target.arch, Arch::X86 | Arch::X86_64)
227                    && ecx.tcx.sess.target.endian == Endian::Little =>
228                    break 'handled x86::EvalContextExt::emulate_x86_intrinsic(
229                        ecx, link_name, args, &dest,
230                    )?,
231                name if name.starts_with("llvm.aarch64.")
232                    && ecx.tcx.sess.target.arch == Arch::AArch64
233                    && ecx.tcx.sess.target.endian == Endian::Little =>
234                    break 'handled aarch64::EvalContextExt::emulate_aarch64_intrinsic(
235                        ecx, link_name, args, &dest,
236                    )?,
237                name if name.starts_with("llvm.loongarch.")
238                    && matches!(
239                        ecx.tcx.sess.target.arch,
240                        Arch::LoongArch32 | Arch::LoongArch64
241                    )
242                    && ecx.tcx.sess.target.endian == Endian::Little =>
243                    break 'handled loongarch::EvalContextExt::emulate_loongarch_intrinsic(
244                        ecx, link_name, args, &dest,
245                    )?,
246                _ => break 'handled EmulateItemResult::NotSupported,
247            }
248            EmulateItemResult::NeedsReturn
249        };
250
251        // The rest either implements the logic, or falls back to `lookup_exported_symbol`.
252        res.jump_to_next_block(ecx, &dest.clone().into(), ret, None, |ecx| {
253            throw_machine_stop!(TerminationInfo::UnsupportedForeignItem(format!(
254                "can't call LLVM intrinsic `{link_name}` on architecture `{arch}`",
255                arch = ecx.tcx.sess.target.arch,
256            )));
257        })
258    }
259}
miri/intrinsics/mod.rs

miri/intrinsics/
mod.rs
