miri/intrinsics/
mod.rs

Help
1#![warn(clippy::arithmetic_side_effects)]
2
3mod atomic;
4mod simd;
5
6use rand::Rng;
7use rustc_abi::Size;
8use rustc_apfloat::{Float, Round};
9use rustc_middle::mir;
10use rustc_middle::ty::{self, FloatTy};
11use rustc_span::{Symbol, sym};
12
13use self::atomic::EvalContextExt as _;
14use self::helpers::{ToHost, ToSoft, check_arg_count};
15use self::simd::EvalContextExt as _;
16use crate::*;
17
18impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {}
19pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
20    fn call_intrinsic(
21        &mut self,
22        instance: ty::Instance<'tcx>,
23        args: &[OpTy<'tcx>],
24        dest: &MPlaceTy<'tcx>,
25        ret: Option<mir::BasicBlock>,
26        unwind: mir::UnwindAction,
27    ) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>> {
28        let this = self.eval_context_mut();
29
30        // See if the core engine can handle this intrinsic.
31        if this.eval_intrinsic(instance, args, dest, ret)? {
32            return interp_ok(None);
33        }
34        let intrinsic_name = this.tcx.item_name(instance.def_id());
35        let intrinsic_name = intrinsic_name.as_str();
36
37        match this.emulate_intrinsic_by_name(intrinsic_name, instance.args, args, dest, ret)? {
38            EmulateItemResult::NotSupported => {
39                // We haven't handled the intrinsic, let's see if we can use a fallback body.
40                if this.tcx.intrinsic(instance.def_id()).unwrap().must_be_overridden {
41                    throw_unsup_format!("unimplemented intrinsic: `{intrinsic_name}`")
42                }
43                let intrinsic_fallback_is_spec = Symbol::intern("intrinsic_fallback_is_spec");
44                if this
45                    .tcx
46                    .get_attrs_by_path(instance.def_id(), &[sym::miri, intrinsic_fallback_is_spec])
47                    .next()
48                    .is_none()
49                {
50                    throw_unsup_format!(
51                        "Miri can only use intrinsic fallback bodies that exactly reflect the specification: they fully check for UB and are as non-deterministic as possible. After verifying that `{intrinsic_name}` does so, add the `#[miri::intrinsic_fallback_is_spec]` attribute to it; also ping @rust-lang/miri when you do that"
52                    );
53                }
54                interp_ok(Some(ty::Instance {
55                    def: ty::InstanceKind::Item(instance.def_id()),
56                    args: instance.args,
57                }))
58            }
59            EmulateItemResult::NeedsReturn => {
60                trace!("{:?}", this.dump_place(&dest.clone().into()));
61                this.return_to_block(ret)?;
62                interp_ok(None)
63            }
64            EmulateItemResult::NeedsUnwind => {
65                // Jump to the unwind block to begin unwinding.
66                this.unwind_to_block(unwind)?;
67                interp_ok(None)
68            }
69            EmulateItemResult::AlreadyJumped => interp_ok(None),
70        }
71    }
72
73    /// Emulates a Miri-supported intrinsic (not supported by the core engine).
74    /// Returns `Ok(true)` if the intrinsic was handled.
75    fn emulate_intrinsic_by_name(
76        &mut self,
77        intrinsic_name: &str,
78        generic_args: ty::GenericArgsRef<'tcx>,
79        args: &[OpTy<'tcx>],
80        dest: &MPlaceTy<'tcx>,
81        ret: Option<mir::BasicBlock>,
82    ) -> InterpResult<'tcx, EmulateItemResult> {
83        let this = self.eval_context_mut();
84
85        if let Some(name) = intrinsic_name.strip_prefix("atomic_") {
86            return this.emulate_atomic_intrinsic(name, args, dest);
87        }
88        if let Some(name) = intrinsic_name.strip_prefix("simd_") {
89            return this.emulate_simd_intrinsic(name, generic_args, args, dest);
90        }
91
92        match intrinsic_name {
93            // Basic control flow
94            "abort" => {
95                throw_machine_stop!(TerminationInfo::Abort(
96                    "the program aborted execution".to_owned()
97                ));
98            }
99            "catch_unwind" => {
100                this.handle_catch_unwind(args, dest, ret)?;
101                // This pushed a stack frame, don't jump to `ret`.
102                return interp_ok(EmulateItemResult::AlreadyJumped);
103            }
104
105            // Raw memory accesses
106            "volatile_load" => {
107                let [place] = check_arg_count(args)?;
108                let place = this.deref_pointer(place)?;
109                this.copy_op(&place, dest)?;
110            }
111            "volatile_store" => {
112                let [place, dest] = check_arg_count(args)?;
113                let place = this.deref_pointer(place)?;
114                this.copy_op(dest, &place)?;
115            }
116
117            "volatile_set_memory" => {
118                let [ptr, val_byte, count] = check_arg_count(args)?;
119                this.write_bytes_intrinsic(ptr, val_byte, count, "volatile_set_memory")?;
120            }
121
122            // Memory model / provenance manipulation
123            "ptr_mask" => {
124                let [ptr, mask] = check_arg_count(args)?;
125
126                let ptr = this.read_pointer(ptr)?;
127                let mask = this.read_target_usize(mask)?;
128
129                let masked_addr = Size::from_bytes(ptr.addr().bytes() & mask);
130
131                this.write_pointer(Pointer::new(ptr.provenance, masked_addr), dest)?;
132            }
133
134            // We want to return either `true` or `false` at random, or else something like
135            // ```
136            // if !is_val_statically_known(0) { unreachable_unchecked(); }
137            // ```
138            // Would not be considered UB, or the other way around (`is_val_statically_known(0)`).
139            "is_val_statically_known" => {
140                let [_arg] = check_arg_count(args)?;
141                // FIXME: should we check for validity here? It's tricky because we do not have a
142                // place. Codegen does not seem to set any attributes like `noundef` for intrinsic
143                // calls, so we don't *have* to do anything.
144                let branch: bool = this.machine.rng.get_mut().random();
145                this.write_scalar(Scalar::from_bool(branch), dest)?;
146            }
147
148            "floorf16" | "ceilf16" | "truncf16" | "roundf16" | "rintf16" => {
149                let [f] = check_arg_count(args)?;
150                let f = this.read_scalar(f)?.to_f16()?;
151                let mode = match intrinsic_name {
152                    "floorf16" => Round::TowardNegative,
153                    "ceilf16" => Round::TowardPositive,
154                    "truncf16" => Round::TowardZero,
155                    "roundf16" => Round::NearestTiesToAway,
156                    "rintf16" => Round::NearestTiesToEven,
157                    _ => bug!(),
158                };
159                let res = f.round_to_integral(mode).value;
160                let res = this.adjust_nan(res, &[f]);
161                this.write_scalar(res, dest)?;
162            }
163            "floorf32" | "ceilf32" | "truncf32" | "roundf32" | "rintf32" => {
164                let [f] = check_arg_count(args)?;
165                let f = this.read_scalar(f)?.to_f32()?;
166                let mode = match intrinsic_name {
167                    "floorf32" => Round::TowardNegative,
168                    "ceilf32" => Round::TowardPositive,
169                    "truncf32" => Round::TowardZero,
170                    "roundf32" => Round::NearestTiesToAway,
171                    "rintf32" => Round::NearestTiesToEven,
172                    _ => bug!(),
173                };
174                let res = f.round_to_integral(mode).value;
175                let res = this.adjust_nan(res, &[f]);
176                this.write_scalar(res, dest)?;
177            }
178            "floorf64" | "ceilf64" | "truncf64" | "roundf64" | "rintf64" => {
179                let [f] = check_arg_count(args)?;
180                let f = this.read_scalar(f)?.to_f64()?;
181                let mode = match intrinsic_name {
182                    "floorf64" => Round::TowardNegative,
183                    "ceilf64" => Round::TowardPositive,
184                    "truncf64" => Round::TowardZero,
185                    "roundf64" => Round::NearestTiesToAway,
186                    "rintf64" => Round::NearestTiesToEven,
187                    _ => bug!(),
188                };
189                let res = f.round_to_integral(mode).value;
190                let res = this.adjust_nan(res, &[f]);
191                this.write_scalar(res, dest)?;
192            }
193            "floorf128" | "ceilf128" | "truncf128" | "roundf128" | "rintf128" => {
194                let [f] = check_arg_count(args)?;
195                let f = this.read_scalar(f)?.to_f128()?;
196                let mode = match intrinsic_name {
197                    "floorf128" => Round::TowardNegative,
198                    "ceilf128" => Round::TowardPositive,
199                    "truncf128" => Round::TowardZero,
200                    "roundf128" => Round::NearestTiesToAway,
201                    "rintf128" => Round::NearestTiesToEven,
202                    _ => bug!(),
203                };
204                let res = f.round_to_integral(mode).value;
205                let res = this.adjust_nan(res, &[f]);
206                this.write_scalar(res, dest)?;
207            }
208
209            #[rustfmt::skip]
210            | "sinf32"
211            | "cosf32"
212            | "sqrtf32"
213            | "expf32"
214            | "exp2f32"
215            | "logf32"
216            | "log10f32"
217            | "log2f32"
218            => {
219                let [f] = check_arg_count(args)?;
220                let f = this.read_scalar(f)?.to_f32()?;
221                // Using host floats except for sqrt (but it's fine, these operations do not have
222                // guaranteed precision).
223                let res = match intrinsic_name {
224                    "sinf32" => f.to_host().sin().to_soft(),
225                    "cosf32" => f.to_host().cos().to_soft(),
226                    "sqrtf32" => math::sqrt(f),
227                    "expf32" => f.to_host().exp().to_soft(),
228                    "exp2f32" => f.to_host().exp2().to_soft(),
229                    "logf32" => f.to_host().ln().to_soft(),
230                    "log10f32" => f.to_host().log10().to_soft(),
231                    "log2f32" => f.to_host().log2().to_soft(),
232                    _ => bug!(),
233                };
234                let res = this.adjust_nan(res, &[f]);
235                this.write_scalar(res, dest)?;
236            }
237            #[rustfmt::skip]
238            | "sinf64"
239            | "cosf64"
240            | "sqrtf64"
241            | "expf64"
242            | "exp2f64"
243            | "logf64"
244            | "log10f64"
245            | "log2f64"
246            => {
247                let [f] = check_arg_count(args)?;
248                let f = this.read_scalar(f)?.to_f64()?;
249                // Using host floats except for sqrt (but it's fine, these operations do not have
250                // guaranteed precision).
251                let res = match intrinsic_name {
252                    "sinf64" => f.to_host().sin().to_soft(),
253                    "cosf64" => f.to_host().cos().to_soft(),
254                    "sqrtf64" => math::sqrt(f),
255                    "expf64" => f.to_host().exp().to_soft(),
256                    "exp2f64" => f.to_host().exp2().to_soft(),
257                    "logf64" => f.to_host().ln().to_soft(),
258                    "log10f64" => f.to_host().log10().to_soft(),
259                    "log2f64" => f.to_host().log2().to_soft(),
260                    _ => bug!(),
261                };
262                let res = this.adjust_nan(res, &[f]);
263                this.write_scalar(res, dest)?;
264            }
265
266            "fmaf32" => {
267                let [a, b, c] = check_arg_count(args)?;
268                let a = this.read_scalar(a)?.to_f32()?;
269                let b = this.read_scalar(b)?.to_f32()?;
270                let c = this.read_scalar(c)?.to_f32()?;
271                // FIXME: Using host floats, to work around https://github.com/rust-lang/rustc_apfloat/issues/11
272                let res = a.to_host().mul_add(b.to_host(), c.to_host()).to_soft();
273                let res = this.adjust_nan(res, &[a, b, c]);
274                this.write_scalar(res, dest)?;
275            }
276            "fmaf64" => {
277                let [a, b, c] = check_arg_count(args)?;
278                let a = this.read_scalar(a)?.to_f64()?;
279                let b = this.read_scalar(b)?.to_f64()?;
280                let c = this.read_scalar(c)?.to_f64()?;
281                // FIXME: Using host floats, to work around https://github.com/rust-lang/rustc_apfloat/issues/11
282                let res = a.to_host().mul_add(b.to_host(), c.to_host()).to_soft();
283                let res = this.adjust_nan(res, &[a, b, c]);
284                this.write_scalar(res, dest)?;
285            }
286
287            "fmuladdf32" => {
288                let [a, b, c] = check_arg_count(args)?;
289                let a = this.read_scalar(a)?.to_f32()?;
290                let b = this.read_scalar(b)?.to_f32()?;
291                let c = this.read_scalar(c)?.to_f32()?;
292                let fuse: bool = this.machine.rng.get_mut().random();
293                let res = if fuse {
294                    // FIXME: Using host floats, to work around https://github.com/rust-lang/rustc_apfloat/issues/11
295                    a.to_host().mul_add(b.to_host(), c.to_host()).to_soft()
296                } else {
297                    ((a * b).value + c).value
298                };
299                let res = this.adjust_nan(res, &[a, b, c]);
300                this.write_scalar(res, dest)?;
301            }
302            "fmuladdf64" => {
303                let [a, b, c] = check_arg_count(args)?;
304                let a = this.read_scalar(a)?.to_f64()?;
305                let b = this.read_scalar(b)?.to_f64()?;
306                let c = this.read_scalar(c)?.to_f64()?;
307                let fuse: bool = this.machine.rng.get_mut().random();
308                let res = if fuse {
309                    // FIXME: Using host floats, to work around https://github.com/rust-lang/rustc_apfloat/issues/11
310                    a.to_host().mul_add(b.to_host(), c.to_host()).to_soft()
311                } else {
312                    ((a * b).value + c).value
313                };
314                let res = this.adjust_nan(res, &[a, b, c]);
315                this.write_scalar(res, dest)?;
316            }
317
318            "powf32" => {
319                let [f1, f2] = check_arg_count(args)?;
320                let f1 = this.read_scalar(f1)?.to_f32()?;
321                let f2 = this.read_scalar(f2)?.to_f32()?;
322                // Using host floats (but it's fine, this operation does not have guaranteed precision).
323                let res = f1.to_host().powf(f2.to_host()).to_soft();
324                let res = this.adjust_nan(res, &[f1, f2]);
325                this.write_scalar(res, dest)?;
326            }
327            "powf64" => {
328                let [f1, f2] = check_arg_count(args)?;
329                let f1 = this.read_scalar(f1)?.to_f64()?;
330                let f2 = this.read_scalar(f2)?.to_f64()?;
331                // Using host floats (but it's fine, this operation does not have guaranteed precision).
332                let res = f1.to_host().powf(f2.to_host()).to_soft();
333                let res = this.adjust_nan(res, &[f1, f2]);
334                this.write_scalar(res, dest)?;
335            }
336
337            "powif32" => {
338                let [f, i] = check_arg_count(args)?;
339                let f = this.read_scalar(f)?.to_f32()?;
340                let i = this.read_scalar(i)?.to_i32()?;
341                // Using host floats (but it's fine, this operation does not have guaranteed precision).
342                let res = f.to_host().powi(i).to_soft();
343                let res = this.adjust_nan(res, &[f]);
344                this.write_scalar(res, dest)?;
345            }
346            "powif64" => {
347                let [f, i] = check_arg_count(args)?;
348                let f = this.read_scalar(f)?.to_f64()?;
349                let i = this.read_scalar(i)?.to_i32()?;
350                // Using host floats (but it's fine, this operation does not have guaranteed precision).
351                let res = f.to_host().powi(i).to_soft();
352                let res = this.adjust_nan(res, &[f]);
353                this.write_scalar(res, dest)?;
354            }
355
356            #[rustfmt::skip]
357            | "fadd_algebraic"
358            | "fsub_algebraic"
359            | "fmul_algebraic"
360            | "fdiv_algebraic"
361            | "frem_algebraic"
362            => {
363                let [a, b] = check_arg_count(args)?;
364                let a = this.read_immediate(a)?;
365                let b = this.read_immediate(b)?;
366                let op = match intrinsic_name {
367                    "fadd_algebraic" => mir::BinOp::Add,
368                    "fsub_algebraic" => mir::BinOp::Sub,
369                    "fmul_algebraic" => mir::BinOp::Mul,
370                    "fdiv_algebraic" => mir::BinOp::Div,
371                    "frem_algebraic" => mir::BinOp::Rem,
372                    _ => bug!(),
373                };
374                let res = this.binary_op(op, &a, &b)?;
375                // `binary_op` already called `generate_nan` if necessary.
376                this.write_immediate(*res, dest)?;
377            }
378
379            #[rustfmt::skip]
380            | "fadd_fast"
381            | "fsub_fast"
382            | "fmul_fast"
383            | "fdiv_fast"
384            | "frem_fast"
385            => {
386                let [a, b] = check_arg_count(args)?;
387                let a = this.read_immediate(a)?;
388                let b = this.read_immediate(b)?;
389                let op = match intrinsic_name {
390                    "fadd_fast" => mir::BinOp::Add,
391                    "fsub_fast" => mir::BinOp::Sub,
392                    "fmul_fast" => mir::BinOp::Mul,
393                    "fdiv_fast" => mir::BinOp::Div,
394                    "frem_fast" => mir::BinOp::Rem,
395                    _ => bug!(),
396                };
397                let float_finite = |x: &ImmTy<'tcx>| -> InterpResult<'tcx, bool> {
398                    let ty::Float(fty) = x.layout.ty.kind() else {
399                        bug!("float_finite: non-float input type {}", x.layout.ty)
400                    };
401                    interp_ok(match fty {
402                        FloatTy::F16 => x.to_scalar().to_f16()?.is_finite(),
403                        FloatTy::F32 => x.to_scalar().to_f32()?.is_finite(),
404                        FloatTy::F64 => x.to_scalar().to_f64()?.is_finite(),
405                        FloatTy::F128 => x.to_scalar().to_f128()?.is_finite(),
406                    })
407                };
408                match (float_finite(&a)?, float_finite(&b)?) {
409                    (false, false) => throw_ub_format!(
410                        "`{intrinsic_name}` intrinsic called with non-finite value as both parameters",
411                    ),
412                    (false, _) => throw_ub_format!(
413                        "`{intrinsic_name}` intrinsic called with non-finite value as first parameter",
414                    ),
415                    (_, false) => throw_ub_format!(
416                        "`{intrinsic_name}` intrinsic called with non-finite value as second parameter",
417                    ),
418                    _ => {}
419                }
420                let res = this.binary_op(op, &a, &b)?;
421                if !float_finite(&res)? {
422                    throw_ub_format!("`{intrinsic_name}` intrinsic produced non-finite value as result");
423                }
424                // This cannot be a NaN so we also don't have to apply any non-determinism.
425                // (Also, `binary_op` already called `generate_nan` if needed.)
426                this.write_immediate(*res, dest)?;
427            }
428
429            "float_to_int_unchecked" => {
430                let [val] = check_arg_count(args)?;
431                let val = this.read_immediate(val)?;
432
433                let res = this
434                    .float_to_int_checked(&val, dest.layout, Round::TowardZero)?
435                    .ok_or_else(|| {
436                        err_ub_format!(
437                            "`float_to_int_unchecked` intrinsic called on {val} which cannot be represented in target type `{:?}`",
438                            dest.layout.ty
439                        )
440                    })?;
441
442                this.write_immediate(*res, dest)?;
443            }
444
445            // Other
446            "breakpoint" => {
447                let [] = check_arg_count(args)?;
448                // normally this would raise a SIGTRAP, which aborts if no debugger is connected
449                throw_machine_stop!(TerminationInfo::Abort(format!("trace/breakpoint trap")))
450            }
451
452            _ => return interp_ok(EmulateItemResult::NotSupported),
453        }
454
455        interp_ok(EmulateItemResult::NeedsReturn)
456    }
457}
miri/intrinsics/mod.rs

miri/intrinsics/
mod.rs
