rustc_const_eval/interpret/
intrinsics.rs

1//! Intrinsics and other functions that the interpreter executes without
2//! looking at their MIR. Intrinsics/functions supported here are shared by CTFE
3//! and miri.
4
5mod simd;
6
7use std::assert_matches::assert_matches;
8
9use rustc_abi::{FieldIdx, HasDataLayout, Size};
10use rustc_apfloat::ieee::{Double, Half, Quad, Single};
11use rustc_middle::mir::interpret::{CTFE_ALLOC_SALT, read_target_uint, write_target_uint};
12use rustc_middle::mir::{self, BinOp, ConstValue, NonDivergingIntrinsic};
13use rustc_middle::ty::layout::TyAndLayout;
14use rustc_middle::ty::{FloatTy, Ty, TyCtxt};
15use rustc_middle::{bug, span_bug, ty};
16use rustc_span::{Symbol, sym};
17use tracing::trace;
18
19use super::memory::MemoryKind;
20use super::util::ensure_monomorphic_enough;
21use super::{
22    AllocId, CheckInAllocMsg, ImmTy, InterpCx, InterpResult, Machine, OpTy, PlaceTy, Pointer,
23    PointerArithmetic, Provenance, Scalar, err_ub_custom, err_unsup_format, interp_ok, throw_inval,
24    throw_ub_custom, throw_ub_format,
25};
26use crate::fluent_generated as fluent;
27
28#[derive(Copy, Clone, Debug, PartialEq, Eq)]
29enum MulAddType {
30    /// Used with `fma` and `simd_fma`, always uses fused-multiply-add
31    Fused,
32    /// Used with `fmuladd` and `simd_relaxed_fma`, nondeterministically determines whether to use
33    /// fma or simple multiply-add
34    Nondeterministic,
35}
36
37#[derive(Copy, Clone)]
38pub(crate) enum MinMax {
39    /// The IEEE `Minimum` operation - see `f32::minimum` etc
40    /// In particular, `-0.0` is considered smaller than `+0.0` and
41    /// if either input is NaN, the result is NaN.
42    Minimum,
43    /// The IEEE `MinNum` operation - see `f32::min` etc
44    /// In particular, if the inputs are `-0.0` and `+0.0`, the result is non-deterministic,
45    /// and is one argument is NaN, the other one is returned.
46    MinNum,
47    /// The IEEE `Maximum` operation - see `f32::maximum` etc
48    /// In particular, `-0.0` is considered smaller than `+0.0` and
49    /// if either input is NaN, the result is NaN.
50    Maximum,
51    /// The IEEE `MaxNum` operation - see `f32::max` etc
52    /// In particular, if the inputs are `-0.0` and `+0.0`, the result is non-deterministic,
53    /// and is one argument is NaN, the other one is returned.
54    MaxNum,
55}
56
57/// Directly returns an `Allocation` containing an absolute path representation of the given type.
58pub(crate) fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> (AllocId, u64) {
59    let path = crate::util::type_name(tcx, ty);
60    let bytes = path.into_bytes();
61    let len = bytes.len().try_into().unwrap();
62    (tcx.allocate_bytes_dedup(bytes, CTFE_ALLOC_SALT), len)
63}
64impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
65    /// Generates a value of `TypeId` for `ty` in-place.
66    fn write_type_id(
67        &mut self,
68        ty: Ty<'tcx>,
69        dest: &PlaceTy<'tcx, M::Provenance>,
70    ) -> InterpResult<'tcx, ()> {
71        let tcx = self.tcx;
72        let type_id_hash = tcx.type_id_hash(ty).as_u128();
73        let op = self.const_val_to_op(
74            ConstValue::Scalar(Scalar::from_u128(type_id_hash)),
75            tcx.types.u128,
76            None,
77        )?;
78        self.copy_op_allow_transmute(&op, dest)?;
79
80        // Give the each pointer-sized chunk provenance that knows about the type id.
81        // Here we rely on `TypeId` being a newtype around an array of pointers, so we
82        // first project to its only field and then the array elements.
83        let alloc_id = tcx.reserve_and_set_type_id_alloc(ty);
84        let arr = self.project_field(dest, FieldIdx::ZERO)?;
85        let mut elem_iter = self.project_array_fields(&arr)?;
86        while let Some((_, elem)) = elem_iter.next(self)? {
87            // Decorate this part of the hash with provenance; leave the integer part unchanged.
88            let hash_fragment = self.read_scalar(&elem)?.to_target_usize(&tcx)?;
89            let ptr = Pointer::new(alloc_id.into(), Size::from_bytes(hash_fragment));
90            let ptr = self.global_root_pointer(ptr)?;
91            let val = Scalar::from_pointer(ptr, &tcx);
92            self.write_scalar(val, &elem)?;
93        }
94        interp_ok(())
95    }
96
97    /// Read a value of type `TypeId`, returning the type it represents.
98    pub(crate) fn read_type_id(
99        &self,
100        op: &OpTy<'tcx, M::Provenance>,
101    ) -> InterpResult<'tcx, Ty<'tcx>> {
102        // `TypeId` is a newtype around an array of pointers. All pointers must have the same
103        // provenance, and that provenance represents the type.
104        let ptr_size = self.pointer_size().bytes_usize();
105        let arr = self.project_field(op, FieldIdx::ZERO)?;
106
107        let mut ty_and_hash = None;
108        let mut elem_iter = self.project_array_fields(&arr)?;
109        while let Some((idx, elem)) = elem_iter.next(self)? {
110            let elem = self.read_pointer(&elem)?;
111            let (elem_ty, elem_hash) = self.get_ptr_type_id(elem)?;
112            // If this is the first element, remember the type and its hash.
113            // If this is not the first element, ensure it is consistent with the previous ones.
114            let full_hash = match ty_and_hash {
115                None => {
116                    let hash = self.tcx.type_id_hash(elem_ty).as_u128();
117                    let mut hash_bytes = [0u8; 16];
118                    write_target_uint(self.data_layout().endian, &mut hash_bytes, hash).unwrap();
119                    ty_and_hash = Some((elem_ty, hash_bytes));
120                    hash_bytes
121                }
122                Some((ty, hash_bytes)) => {
123                    if ty != elem_ty {
124                        throw_ub_format!(
125                            "invalid `TypeId` value: not all bytes carry the same type id metadata"
126                        );
127                    }
128                    hash_bytes
129                }
130            };
131            // Ensure the elem_hash matches the corresponding part of the full hash.
132            let hash_frag = &full_hash[(idx as usize) * ptr_size..][..ptr_size];
133            if read_target_uint(self.data_layout().endian, hash_frag).unwrap() != elem_hash.into() {
134                throw_ub_format!(
135                    "invalid `TypeId` value: the hash does not match the type id metadata"
136                );
137            }
138        }
139
140        interp_ok(ty_and_hash.unwrap().0)
141    }
142
143    /// Returns `true` if emulation happened.
144    /// Here we implement the intrinsics that are common to all Miri instances; individual machines can add their own
145    /// intrinsic handling.
146    pub fn eval_intrinsic(
147        &mut self,
148        instance: ty::Instance<'tcx>,
149        args: &[OpTy<'tcx, M::Provenance>],
150        dest: &PlaceTy<'tcx, M::Provenance>,
151        ret: Option<mir::BasicBlock>,
152    ) -> InterpResult<'tcx, bool> {
153        let instance_args = instance.args;
154        let intrinsic_name = self.tcx.item_name(instance.def_id());
155
156        if intrinsic_name.as_str().starts_with("simd_") {
157            return self.eval_simd_intrinsic(intrinsic_name, instance_args, args, dest, ret);
158        }
159
160        let tcx = self.tcx.tcx;
161
162        match intrinsic_name {
163            sym::type_name => {
164                let tp_ty = instance.args.type_at(0);
165                ensure_monomorphic_enough(tcx, tp_ty)?;
166                let (alloc_id, meta) = alloc_type_name(tcx, tp_ty);
167                let val = ConstValue::Slice { alloc_id, meta };
168                let val = self.const_val_to_op(val, dest.layout.ty, Some(dest.layout))?;
169                self.copy_op(&val, dest)?;
170            }
171            sym::needs_drop => {
172                let tp_ty = instance.args.type_at(0);
173                ensure_monomorphic_enough(tcx, tp_ty)?;
174                let val = ConstValue::from_bool(tp_ty.needs_drop(tcx, self.typing_env));
175                let val = self.const_val_to_op(val, tcx.types.bool, Some(dest.layout))?;
176                self.copy_op(&val, dest)?;
177            }
178            sym::type_id => {
179                let tp_ty = instance.args.type_at(0);
180                ensure_monomorphic_enough(tcx, tp_ty)?;
181                self.write_type_id(tp_ty, dest)?;
182            }
183            sym::type_id_eq => {
184                let a_ty = self.read_type_id(&args[0])?;
185                let b_ty = self.read_type_id(&args[1])?;
186                self.write_scalar(Scalar::from_bool(a_ty == b_ty), dest)?;
187            }
188            sym::size_of => {
189                let tp_ty = instance.args.type_at(0);
190                let layout = self.layout_of(tp_ty)?;
191                if !layout.is_sized() {
192                    span_bug!(self.cur_span(), "unsized type for `size_of`");
193                }
194                let val = layout.size.bytes();
195                self.write_scalar(Scalar::from_target_usize(val, self), dest)?;
196            }
197            sym::align_of => {
198                let tp_ty = instance.args.type_at(0);
199                let layout = self.layout_of(tp_ty)?;
200                if !layout.is_sized() {
201                    span_bug!(self.cur_span(), "unsized type for `align_of`");
202                }
203                let val = layout.align.bytes();
204                self.write_scalar(Scalar::from_target_usize(val, self), dest)?;
205            }
206            sym::variant_count => {
207                let tp_ty = instance.args.type_at(0);
208                let ty = match tp_ty.kind() {
209                    // Pattern types have the same number of variants as their base type.
210                    // Even if we restrict e.g. which variants are valid, the variants are essentially just uninhabited.
211                    // And `Result<(), !>` still has two variants according to `variant_count`.
212                    ty::Pat(base, _) => *base,
213                    _ => tp_ty,
214                };
215                let val = match ty.kind() {
216                    // Correctly handles non-monomorphic calls, so there is no need for ensure_monomorphic_enough.
217                    ty::Adt(adt, _) => {
218                        ConstValue::from_target_usize(adt.variants().len() as u64, &tcx)
219                    }
220                    ty::Alias(..) | ty::Param(_) | ty::Placeholder(_) | ty::Infer(_) => {
221                        throw_inval!(TooGeneric)
222                    }
223                    ty::Pat(..) => unreachable!(),
224                    ty::Bound(_, _) => bug!("bound ty during ctfe"),
225                    ty::Bool
226                    | ty::Char
227                    | ty::Int(_)
228                    | ty::Uint(_)
229                    | ty::Float(_)
230                    | ty::Foreign(_)
231                    | ty::Str
232                    | ty::Array(_, _)
233                    | ty::Slice(_)
234                    | ty::RawPtr(_, _)
235                    | ty::Ref(_, _, _)
236                    | ty::FnDef(_, _)
237                    | ty::FnPtr(..)
238                    | ty::Dynamic(_, _)
239                    | ty::Closure(_, _)
240                    | ty::CoroutineClosure(_, _)
241                    | ty::Coroutine(_, _)
242                    | ty::CoroutineWitness(..)
243                    | ty::UnsafeBinder(_)
244                    | ty::Never
245                    | ty::Tuple(_)
246                    | ty::Error(_) => ConstValue::from_target_usize(0u64, &tcx),
247                };
248                let val = self.const_val_to_op(val, dest.layout.ty, Some(dest.layout))?;
249                self.copy_op(&val, dest)?;
250            }
251
252            sym::caller_location => {
253                let span = self.find_closest_untracked_caller_location();
254                let val = self.tcx.span_as_caller_location(span);
255                let val =
256                    self.const_val_to_op(val, self.tcx.caller_location_ty(), Some(dest.layout))?;
257                self.copy_op(&val, dest)?;
258            }
259
260            sym::align_of_val | sym::size_of_val => {
261                // Avoid `deref_pointer` -- this is not a deref, the ptr does not have to be
262                // dereferenceable!
263                let place = self.ref_to_mplace(&self.read_immediate(&args[0])?)?;
264                let (size, align) = self
265                    .size_and_align_of_val(&place)?
266                    .ok_or_else(|| err_unsup_format!("`extern type` does not have known layout"))?;
267
268                let result = match intrinsic_name {
269                    sym::align_of_val => align.bytes(),
270                    sym::size_of_val => size.bytes(),
271                    _ => bug!(),
272                };
273
274                self.write_scalar(Scalar::from_target_usize(result, self), dest)?;
275            }
276
277            sym::fadd_algebraic
278            | sym::fsub_algebraic
279            | sym::fmul_algebraic
280            | sym::fdiv_algebraic
281            | sym::frem_algebraic => {
282                let a = self.read_immediate(&args[0])?;
283                let b = self.read_immediate(&args[1])?;
284
285                let op = match intrinsic_name {
286                    sym::fadd_algebraic => BinOp::Add,
287                    sym::fsub_algebraic => BinOp::Sub,
288                    sym::fmul_algebraic => BinOp::Mul,
289                    sym::fdiv_algebraic => BinOp::Div,
290                    sym::frem_algebraic => BinOp::Rem,
291
292                    _ => bug!(),
293                };
294
295                let res = self.binary_op(op, &a, &b)?;
296                // `binary_op` already called `generate_nan` if needed.
297                let res = M::apply_float_nondet(self, res)?;
298                self.write_immediate(*res, dest)?;
299            }
300
301            sym::ctpop
302            | sym::cttz
303            | sym::cttz_nonzero
304            | sym::ctlz
305            | sym::ctlz_nonzero
306            | sym::bswap
307            | sym::bitreverse => {
308                let ty = instance_args.type_at(0);
309                let layout = self.layout_of(ty)?;
310                let val = self.read_scalar(&args[0])?;
311
312                let out_val = self.numeric_intrinsic(intrinsic_name, val, layout, dest.layout)?;
313                self.write_scalar(out_val, dest)?;
314            }
315            sym::saturating_add | sym::saturating_sub => {
316                let l = self.read_immediate(&args[0])?;
317                let r = self.read_immediate(&args[1])?;
318                let val = self.saturating_arith(
319                    if intrinsic_name == sym::saturating_add { BinOp::Add } else { BinOp::Sub },
320                    &l,
321                    &r,
322                )?;
323                self.write_scalar(val, dest)?;
324            }
325            sym::discriminant_value => {
326                let place = self.deref_pointer(&args[0])?;
327                let variant = self.read_discriminant(&place)?;
328                let discr = self.discriminant_for_variant(place.layout.ty, variant)?;
329                self.write_immediate(*discr, dest)?;
330            }
331            sym::exact_div => {
332                let l = self.read_immediate(&args[0])?;
333                let r = self.read_immediate(&args[1])?;
334                self.exact_div(&l, &r, dest)?;
335            }
336            sym::rotate_left | sym::rotate_right => {
337                // rotate_left: (X << (S % BW)) | (X >> ((BW - S) % BW))
338                // rotate_right: (X << ((BW - S) % BW)) | (X >> (S % BW))
339                let layout_val = self.layout_of(instance_args.type_at(0))?;
340                let val = self.read_scalar(&args[0])?;
341                let val_bits = val.to_bits(layout_val.size)?; // sign is ignored here
342
343                let layout_raw_shift = self.layout_of(self.tcx.types.u32)?;
344                let raw_shift = self.read_scalar(&args[1])?;
345                let raw_shift_bits = raw_shift.to_bits(layout_raw_shift.size)?;
346
347                let width_bits = u128::from(layout_val.size.bits());
348                let shift_bits = raw_shift_bits % width_bits;
349                let inv_shift_bits = (width_bits - shift_bits) % width_bits;
350                let result_bits = if intrinsic_name == sym::rotate_left {
351                    (val_bits << shift_bits) | (val_bits >> inv_shift_bits)
352                } else {
353                    (val_bits >> shift_bits) | (val_bits << inv_shift_bits)
354                };
355                let truncated_bits = layout_val.size.truncate(result_bits);
356                let result = Scalar::from_uint(truncated_bits, layout_val.size);
357                self.write_scalar(result, dest)?;
358            }
359            sym::copy => {
360                self.copy_intrinsic(&args[0], &args[1], &args[2], /*nonoverlapping*/ false)?;
361            }
362            sym::write_bytes => {
363                self.write_bytes_intrinsic(&args[0], &args[1], &args[2], "write_bytes")?;
364            }
365            sym::compare_bytes => {
366                let result = self.compare_bytes_intrinsic(&args[0], &args[1], &args[2])?;
367                self.write_scalar(result, dest)?;
368            }
369            sym::arith_offset => {
370                let ptr = self.read_pointer(&args[0])?;
371                let offset_count = self.read_target_isize(&args[1])?;
372                let pointee_ty = instance_args.type_at(0);
373
374                let pointee_size = i64::try_from(self.layout_of(pointee_ty)?.size.bytes()).unwrap();
375                let offset_bytes = offset_count.wrapping_mul(pointee_size);
376                let offset_ptr = ptr.wrapping_signed_offset(offset_bytes, self);
377                self.write_pointer(offset_ptr, dest)?;
378            }
379            sym::ptr_offset_from | sym::ptr_offset_from_unsigned => {
380                let a = self.read_pointer(&args[0])?;
381                let b = self.read_pointer(&args[1])?;
382
383                let usize_layout = self.layout_of(self.tcx.types.usize)?;
384                let isize_layout = self.layout_of(self.tcx.types.isize)?;
385
386                // Get offsets for both that are at least relative to the same base.
387                // With `OFFSET_IS_ADDR` this is trivial; without it we need either
388                // two integers or two pointers into the same allocation.
389                let (a_offset, b_offset, is_addr) = if M::Provenance::OFFSET_IS_ADDR {
390                    (a.addr().bytes(), b.addr().bytes(), /*is_addr*/ true)
391                } else {
392                    match (self.ptr_try_get_alloc_id(a, 0), self.ptr_try_get_alloc_id(b, 0)) {
393                        (Err(a), Err(b)) => {
394                            // Neither pointer points to an allocation, so they are both absolute.
395                            (a, b, /*is_addr*/ true)
396                        }
397                        (Ok((a_alloc_id, a_offset, _)), Ok((b_alloc_id, b_offset, _)))
398                            if a_alloc_id == b_alloc_id =>
399                        {
400                            // Found allocation for both, and it's the same.
401                            // Use these offsets for distance calculation.
402                            (a_offset.bytes(), b_offset.bytes(), /*is_addr*/ false)
403                        }
404                        _ => {
405                            // Not into the same allocation -- this is UB.
406                            throw_ub_custom!(
407                                fluent::const_eval_offset_from_different_allocations,
408                                name = intrinsic_name,
409                            );
410                        }
411                    }
412                };
413
414                // Compute distance: a - b.
415                let dist = {
416                    // Addresses are unsigned, so this is a `usize` computation. We have to do the
417                    // overflow check separately anyway.
418                    let (val, overflowed) = {
419                        let a_offset = ImmTy::from_uint(a_offset, usize_layout);
420                        let b_offset = ImmTy::from_uint(b_offset, usize_layout);
421                        self.binary_op(BinOp::SubWithOverflow, &a_offset, &b_offset)?
422                            .to_scalar_pair()
423                    };
424                    if overflowed.to_bool()? {
425                        // a < b
426                        if intrinsic_name == sym::ptr_offset_from_unsigned {
427                            throw_ub_custom!(
428                                fluent::const_eval_offset_from_unsigned_overflow,
429                                a_offset = a_offset,
430                                b_offset = b_offset,
431                                is_addr = is_addr,
432                            );
433                        }
434                        // The signed form of the intrinsic allows this. If we interpret the
435                        // difference as isize, we'll get the proper signed difference. If that
436                        // seems *positive* or equal to isize::MIN, they were more than isize::MAX apart.
437                        let dist = val.to_target_isize(self)?;
438                        if dist >= 0 || i128::from(dist) == self.pointer_size().signed_int_min() {
439                            throw_ub_custom!(
440                                fluent::const_eval_offset_from_underflow,
441                                name = intrinsic_name,
442                            );
443                        }
444                        dist
445                    } else {
446                        // b >= a
447                        let dist = val.to_target_isize(self)?;
448                        // If converting to isize produced a *negative* result, we had an overflow
449                        // because they were more than isize::MAX apart.
450                        if dist < 0 {
451                            throw_ub_custom!(
452                                fluent::const_eval_offset_from_overflow,
453                                name = intrinsic_name,
454                            );
455                        }
456                        dist
457                    }
458                };
459
460                // Check that the memory between them is dereferenceable at all, starting from the
461                // origin pointer: `dist` is `a - b`, so it is based on `b`.
462                self.check_ptr_access_signed(b, dist, CheckInAllocMsg::Dereferenceable)
463                    .map_err_kind(|_| {
464                        // This could mean they point to different allocations, or they point to the same allocation
465                        // but not the entire range between the pointers is in-bounds.
466                        if let Ok((a_alloc_id, ..)) = self.ptr_try_get_alloc_id(a, 0)
467                            && let Ok((b_alloc_id, ..)) = self.ptr_try_get_alloc_id(b, 0)
468                            && a_alloc_id == b_alloc_id
469                        {
470                            err_ub_custom!(
471                                fluent::const_eval_offset_from_out_of_bounds,
472                                name = intrinsic_name,
473                            )
474                        } else {
475                            err_ub_custom!(
476                                fluent::const_eval_offset_from_different_allocations,
477                                name = intrinsic_name,
478                            )
479                        }
480                    })?;
481                // Then check that this is also dereferenceable from `a`. This ensures that they are
482                // derived from the same allocation.
483                self.check_ptr_access_signed(
484                    a,
485                    dist.checked_neg().unwrap(), // i64::MIN is impossible as no allocation can be that large
486                    CheckInAllocMsg::Dereferenceable,
487                )
488                .map_err_kind(|_| {
489                    // Make the error more specific.
490                    err_ub_custom!(
491                        fluent::const_eval_offset_from_different_allocations,
492                        name = intrinsic_name,
493                    )
494                })?;
495
496                // Perform division by size to compute return value.
497                let ret_layout = if intrinsic_name == sym::ptr_offset_from_unsigned {
498                    assert!(0 <= dist && dist <= self.target_isize_max());
499                    usize_layout
500                } else {
501                    assert!(self.target_isize_min() <= dist && dist <= self.target_isize_max());
502                    isize_layout
503                };
504                let pointee_layout = self.layout_of(instance_args.type_at(0))?;
505                // If ret_layout is unsigned, we checked that so is the distance, so we are good.
506                let val = ImmTy::from_int(dist, ret_layout);
507                let size = ImmTy::from_int(pointee_layout.size.bytes(), ret_layout);
508                self.exact_div(&val, &size, dest)?;
509            }
510
511            sym::black_box => {
512                // These just return their argument
513                self.copy_op(&args[0], dest)?;
514            }
515            sym::raw_eq => {
516                let result = self.raw_eq_intrinsic(&args[0], &args[1])?;
517                self.write_scalar(result, dest)?;
518            }
519            sym::typed_swap_nonoverlapping => {
520                self.typed_swap_nonoverlapping_intrinsic(&args[0], &args[1])?;
521            }
522
523            sym::vtable_size => {
524                let ptr = self.read_pointer(&args[0])?;
525                // `None` because we don't know which trait to expect here; any vtable is okay.
526                let (size, _align) = self.get_vtable_size_and_align(ptr, None)?;
527                self.write_scalar(Scalar::from_target_usize(size.bytes(), self), dest)?;
528            }
529            sym::vtable_align => {
530                let ptr = self.read_pointer(&args[0])?;
531                // `None` because we don't know which trait to expect here; any vtable is okay.
532                let (_size, align) = self.get_vtable_size_and_align(ptr, None)?;
533                self.write_scalar(Scalar::from_target_usize(align.bytes(), self), dest)?;
534            }
535
536            sym::minnumf16 => self.float_minmax_intrinsic::<Half>(args, MinMax::MinNum, dest)?,
537            sym::minnumf32 => self.float_minmax_intrinsic::<Single>(args, MinMax::MinNum, dest)?,
538            sym::minnumf64 => self.float_minmax_intrinsic::<Double>(args, MinMax::MinNum, dest)?,
539            sym::minnumf128 => self.float_minmax_intrinsic::<Quad>(args, MinMax::MinNum, dest)?,
540
541            sym::minimumf16 => self.float_minmax_intrinsic::<Half>(args, MinMax::Minimum, dest)?,
542            sym::minimumf32 => {
543                self.float_minmax_intrinsic::<Single>(args, MinMax::Minimum, dest)?
544            }
545            sym::minimumf64 => {
546                self.float_minmax_intrinsic::<Double>(args, MinMax::Minimum, dest)?
547            }
548            sym::minimumf128 => self.float_minmax_intrinsic::<Quad>(args, MinMax::Minimum, dest)?,
549
550            sym::maxnumf16 => self.float_minmax_intrinsic::<Half>(args, MinMax::MaxNum, dest)?,
551            sym::maxnumf32 => self.float_minmax_intrinsic::<Single>(args, MinMax::MaxNum, dest)?,
552            sym::maxnumf64 => self.float_minmax_intrinsic::<Double>(args, MinMax::MaxNum, dest)?,
553            sym::maxnumf128 => self.float_minmax_intrinsic::<Quad>(args, MinMax::MaxNum, dest)?,
554
555            sym::maximumf16 => self.float_minmax_intrinsic::<Half>(args, MinMax::Maximum, dest)?,
556            sym::maximumf32 => {
557                self.float_minmax_intrinsic::<Single>(args, MinMax::Maximum, dest)?
558            }
559            sym::maximumf64 => {
560                self.float_minmax_intrinsic::<Double>(args, MinMax::Maximum, dest)?
561            }
562            sym::maximumf128 => self.float_minmax_intrinsic::<Quad>(args, MinMax::Maximum, dest)?,
563
564            sym::copysignf16 => self.float_copysign_intrinsic::<Half>(args, dest)?,
565            sym::copysignf32 => self.float_copysign_intrinsic::<Single>(args, dest)?,
566            sym::copysignf64 => self.float_copysign_intrinsic::<Double>(args, dest)?,
567            sym::copysignf128 => self.float_copysign_intrinsic::<Quad>(args, dest)?,
568
569            sym::fabsf16 => self.float_abs_intrinsic::<Half>(args, dest)?,
570            sym::fabsf32 => self.float_abs_intrinsic::<Single>(args, dest)?,
571            sym::fabsf64 => self.float_abs_intrinsic::<Double>(args, dest)?,
572            sym::fabsf128 => self.float_abs_intrinsic::<Quad>(args, dest)?,
573
574            sym::floorf16 => self.float_round_intrinsic::<Half>(
575                args,
576                dest,
577                rustc_apfloat::Round::TowardNegative,
578            )?,
579            sym::floorf32 => self.float_round_intrinsic::<Single>(
580                args,
581                dest,
582                rustc_apfloat::Round::TowardNegative,
583            )?,
584            sym::floorf64 => self.float_round_intrinsic::<Double>(
585                args,
586                dest,
587                rustc_apfloat::Round::TowardNegative,
588            )?,
589            sym::floorf128 => self.float_round_intrinsic::<Quad>(
590                args,
591                dest,
592                rustc_apfloat::Round::TowardNegative,
593            )?,
594
595            sym::ceilf16 => self.float_round_intrinsic::<Half>(
596                args,
597                dest,
598                rustc_apfloat::Round::TowardPositive,
599            )?,
600            sym::ceilf32 => self.float_round_intrinsic::<Single>(
601                args,
602                dest,
603                rustc_apfloat::Round::TowardPositive,
604            )?,
605            sym::ceilf64 => self.float_round_intrinsic::<Double>(
606                args,
607                dest,
608                rustc_apfloat::Round::TowardPositive,
609            )?,
610            sym::ceilf128 => self.float_round_intrinsic::<Quad>(
611                args,
612                dest,
613                rustc_apfloat::Round::TowardPositive,
614            )?,
615
616            sym::truncf16 => {
617                self.float_round_intrinsic::<Half>(args, dest, rustc_apfloat::Round::TowardZero)?
618            }
619            sym::truncf32 => {
620                self.float_round_intrinsic::<Single>(args, dest, rustc_apfloat::Round::TowardZero)?
621            }
622            sym::truncf64 => {
623                self.float_round_intrinsic::<Double>(args, dest, rustc_apfloat::Round::TowardZero)?
624            }
625            sym::truncf128 => {
626                self.float_round_intrinsic::<Quad>(args, dest, rustc_apfloat::Round::TowardZero)?
627            }
628
629            sym::roundf16 => self.float_round_intrinsic::<Half>(
630                args,
631                dest,
632                rustc_apfloat::Round::NearestTiesToAway,
633            )?,
634            sym::roundf32 => self.float_round_intrinsic::<Single>(
635                args,
636                dest,
637                rustc_apfloat::Round::NearestTiesToAway,
638            )?,
639            sym::roundf64 => self.float_round_intrinsic::<Double>(
640                args,
641                dest,
642                rustc_apfloat::Round::NearestTiesToAway,
643            )?,
644            sym::roundf128 => self.float_round_intrinsic::<Quad>(
645                args,
646                dest,
647                rustc_apfloat::Round::NearestTiesToAway,
648            )?,
649
650            sym::round_ties_even_f16 => self.float_round_intrinsic::<Half>(
651                args,
652                dest,
653                rustc_apfloat::Round::NearestTiesToEven,
654            )?,
655            sym::round_ties_even_f32 => self.float_round_intrinsic::<Single>(
656                args,
657                dest,
658                rustc_apfloat::Round::NearestTiesToEven,
659            )?,
660            sym::round_ties_even_f64 => self.float_round_intrinsic::<Double>(
661                args,
662                dest,
663                rustc_apfloat::Round::NearestTiesToEven,
664            )?,
665            sym::round_ties_even_f128 => self.float_round_intrinsic::<Quad>(
666                args,
667                dest,
668                rustc_apfloat::Round::NearestTiesToEven,
669            )?,
670            sym::fmaf16 => self.float_muladd_intrinsic::<Half>(args, dest, MulAddType::Fused)?,
671            sym::fmaf32 => self.float_muladd_intrinsic::<Single>(args, dest, MulAddType::Fused)?,
672            sym::fmaf64 => self.float_muladd_intrinsic::<Double>(args, dest, MulAddType::Fused)?,
673            sym::fmaf128 => self.float_muladd_intrinsic::<Quad>(args, dest, MulAddType::Fused)?,
674            sym::fmuladdf16 => {
675                self.float_muladd_intrinsic::<Half>(args, dest, MulAddType::Nondeterministic)?
676            }
677            sym::fmuladdf32 => {
678                self.float_muladd_intrinsic::<Single>(args, dest, MulAddType::Nondeterministic)?
679            }
680            sym::fmuladdf64 => {
681                self.float_muladd_intrinsic::<Double>(args, dest, MulAddType::Nondeterministic)?
682            }
683            sym::fmuladdf128 => {
684                self.float_muladd_intrinsic::<Quad>(args, dest, MulAddType::Nondeterministic)?
685            }
686
687            // Unsupported intrinsic: skip the return_to_block below.
688            _ => return interp_ok(false),
689        }
690
691        trace!("{:?}", self.dump_place(&dest.clone().into()));
692        self.return_to_block(ret)?;
693        interp_ok(true)
694    }
695
696    pub(super) fn eval_nondiverging_intrinsic(
697        &mut self,
698        intrinsic: &NonDivergingIntrinsic<'tcx>,
699    ) -> InterpResult<'tcx> {
700        match intrinsic {
701            NonDivergingIntrinsic::Assume(op) => {
702                let op = self.eval_operand(op, None)?;
703                let cond = self.read_scalar(&op)?.to_bool()?;
704                if !cond {
705                    throw_ub_custom!(fluent::const_eval_assume_false);
706                }
707                interp_ok(())
708            }
709            NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping {
710                count,
711                src,
712                dst,
713            }) => {
714                let src = self.eval_operand(src, None)?;
715                let dst = self.eval_operand(dst, None)?;
716                let count = self.eval_operand(count, None)?;
717                self.copy_intrinsic(&src, &dst, &count, /* nonoverlapping */ true)
718            }
719        }
720    }
721
722    pub fn numeric_intrinsic(
723        &self,
724        name: Symbol,
725        val: Scalar<M::Provenance>,
726        layout: TyAndLayout<'tcx>,
727        ret_layout: TyAndLayout<'tcx>,
728    ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
729        assert!(layout.ty.is_integral(), "invalid type for numeric intrinsic: {}", layout.ty);
730        let bits = val.to_bits(layout.size)?; // these operations all ignore the sign
731        let extra = 128 - u128::from(layout.size.bits());
732        let bits_out = match name {
733            sym::ctpop => u128::from(bits.count_ones()),
734            sym::ctlz_nonzero | sym::cttz_nonzero if bits == 0 => {
735                throw_ub_custom!(fluent::const_eval_call_nonzero_intrinsic, name = name,);
736            }
737            sym::ctlz | sym::ctlz_nonzero => u128::from(bits.leading_zeros()) - extra,
738            sym::cttz | sym::cttz_nonzero => u128::from((bits << extra).trailing_zeros()) - extra,
739            sym::bswap => {
740                assert_eq!(layout, ret_layout);
741                (bits << extra).swap_bytes()
742            }
743            sym::bitreverse => {
744                assert_eq!(layout, ret_layout);
745                (bits << extra).reverse_bits()
746            }
747            _ => bug!("not a numeric intrinsic: {}", name),
748        };
749        interp_ok(Scalar::from_uint(bits_out, ret_layout.size))
750    }
751
752    pub fn exact_div(
753        &mut self,
754        a: &ImmTy<'tcx, M::Provenance>,
755        b: &ImmTy<'tcx, M::Provenance>,
756        dest: &PlaceTy<'tcx, M::Provenance>,
757    ) -> InterpResult<'tcx> {
758        assert_eq!(a.layout.ty, b.layout.ty);
759        assert_matches!(a.layout.ty.kind(), ty::Int(..) | ty::Uint(..));
760
761        // Performs an exact division, resulting in undefined behavior where
762        // `x % y != 0` or `y == 0` or `x == T::MIN && y == -1`.
763        // First, check x % y != 0 (or if that computation overflows).
764        let rem = self.binary_op(BinOp::Rem, a, b)?;
765        // sign does not matter for 0 test, so `to_bits` is fine
766        if rem.to_scalar().to_bits(a.layout.size)? != 0 {
767            throw_ub_custom!(
768                fluent::const_eval_exact_div_has_remainder,
769                a = format!("{a}"),
770                b = format!("{b}")
771            )
772        }
773        // `Rem` says this is all right, so we can let `Div` do its job.
774        let res = self.binary_op(BinOp::Div, a, b)?;
775        self.write_immediate(*res, dest)
776    }
777
778    pub fn saturating_arith(
779        &self,
780        mir_op: BinOp,
781        l: &ImmTy<'tcx, M::Provenance>,
782        r: &ImmTy<'tcx, M::Provenance>,
783    ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
784        assert_eq!(l.layout.ty, r.layout.ty);
785        assert_matches!(l.layout.ty.kind(), ty::Int(..) | ty::Uint(..));
786        assert_matches!(mir_op, BinOp::Add | BinOp::Sub);
787
788        let (val, overflowed) =
789            self.binary_op(mir_op.wrapping_to_overflowing().unwrap(), l, r)?.to_scalar_pair();
790        interp_ok(if overflowed.to_bool()? {
791            let size = l.layout.size;
792            if l.layout.backend_repr.is_signed() {
793                // For signed ints the saturated value depends on the sign of the first
794                // term since the sign of the second term can be inferred from this and
795                // the fact that the operation has overflowed (if either is 0 no
796                // overflow can occur)
797                let first_term: i128 = l.to_scalar().to_int(l.layout.size)?;
798                if first_term >= 0 {
799                    // Negative overflow not possible since the positive first term
800                    // can only increase an (in range) negative term for addition
801                    // or corresponding negated positive term for subtraction.
802                    Scalar::from_int(size.signed_int_max(), size)
803                } else {
804                    // Positive overflow not possible for similar reason.
805                    Scalar::from_int(size.signed_int_min(), size)
806                }
807            } else {
808                // unsigned
809                if matches!(mir_op, BinOp::Add) {
810                    // max unsigned
811                    Scalar::from_uint(size.unsigned_int_max(), size)
812                } else {
813                    // underflow to 0
814                    Scalar::from_uint(0u128, size)
815                }
816            }
817        } else {
818            val
819        })
820    }
821
822    /// Offsets a pointer by some multiple of its type, returning an error if the pointer leaves its
823    /// allocation.
824    pub fn ptr_offset_inbounds(
825        &self,
826        ptr: Pointer<Option<M::Provenance>>,
827        offset_bytes: i64,
828    ) -> InterpResult<'tcx, Pointer<Option<M::Provenance>>> {
829        // The offset must be in bounds starting from `ptr`.
830        self.check_ptr_access_signed(
831            ptr,
832            offset_bytes,
833            CheckInAllocMsg::InboundsPointerArithmetic,
834        )?;
835        // This also implies that there is no overflow, so we are done.
836        interp_ok(ptr.wrapping_signed_offset(offset_bytes, self))
837    }
838
839    /// Copy `count*size_of::<T>()` many bytes from `*src` to `*dst`.
840    pub(crate) fn copy_intrinsic(
841        &mut self,
842        src: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
843        dst: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
844        count: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
845        nonoverlapping: bool,
846    ) -> InterpResult<'tcx> {
847        let count = self.read_target_usize(count)?;
848        let layout = self.layout_of(src.layout.ty.builtin_deref(true).unwrap())?;
849        let (size, align) = (layout.size, layout.align.abi);
850
851        let size = self.compute_size_in_bytes(size, count).ok_or_else(|| {
852            err_ub_custom!(
853                fluent::const_eval_size_overflow,
854                name = if nonoverlapping { "copy_nonoverlapping" } else { "copy" }
855            )
856        })?;
857
858        let src = self.read_pointer(src)?;
859        let dst = self.read_pointer(dst)?;
860
861        self.check_ptr_align(src, align)?;
862        self.check_ptr_align(dst, align)?;
863
864        self.mem_copy(src, dst, size, nonoverlapping)
865    }
866
867    /// Does a *typed* swap of `*left` and `*right`.
868    fn typed_swap_nonoverlapping_intrinsic(
869        &mut self,
870        left: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
871        right: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
872    ) -> InterpResult<'tcx> {
873        let left = self.deref_pointer(left)?;
874        let right = self.deref_pointer(right)?;
875        assert_eq!(left.layout, right.layout);
876        assert!(left.layout.is_sized());
877        let kind = MemoryKind::Stack;
878        let temp = self.allocate(left.layout, kind)?;
879        self.copy_op(&left, &temp)?; // checks alignment of `left`
880
881        // We want to always enforce non-overlapping, even if this is a scalar type.
882        // Therefore we directly use the underlying `mem_copy` here.
883        self.mem_copy(right.ptr(), left.ptr(), left.layout.size, /*nonoverlapping*/ true)?;
884        // This means we also need to do the validation of the value that used to be in `right`
885        // ourselves. This value is now in `left.` The one that started out in `left` already got
886        // validated by the copy above.
887        if M::enforce_validity(self, left.layout) {
888            self.validate_operand(
889                &left.clone().into(),
890                M::enforce_validity_recursively(self, left.layout),
891                /*reset_provenance_and_padding*/ true,
892            )?;
893        }
894
895        self.copy_op(&temp, &right)?; // checks alignment of `right`
896
897        self.deallocate_ptr(temp.ptr(), None, kind)?;
898        interp_ok(())
899    }
900
901    pub fn write_bytes_intrinsic(
902        &mut self,
903        dst: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
904        byte: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
905        count: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
906        name: &'static str,
907    ) -> InterpResult<'tcx> {
908        let layout = self.layout_of(dst.layout.ty.builtin_deref(true).unwrap())?;
909
910        let dst = self.read_pointer(dst)?;
911        let byte = self.read_scalar(byte)?.to_u8()?;
912        let count = self.read_target_usize(count)?;
913
914        // `checked_mul` enforces a too small bound (the correct one would probably be target_isize_max),
915        // but no actual allocation can be big enough for the difference to be noticeable.
916        let len = self
917            .compute_size_in_bytes(layout.size, count)
918            .ok_or_else(|| err_ub_custom!(fluent::const_eval_size_overflow, name = name))?;
919
920        let bytes = std::iter::repeat_n(byte, len.bytes_usize());
921        self.write_bytes_ptr(dst, bytes)
922    }
923
924    pub(crate) fn compare_bytes_intrinsic(
925        &mut self,
926        left: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
927        right: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
928        byte_count: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
929    ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
930        let left = self.read_pointer(left)?;
931        let right = self.read_pointer(right)?;
932        let n = Size::from_bytes(self.read_target_usize(byte_count)?);
933
934        let left_bytes = self.read_bytes_ptr_strip_provenance(left, n)?;
935        let right_bytes = self.read_bytes_ptr_strip_provenance(right, n)?;
936
937        // `Ordering`'s discriminants are -1/0/+1, so casting does the right thing.
938        let result = Ord::cmp(left_bytes, right_bytes) as i32;
939        interp_ok(Scalar::from_i32(result))
940    }
941
942    pub(crate) fn raw_eq_intrinsic(
943        &mut self,
944        lhs: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
945        rhs: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
946    ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
947        let layout = self.layout_of(lhs.layout.ty.builtin_deref(true).unwrap())?;
948        assert!(layout.is_sized());
949
950        let get_bytes = |this: &InterpCx<'tcx, M>,
951                         op: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>|
952         -> InterpResult<'tcx, &[u8]> {
953            let ptr = this.read_pointer(op)?;
954            this.check_ptr_align(ptr, layout.align.abi)?;
955            let Some(alloc_ref) = self.get_ptr_alloc(ptr, layout.size)? else {
956                // zero-sized access
957                return interp_ok(&[]);
958            };
959            alloc_ref.get_bytes_strip_provenance()
960        };
961
962        let lhs_bytes = get_bytes(self, lhs)?;
963        let rhs_bytes = get_bytes(self, rhs)?;
964        interp_ok(Scalar::from_bool(lhs_bytes == rhs_bytes))
965    }
966
967    fn float_minmax<F>(
968        &self,
969        a: Scalar<M::Provenance>,
970        b: Scalar<M::Provenance>,
971        op: MinMax,
972    ) -> InterpResult<'tcx, Scalar<M::Provenance>>
973    where
974        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
975    {
976        let a: F = a.to_float()?;
977        let b: F = b.to_float()?;
978        let res = if matches!(op, MinMax::MinNum | MinMax::MaxNum) && a == b {
979            // They are definitely not NaN (those are never equal), but they could be `+0` and `-0`.
980            // Let the machine decide which one to return.
981            M::equal_float_min_max(self, a, b)
982        } else {
983            let result = match op {
984                MinMax::Minimum => a.minimum(b),
985                MinMax::MinNum => a.min(b),
986                MinMax::Maximum => a.maximum(b),
987                MinMax::MaxNum => a.max(b),
988            };
989            self.adjust_nan(result, &[a, b])
990        };
991
992        interp_ok(res.into())
993    }
994
995    fn float_minmax_intrinsic<F>(
996        &mut self,
997        args: &[OpTy<'tcx, M::Provenance>],
998        op: MinMax,
999        dest: &PlaceTy<'tcx, M::Provenance>,
1000    ) -> InterpResult<'tcx, ()>
1001    where
1002        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1003    {
1004        let res =
1005            self.float_minmax::<F>(self.read_scalar(&args[0])?, self.read_scalar(&args[1])?, op)?;
1006        self.write_scalar(res, dest)?;
1007        interp_ok(())
1008    }
1009
1010    fn float_copysign_intrinsic<F>(
1011        &mut self,
1012        args: &[OpTy<'tcx, M::Provenance>],
1013        dest: &PlaceTy<'tcx, M::Provenance>,
1014    ) -> InterpResult<'tcx, ()>
1015    where
1016        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1017    {
1018        let a: F = self.read_scalar(&args[0])?.to_float()?;
1019        let b: F = self.read_scalar(&args[1])?.to_float()?;
1020        // bitwise, no NaN adjustments
1021        self.write_scalar(a.copy_sign(b), dest)?;
1022        interp_ok(())
1023    }
1024
1025    fn float_abs_intrinsic<F>(
1026        &mut self,
1027        args: &[OpTy<'tcx, M::Provenance>],
1028        dest: &PlaceTy<'tcx, M::Provenance>,
1029    ) -> InterpResult<'tcx, ()>
1030    where
1031        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1032    {
1033        let x: F = self.read_scalar(&args[0])?.to_float()?;
1034        // bitwise, no NaN adjustments
1035        self.write_scalar(x.abs(), dest)?;
1036        interp_ok(())
1037    }
1038
1039    fn float_round<F>(
1040        &mut self,
1041        x: Scalar<M::Provenance>,
1042        mode: rustc_apfloat::Round,
1043    ) -> InterpResult<'tcx, Scalar<M::Provenance>>
1044    where
1045        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1046    {
1047        let x: F = x.to_float()?;
1048        let res = x.round_to_integral(mode).value;
1049        let res = self.adjust_nan(res, &[x]);
1050        interp_ok(res.into())
1051    }
1052
1053    fn float_round_intrinsic<F>(
1054        &mut self,
1055        args: &[OpTy<'tcx, M::Provenance>],
1056        dest: &PlaceTy<'tcx, M::Provenance>,
1057        mode: rustc_apfloat::Round,
1058    ) -> InterpResult<'tcx, ()>
1059    where
1060        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1061    {
1062        let res = self.float_round::<F>(self.read_scalar(&args[0])?, mode)?;
1063        self.write_scalar(res, dest)?;
1064        interp_ok(())
1065    }
1066
1067    fn float_muladd<F>(
1068        &self,
1069        a: Scalar<M::Provenance>,
1070        b: Scalar<M::Provenance>,
1071        c: Scalar<M::Provenance>,
1072        typ: MulAddType,
1073    ) -> InterpResult<'tcx, Scalar<M::Provenance>>
1074    where
1075        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1076    {
1077        let a: F = a.to_float()?;
1078        let b: F = b.to_float()?;
1079        let c: F = c.to_float()?;
1080
1081        let fuse = typ == MulAddType::Fused || M::float_fuse_mul_add(self);
1082
1083        let res = if fuse { a.mul_add(b, c).value } else { ((a * b).value + c).value };
1084        let res = self.adjust_nan(res, &[a, b, c]);
1085        interp_ok(res.into())
1086    }
1087
1088    fn float_muladd_intrinsic<F>(
1089        &mut self,
1090        args: &[OpTy<'tcx, M::Provenance>],
1091        dest: &PlaceTy<'tcx, M::Provenance>,
1092        typ: MulAddType,
1093    ) -> InterpResult<'tcx, ()>
1094    where
1095        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1096    {
1097        let a = self.read_scalar(&args[0])?;
1098        let b = self.read_scalar(&args[1])?;
1099        let c = self.read_scalar(&args[2])?;
1100
1101        let res = self.float_muladd::<F>(a, b, c, typ)?;
1102        self.write_scalar(res, dest)?;
1103        interp_ok(())
1104    }
1105
1106    /// Converts `src` from floating point to integer type `dest_ty`
1107    /// after rounding with mode `round`.
1108    /// Returns `None` if `f` is NaN or out of range.
1109    pub fn float_to_int_checked(
1110        &self,
1111        src: &ImmTy<'tcx, M::Provenance>,
1112        cast_to: TyAndLayout<'tcx>,
1113        round: rustc_apfloat::Round,
1114    ) -> InterpResult<'tcx, Option<ImmTy<'tcx, M::Provenance>>> {
1115        fn float_to_int_inner<'tcx, F: rustc_apfloat::Float, M: Machine<'tcx>>(
1116            ecx: &InterpCx<'tcx, M>,
1117            src: F,
1118            cast_to: TyAndLayout<'tcx>,
1119            round: rustc_apfloat::Round,
1120        ) -> (Scalar<M::Provenance>, rustc_apfloat::Status) {
1121            let int_size = cast_to.layout.size;
1122            match cast_to.ty.kind() {
1123                // Unsigned
1124                ty::Uint(_) => {
1125                    let res = src.to_u128_r(int_size.bits_usize(), round, &mut false);
1126                    (Scalar::from_uint(res.value, int_size), res.status)
1127                }
1128                // Signed
1129                ty::Int(_) => {
1130                    let res = src.to_i128_r(int_size.bits_usize(), round, &mut false);
1131                    (Scalar::from_int(res.value, int_size), res.status)
1132                }
1133                // Nothing else
1134                _ => span_bug!(
1135                    ecx.cur_span(),
1136                    "attempted float-to-int conversion with non-int output type {}",
1137                    cast_to.ty,
1138                ),
1139            }
1140        }
1141
1142        let ty::Float(fty) = src.layout.ty.kind() else {
1143            bug!("float_to_int_checked: non-float input type {}", src.layout.ty)
1144        };
1145
1146        let (val, status) = match fty {
1147            FloatTy::F16 => float_to_int_inner(self, src.to_scalar().to_f16()?, cast_to, round),
1148            FloatTy::F32 => float_to_int_inner(self, src.to_scalar().to_f32()?, cast_to, round),
1149            FloatTy::F64 => float_to_int_inner(self, src.to_scalar().to_f64()?, cast_to, round),
1150            FloatTy::F128 => float_to_int_inner(self, src.to_scalar().to_f128()?, cast_to, round),
1151        };
1152
1153        if status.intersects(
1154            rustc_apfloat::Status::INVALID_OP
1155                | rustc_apfloat::Status::OVERFLOW
1156                | rustc_apfloat::Status::UNDERFLOW,
1157        ) {
1158            // Floating point value is NaN (flagged with INVALID_OP) or outside the range
1159            // of values of the integer type (flagged with OVERFLOW or UNDERFLOW).
1160            interp_ok(None)
1161        } else {
1162            // Floating point value can be represented by the integer type after rounding.
1163            // The INEXACT flag is ignored on purpose to allow rounding.
1164            interp_ok(Some(ImmTy::from_scalar(val, cast_to)))
1165        }
1166    }
1167}
rustc_const_eval/interpret/intrinsics.rs

rustc_const_eval/interpret/
intrinsics.rs
