rustc_const_eval/interpret/
intrinsics.rs

1//! Intrinsics and other functions that the interpreter executes without
2//! looking at their MIR. Intrinsics/functions supported here are shared by CTFE
3//! and miri.
4
5mod simd;
6
7use std::assert_matches::assert_matches;
8
9use rustc_abi::{FieldIdx, HasDataLayout, Size};
10use rustc_apfloat::ieee::{Double, Half, Quad, Single};
11use rustc_middle::mir::interpret::{CTFE_ALLOC_SALT, read_target_uint, write_target_uint};
12use rustc_middle::mir::{self, BinOp, ConstValue, NonDivergingIntrinsic};
13use rustc_middle::ty::layout::TyAndLayout;
14use rustc_middle::ty::{FloatTy, Ty, TyCtxt};
15use rustc_middle::{bug, span_bug, ty};
16use rustc_span::{Symbol, sym};
17use tracing::trace;
18
19use super::memory::MemoryKind;
20use super::util::ensure_monomorphic_enough;
21use super::{
22    AllocId, CheckInAllocMsg, ImmTy, InterpCx, InterpResult, Machine, OpTy, PlaceTy, Pointer,
23    PointerArithmetic, Provenance, Scalar, err_ub_custom, err_unsup_format, interp_ok, throw_inval,
24    throw_ub_custom, throw_ub_format,
25};
26use crate::fluent_generated as fluent;
27
28/// Directly returns an `Allocation` containing an absolute path representation of the given type.
29pub(crate) fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> (AllocId, u64) {
30    let path = crate::util::type_name(tcx, ty);
31    let bytes = path.into_bytes();
32    let len = bytes.len().try_into().unwrap();
33    (tcx.allocate_bytes_dedup(bytes, CTFE_ALLOC_SALT), len)
34}
35impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
36    /// Generates a value of `TypeId` for `ty` in-place.
37    fn write_type_id(
38        &mut self,
39        ty: Ty<'tcx>,
40        dest: &PlaceTy<'tcx, M::Provenance>,
41    ) -> InterpResult<'tcx, ()> {
42        let tcx = self.tcx;
43        let type_id_hash = tcx.type_id_hash(ty).as_u128();
44        let op = self.const_val_to_op(
45            ConstValue::Scalar(Scalar::from_u128(type_id_hash)),
46            tcx.types.u128,
47            None,
48        )?;
49        self.copy_op_allow_transmute(&op, dest)?;
50
51        // Give the each pointer-sized chunk provenance that knows about the type id.
52        // Here we rely on `TypeId` being a newtype around an array of pointers, so we
53        // first project to its only field and then the array elements.
54        let alloc_id = tcx.reserve_and_set_type_id_alloc(ty);
55        let arr = self.project_field(dest, FieldIdx::ZERO)?;
56        let mut elem_iter = self.project_array_fields(&arr)?;
57        while let Some((_, elem)) = elem_iter.next(self)? {
58            // Decorate this part of the hash with provenance; leave the integer part unchanged.
59            let hash_fragment = self.read_scalar(&elem)?.to_target_usize(&tcx)?;
60            let ptr = Pointer::new(alloc_id.into(), Size::from_bytes(hash_fragment));
61            let ptr = self.global_root_pointer(ptr)?;
62            let val = Scalar::from_pointer(ptr, &tcx);
63            self.write_scalar(val, &elem)?;
64        }
65        interp_ok(())
66    }
67
68    /// Read a value of type `TypeId`, returning the type it represents.
69    pub(crate) fn read_type_id(
70        &self,
71        op: &OpTy<'tcx, M::Provenance>,
72    ) -> InterpResult<'tcx, Ty<'tcx>> {
73        // `TypeId` is a newtype around an array of pointers. All pointers must have the same
74        // provenance, and that provenance represents the type.
75        let ptr_size = self.pointer_size().bytes_usize();
76        let arr = self.project_field(op, FieldIdx::ZERO)?;
77
78        let mut ty_and_hash = None;
79        let mut elem_iter = self.project_array_fields(&arr)?;
80        while let Some((idx, elem)) = elem_iter.next(self)? {
81            let elem = self.read_pointer(&elem)?;
82            let (elem_ty, elem_hash) = self.get_ptr_type_id(elem)?;
83            // If this is the first element, remember the type and its hash.
84            // If this is not the first element, ensure it is consistent with the previous ones.
85            let full_hash = match ty_and_hash {
86                None => {
87                    let hash = self.tcx.type_id_hash(elem_ty).as_u128();
88                    let mut hash_bytes = [0u8; 16];
89                    write_target_uint(self.data_layout().endian, &mut hash_bytes, hash).unwrap();
90                    ty_and_hash = Some((elem_ty, hash_bytes));
91                    hash_bytes
92                }
93                Some((ty, hash_bytes)) => {
94                    if ty != elem_ty {
95                        throw_ub_format!(
96                            "invalid `TypeId` value: not all bytes carry the same type id metadata"
97                        );
98                    }
99                    hash_bytes
100                }
101            };
102            // Ensure the elem_hash matches the corresponding part of the full hash.
103            let hash_frag = &full_hash[(idx as usize) * ptr_size..][..ptr_size];
104            if read_target_uint(self.data_layout().endian, hash_frag).unwrap() != elem_hash.into() {
105                throw_ub_format!(
106                    "invalid `TypeId` value: the hash does not match the type id metadata"
107                );
108            }
109        }
110
111        interp_ok(ty_and_hash.unwrap().0)
112    }
113
114    /// Returns `true` if emulation happened.
115    /// Here we implement the intrinsics that are common to all Miri instances; individual machines can add their own
116    /// intrinsic handling.
117    pub fn eval_intrinsic(
118        &mut self,
119        instance: ty::Instance<'tcx>,
120        args: &[OpTy<'tcx, M::Provenance>],
121        dest: &PlaceTy<'tcx, M::Provenance>,
122        ret: Option<mir::BasicBlock>,
123    ) -> InterpResult<'tcx, bool> {
124        let instance_args = instance.args;
125        let intrinsic_name = self.tcx.item_name(instance.def_id());
126
127        if intrinsic_name.as_str().starts_with("simd_") {
128            return self.eval_simd_intrinsic(intrinsic_name, instance_args, args, dest, ret);
129        }
130
131        let tcx = self.tcx.tcx;
132
133        match intrinsic_name {
134            sym::type_name => {
135                let tp_ty = instance.args.type_at(0);
136                ensure_monomorphic_enough(tcx, tp_ty)?;
137                let (alloc_id, meta) = alloc_type_name(tcx, tp_ty);
138                let val = ConstValue::Slice { alloc_id, meta };
139                let val = self.const_val_to_op(val, dest.layout.ty, Some(dest.layout))?;
140                self.copy_op(&val, dest)?;
141            }
142            sym::needs_drop => {
143                let tp_ty = instance.args.type_at(0);
144                ensure_monomorphic_enough(tcx, tp_ty)?;
145                let val = ConstValue::from_bool(tp_ty.needs_drop(tcx, self.typing_env));
146                let val = self.const_val_to_op(val, tcx.types.bool, Some(dest.layout))?;
147                self.copy_op(&val, dest)?;
148            }
149            sym::type_id => {
150                let tp_ty = instance.args.type_at(0);
151                ensure_monomorphic_enough(tcx, tp_ty)?;
152                self.write_type_id(tp_ty, dest)?;
153            }
154            sym::type_id_eq => {
155                let a_ty = self.read_type_id(&args[0])?;
156                let b_ty = self.read_type_id(&args[1])?;
157                self.write_scalar(Scalar::from_bool(a_ty == b_ty), dest)?;
158            }
159            sym::size_of => {
160                let tp_ty = instance.args.type_at(0);
161                let layout = self.layout_of(tp_ty)?;
162                if !layout.is_sized() {
163                    span_bug!(self.cur_span(), "unsized type for `size_of`");
164                }
165                let val = layout.size.bytes();
166                self.write_scalar(Scalar::from_target_usize(val, self), dest)?;
167            }
168            sym::align_of => {
169                let tp_ty = instance.args.type_at(0);
170                let layout = self.layout_of(tp_ty)?;
171                if !layout.is_sized() {
172                    span_bug!(self.cur_span(), "unsized type for `align_of`");
173                }
174                let val = layout.align.bytes();
175                self.write_scalar(Scalar::from_target_usize(val, self), dest)?;
176            }
177            sym::variant_count => {
178                let tp_ty = instance.args.type_at(0);
179                let ty = match tp_ty.kind() {
180                    // Pattern types have the same number of variants as their base type.
181                    // Even if we restrict e.g. which variants are valid, the variants are essentially just uninhabited.
182                    // And `Result<(), !>` still has two variants according to `variant_count`.
183                    ty::Pat(base, _) => *base,
184                    _ => tp_ty,
185                };
186                let val = match ty.kind() {
187                    // Correctly handles non-monomorphic calls, so there is no need for ensure_monomorphic_enough.
188                    ty::Adt(adt, _) => {
189                        ConstValue::from_target_usize(adt.variants().len() as u64, &tcx)
190                    }
191                    ty::Alias(..) | ty::Param(_) | ty::Placeholder(_) | ty::Infer(_) => {
192                        throw_inval!(TooGeneric)
193                    }
194                    ty::Pat(..) => unreachable!(),
195                    ty::Bound(_, _) => bug!("bound ty during ctfe"),
196                    ty::Bool
197                    | ty::Char
198                    | ty::Int(_)
199                    | ty::Uint(_)
200                    | ty::Float(_)
201                    | ty::Foreign(_)
202                    | ty::Str
203                    | ty::Array(_, _)
204                    | ty::Slice(_)
205                    | ty::RawPtr(_, _)
206                    | ty::Ref(_, _, _)
207                    | ty::FnDef(_, _)
208                    | ty::FnPtr(..)
209                    | ty::Dynamic(_, _)
210                    | ty::Closure(_, _)
211                    | ty::CoroutineClosure(_, _)
212                    | ty::Coroutine(_, _)
213                    | ty::CoroutineWitness(..)
214                    | ty::UnsafeBinder(_)
215                    | ty::Never
216                    | ty::Tuple(_)
217                    | ty::Error(_) => ConstValue::from_target_usize(0u64, &tcx),
218                };
219                let val = self.const_val_to_op(val, dest.layout.ty, Some(dest.layout))?;
220                self.copy_op(&val, dest)?;
221            }
222
223            sym::caller_location => {
224                let span = self.find_closest_untracked_caller_location();
225                let val = self.tcx.span_as_caller_location(span);
226                let val =
227                    self.const_val_to_op(val, self.tcx.caller_location_ty(), Some(dest.layout))?;
228                self.copy_op(&val, dest)?;
229            }
230
231            sym::align_of_val | sym::size_of_val => {
232                // Avoid `deref_pointer` -- this is not a deref, the ptr does not have to be
233                // dereferenceable!
234                let place = self.ref_to_mplace(&self.read_immediate(&args[0])?)?;
235                let (size, align) = self
236                    .size_and_align_of_val(&place)?
237                    .ok_or_else(|| err_unsup_format!("`extern type` does not have known layout"))?;
238
239                let result = match intrinsic_name {
240                    sym::align_of_val => align.bytes(),
241                    sym::size_of_val => size.bytes(),
242                    _ => bug!(),
243                };
244
245                self.write_scalar(Scalar::from_target_usize(result, self), dest)?;
246            }
247
248            sym::fadd_algebraic
249            | sym::fsub_algebraic
250            | sym::fmul_algebraic
251            | sym::fdiv_algebraic
252            | sym::frem_algebraic => {
253                let a = self.read_immediate(&args[0])?;
254                let b = self.read_immediate(&args[1])?;
255
256                let op = match intrinsic_name {
257                    sym::fadd_algebraic => BinOp::Add,
258                    sym::fsub_algebraic => BinOp::Sub,
259                    sym::fmul_algebraic => BinOp::Mul,
260                    sym::fdiv_algebraic => BinOp::Div,
261                    sym::frem_algebraic => BinOp::Rem,
262
263                    _ => bug!(),
264                };
265
266                let res = self.binary_op(op, &a, &b)?;
267                // `binary_op` already called `generate_nan` if needed.
268                let res = M::apply_float_nondet(self, res)?;
269                self.write_immediate(*res, dest)?;
270            }
271
272            sym::ctpop
273            | sym::cttz
274            | sym::cttz_nonzero
275            | sym::ctlz
276            | sym::ctlz_nonzero
277            | sym::bswap
278            | sym::bitreverse => {
279                let ty = instance_args.type_at(0);
280                let layout = self.layout_of(ty)?;
281                let val = self.read_scalar(&args[0])?;
282
283                let out_val = self.numeric_intrinsic(intrinsic_name, val, layout, dest.layout)?;
284                self.write_scalar(out_val, dest)?;
285            }
286            sym::saturating_add | sym::saturating_sub => {
287                let l = self.read_immediate(&args[0])?;
288                let r = self.read_immediate(&args[1])?;
289                let val = self.saturating_arith(
290                    if intrinsic_name == sym::saturating_add { BinOp::Add } else { BinOp::Sub },
291                    &l,
292                    &r,
293                )?;
294                self.write_scalar(val, dest)?;
295            }
296            sym::discriminant_value => {
297                let place = self.deref_pointer(&args[0])?;
298                let variant = self.read_discriminant(&place)?;
299                let discr = self.discriminant_for_variant(place.layout.ty, variant)?;
300                self.write_immediate(*discr, dest)?;
301            }
302            sym::exact_div => {
303                let l = self.read_immediate(&args[0])?;
304                let r = self.read_immediate(&args[1])?;
305                self.exact_div(&l, &r, dest)?;
306            }
307            sym::rotate_left | sym::rotate_right => {
308                // rotate_left: (X << (S % BW)) | (X >> ((BW - S) % BW))
309                // rotate_right: (X << ((BW - S) % BW)) | (X >> (S % BW))
310                let layout_val = self.layout_of(instance_args.type_at(0))?;
311                let val = self.read_scalar(&args[0])?;
312                let val_bits = val.to_bits(layout_val.size)?; // sign is ignored here
313
314                let layout_raw_shift = self.layout_of(self.tcx.types.u32)?;
315                let raw_shift = self.read_scalar(&args[1])?;
316                let raw_shift_bits = raw_shift.to_bits(layout_raw_shift.size)?;
317
318                let width_bits = u128::from(layout_val.size.bits());
319                let shift_bits = raw_shift_bits % width_bits;
320                let inv_shift_bits = (width_bits - shift_bits) % width_bits;
321                let result_bits = if intrinsic_name == sym::rotate_left {
322                    (val_bits << shift_bits) | (val_bits >> inv_shift_bits)
323                } else {
324                    (val_bits >> shift_bits) | (val_bits << inv_shift_bits)
325                };
326                let truncated_bits = layout_val.size.truncate(result_bits);
327                let result = Scalar::from_uint(truncated_bits, layout_val.size);
328                self.write_scalar(result, dest)?;
329            }
330            sym::copy => {
331                self.copy_intrinsic(&args[0], &args[1], &args[2], /*nonoverlapping*/ false)?;
332            }
333            sym::write_bytes => {
334                self.write_bytes_intrinsic(&args[0], &args[1], &args[2], "write_bytes")?;
335            }
336            sym::compare_bytes => {
337                let result = self.compare_bytes_intrinsic(&args[0], &args[1], &args[2])?;
338                self.write_scalar(result, dest)?;
339            }
340            sym::arith_offset => {
341                let ptr = self.read_pointer(&args[0])?;
342                let offset_count = self.read_target_isize(&args[1])?;
343                let pointee_ty = instance_args.type_at(0);
344
345                let pointee_size = i64::try_from(self.layout_of(pointee_ty)?.size.bytes()).unwrap();
346                let offset_bytes = offset_count.wrapping_mul(pointee_size);
347                let offset_ptr = ptr.wrapping_signed_offset(offset_bytes, self);
348                self.write_pointer(offset_ptr, dest)?;
349            }
350            sym::ptr_offset_from | sym::ptr_offset_from_unsigned => {
351                let a = self.read_pointer(&args[0])?;
352                let b = self.read_pointer(&args[1])?;
353
354                let usize_layout = self.layout_of(self.tcx.types.usize)?;
355                let isize_layout = self.layout_of(self.tcx.types.isize)?;
356
357                // Get offsets for both that are at least relative to the same base.
358                // With `OFFSET_IS_ADDR` this is trivial; without it we need either
359                // two integers or two pointers into the same allocation.
360                let (a_offset, b_offset, is_addr) = if M::Provenance::OFFSET_IS_ADDR {
361                    (a.addr().bytes(), b.addr().bytes(), /*is_addr*/ true)
362                } else {
363                    match (self.ptr_try_get_alloc_id(a, 0), self.ptr_try_get_alloc_id(b, 0)) {
364                        (Err(a), Err(b)) => {
365                            // Neither pointer points to an allocation, so they are both absolute.
366                            (a, b, /*is_addr*/ true)
367                        }
368                        (Ok((a_alloc_id, a_offset, _)), Ok((b_alloc_id, b_offset, _)))
369                            if a_alloc_id == b_alloc_id =>
370                        {
371                            // Found allocation for both, and it's the same.
372                            // Use these offsets for distance calculation.
373                            (a_offset.bytes(), b_offset.bytes(), /*is_addr*/ false)
374                        }
375                        _ => {
376                            // Not into the same allocation -- this is UB.
377                            throw_ub_custom!(
378                                fluent::const_eval_offset_from_different_allocations,
379                                name = intrinsic_name,
380                            );
381                        }
382                    }
383                };
384
385                // Compute distance: a - b.
386                let dist = {
387                    // Addresses are unsigned, so this is a `usize` computation. We have to do the
388                    // overflow check separately anyway.
389                    let (val, overflowed) = {
390                        let a_offset = ImmTy::from_uint(a_offset, usize_layout);
391                        let b_offset = ImmTy::from_uint(b_offset, usize_layout);
392                        self.binary_op(BinOp::SubWithOverflow, &a_offset, &b_offset)?
393                            .to_scalar_pair()
394                    };
395                    if overflowed.to_bool()? {
396                        // a < b
397                        if intrinsic_name == sym::ptr_offset_from_unsigned {
398                            throw_ub_custom!(
399                                fluent::const_eval_offset_from_unsigned_overflow,
400                                a_offset = a_offset,
401                                b_offset = b_offset,
402                                is_addr = is_addr,
403                            );
404                        }
405                        // The signed form of the intrinsic allows this. If we interpret the
406                        // difference as isize, we'll get the proper signed difference. If that
407                        // seems *positive* or equal to isize::MIN, they were more than isize::MAX apart.
408                        let dist = val.to_target_isize(self)?;
409                        if dist >= 0 || i128::from(dist) == self.pointer_size().signed_int_min() {
410                            throw_ub_custom!(
411                                fluent::const_eval_offset_from_underflow,
412                                name = intrinsic_name,
413                            );
414                        }
415                        dist
416                    } else {
417                        // b >= a
418                        let dist = val.to_target_isize(self)?;
419                        // If converting to isize produced a *negative* result, we had an overflow
420                        // because they were more than isize::MAX apart.
421                        if dist < 0 {
422                            throw_ub_custom!(
423                                fluent::const_eval_offset_from_overflow,
424                                name = intrinsic_name,
425                            );
426                        }
427                        dist
428                    }
429                };
430
431                // Check that the memory between them is dereferenceable at all, starting from the
432                // origin pointer: `dist` is `a - b`, so it is based on `b`.
433                self.check_ptr_access_signed(b, dist, CheckInAllocMsg::Dereferenceable)
434                    .map_err_kind(|_| {
435                        // This could mean they point to different allocations, or they point to the same allocation
436                        // but not the entire range between the pointers is in-bounds.
437                        if let Ok((a_alloc_id, ..)) = self.ptr_try_get_alloc_id(a, 0)
438                            && let Ok((b_alloc_id, ..)) = self.ptr_try_get_alloc_id(b, 0)
439                            && a_alloc_id == b_alloc_id
440                        {
441                            err_ub_custom!(
442                                fluent::const_eval_offset_from_out_of_bounds,
443                                name = intrinsic_name,
444                            )
445                        } else {
446                            err_ub_custom!(
447                                fluent::const_eval_offset_from_different_allocations,
448                                name = intrinsic_name,
449                            )
450                        }
451                    })?;
452                // Then check that this is also dereferenceable from `a`. This ensures that they are
453                // derived from the same allocation.
454                self.check_ptr_access_signed(
455                    a,
456                    dist.checked_neg().unwrap(), // i64::MIN is impossible as no allocation can be that large
457                    CheckInAllocMsg::Dereferenceable,
458                )
459                .map_err_kind(|_| {
460                    // Make the error more specific.
461                    err_ub_custom!(
462                        fluent::const_eval_offset_from_different_allocations,
463                        name = intrinsic_name,
464                    )
465                })?;
466
467                // Perform division by size to compute return value.
468                let ret_layout = if intrinsic_name == sym::ptr_offset_from_unsigned {
469                    assert!(0 <= dist && dist <= self.target_isize_max());
470                    usize_layout
471                } else {
472                    assert!(self.target_isize_min() <= dist && dist <= self.target_isize_max());
473                    isize_layout
474                };
475                let pointee_layout = self.layout_of(instance_args.type_at(0))?;
476                // If ret_layout is unsigned, we checked that so is the distance, so we are good.
477                let val = ImmTy::from_int(dist, ret_layout);
478                let size = ImmTy::from_int(pointee_layout.size.bytes(), ret_layout);
479                self.exact_div(&val, &size, dest)?;
480            }
481
482            sym::black_box => {
483                // These just return their argument
484                self.copy_op(&args[0], dest)?;
485            }
486            sym::raw_eq => {
487                let result = self.raw_eq_intrinsic(&args[0], &args[1])?;
488                self.write_scalar(result, dest)?;
489            }
490            sym::typed_swap_nonoverlapping => {
491                self.typed_swap_nonoverlapping_intrinsic(&args[0], &args[1])?;
492            }
493
494            sym::vtable_size => {
495                let ptr = self.read_pointer(&args[0])?;
496                // `None` because we don't know which trait to expect here; any vtable is okay.
497                let (size, _align) = self.get_vtable_size_and_align(ptr, None)?;
498                self.write_scalar(Scalar::from_target_usize(size.bytes(), self), dest)?;
499            }
500            sym::vtable_align => {
501                let ptr = self.read_pointer(&args[0])?;
502                // `None` because we don't know which trait to expect here; any vtable is okay.
503                let (_size, align) = self.get_vtable_size_and_align(ptr, None)?;
504                self.write_scalar(Scalar::from_target_usize(align.bytes(), self), dest)?;
505            }
506
507            sym::minnumf16 => self.float_min_intrinsic::<Half>(args, dest)?,
508            sym::minnumf32 => self.float_min_intrinsic::<Single>(args, dest)?,
509            sym::minnumf64 => self.float_min_intrinsic::<Double>(args, dest)?,
510            sym::minnumf128 => self.float_min_intrinsic::<Quad>(args, dest)?,
511
512            sym::minimumf16 => self.float_minimum_intrinsic::<Half>(args, dest)?,
513            sym::minimumf32 => self.float_minimum_intrinsic::<Single>(args, dest)?,
514            sym::minimumf64 => self.float_minimum_intrinsic::<Double>(args, dest)?,
515            sym::minimumf128 => self.float_minimum_intrinsic::<Quad>(args, dest)?,
516
517            sym::maxnumf16 => self.float_max_intrinsic::<Half>(args, dest)?,
518            sym::maxnumf32 => self.float_max_intrinsic::<Single>(args, dest)?,
519            sym::maxnumf64 => self.float_max_intrinsic::<Double>(args, dest)?,
520            sym::maxnumf128 => self.float_max_intrinsic::<Quad>(args, dest)?,
521
522            sym::maximumf16 => self.float_maximum_intrinsic::<Half>(args, dest)?,
523            sym::maximumf32 => self.float_maximum_intrinsic::<Single>(args, dest)?,
524            sym::maximumf64 => self.float_maximum_intrinsic::<Double>(args, dest)?,
525            sym::maximumf128 => self.float_maximum_intrinsic::<Quad>(args, dest)?,
526
527            sym::copysignf16 => self.float_copysign_intrinsic::<Half>(args, dest)?,
528            sym::copysignf32 => self.float_copysign_intrinsic::<Single>(args, dest)?,
529            sym::copysignf64 => self.float_copysign_intrinsic::<Double>(args, dest)?,
530            sym::copysignf128 => self.float_copysign_intrinsic::<Quad>(args, dest)?,
531
532            sym::fabsf16 => self.float_abs_intrinsic::<Half>(args, dest)?,
533            sym::fabsf32 => self.float_abs_intrinsic::<Single>(args, dest)?,
534            sym::fabsf64 => self.float_abs_intrinsic::<Double>(args, dest)?,
535            sym::fabsf128 => self.float_abs_intrinsic::<Quad>(args, dest)?,
536
537            sym::floorf16 => self.float_round_intrinsic::<Half>(
538                args,
539                dest,
540                rustc_apfloat::Round::TowardNegative,
541            )?,
542            sym::floorf32 => self.float_round_intrinsic::<Single>(
543                args,
544                dest,
545                rustc_apfloat::Round::TowardNegative,
546            )?,
547            sym::floorf64 => self.float_round_intrinsic::<Double>(
548                args,
549                dest,
550                rustc_apfloat::Round::TowardNegative,
551            )?,
552            sym::floorf128 => self.float_round_intrinsic::<Quad>(
553                args,
554                dest,
555                rustc_apfloat::Round::TowardNegative,
556            )?,
557
558            sym::ceilf16 => self.float_round_intrinsic::<Half>(
559                args,
560                dest,
561                rustc_apfloat::Round::TowardPositive,
562            )?,
563            sym::ceilf32 => self.float_round_intrinsic::<Single>(
564                args,
565                dest,
566                rustc_apfloat::Round::TowardPositive,
567            )?,
568            sym::ceilf64 => self.float_round_intrinsic::<Double>(
569                args,
570                dest,
571                rustc_apfloat::Round::TowardPositive,
572            )?,
573            sym::ceilf128 => self.float_round_intrinsic::<Quad>(
574                args,
575                dest,
576                rustc_apfloat::Round::TowardPositive,
577            )?,
578
579            sym::truncf16 => {
580                self.float_round_intrinsic::<Half>(args, dest, rustc_apfloat::Round::TowardZero)?
581            }
582            sym::truncf32 => {
583                self.float_round_intrinsic::<Single>(args, dest, rustc_apfloat::Round::TowardZero)?
584            }
585            sym::truncf64 => {
586                self.float_round_intrinsic::<Double>(args, dest, rustc_apfloat::Round::TowardZero)?
587            }
588            sym::truncf128 => {
589                self.float_round_intrinsic::<Quad>(args, dest, rustc_apfloat::Round::TowardZero)?
590            }
591
592            sym::roundf16 => self.float_round_intrinsic::<Half>(
593                args,
594                dest,
595                rustc_apfloat::Round::NearestTiesToAway,
596            )?,
597            sym::roundf32 => self.float_round_intrinsic::<Single>(
598                args,
599                dest,
600                rustc_apfloat::Round::NearestTiesToAway,
601            )?,
602            sym::roundf64 => self.float_round_intrinsic::<Double>(
603                args,
604                dest,
605                rustc_apfloat::Round::NearestTiesToAway,
606            )?,
607            sym::roundf128 => self.float_round_intrinsic::<Quad>(
608                args,
609                dest,
610                rustc_apfloat::Round::NearestTiesToAway,
611            )?,
612
613            sym::round_ties_even_f16 => self.float_round_intrinsic::<Half>(
614                args,
615                dest,
616                rustc_apfloat::Round::NearestTiesToEven,
617            )?,
618            sym::round_ties_even_f32 => self.float_round_intrinsic::<Single>(
619                args,
620                dest,
621                rustc_apfloat::Round::NearestTiesToEven,
622            )?,
623            sym::round_ties_even_f64 => self.float_round_intrinsic::<Double>(
624                args,
625                dest,
626                rustc_apfloat::Round::NearestTiesToEven,
627            )?,
628            sym::round_ties_even_f128 => self.float_round_intrinsic::<Quad>(
629                args,
630                dest,
631                rustc_apfloat::Round::NearestTiesToEven,
632            )?,
633            sym::fmaf16 => self.fma_intrinsic::<Half>(args, dest)?,
634            sym::fmaf32 => self.fma_intrinsic::<Single>(args, dest)?,
635            sym::fmaf64 => self.fma_intrinsic::<Double>(args, dest)?,
636            sym::fmaf128 => self.fma_intrinsic::<Quad>(args, dest)?,
637            sym::fmuladdf16 => self.float_muladd_intrinsic::<Half>(args, dest)?,
638            sym::fmuladdf32 => self.float_muladd_intrinsic::<Single>(args, dest)?,
639            sym::fmuladdf64 => self.float_muladd_intrinsic::<Double>(args, dest)?,
640            sym::fmuladdf128 => self.float_muladd_intrinsic::<Quad>(args, dest)?,
641
642            // Unsupported intrinsic: skip the return_to_block below.
643            _ => return interp_ok(false),
644        }
645
646        trace!("{:?}", self.dump_place(&dest.clone().into()));
647        self.return_to_block(ret)?;
648        interp_ok(true)
649    }
650
651    pub(super) fn eval_nondiverging_intrinsic(
652        &mut self,
653        intrinsic: &NonDivergingIntrinsic<'tcx>,
654    ) -> InterpResult<'tcx> {
655        match intrinsic {
656            NonDivergingIntrinsic::Assume(op) => {
657                let op = self.eval_operand(op, None)?;
658                let cond = self.read_scalar(&op)?.to_bool()?;
659                if !cond {
660                    throw_ub_custom!(fluent::const_eval_assume_false);
661                }
662                interp_ok(())
663            }
664            NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping {
665                count,
666                src,
667                dst,
668            }) => {
669                let src = self.eval_operand(src, None)?;
670                let dst = self.eval_operand(dst, None)?;
671                let count = self.eval_operand(count, None)?;
672                self.copy_intrinsic(&src, &dst, &count, /* nonoverlapping */ true)
673            }
674        }
675    }
676
677    pub fn numeric_intrinsic(
678        &self,
679        name: Symbol,
680        val: Scalar<M::Provenance>,
681        layout: TyAndLayout<'tcx>,
682        ret_layout: TyAndLayout<'tcx>,
683    ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
684        assert!(layout.ty.is_integral(), "invalid type for numeric intrinsic: {}", layout.ty);
685        let bits = val.to_bits(layout.size)?; // these operations all ignore the sign
686        let extra = 128 - u128::from(layout.size.bits());
687        let bits_out = match name {
688            sym::ctpop => u128::from(bits.count_ones()),
689            sym::ctlz_nonzero | sym::cttz_nonzero if bits == 0 => {
690                throw_ub_custom!(fluent::const_eval_call_nonzero_intrinsic, name = name,);
691            }
692            sym::ctlz | sym::ctlz_nonzero => u128::from(bits.leading_zeros()) - extra,
693            sym::cttz | sym::cttz_nonzero => u128::from((bits << extra).trailing_zeros()) - extra,
694            sym::bswap => {
695                assert_eq!(layout, ret_layout);
696                (bits << extra).swap_bytes()
697            }
698            sym::bitreverse => {
699                assert_eq!(layout, ret_layout);
700                (bits << extra).reverse_bits()
701            }
702            _ => bug!("not a numeric intrinsic: {}", name),
703        };
704        interp_ok(Scalar::from_uint(bits_out, ret_layout.size))
705    }
706
707    pub fn exact_div(
708        &mut self,
709        a: &ImmTy<'tcx, M::Provenance>,
710        b: &ImmTy<'tcx, M::Provenance>,
711        dest: &PlaceTy<'tcx, M::Provenance>,
712    ) -> InterpResult<'tcx> {
713        assert_eq!(a.layout.ty, b.layout.ty);
714        assert_matches!(a.layout.ty.kind(), ty::Int(..) | ty::Uint(..));
715
716        // Performs an exact division, resulting in undefined behavior where
717        // `x % y != 0` or `y == 0` or `x == T::MIN && y == -1`.
718        // First, check x % y != 0 (or if that computation overflows).
719        let rem = self.binary_op(BinOp::Rem, a, b)?;
720        // sign does not matter for 0 test, so `to_bits` is fine
721        if rem.to_scalar().to_bits(a.layout.size)? != 0 {
722            throw_ub_custom!(
723                fluent::const_eval_exact_div_has_remainder,
724                a = format!("{a}"),
725                b = format!("{b}")
726            )
727        }
728        // `Rem` says this is all right, so we can let `Div` do its job.
729        let res = self.binary_op(BinOp::Div, a, b)?;
730        self.write_immediate(*res, dest)
731    }
732
733    pub fn saturating_arith(
734        &self,
735        mir_op: BinOp,
736        l: &ImmTy<'tcx, M::Provenance>,
737        r: &ImmTy<'tcx, M::Provenance>,
738    ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
739        assert_eq!(l.layout.ty, r.layout.ty);
740        assert_matches!(l.layout.ty.kind(), ty::Int(..) | ty::Uint(..));
741        assert_matches!(mir_op, BinOp::Add | BinOp::Sub);
742
743        let (val, overflowed) =
744            self.binary_op(mir_op.wrapping_to_overflowing().unwrap(), l, r)?.to_scalar_pair();
745        interp_ok(if overflowed.to_bool()? {
746            let size = l.layout.size;
747            if l.layout.backend_repr.is_signed() {
748                // For signed ints the saturated value depends on the sign of the first
749                // term since the sign of the second term can be inferred from this and
750                // the fact that the operation has overflowed (if either is 0 no
751                // overflow can occur)
752                let first_term: i128 = l.to_scalar().to_int(l.layout.size)?;
753                if first_term >= 0 {
754                    // Negative overflow not possible since the positive first term
755                    // can only increase an (in range) negative term for addition
756                    // or corresponding negated positive term for subtraction.
757                    Scalar::from_int(size.signed_int_max(), size)
758                } else {
759                    // Positive overflow not possible for similar reason.
760                    Scalar::from_int(size.signed_int_min(), size)
761                }
762            } else {
763                // unsigned
764                if matches!(mir_op, BinOp::Add) {
765                    // max unsigned
766                    Scalar::from_uint(size.unsigned_int_max(), size)
767                } else {
768                    // underflow to 0
769                    Scalar::from_uint(0u128, size)
770                }
771            }
772        } else {
773            val
774        })
775    }
776
777    /// Offsets a pointer by some multiple of its type, returning an error if the pointer leaves its
778    /// allocation.
779    pub fn ptr_offset_inbounds(
780        &self,
781        ptr: Pointer<Option<M::Provenance>>,
782        offset_bytes: i64,
783    ) -> InterpResult<'tcx, Pointer<Option<M::Provenance>>> {
784        // The offset must be in bounds starting from `ptr`.
785        self.check_ptr_access_signed(
786            ptr,
787            offset_bytes,
788            CheckInAllocMsg::InboundsPointerArithmetic,
789        )?;
790        // This also implies that there is no overflow, so we are done.
791        interp_ok(ptr.wrapping_signed_offset(offset_bytes, self))
792    }
793
794    /// Copy `count*size_of::<T>()` many bytes from `*src` to `*dst`.
795    pub(crate) fn copy_intrinsic(
796        &mut self,
797        src: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
798        dst: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
799        count: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
800        nonoverlapping: bool,
801    ) -> InterpResult<'tcx> {
802        let count = self.read_target_usize(count)?;
803        let layout = self.layout_of(src.layout.ty.builtin_deref(true).unwrap())?;
804        let (size, align) = (layout.size, layout.align.abi);
805
806        let size = self.compute_size_in_bytes(size, count).ok_or_else(|| {
807            err_ub_custom!(
808                fluent::const_eval_size_overflow,
809                name = if nonoverlapping { "copy_nonoverlapping" } else { "copy" }
810            )
811        })?;
812
813        let src = self.read_pointer(src)?;
814        let dst = self.read_pointer(dst)?;
815
816        self.check_ptr_align(src, align)?;
817        self.check_ptr_align(dst, align)?;
818
819        self.mem_copy(src, dst, size, nonoverlapping)
820    }
821
822    /// Does a *typed* swap of `*left` and `*right`.
823    fn typed_swap_nonoverlapping_intrinsic(
824        &mut self,
825        left: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
826        right: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
827    ) -> InterpResult<'tcx> {
828        let left = self.deref_pointer(left)?;
829        let right = self.deref_pointer(right)?;
830        assert_eq!(left.layout, right.layout);
831        assert!(left.layout.is_sized());
832        let kind = MemoryKind::Stack;
833        let temp = self.allocate(left.layout, kind)?;
834        self.copy_op(&left, &temp)?; // checks alignment of `left`
835
836        // We want to always enforce non-overlapping, even if this is a scalar type.
837        // Therefore we directly use the underlying `mem_copy` here.
838        self.mem_copy(right.ptr(), left.ptr(), left.layout.size, /*nonoverlapping*/ true)?;
839        // This means we also need to do the validation of the value that used to be in `right`
840        // ourselves. This value is now in `left.` The one that started out in `left` already got
841        // validated by the copy above.
842        if M::enforce_validity(self, left.layout) {
843            self.validate_operand(
844                &left.clone().into(),
845                M::enforce_validity_recursively(self, left.layout),
846                /*reset_provenance_and_padding*/ true,
847            )?;
848        }
849
850        self.copy_op(&temp, &right)?; // checks alignment of `right`
851
852        self.deallocate_ptr(temp.ptr(), None, kind)?;
853        interp_ok(())
854    }
855
856    pub fn write_bytes_intrinsic(
857        &mut self,
858        dst: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
859        byte: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
860        count: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
861        name: &'static str,
862    ) -> InterpResult<'tcx> {
863        let layout = self.layout_of(dst.layout.ty.builtin_deref(true).unwrap())?;
864
865        let dst = self.read_pointer(dst)?;
866        let byte = self.read_scalar(byte)?.to_u8()?;
867        let count = self.read_target_usize(count)?;
868
869        // `checked_mul` enforces a too small bound (the correct one would probably be target_isize_max),
870        // but no actual allocation can be big enough for the difference to be noticeable.
871        let len = self
872            .compute_size_in_bytes(layout.size, count)
873            .ok_or_else(|| err_ub_custom!(fluent::const_eval_size_overflow, name = name))?;
874
875        let bytes = std::iter::repeat_n(byte, len.bytes_usize());
876        self.write_bytes_ptr(dst, bytes)
877    }
878
879    pub(crate) fn compare_bytes_intrinsic(
880        &mut self,
881        left: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
882        right: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
883        byte_count: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
884    ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
885        let left = self.read_pointer(left)?;
886        let right = self.read_pointer(right)?;
887        let n = Size::from_bytes(self.read_target_usize(byte_count)?);
888
889        let left_bytes = self.read_bytes_ptr_strip_provenance(left, n)?;
890        let right_bytes = self.read_bytes_ptr_strip_provenance(right, n)?;
891
892        // `Ordering`'s discriminants are -1/0/+1, so casting does the right thing.
893        let result = Ord::cmp(left_bytes, right_bytes) as i32;
894        interp_ok(Scalar::from_i32(result))
895    }
896
897    pub(crate) fn raw_eq_intrinsic(
898        &mut self,
899        lhs: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
900        rhs: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
901    ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
902        let layout = self.layout_of(lhs.layout.ty.builtin_deref(true).unwrap())?;
903        assert!(layout.is_sized());
904
905        let get_bytes = |this: &InterpCx<'tcx, M>,
906                         op: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>|
907         -> InterpResult<'tcx, &[u8]> {
908            let ptr = this.read_pointer(op)?;
909            this.check_ptr_align(ptr, layout.align.abi)?;
910            let Some(alloc_ref) = self.get_ptr_alloc(ptr, layout.size)? else {
911                // zero-sized access
912                return interp_ok(&[]);
913            };
914            alloc_ref.get_bytes_strip_provenance()
915        };
916
917        let lhs_bytes = get_bytes(self, lhs)?;
918        let rhs_bytes = get_bytes(self, rhs)?;
919        interp_ok(Scalar::from_bool(lhs_bytes == rhs_bytes))
920    }
921
922    fn float_min_intrinsic<F>(
923        &mut self,
924        args: &[OpTy<'tcx, M::Provenance>],
925        dest: &PlaceTy<'tcx, M::Provenance>,
926    ) -> InterpResult<'tcx, ()>
927    where
928        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
929    {
930        let a: F = self.read_scalar(&args[0])?.to_float()?;
931        let b: F = self.read_scalar(&args[1])?.to_float()?;
932        let res = if a == b {
933            // They are definitely not NaN (those are never equal), but they could be `+0` and `-0`.
934            // Let the machine decide which one to return.
935            M::equal_float_min_max(self, a, b)
936        } else {
937            self.adjust_nan(a.min(b), &[a, b])
938        };
939        self.write_scalar(res, dest)?;
940        interp_ok(())
941    }
942
943    fn float_max_intrinsic<F>(
944        &mut self,
945        args: &[OpTy<'tcx, M::Provenance>],
946        dest: &PlaceTy<'tcx, M::Provenance>,
947    ) -> InterpResult<'tcx, ()>
948    where
949        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
950    {
951        let a: F = self.read_scalar(&args[0])?.to_float()?;
952        let b: F = self.read_scalar(&args[1])?.to_float()?;
953        let res = if a == b {
954            // They are definitely not NaN (those are never equal), but they could be `+0` and `-0`.
955            // Let the machine decide which one to return.
956            M::equal_float_min_max(self, a, b)
957        } else {
958            self.adjust_nan(a.max(b), &[a, b])
959        };
960        self.write_scalar(res, dest)?;
961        interp_ok(())
962    }
963
964    fn float_minimum_intrinsic<F>(
965        &mut self,
966        args: &[OpTy<'tcx, M::Provenance>],
967        dest: &PlaceTy<'tcx, M::Provenance>,
968    ) -> InterpResult<'tcx, ()>
969    where
970        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
971    {
972        let a: F = self.read_scalar(&args[0])?.to_float()?;
973        let b: F = self.read_scalar(&args[1])?.to_float()?;
974        let res = a.minimum(b);
975        let res = self.adjust_nan(res, &[a, b]);
976        self.write_scalar(res, dest)?;
977        interp_ok(())
978    }
979
980    fn float_maximum_intrinsic<F>(
981        &mut self,
982        args: &[OpTy<'tcx, M::Provenance>],
983        dest: &PlaceTy<'tcx, M::Provenance>,
984    ) -> InterpResult<'tcx, ()>
985    where
986        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
987    {
988        let a: F = self.read_scalar(&args[0])?.to_float()?;
989        let b: F = self.read_scalar(&args[1])?.to_float()?;
990        let res = a.maximum(b);
991        let res = self.adjust_nan(res, &[a, b]);
992        self.write_scalar(res, dest)?;
993        interp_ok(())
994    }
995
996    fn float_copysign_intrinsic<F>(
997        &mut self,
998        args: &[OpTy<'tcx, M::Provenance>],
999        dest: &PlaceTy<'tcx, M::Provenance>,
1000    ) -> InterpResult<'tcx, ()>
1001    where
1002        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1003    {
1004        let a: F = self.read_scalar(&args[0])?.to_float()?;
1005        let b: F = self.read_scalar(&args[1])?.to_float()?;
1006        // bitwise, no NaN adjustments
1007        self.write_scalar(a.copy_sign(b), dest)?;
1008        interp_ok(())
1009    }
1010
1011    fn float_abs_intrinsic<F>(
1012        &mut self,
1013        args: &[OpTy<'tcx, M::Provenance>],
1014        dest: &PlaceTy<'tcx, M::Provenance>,
1015    ) -> InterpResult<'tcx, ()>
1016    where
1017        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1018    {
1019        let x: F = self.read_scalar(&args[0])?.to_float()?;
1020        // bitwise, no NaN adjustments
1021        self.write_scalar(x.abs(), dest)?;
1022        interp_ok(())
1023    }
1024
1025    fn float_round_intrinsic<F>(
1026        &mut self,
1027        args: &[OpTy<'tcx, M::Provenance>],
1028        dest: &PlaceTy<'tcx, M::Provenance>,
1029        mode: rustc_apfloat::Round,
1030    ) -> InterpResult<'tcx, ()>
1031    where
1032        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1033    {
1034        let x: F = self.read_scalar(&args[0])?.to_float()?;
1035        let res = x.round_to_integral(mode).value;
1036        let res = self.adjust_nan(res, &[x]);
1037        self.write_scalar(res, dest)?;
1038        interp_ok(())
1039    }
1040
1041    fn fma_intrinsic<F>(
1042        &mut self,
1043        args: &[OpTy<'tcx, M::Provenance>],
1044        dest: &PlaceTy<'tcx, M::Provenance>,
1045    ) -> InterpResult<'tcx, ()>
1046    where
1047        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1048    {
1049        let a: F = self.read_scalar(&args[0])?.to_float()?;
1050        let b: F = self.read_scalar(&args[1])?.to_float()?;
1051        let c: F = self.read_scalar(&args[2])?.to_float()?;
1052
1053        let res = a.mul_add(b, c).value;
1054        let res = self.adjust_nan(res, &[a, b, c]);
1055        self.write_scalar(res, dest)?;
1056        interp_ok(())
1057    }
1058
1059    fn float_muladd_intrinsic<F>(
1060        &mut self,
1061        args: &[OpTy<'tcx, M::Provenance>],
1062        dest: &PlaceTy<'tcx, M::Provenance>,
1063    ) -> InterpResult<'tcx, ()>
1064    where
1065        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1066    {
1067        let a: F = self.read_scalar(&args[0])?.to_float()?;
1068        let b: F = self.read_scalar(&args[1])?.to_float()?;
1069        let c: F = self.read_scalar(&args[2])?.to_float()?;
1070
1071        let fuse = M::float_fuse_mul_add(self);
1072
1073        let res = if fuse { a.mul_add(b, c).value } else { ((a * b).value + c).value };
1074        let res = self.adjust_nan(res, &[a, b, c]);
1075        self.write_scalar(res, dest)?;
1076        interp_ok(())
1077    }
1078
1079    /// Converts `src` from floating point to integer type `dest_ty`
1080    /// after rounding with mode `round`.
1081    /// Returns `None` if `f` is NaN or out of range.
1082    pub fn float_to_int_checked(
1083        &self,
1084        src: &ImmTy<'tcx, M::Provenance>,
1085        cast_to: TyAndLayout<'tcx>,
1086        round: rustc_apfloat::Round,
1087    ) -> InterpResult<'tcx, Option<ImmTy<'tcx, M::Provenance>>> {
1088        fn float_to_int_inner<'tcx, F: rustc_apfloat::Float, M: Machine<'tcx>>(
1089            ecx: &InterpCx<'tcx, M>,
1090            src: F,
1091            cast_to: TyAndLayout<'tcx>,
1092            round: rustc_apfloat::Round,
1093        ) -> (Scalar<M::Provenance>, rustc_apfloat::Status) {
1094            let int_size = cast_to.layout.size;
1095            match cast_to.ty.kind() {
1096                // Unsigned
1097                ty::Uint(_) => {
1098                    let res = src.to_u128_r(int_size.bits_usize(), round, &mut false);
1099                    (Scalar::from_uint(res.value, int_size), res.status)
1100                }
1101                // Signed
1102                ty::Int(_) => {
1103                    let res = src.to_i128_r(int_size.bits_usize(), round, &mut false);
1104                    (Scalar::from_int(res.value, int_size), res.status)
1105                }
1106                // Nothing else
1107                _ => span_bug!(
1108                    ecx.cur_span(),
1109                    "attempted float-to-int conversion with non-int output type {}",
1110                    cast_to.ty,
1111                ),
1112            }
1113        }
1114
1115        let ty::Float(fty) = src.layout.ty.kind() else {
1116            bug!("float_to_int_checked: non-float input type {}", src.layout.ty)
1117        };
1118
1119        let (val, status) = match fty {
1120            FloatTy::F16 => float_to_int_inner(self, src.to_scalar().to_f16()?, cast_to, round),
1121            FloatTy::F32 => float_to_int_inner(self, src.to_scalar().to_f32()?, cast_to, round),
1122            FloatTy::F64 => float_to_int_inner(self, src.to_scalar().to_f64()?, cast_to, round),
1123            FloatTy::F128 => float_to_int_inner(self, src.to_scalar().to_f128()?, cast_to, round),
1124        };
1125
1126        if status.intersects(
1127            rustc_apfloat::Status::INVALID_OP
1128                | rustc_apfloat::Status::OVERFLOW
1129                | rustc_apfloat::Status::UNDERFLOW,
1130        ) {
1131            // Floating point value is NaN (flagged with INVALID_OP) or outside the range
1132            // of values of the integer type (flagged with OVERFLOW or UNDERFLOW).
1133            interp_ok(None)
1134        } else {
1135            // Floating point value can be represented by the integer type after rounding.
1136            // The INEXACT flag is ignored on purpose to allow rounding.
1137            interp_ok(Some(ImmTy::from_scalar(val, cast_to)))
1138        }
1139    }
1140}