rustc_const_eval/interpret/
intrinsics.rs

1//! Intrinsics and other functions that the interpreter executes without
2//! looking at their MIR. Intrinsics/functions supported here are shared by CTFE
3//! and miri.
4
5mod simd;
6
7use std::assert_matches::assert_matches;
8
9use rustc_abi::{FIRST_VARIANT, FieldIdx, HasDataLayout, Size, VariantIdx};
10use rustc_apfloat::ieee::{Double, Half, Quad, Single};
11use rustc_hir::def_id::CRATE_DEF_ID;
12use rustc_infer::infer::TyCtxtInferExt;
13use rustc_middle::mir::interpret::{CTFE_ALLOC_SALT, read_target_uint, write_target_uint};
14use rustc_middle::mir::{self, BinOp, ConstValue, NonDivergingIntrinsic};
15use rustc_middle::ty::layout::TyAndLayout;
16use rustc_middle::ty::{FloatTy, PolyExistentialPredicate, Ty, TyCtxt, TypeFoldable};
17use rustc_middle::{bug, span_bug, ty};
18use rustc_span::{Symbol, sym};
19use rustc_trait_selection::traits::{Obligation, ObligationCause, ObligationCtxt};
20use tracing::trace;
21
22use super::memory::MemoryKind;
23use super::util::ensure_monomorphic_enough;
24use super::{
25    AllocId, CheckInAllocMsg, ImmTy, InterpCx, InterpResult, Machine, OpTy, PlaceTy, Pointer,
26    PointerArithmetic, Provenance, Scalar, err_ub_custom, err_unsup_format, interp_ok, throw_inval,
27    throw_ub_custom, throw_ub_format,
28};
29use crate::fluent_generated as fluent;
30
31#[derive(Copy, Clone, Debug, PartialEq, Eq)]
32enum MulAddType {
33    /// Used with `fma` and `simd_fma`, always uses fused-multiply-add
34    Fused,
35    /// Used with `fmuladd` and `simd_relaxed_fma`, nondeterministically determines whether to use
36    /// fma or simple multiply-add
37    Nondeterministic,
38}
39
40#[derive(Copy, Clone)]
41pub(crate) enum MinMax {
42    /// The IEEE-2019 `minimum` operation - see `f32::minimum` etc.
43    /// In particular, `-0.0` is considered smaller than `+0.0` and
44    /// if either input is NaN, the result is NaN.
45    Minimum,
46    /// The IEEE-2008 `minNum` operation with the SNaN handling of the
47    /// IEEE-2019 `minimumNumber` operation - see `f32::min` etc.
48    /// In particular, if the inputs are `-0.0` and `+0.0`, the result is non-deterministic,
49    /// and if one argument is NaN (quiet or signaling), the other one is returned.
50    MinimumNumber,
51    /// The IEEE-2019 `maximum` operation - see `f32::maximum` etc.
52    /// In particular, `-0.0` is considered smaller than `+0.0` and
53    /// if either input is NaN, the result is NaN.
54    Maximum,
55    /// The IEEE-2008 `maxNum` operation with the SNaN handling of the
56    /// IEEE-2019 `maximumNumber` operation - see `f32::max` etc.
57    /// In particular, if the inputs are `-0.0` and `+0.0`, the result is non-deterministic,
58    /// and if one argument is NaN (quiet or signaling), the other one is returned.
59    MaximumNumber,
60}
61
62/// Directly returns an `Allocation` containing an absolute path representation of the given type.
63pub(crate) fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> (AllocId, u64) {
64    let path = crate::util::type_name(tcx, ty);
65    let bytes = path.into_bytes();
66    let len = bytes.len().try_into().unwrap();
67    (tcx.allocate_bytes_dedup(bytes, CTFE_ALLOC_SALT), len)
68}
69impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
70    /// Generates a value of `TypeId` for `ty` in-place.
71    fn write_type_id(
72        &mut self,
73        ty: Ty<'tcx>,
74        dest: &PlaceTy<'tcx, M::Provenance>,
75    ) -> InterpResult<'tcx, ()> {
76        let tcx = self.tcx;
77        let type_id_hash = tcx.type_id_hash(ty).as_u128();
78        let op = self.const_val_to_op(
79            ConstValue::Scalar(Scalar::from_u128(type_id_hash)),
80            tcx.types.u128,
81            None,
82        )?;
83        self.copy_op_allow_transmute(&op, dest)?;
84
85        // Give the each pointer-sized chunk provenance that knows about the type id.
86        // Here we rely on `TypeId` being a newtype around an array of pointers, so we
87        // first project to its only field and then the array elements.
88        let alloc_id = tcx.reserve_and_set_type_id_alloc(ty);
89        let arr = self.project_field(dest, FieldIdx::ZERO)?;
90        let mut elem_iter = self.project_array_fields(&arr)?;
91        while let Some((_, elem)) = elem_iter.next(self)? {
92            // Decorate this part of the hash with provenance; leave the integer part unchanged.
93            let hash_fragment = self.read_scalar(&elem)?.to_target_usize(&tcx)?;
94            let ptr = Pointer::new(alloc_id.into(), Size::from_bytes(hash_fragment));
95            let ptr = self.global_root_pointer(ptr)?;
96            let val = Scalar::from_pointer(ptr, &tcx);
97            self.write_scalar(val, &elem)?;
98        }
99        interp_ok(())
100    }
101
102    /// Read a value of type `TypeId`, returning the type it represents.
103    pub(crate) fn read_type_id(
104        &self,
105        op: &OpTy<'tcx, M::Provenance>,
106    ) -> InterpResult<'tcx, Ty<'tcx>> {
107        // `TypeId` is a newtype around an array of pointers. All pointers must have the same
108        // provenance, and that provenance represents the type.
109        let ptr_size = self.pointer_size().bytes_usize();
110        let arr = self.project_field(op, FieldIdx::ZERO)?;
111
112        let mut ty_and_hash = None;
113        let mut elem_iter = self.project_array_fields(&arr)?;
114        while let Some((idx, elem)) = elem_iter.next(self)? {
115            let elem = self.read_pointer(&elem)?;
116            let (elem_ty, elem_hash) = self.get_ptr_type_id(elem)?;
117            // If this is the first element, remember the type and its hash.
118            // If this is not the first element, ensure it is consistent with the previous ones.
119            let full_hash = match ty_and_hash {
120                None => {
121                    let hash = self.tcx.type_id_hash(elem_ty).as_u128();
122                    let mut hash_bytes = [0u8; 16];
123                    write_target_uint(self.data_layout().endian, &mut hash_bytes, hash).unwrap();
124                    ty_and_hash = Some((elem_ty, hash_bytes));
125                    hash_bytes
126                }
127                Some((ty, hash_bytes)) => {
128                    if ty != elem_ty {
129                        throw_ub_format!(
130                            "invalid `TypeId` value: not all bytes carry the same type id metadata"
131                        );
132                    }
133                    hash_bytes
134                }
135            };
136            // Ensure the elem_hash matches the corresponding part of the full hash.
137            let hash_frag = &full_hash[(idx as usize) * ptr_size..][..ptr_size];
138            if read_target_uint(self.data_layout().endian, hash_frag).unwrap() != elem_hash.into() {
139                throw_ub_format!(
140                    "invalid `TypeId` value: the hash does not match the type id metadata"
141                );
142            }
143        }
144
145        interp_ok(ty_and_hash.unwrap().0)
146    }
147
148    /// Returns `true` if emulation happened.
149    /// Here we implement the intrinsics that are common to all Miri instances; individual machines can add their own
150    /// intrinsic handling.
151    pub fn eval_intrinsic(
152        &mut self,
153        instance: ty::Instance<'tcx>,
154        args: &[OpTy<'tcx, M::Provenance>],
155        dest: &PlaceTy<'tcx, M::Provenance>,
156        ret: Option<mir::BasicBlock>,
157    ) -> InterpResult<'tcx, bool> {
158        let instance_args = instance.args;
159        let intrinsic_name = self.tcx.item_name(instance.def_id());
160
161        if intrinsic_name.as_str().starts_with("simd_") {
162            return self.eval_simd_intrinsic(intrinsic_name, instance_args, args, dest, ret);
163        }
164
165        let tcx = self.tcx.tcx;
166
167        match intrinsic_name {
168            sym::type_name => {
169                let tp_ty = instance.args.type_at(0);
170                ensure_monomorphic_enough(tcx, tp_ty)?;
171                let (alloc_id, meta) = alloc_type_name(tcx, tp_ty);
172                let val = ConstValue::Slice { alloc_id, meta };
173                let val = self.const_val_to_op(val, dest.layout.ty, Some(dest.layout))?;
174                self.copy_op(&val, dest)?;
175            }
176            sym::needs_drop => {
177                let tp_ty = instance.args.type_at(0);
178                ensure_monomorphic_enough(tcx, tp_ty)?;
179                let val = ConstValue::from_bool(tp_ty.needs_drop(tcx, self.typing_env));
180                let val = self.const_val_to_op(val, tcx.types.bool, Some(dest.layout))?;
181                self.copy_op(&val, dest)?;
182            }
183            sym::type_id => {
184                let tp_ty = instance.args.type_at(0);
185                ensure_monomorphic_enough(tcx, tp_ty)?;
186                self.write_type_id(tp_ty, dest)?;
187            }
188            sym::type_id_eq => {
189                let a_ty = self.read_type_id(&args[0])?;
190                let b_ty = self.read_type_id(&args[1])?;
191                self.write_scalar(Scalar::from_bool(a_ty == b_ty), dest)?;
192            }
193            sym::size_of => {
194                let tp_ty = instance.args.type_at(0);
195                let layout = self.layout_of(tp_ty)?;
196                if !layout.is_sized() {
197                    span_bug!(self.cur_span(), "unsized type for `size_of`");
198                }
199                let val = layout.size.bytes();
200                self.write_scalar(Scalar::from_target_usize(val, self), dest)?;
201            }
202            sym::align_of => {
203                let tp_ty = instance.args.type_at(0);
204                let layout = self.layout_of(tp_ty)?;
205                if !layout.is_sized() {
206                    span_bug!(self.cur_span(), "unsized type for `align_of`");
207                }
208                let val = layout.align.bytes();
209                self.write_scalar(Scalar::from_target_usize(val, self), dest)?;
210            }
211            sym::offset_of => {
212                let tp_ty = instance.args.type_at(0);
213
214                let variant = self.read_scalar(&args[0])?.to_u32()?;
215                let field = self.read_scalar(&args[1])?.to_u32()? as usize;
216
217                let layout = self.layout_of(tp_ty)?;
218                let cx = ty::layout::LayoutCx::new(*self.tcx, self.typing_env);
219
220                let layout = layout.for_variant(&cx, VariantIdx::from_u32(variant));
221                let offset = layout.fields.offset(field).bytes();
222
223                self.write_scalar(Scalar::from_target_usize(offset, self), dest)?;
224            }
225            sym::vtable_for => {
226                let tp_ty = instance.args.type_at(0);
227                let result_ty = instance.args.type_at(1);
228
229                ensure_monomorphic_enough(tcx, tp_ty)?;
230                ensure_monomorphic_enough(tcx, result_ty)?;
231                let ty::Dynamic(preds, _) = result_ty.kind() else {
232                    span_bug!(
233                        self.find_closest_untracked_caller_location(),
234                        "Invalid type provided to vtable_for::<T, U>. U must be dyn Trait, got {result_ty}."
235                    );
236                };
237
238                let (infcx, param_env) =
239                    self.tcx.infer_ctxt().build_with_typing_env(self.typing_env);
240
241                let ocx = ObligationCtxt::new(&infcx);
242                ocx.register_obligations(preds.iter().map(|pred: PolyExistentialPredicate<'_>| {
243                    let pred = pred.with_self_ty(tcx, tp_ty);
244                    // Lifetimes can only be 'static because of the bound on T
245                    let pred = pred.fold_with(&mut ty::BottomUpFolder {
246                        tcx,
247                        ty_op: |ty| ty,
248                        lt_op: |lt| {
249                            if lt == tcx.lifetimes.re_erased { tcx.lifetimes.re_static } else { lt }
250                        },
251                        ct_op: |ct| ct,
252                    });
253                    Obligation::new(tcx, ObligationCause::dummy(), param_env, pred)
254                }));
255                let type_impls_trait = ocx.evaluate_obligations_error_on_ambiguity().is_empty();
256                // Since `assumed_wf_tys=[]` the choice of LocalDefId is irrelevant, so using the "default"
257                let regions_are_valid = ocx.resolve_regions(CRATE_DEF_ID, param_env, []).is_empty();
258
259                if regions_are_valid && type_impls_trait {
260                    let vtable_ptr = self.get_vtable_ptr(tp_ty, preds)?;
261                    // Writing a non-null pointer into an `Option<NonNull>` will automatically make it `Some`.
262                    self.write_pointer(vtable_ptr, dest)?;
263                } else {
264                    // Write `None`
265                    self.write_discriminant(FIRST_VARIANT, dest)?;
266                }
267            }
268            sym::variant_count => {
269                let tp_ty = instance.args.type_at(0);
270                let ty = match tp_ty.kind() {
271                    // Pattern types have the same number of variants as their base type.
272                    // Even if we restrict e.g. which variants are valid, the variants are essentially just uninhabited.
273                    // And `Result<(), !>` still has two variants according to `variant_count`.
274                    ty::Pat(base, _) => *base,
275                    _ => tp_ty,
276                };
277                let val = match ty.kind() {
278                    // Correctly handles non-monomorphic calls, so there is no need for ensure_monomorphic_enough.
279                    ty::Adt(adt, _) => {
280                        ConstValue::from_target_usize(adt.variants().len() as u64, &tcx)
281                    }
282                    ty::Alias(..) | ty::Param(_) | ty::Placeholder(_) | ty::Infer(_) => {
283                        throw_inval!(TooGeneric)
284                    }
285                    ty::Pat(..) => unreachable!(),
286                    ty::Bound(_, _) => bug!("bound ty during ctfe"),
287                    ty::Bool
288                    | ty::Char
289                    | ty::Int(_)
290                    | ty::Uint(_)
291                    | ty::Float(_)
292                    | ty::Foreign(_)
293                    | ty::Str
294                    | ty::Array(_, _)
295                    | ty::Slice(_)
296                    | ty::RawPtr(_, _)
297                    | ty::Ref(_, _, _)
298                    | ty::FnDef(_, _)
299                    | ty::FnPtr(..)
300                    | ty::Dynamic(_, _)
301                    | ty::Closure(_, _)
302                    | ty::CoroutineClosure(_, _)
303                    | ty::Coroutine(_, _)
304                    | ty::CoroutineWitness(..)
305                    | ty::UnsafeBinder(_)
306                    | ty::Never
307                    | ty::Tuple(_)
308                    | ty::Error(_) => ConstValue::from_target_usize(0u64, &tcx),
309                };
310                let val = self.const_val_to_op(val, dest.layout.ty, Some(dest.layout))?;
311                self.copy_op(&val, dest)?;
312            }
313
314            sym::caller_location => {
315                let span = self.find_closest_untracked_caller_location();
316                let val = self.tcx.span_as_caller_location(span);
317                let val =
318                    self.const_val_to_op(val, self.tcx.caller_location_ty(), Some(dest.layout))?;
319                self.copy_op(&val, dest)?;
320            }
321
322            sym::align_of_val | sym::size_of_val => {
323                // Avoid `deref_pointer` -- this is not a deref, the ptr does not have to be
324                // dereferenceable!
325                let place = self.ref_to_mplace(&self.read_immediate(&args[0])?)?;
326                let (size, align) = self
327                    .size_and_align_of_val(&place)?
328                    .ok_or_else(|| err_unsup_format!("`extern type` does not have known layout"))?;
329
330                let result = match intrinsic_name {
331                    sym::align_of_val => align.bytes(),
332                    sym::size_of_val => size.bytes(),
333                    _ => bug!(),
334                };
335
336                self.write_scalar(Scalar::from_target_usize(result, self), dest)?;
337            }
338
339            sym::fadd_algebraic
340            | sym::fsub_algebraic
341            | sym::fmul_algebraic
342            | sym::fdiv_algebraic
343            | sym::frem_algebraic => {
344                let a = self.read_immediate(&args[0])?;
345                let b = self.read_immediate(&args[1])?;
346
347                let op = match intrinsic_name {
348                    sym::fadd_algebraic => BinOp::Add,
349                    sym::fsub_algebraic => BinOp::Sub,
350                    sym::fmul_algebraic => BinOp::Mul,
351                    sym::fdiv_algebraic => BinOp::Div,
352                    sym::frem_algebraic => BinOp::Rem,
353
354                    _ => bug!(),
355                };
356
357                let res = self.binary_op(op, &a, &b)?;
358                // `binary_op` already called `generate_nan` if needed.
359                let res = M::apply_float_nondet(self, res)?;
360                self.write_immediate(*res, dest)?;
361            }
362
363            sym::ctpop
364            | sym::cttz
365            | sym::cttz_nonzero
366            | sym::ctlz
367            | sym::ctlz_nonzero
368            | sym::bswap
369            | sym::bitreverse => {
370                let ty = instance_args.type_at(0);
371                let layout = self.layout_of(ty)?;
372                let val = self.read_scalar(&args[0])?;
373
374                let out_val = self.numeric_intrinsic(intrinsic_name, val, layout, dest.layout)?;
375                self.write_scalar(out_val, dest)?;
376            }
377            sym::saturating_add | sym::saturating_sub => {
378                let l = self.read_immediate(&args[0])?;
379                let r = self.read_immediate(&args[1])?;
380                let val = self.saturating_arith(
381                    if intrinsic_name == sym::saturating_add { BinOp::Add } else { BinOp::Sub },
382                    &l,
383                    &r,
384                )?;
385                self.write_scalar(val, dest)?;
386            }
387            sym::discriminant_value => {
388                let place = self.deref_pointer(&args[0])?;
389                let variant = self.read_discriminant(&place)?;
390                let discr = self.discriminant_for_variant(place.layout.ty, variant)?;
391                self.write_immediate(*discr, dest)?;
392            }
393            sym::exact_div => {
394                let l = self.read_immediate(&args[0])?;
395                let r = self.read_immediate(&args[1])?;
396                self.exact_div(&l, &r, dest)?;
397            }
398            sym::copy => {
399                self.copy_intrinsic(&args[0], &args[1], &args[2], /*nonoverlapping*/ false)?;
400            }
401            sym::write_bytes => {
402                self.write_bytes_intrinsic(&args[0], &args[1], &args[2], "write_bytes")?;
403            }
404            sym::compare_bytes => {
405                let result = self.compare_bytes_intrinsic(&args[0], &args[1], &args[2])?;
406                self.write_scalar(result, dest)?;
407            }
408            sym::arith_offset => {
409                let ptr = self.read_pointer(&args[0])?;
410                let offset_count = self.read_target_isize(&args[1])?;
411                let pointee_ty = instance_args.type_at(0);
412
413                let pointee_size = i64::try_from(self.layout_of(pointee_ty)?.size.bytes()).unwrap();
414                let offset_bytes = offset_count.wrapping_mul(pointee_size);
415                let offset_ptr = ptr.wrapping_signed_offset(offset_bytes, self);
416                self.write_pointer(offset_ptr, dest)?;
417            }
418            sym::ptr_offset_from | sym::ptr_offset_from_unsigned => {
419                let a = self.read_pointer(&args[0])?;
420                let b = self.read_pointer(&args[1])?;
421
422                let usize_layout = self.layout_of(self.tcx.types.usize)?;
423                let isize_layout = self.layout_of(self.tcx.types.isize)?;
424
425                // Get offsets for both that are at least relative to the same base.
426                // With `OFFSET_IS_ADDR` this is trivial; without it we need either
427                // two integers or two pointers into the same allocation.
428                let (a_offset, b_offset, is_addr) = if M::Provenance::OFFSET_IS_ADDR {
429                    (a.addr().bytes(), b.addr().bytes(), /*is_addr*/ true)
430                } else {
431                    match (self.ptr_try_get_alloc_id(a, 0), self.ptr_try_get_alloc_id(b, 0)) {
432                        (Err(a), Err(b)) => {
433                            // Neither pointer points to an allocation, so they are both absolute.
434                            (a, b, /*is_addr*/ true)
435                        }
436                        (Ok((a_alloc_id, a_offset, _)), Ok((b_alloc_id, b_offset, _)))
437                            if a_alloc_id == b_alloc_id =>
438                        {
439                            // Found allocation for both, and it's the same.
440                            // Use these offsets for distance calculation.
441                            (a_offset.bytes(), b_offset.bytes(), /*is_addr*/ false)
442                        }
443                        _ => {
444                            // Not into the same allocation -- this is UB.
445                            throw_ub_custom!(
446                                fluent::const_eval_offset_from_different_allocations,
447                                name = intrinsic_name,
448                            );
449                        }
450                    }
451                };
452
453                // Compute distance: a - b.
454                let dist = {
455                    // Addresses are unsigned, so this is a `usize` computation. We have to do the
456                    // overflow check separately anyway.
457                    let (val, overflowed) = {
458                        let a_offset = ImmTy::from_uint(a_offset, usize_layout);
459                        let b_offset = ImmTy::from_uint(b_offset, usize_layout);
460                        self.binary_op(BinOp::SubWithOverflow, &a_offset, &b_offset)?
461                            .to_scalar_pair()
462                    };
463                    if overflowed.to_bool()? {
464                        // a < b
465                        if intrinsic_name == sym::ptr_offset_from_unsigned {
466                            throw_ub_custom!(
467                                fluent::const_eval_offset_from_unsigned_overflow,
468                                a_offset = a_offset,
469                                b_offset = b_offset,
470                                is_addr = is_addr,
471                            );
472                        }
473                        // The signed form of the intrinsic allows this. If we interpret the
474                        // difference as isize, we'll get the proper signed difference. If that
475                        // seems *positive* or equal to isize::MIN, they were more than isize::MAX apart.
476                        let dist = val.to_target_isize(self)?;
477                        if dist >= 0 || i128::from(dist) == self.pointer_size().signed_int_min() {
478                            throw_ub_custom!(
479                                fluent::const_eval_offset_from_underflow,
480                                name = intrinsic_name,
481                            );
482                        }
483                        dist
484                    } else {
485                        // b >= a
486                        let dist = val.to_target_isize(self)?;
487                        // If converting to isize produced a *negative* result, we had an overflow
488                        // because they were more than isize::MAX apart.
489                        if dist < 0 {
490                            throw_ub_custom!(
491                                fluent::const_eval_offset_from_overflow,
492                                name = intrinsic_name,
493                            );
494                        }
495                        dist
496                    }
497                };
498
499                // Check that the memory between them is dereferenceable at all, starting from the
500                // origin pointer: `dist` is `a - b`, so it is based on `b`.
501                self.check_ptr_access_signed(b, dist, CheckInAllocMsg::Dereferenceable)
502                    .map_err_kind(|_| {
503                        // This could mean they point to different allocations, or they point to the same allocation
504                        // but not the entire range between the pointers is in-bounds.
505                        if let Ok((a_alloc_id, ..)) = self.ptr_try_get_alloc_id(a, 0)
506                            && let Ok((b_alloc_id, ..)) = self.ptr_try_get_alloc_id(b, 0)
507                            && a_alloc_id == b_alloc_id
508                        {
509                            err_ub_custom!(
510                                fluent::const_eval_offset_from_out_of_bounds,
511                                name = intrinsic_name,
512                            )
513                        } else {
514                            err_ub_custom!(
515                                fluent::const_eval_offset_from_different_allocations,
516                                name = intrinsic_name,
517                            )
518                        }
519                    })?;
520                // Then check that this is also dereferenceable from `a`. This ensures that they are
521                // derived from the same allocation.
522                self.check_ptr_access_signed(
523                    a,
524                    dist.checked_neg().unwrap(), // i64::MIN is impossible as no allocation can be that large
525                    CheckInAllocMsg::Dereferenceable,
526                )
527                .map_err_kind(|_| {
528                    // Make the error more specific.
529                    err_ub_custom!(
530                        fluent::const_eval_offset_from_different_allocations,
531                        name = intrinsic_name,
532                    )
533                })?;
534
535                // Perform division by size to compute return value.
536                let ret_layout = if intrinsic_name == sym::ptr_offset_from_unsigned {
537                    assert!(0 <= dist && dist <= self.target_isize_max());
538                    usize_layout
539                } else {
540                    assert!(self.target_isize_min() <= dist && dist <= self.target_isize_max());
541                    isize_layout
542                };
543                let pointee_layout = self.layout_of(instance_args.type_at(0))?;
544                // If ret_layout is unsigned, we checked that so is the distance, so we are good.
545                let val = ImmTy::from_int(dist, ret_layout);
546                let size = ImmTy::from_int(pointee_layout.size.bytes(), ret_layout);
547                self.exact_div(&val, &size, dest)?;
548            }
549
550            sym::black_box => {
551                // These just return their argument
552                self.copy_op(&args[0], dest)?;
553            }
554            sym::raw_eq => {
555                let result = self.raw_eq_intrinsic(&args[0], &args[1])?;
556                self.write_scalar(result, dest)?;
557            }
558            sym::typed_swap_nonoverlapping => {
559                self.typed_swap_nonoverlapping_intrinsic(&args[0], &args[1])?;
560            }
561
562            sym::vtable_size => {
563                let ptr = self.read_pointer(&args[0])?;
564                // `None` because we don't know which trait to expect here; any vtable is okay.
565                let (size, _align) = self.get_vtable_size_and_align(ptr, None)?;
566                self.write_scalar(Scalar::from_target_usize(size.bytes(), self), dest)?;
567            }
568            sym::vtable_align => {
569                let ptr = self.read_pointer(&args[0])?;
570                // `None` because we don't know which trait to expect here; any vtable is okay.
571                let (_size, align) = self.get_vtable_size_and_align(ptr, None)?;
572                self.write_scalar(Scalar::from_target_usize(align.bytes(), self), dest)?;
573            }
574
575            sym::minnumf16 => {
576                self.float_minmax_intrinsic::<Half>(args, MinMax::MinimumNumber, dest)?
577            }
578            sym::minnumf32 => {
579                self.float_minmax_intrinsic::<Single>(args, MinMax::MinimumNumber, dest)?
580            }
581            sym::minnumf64 => {
582                self.float_minmax_intrinsic::<Double>(args, MinMax::MinimumNumber, dest)?
583            }
584            sym::minnumf128 => {
585                self.float_minmax_intrinsic::<Quad>(args, MinMax::MinimumNumber, dest)?
586            }
587
588            sym::minimumf16 => self.float_minmax_intrinsic::<Half>(args, MinMax::Minimum, dest)?,
589            sym::minimumf32 => {
590                self.float_minmax_intrinsic::<Single>(args, MinMax::Minimum, dest)?
591            }
592            sym::minimumf64 => {
593                self.float_minmax_intrinsic::<Double>(args, MinMax::Minimum, dest)?
594            }
595            sym::minimumf128 => self.float_minmax_intrinsic::<Quad>(args, MinMax::Minimum, dest)?,
596
597            sym::maxnumf16 => {
598                self.float_minmax_intrinsic::<Half>(args, MinMax::MaximumNumber, dest)?
599            }
600            sym::maxnumf32 => {
601                self.float_minmax_intrinsic::<Single>(args, MinMax::MaximumNumber, dest)?
602            }
603            sym::maxnumf64 => {
604                self.float_minmax_intrinsic::<Double>(args, MinMax::MaximumNumber, dest)?
605            }
606            sym::maxnumf128 => {
607                self.float_minmax_intrinsic::<Quad>(args, MinMax::MaximumNumber, dest)?
608            }
609
610            sym::maximumf16 => self.float_minmax_intrinsic::<Half>(args, MinMax::Maximum, dest)?,
611            sym::maximumf32 => {
612                self.float_minmax_intrinsic::<Single>(args, MinMax::Maximum, dest)?
613            }
614            sym::maximumf64 => {
615                self.float_minmax_intrinsic::<Double>(args, MinMax::Maximum, dest)?
616            }
617            sym::maximumf128 => self.float_minmax_intrinsic::<Quad>(args, MinMax::Maximum, dest)?,
618
619            sym::copysignf16 => self.float_copysign_intrinsic::<Half>(args, dest)?,
620            sym::copysignf32 => self.float_copysign_intrinsic::<Single>(args, dest)?,
621            sym::copysignf64 => self.float_copysign_intrinsic::<Double>(args, dest)?,
622            sym::copysignf128 => self.float_copysign_intrinsic::<Quad>(args, dest)?,
623
624            sym::fabsf16 => self.float_abs_intrinsic::<Half>(args, dest)?,
625            sym::fabsf32 => self.float_abs_intrinsic::<Single>(args, dest)?,
626            sym::fabsf64 => self.float_abs_intrinsic::<Double>(args, dest)?,
627            sym::fabsf128 => self.float_abs_intrinsic::<Quad>(args, dest)?,
628
629            sym::floorf16 => self.float_round_intrinsic::<Half>(
630                args,
631                dest,
632                rustc_apfloat::Round::TowardNegative,
633            )?,
634            sym::floorf32 => self.float_round_intrinsic::<Single>(
635                args,
636                dest,
637                rustc_apfloat::Round::TowardNegative,
638            )?,
639            sym::floorf64 => self.float_round_intrinsic::<Double>(
640                args,
641                dest,
642                rustc_apfloat::Round::TowardNegative,
643            )?,
644            sym::floorf128 => self.float_round_intrinsic::<Quad>(
645                args,
646                dest,
647                rustc_apfloat::Round::TowardNegative,
648            )?,
649
650            sym::ceilf16 => self.float_round_intrinsic::<Half>(
651                args,
652                dest,
653                rustc_apfloat::Round::TowardPositive,
654            )?,
655            sym::ceilf32 => self.float_round_intrinsic::<Single>(
656                args,
657                dest,
658                rustc_apfloat::Round::TowardPositive,
659            )?,
660            sym::ceilf64 => self.float_round_intrinsic::<Double>(
661                args,
662                dest,
663                rustc_apfloat::Round::TowardPositive,
664            )?,
665            sym::ceilf128 => self.float_round_intrinsic::<Quad>(
666                args,
667                dest,
668                rustc_apfloat::Round::TowardPositive,
669            )?,
670
671            sym::truncf16 => {
672                self.float_round_intrinsic::<Half>(args, dest, rustc_apfloat::Round::TowardZero)?
673            }
674            sym::truncf32 => {
675                self.float_round_intrinsic::<Single>(args, dest, rustc_apfloat::Round::TowardZero)?
676            }
677            sym::truncf64 => {
678                self.float_round_intrinsic::<Double>(args, dest, rustc_apfloat::Round::TowardZero)?
679            }
680            sym::truncf128 => {
681                self.float_round_intrinsic::<Quad>(args, dest, rustc_apfloat::Round::TowardZero)?
682            }
683
684            sym::roundf16 => self.float_round_intrinsic::<Half>(
685                args,
686                dest,
687                rustc_apfloat::Round::NearestTiesToAway,
688            )?,
689            sym::roundf32 => self.float_round_intrinsic::<Single>(
690                args,
691                dest,
692                rustc_apfloat::Round::NearestTiesToAway,
693            )?,
694            sym::roundf64 => self.float_round_intrinsic::<Double>(
695                args,
696                dest,
697                rustc_apfloat::Round::NearestTiesToAway,
698            )?,
699            sym::roundf128 => self.float_round_intrinsic::<Quad>(
700                args,
701                dest,
702                rustc_apfloat::Round::NearestTiesToAway,
703            )?,
704
705            sym::round_ties_even_f16 => self.float_round_intrinsic::<Half>(
706                args,
707                dest,
708                rustc_apfloat::Round::NearestTiesToEven,
709            )?,
710            sym::round_ties_even_f32 => self.float_round_intrinsic::<Single>(
711                args,
712                dest,
713                rustc_apfloat::Round::NearestTiesToEven,
714            )?,
715            sym::round_ties_even_f64 => self.float_round_intrinsic::<Double>(
716                args,
717                dest,
718                rustc_apfloat::Round::NearestTiesToEven,
719            )?,
720            sym::round_ties_even_f128 => self.float_round_intrinsic::<Quad>(
721                args,
722                dest,
723                rustc_apfloat::Round::NearestTiesToEven,
724            )?,
725            sym::fmaf16 => self.float_muladd_intrinsic::<Half>(args, dest, MulAddType::Fused)?,
726            sym::fmaf32 => self.float_muladd_intrinsic::<Single>(args, dest, MulAddType::Fused)?,
727            sym::fmaf64 => self.float_muladd_intrinsic::<Double>(args, dest, MulAddType::Fused)?,
728            sym::fmaf128 => self.float_muladd_intrinsic::<Quad>(args, dest, MulAddType::Fused)?,
729            sym::fmuladdf16 => {
730                self.float_muladd_intrinsic::<Half>(args, dest, MulAddType::Nondeterministic)?
731            }
732            sym::fmuladdf32 => {
733                self.float_muladd_intrinsic::<Single>(args, dest, MulAddType::Nondeterministic)?
734            }
735            sym::fmuladdf64 => {
736                self.float_muladd_intrinsic::<Double>(args, dest, MulAddType::Nondeterministic)?
737            }
738            sym::fmuladdf128 => {
739                self.float_muladd_intrinsic::<Quad>(args, dest, MulAddType::Nondeterministic)?
740            }
741
742            // Unsupported intrinsic: skip the return_to_block below.
743            _ => return interp_ok(false),
744        }
745
746        trace!("{:?}", self.dump_place(&dest.clone().into()));
747        self.return_to_block(ret)?;
748        interp_ok(true)
749    }
750
751    pub(super) fn eval_nondiverging_intrinsic(
752        &mut self,
753        intrinsic: &NonDivergingIntrinsic<'tcx>,
754    ) -> InterpResult<'tcx> {
755        match intrinsic {
756            NonDivergingIntrinsic::Assume(op) => {
757                let op = self.eval_operand(op, None)?;
758                let cond = self.read_scalar(&op)?.to_bool()?;
759                if !cond {
760                    throw_ub_custom!(fluent::const_eval_assume_false);
761                }
762                interp_ok(())
763            }
764            NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping {
765                count,
766                src,
767                dst,
768            }) => {
769                let src = self.eval_operand(src, None)?;
770                let dst = self.eval_operand(dst, None)?;
771                let count = self.eval_operand(count, None)?;
772                self.copy_intrinsic(&src, &dst, &count, /* nonoverlapping */ true)
773            }
774        }
775    }
776
777    pub fn numeric_intrinsic(
778        &self,
779        name: Symbol,
780        val: Scalar<M::Provenance>,
781        layout: TyAndLayout<'tcx>,
782        ret_layout: TyAndLayout<'tcx>,
783    ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
784        assert!(layout.ty.is_integral(), "invalid type for numeric intrinsic: {}", layout.ty);
785        let bits = val.to_bits(layout.size)?; // these operations all ignore the sign
786        let extra = 128 - u128::from(layout.size.bits());
787        let bits_out = match name {
788            sym::ctpop => u128::from(bits.count_ones()),
789            sym::ctlz_nonzero | sym::cttz_nonzero if bits == 0 => {
790                throw_ub_custom!(fluent::const_eval_call_nonzero_intrinsic, name = name,);
791            }
792            sym::ctlz | sym::ctlz_nonzero => u128::from(bits.leading_zeros()) - extra,
793            sym::cttz | sym::cttz_nonzero => u128::from((bits << extra).trailing_zeros()) - extra,
794            sym::bswap => {
795                assert_eq!(layout, ret_layout);
796                (bits << extra).swap_bytes()
797            }
798            sym::bitreverse => {
799                assert_eq!(layout, ret_layout);
800                (bits << extra).reverse_bits()
801            }
802            _ => bug!("not a numeric intrinsic: {}", name),
803        };
804        interp_ok(Scalar::from_uint(bits_out, ret_layout.size))
805    }
806
807    pub fn exact_div(
808        &mut self,
809        a: &ImmTy<'tcx, M::Provenance>,
810        b: &ImmTy<'tcx, M::Provenance>,
811        dest: &PlaceTy<'tcx, M::Provenance>,
812    ) -> InterpResult<'tcx> {
813        assert_eq!(a.layout.ty, b.layout.ty);
814        assert_matches!(a.layout.ty.kind(), ty::Int(..) | ty::Uint(..));
815
816        // Performs an exact division, resulting in undefined behavior where
817        // `x % y != 0` or `y == 0` or `x == T::MIN && y == -1`.
818        // First, check x % y != 0 (or if that computation overflows).
819        let rem = self.binary_op(BinOp::Rem, a, b)?;
820        // sign does not matter for 0 test, so `to_bits` is fine
821        if rem.to_scalar().to_bits(a.layout.size)? != 0 {
822            throw_ub_custom!(
823                fluent::const_eval_exact_div_has_remainder,
824                a = format!("{a}"),
825                b = format!("{b}")
826            )
827        }
828        // `Rem` says this is all right, so we can let `Div` do its job.
829        let res = self.binary_op(BinOp::Div, a, b)?;
830        self.write_immediate(*res, dest)
831    }
832
833    pub fn saturating_arith(
834        &self,
835        mir_op: BinOp,
836        l: &ImmTy<'tcx, M::Provenance>,
837        r: &ImmTy<'tcx, M::Provenance>,
838    ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
839        assert_eq!(l.layout.ty, r.layout.ty);
840        assert_matches!(l.layout.ty.kind(), ty::Int(..) | ty::Uint(..));
841        assert_matches!(mir_op, BinOp::Add | BinOp::Sub);
842
843        let (val, overflowed) =
844            self.binary_op(mir_op.wrapping_to_overflowing().unwrap(), l, r)?.to_scalar_pair();
845        interp_ok(if overflowed.to_bool()? {
846            let size = l.layout.size;
847            if l.layout.backend_repr.is_signed() {
848                // For signed ints the saturated value depends on the sign of the first
849                // term since the sign of the second term can be inferred from this and
850                // the fact that the operation has overflowed (if either is 0 no
851                // overflow can occur)
852                let first_term: i128 = l.to_scalar().to_int(l.layout.size)?;
853                if first_term >= 0 {
854                    // Negative overflow not possible since the positive first term
855                    // can only increase an (in range) negative term for addition
856                    // or corresponding negated positive term for subtraction.
857                    Scalar::from_int(size.signed_int_max(), size)
858                } else {
859                    // Positive overflow not possible for similar reason.
860                    Scalar::from_int(size.signed_int_min(), size)
861                }
862            } else {
863                // unsigned
864                if matches!(mir_op, BinOp::Add) {
865                    // max unsigned
866                    Scalar::from_uint(size.unsigned_int_max(), size)
867                } else {
868                    // underflow to 0
869                    Scalar::from_uint(0u128, size)
870                }
871            }
872        } else {
873            val
874        })
875    }
876
877    /// Offsets a pointer by some multiple of its type, returning an error if the pointer leaves its
878    /// allocation.
879    pub fn ptr_offset_inbounds(
880        &self,
881        ptr: Pointer<Option<M::Provenance>>,
882        offset_bytes: i64,
883    ) -> InterpResult<'tcx, Pointer<Option<M::Provenance>>> {
884        // The offset must be in bounds starting from `ptr`.
885        self.check_ptr_access_signed(
886            ptr,
887            offset_bytes,
888            CheckInAllocMsg::InboundsPointerArithmetic,
889        )?;
890        // This also implies that there is no overflow, so we are done.
891        interp_ok(ptr.wrapping_signed_offset(offset_bytes, self))
892    }
893
894    /// Copy `count*size_of::<T>()` many bytes from `*src` to `*dst`.
895    pub(crate) fn copy_intrinsic(
896        &mut self,
897        src: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
898        dst: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
899        count: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
900        nonoverlapping: bool,
901    ) -> InterpResult<'tcx> {
902        let count = self.read_target_usize(count)?;
903        let layout = self.layout_of(src.layout.ty.builtin_deref(true).unwrap())?;
904        let (size, align) = (layout.size, layout.align.abi);
905
906        let size = self.compute_size_in_bytes(size, count).ok_or_else(|| {
907            err_ub_custom!(
908                fluent::const_eval_size_overflow,
909                name = if nonoverlapping { "copy_nonoverlapping" } else { "copy" }
910            )
911        })?;
912
913        let src = self.read_pointer(src)?;
914        let dst = self.read_pointer(dst)?;
915
916        self.check_ptr_align(src, align)?;
917        self.check_ptr_align(dst, align)?;
918
919        self.mem_copy(src, dst, size, nonoverlapping)
920    }
921
922    /// Does a *typed* swap of `*left` and `*right`.
923    fn typed_swap_nonoverlapping_intrinsic(
924        &mut self,
925        left: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
926        right: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
927    ) -> InterpResult<'tcx> {
928        let left = self.deref_pointer(left)?;
929        let right = self.deref_pointer(right)?;
930        assert_eq!(left.layout, right.layout);
931        assert!(left.layout.is_sized());
932        let kind = MemoryKind::Stack;
933        let temp = self.allocate(left.layout, kind)?;
934        self.copy_op(&left, &temp)?; // checks alignment of `left`
935
936        // We want to always enforce non-overlapping, even if this is a scalar type.
937        // Therefore we directly use the underlying `mem_copy` here.
938        self.mem_copy(right.ptr(), left.ptr(), left.layout.size, /*nonoverlapping*/ true)?;
939        // This means we also need to do the validation of the value that used to be in `right`
940        // ourselves. This value is now in `left.` The one that started out in `left` already got
941        // validated by the copy above.
942        if M::enforce_validity(self, left.layout) {
943            self.validate_operand(
944                &left.clone().into(),
945                M::enforce_validity_recursively(self, left.layout),
946                /*reset_provenance_and_padding*/ true,
947            )?;
948        }
949
950        self.copy_op(&temp, &right)?; // checks alignment of `right`
951
952        self.deallocate_ptr(temp.ptr(), None, kind)?;
953        interp_ok(())
954    }
955
956    pub fn write_bytes_intrinsic(
957        &mut self,
958        dst: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
959        byte: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
960        count: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
961        name: &'static str,
962    ) -> InterpResult<'tcx> {
963        let layout = self.layout_of(dst.layout.ty.builtin_deref(true).unwrap())?;
964
965        let dst = self.read_pointer(dst)?;
966        let byte = self.read_scalar(byte)?.to_u8()?;
967        let count = self.read_target_usize(count)?;
968
969        // `checked_mul` enforces a too small bound (the correct one would probably be target_isize_max),
970        // but no actual allocation can be big enough for the difference to be noticeable.
971        let len = self
972            .compute_size_in_bytes(layout.size, count)
973            .ok_or_else(|| err_ub_custom!(fluent::const_eval_size_overflow, name = name))?;
974
975        let bytes = std::iter::repeat_n(byte, len.bytes_usize());
976        self.write_bytes_ptr(dst, bytes)
977    }
978
979    pub(crate) fn compare_bytes_intrinsic(
980        &mut self,
981        left: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
982        right: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
983        byte_count: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
984    ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
985        let left = self.read_pointer(left)?;
986        let right = self.read_pointer(right)?;
987        let n = Size::from_bytes(self.read_target_usize(byte_count)?);
988
989        let left_bytes = self.read_bytes_ptr_strip_provenance(left, n)?;
990        let right_bytes = self.read_bytes_ptr_strip_provenance(right, n)?;
991
992        // `Ordering`'s discriminants are -1/0/+1, so casting does the right thing.
993        let result = Ord::cmp(left_bytes, right_bytes) as i32;
994        interp_ok(Scalar::from_i32(result))
995    }
996
997    pub(crate) fn raw_eq_intrinsic(
998        &mut self,
999        lhs: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
1000        rhs: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
1001    ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
1002        let layout = self.layout_of(lhs.layout.ty.builtin_deref(true).unwrap())?;
1003        assert!(layout.is_sized());
1004
1005        let get_bytes = |this: &InterpCx<'tcx, M>,
1006                         op: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>|
1007         -> InterpResult<'tcx, &[u8]> {
1008            let ptr = this.read_pointer(op)?;
1009            this.check_ptr_align(ptr, layout.align.abi)?;
1010            let Some(alloc_ref) = self.get_ptr_alloc(ptr, layout.size)? else {
1011                // zero-sized access
1012                return interp_ok(&[]);
1013            };
1014            alloc_ref.get_bytes_strip_provenance()
1015        };
1016
1017        let lhs_bytes = get_bytes(self, lhs)?;
1018        let rhs_bytes = get_bytes(self, rhs)?;
1019        interp_ok(Scalar::from_bool(lhs_bytes == rhs_bytes))
1020    }
1021
1022    fn float_minmax<F>(
1023        &self,
1024        a: Scalar<M::Provenance>,
1025        b: Scalar<M::Provenance>,
1026        op: MinMax,
1027    ) -> InterpResult<'tcx, Scalar<M::Provenance>>
1028    where
1029        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1030    {
1031        let a: F = a.to_float()?;
1032        let b: F = b.to_float()?;
1033        let res = if matches!(op, MinMax::MinimumNumber | MinMax::MaximumNumber) && a == b {
1034            // They are definitely not NaN (those are never equal), but they could be `+0` and `-0`.
1035            // Let the machine decide which one to return.
1036            M::equal_float_min_max(self, a, b)
1037        } else {
1038            let result = match op {
1039                MinMax::Minimum => a.minimum(b),
1040                MinMax::MinimumNumber => a.min(b),
1041                MinMax::Maximum => a.maximum(b),
1042                MinMax::MaximumNumber => a.max(b),
1043            };
1044            self.adjust_nan(result, &[a, b])
1045        };
1046
1047        interp_ok(res.into())
1048    }
1049
1050    fn float_minmax_intrinsic<F>(
1051        &mut self,
1052        args: &[OpTy<'tcx, M::Provenance>],
1053        op: MinMax,
1054        dest: &PlaceTy<'tcx, M::Provenance>,
1055    ) -> InterpResult<'tcx, ()>
1056    where
1057        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1058    {
1059        let res =
1060            self.float_minmax::<F>(self.read_scalar(&args[0])?, self.read_scalar(&args[1])?, op)?;
1061        self.write_scalar(res, dest)?;
1062        interp_ok(())
1063    }
1064
1065    fn float_copysign_intrinsic<F>(
1066        &mut self,
1067        args: &[OpTy<'tcx, M::Provenance>],
1068        dest: &PlaceTy<'tcx, M::Provenance>,
1069    ) -> InterpResult<'tcx, ()>
1070    where
1071        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1072    {
1073        let a: F = self.read_scalar(&args[0])?.to_float()?;
1074        let b: F = self.read_scalar(&args[1])?.to_float()?;
1075        // bitwise, no NaN adjustments
1076        self.write_scalar(a.copy_sign(b), dest)?;
1077        interp_ok(())
1078    }
1079
1080    fn float_abs_intrinsic<F>(
1081        &mut self,
1082        args: &[OpTy<'tcx, M::Provenance>],
1083        dest: &PlaceTy<'tcx, M::Provenance>,
1084    ) -> InterpResult<'tcx, ()>
1085    where
1086        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1087    {
1088        let x: F = self.read_scalar(&args[0])?.to_float()?;
1089        // bitwise, no NaN adjustments
1090        self.write_scalar(x.abs(), dest)?;
1091        interp_ok(())
1092    }
1093
1094    fn float_round<F>(
1095        &mut self,
1096        x: Scalar<M::Provenance>,
1097        mode: rustc_apfloat::Round,
1098    ) -> InterpResult<'tcx, Scalar<M::Provenance>>
1099    where
1100        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1101    {
1102        let x: F = x.to_float()?;
1103        let res = x.round_to_integral(mode).value;
1104        let res = self.adjust_nan(res, &[x]);
1105        interp_ok(res.into())
1106    }
1107
1108    fn float_round_intrinsic<F>(
1109        &mut self,
1110        args: &[OpTy<'tcx, M::Provenance>],
1111        dest: &PlaceTy<'tcx, M::Provenance>,
1112        mode: rustc_apfloat::Round,
1113    ) -> InterpResult<'tcx, ()>
1114    where
1115        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1116    {
1117        let res = self.float_round::<F>(self.read_scalar(&args[0])?, mode)?;
1118        self.write_scalar(res, dest)?;
1119        interp_ok(())
1120    }
1121
1122    fn float_muladd<F>(
1123        &self,
1124        a: Scalar<M::Provenance>,
1125        b: Scalar<M::Provenance>,
1126        c: Scalar<M::Provenance>,
1127        typ: MulAddType,
1128    ) -> InterpResult<'tcx, Scalar<M::Provenance>>
1129    where
1130        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1131    {
1132        let a: F = a.to_float()?;
1133        let b: F = b.to_float()?;
1134        let c: F = c.to_float()?;
1135
1136        let fuse = typ == MulAddType::Fused || M::float_fuse_mul_add(self);
1137
1138        let res = if fuse { a.mul_add(b, c).value } else { ((a * b).value + c).value };
1139        let res = self.adjust_nan(res, &[a, b, c]);
1140        interp_ok(res.into())
1141    }
1142
1143    fn float_muladd_intrinsic<F>(
1144        &mut self,
1145        args: &[OpTy<'tcx, M::Provenance>],
1146        dest: &PlaceTy<'tcx, M::Provenance>,
1147        typ: MulAddType,
1148    ) -> InterpResult<'tcx, ()>
1149    where
1150        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1151    {
1152        let a = self.read_scalar(&args[0])?;
1153        let b = self.read_scalar(&args[1])?;
1154        let c = self.read_scalar(&args[2])?;
1155
1156        let res = self.float_muladd::<F>(a, b, c, typ)?;
1157        self.write_scalar(res, dest)?;
1158        interp_ok(())
1159    }
1160
1161    /// Converts `src` from floating point to integer type `dest_ty`
1162    /// after rounding with mode `round`.
1163    /// Returns `None` if `f` is NaN or out of range.
1164    pub fn float_to_int_checked(
1165        &self,
1166        src: &ImmTy<'tcx, M::Provenance>,
1167        cast_to: TyAndLayout<'tcx>,
1168        round: rustc_apfloat::Round,
1169    ) -> InterpResult<'tcx, Option<ImmTy<'tcx, M::Provenance>>> {
1170        fn float_to_int_inner<'tcx, F: rustc_apfloat::Float, M: Machine<'tcx>>(
1171            ecx: &InterpCx<'tcx, M>,
1172            src: F,
1173            cast_to: TyAndLayout<'tcx>,
1174            round: rustc_apfloat::Round,
1175        ) -> (Scalar<M::Provenance>, rustc_apfloat::Status) {
1176            let int_size = cast_to.layout.size;
1177            match cast_to.ty.kind() {
1178                // Unsigned
1179                ty::Uint(_) => {
1180                    let res = src.to_u128_r(int_size.bits_usize(), round, &mut false);
1181                    (Scalar::from_uint(res.value, int_size), res.status)
1182                }
1183                // Signed
1184                ty::Int(_) => {
1185                    let res = src.to_i128_r(int_size.bits_usize(), round, &mut false);
1186                    (Scalar::from_int(res.value, int_size), res.status)
1187                }
1188                // Nothing else
1189                _ => span_bug!(
1190                    ecx.cur_span(),
1191                    "attempted float-to-int conversion with non-int output type {}",
1192                    cast_to.ty,
1193                ),
1194            }
1195        }
1196
1197        let ty::Float(fty) = src.layout.ty.kind() else {
1198            bug!("float_to_int_checked: non-float input type {}", src.layout.ty)
1199        };
1200
1201        let (val, status) = match fty {
1202            FloatTy::F16 => float_to_int_inner(self, src.to_scalar().to_f16()?, cast_to, round),
1203            FloatTy::F32 => float_to_int_inner(self, src.to_scalar().to_f32()?, cast_to, round),
1204            FloatTy::F64 => float_to_int_inner(self, src.to_scalar().to_f64()?, cast_to, round),
1205            FloatTy::F128 => float_to_int_inner(self, src.to_scalar().to_f128()?, cast_to, round),
1206        };
1207
1208        if status.intersects(
1209            rustc_apfloat::Status::INVALID_OP
1210                | rustc_apfloat::Status::OVERFLOW
1211                | rustc_apfloat::Status::UNDERFLOW,
1212        ) {
1213            // Floating point value is NaN (flagged with INVALID_OP) or outside the range
1214            // of values of the integer type (flagged with OVERFLOW or UNDERFLOW).
1215            interp_ok(None)
1216        } else {
1217            // Floating point value can be represented by the integer type after rounding.
1218            // The INEXACT flag is ignored on purpose to allow rounding.
1219            interp_ok(Some(ImmTy::from_scalar(val, cast_to)))
1220        }
1221    }
1222}
rustc_const_eval/interpret/intrinsics.rs

rustc_const_eval/interpret/
intrinsics.rs
