rustc_const_eval/interpret/
intrinsics.rs

1//! Intrinsics and other functions that the interpreter executes without
2//! looking at their MIR. Intrinsics/functions supported here are shared by CTFE
3//! and miri.
4
5use std::assert_matches::assert_matches;
6
7use rustc_abi::{FieldIdx, HasDataLayout, Size};
8use rustc_apfloat::ieee::{Double, Half, Quad, Single};
9use rustc_middle::mir::interpret::{read_target_uint, write_target_uint};
10use rustc_middle::mir::{self, BinOp, ConstValue, NonDivergingIntrinsic};
11use rustc_middle::ty::layout::TyAndLayout;
12use rustc_middle::ty::{Ty, TyCtxt};
13use rustc_middle::{bug, ty};
14use rustc_span::{Symbol, sym};
15use tracing::trace;
16
17use super::memory::MemoryKind;
18use super::util::ensure_monomorphic_enough;
19use super::{
20    Allocation, CheckInAllocMsg, ConstAllocation, ImmTy, InterpCx, InterpResult, Machine, OpTy,
21    PlaceTy, Pointer, PointerArithmetic, Provenance, Scalar, err_ub_custom, err_unsup_format,
22    interp_ok, throw_inval, throw_ub_custom, throw_ub_format,
23};
24use crate::fluent_generated as fluent;
25
26/// Directly returns an `Allocation` containing an absolute path representation of the given type.
27pub(crate) fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ConstAllocation<'tcx> {
28    let path = crate::util::type_name(tcx, ty);
29    let alloc = Allocation::from_bytes_byte_aligned_immutable(path.into_bytes(), ());
30    tcx.mk_const_alloc(alloc)
31}
32impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
33    /// Generates a value of `TypeId` for `ty` in-place.
34    fn write_type_id(
35        &mut self,
36        ty: Ty<'tcx>,
37        dest: &PlaceTy<'tcx, M::Provenance>,
38    ) -> InterpResult<'tcx, ()> {
39        let tcx = self.tcx;
40        let type_id_hash = tcx.type_id_hash(ty).as_u128();
41        let op = self.const_val_to_op(
42            ConstValue::Scalar(Scalar::from_u128(type_id_hash)),
43            tcx.types.u128,
44            None,
45        )?;
46        self.copy_op_allow_transmute(&op, dest)?;
47
48        // Give the each pointer-sized chunk provenance that knows about the type id.
49        // Here we rely on `TypeId` being a newtype around an array of pointers, so we
50        // first project to its only field and then the array elements.
51        let alloc_id = tcx.reserve_and_set_type_id_alloc(ty);
52        let arr = self.project_field(dest, FieldIdx::ZERO)?;
53        let mut elem_iter = self.project_array_fields(&arr)?;
54        while let Some((_, elem)) = elem_iter.next(self)? {
55            // Decorate this part of the hash with provenance; leave the integer part unchanged.
56            let hash_fragment = self.read_scalar(&elem)?.to_target_usize(&tcx)?;
57            let ptr = Pointer::new(alloc_id.into(), Size::from_bytes(hash_fragment));
58            let ptr = self.global_root_pointer(ptr)?;
59            let val = Scalar::from_pointer(ptr, &tcx);
60            self.write_scalar(val, &elem)?;
61        }
62        interp_ok(())
63    }
64
65    /// Read a value of type `TypeId`, returning the type it represents.
66    pub(crate) fn read_type_id(
67        &self,
68        op: &OpTy<'tcx, M::Provenance>,
69    ) -> InterpResult<'tcx, Ty<'tcx>> {
70        // `TypeId` is a newtype around an array of pointers. All pointers must have the same
71        // provenance, and that provenance represents the type.
72        let ptr_size = self.pointer_size().bytes_usize();
73        let arr = self.project_field(op, FieldIdx::ZERO)?;
74
75        let mut ty_and_hash = None;
76        let mut elem_iter = self.project_array_fields(&arr)?;
77        while let Some((idx, elem)) = elem_iter.next(self)? {
78            let elem = self.read_pointer(&elem)?;
79            let (elem_ty, elem_hash) = self.get_ptr_type_id(elem)?;
80            // If this is the first element, remember the type and its hash.
81            // If this is not the first element, ensure it is consistent with the previous ones.
82            let full_hash = match ty_and_hash {
83                None => {
84                    let hash = self.tcx.type_id_hash(elem_ty).as_u128();
85                    let mut hash_bytes = [0u8; 16];
86                    write_target_uint(self.data_layout().endian, &mut hash_bytes, hash).unwrap();
87                    ty_and_hash = Some((elem_ty, hash_bytes));
88                    hash_bytes
89                }
90                Some((ty, hash_bytes)) => {
91                    if ty != elem_ty {
92                        throw_ub_format!(
93                            "invalid `TypeId` value: not all bytes carry the same type id metadata"
94                        );
95                    }
96                    hash_bytes
97                }
98            };
99            // Ensure the elem_hash matches the corresponding part of the full hash.
100            let hash_frag = &full_hash[(idx as usize) * ptr_size..][..ptr_size];
101            if read_target_uint(self.data_layout().endian, hash_frag).unwrap() != elem_hash.into() {
102                throw_ub_format!(
103                    "invalid `TypeId` value: the hash does not match the type id metadata"
104                );
105            }
106        }
107
108        interp_ok(ty_and_hash.unwrap().0)
109    }
110
111    /// Returns `true` if emulation happened.
112    /// Here we implement the intrinsics that are common to all Miri instances; individual machines can add their own
113    /// intrinsic handling.
114    pub fn eval_intrinsic(
115        &mut self,
116        instance: ty::Instance<'tcx>,
117        args: &[OpTy<'tcx, M::Provenance>],
118        dest: &PlaceTy<'tcx, M::Provenance>,
119        ret: Option<mir::BasicBlock>,
120    ) -> InterpResult<'tcx, bool> {
121        let instance_args = instance.args;
122        let intrinsic_name = self.tcx.item_name(instance.def_id());
123        let tcx = self.tcx.tcx;
124
125        match intrinsic_name {
126            sym::type_name => {
127                let tp_ty = instance.args.type_at(0);
128                ensure_monomorphic_enough(tcx, tp_ty)?;
129                let alloc = alloc_type_name(tcx, tp_ty);
130                let val = ConstValue::Slice { data: alloc, meta: alloc.inner().size().bytes() };
131                let val = self.const_val_to_op(val, dest.layout.ty, Some(dest.layout))?;
132                self.copy_op(&val, dest)?;
133            }
134            sym::needs_drop => {
135                let tp_ty = instance.args.type_at(0);
136                ensure_monomorphic_enough(tcx, tp_ty)?;
137                let val = ConstValue::from_bool(tp_ty.needs_drop(tcx, self.typing_env));
138                let val = self.const_val_to_op(val, tcx.types.bool, Some(dest.layout))?;
139                self.copy_op(&val, dest)?;
140            }
141            sym::type_id => {
142                let tp_ty = instance.args.type_at(0);
143                ensure_monomorphic_enough(tcx, tp_ty)?;
144                self.write_type_id(tp_ty, dest)?;
145            }
146            sym::type_id_eq => {
147                let a_ty = self.read_type_id(&args[0])?;
148                let b_ty = self.read_type_id(&args[1])?;
149                self.write_scalar(Scalar::from_bool(a_ty == b_ty), dest)?;
150            }
151            sym::variant_count => {
152                let tp_ty = instance.args.type_at(0);
153                let ty = match tp_ty.kind() {
154                    // Pattern types have the same number of variants as their base type.
155                    // Even if we restrict e.g. which variants are valid, the variants are essentially just uninhabited.
156                    // And `Result<(), !>` still has two variants according to `variant_count`.
157                    ty::Pat(base, _) => *base,
158                    _ => tp_ty,
159                };
160                let val = match ty.kind() {
161                    // Correctly handles non-monomorphic calls, so there is no need for ensure_monomorphic_enough.
162                    ty::Adt(adt, _) => {
163                        ConstValue::from_target_usize(adt.variants().len() as u64, &tcx)
164                    }
165                    ty::Alias(..) | ty::Param(_) | ty::Placeholder(_) | ty::Infer(_) => {
166                        throw_inval!(TooGeneric)
167                    }
168                    ty::Pat(..) => unreachable!(),
169                    ty::Bound(_, _) => bug!("bound ty during ctfe"),
170                    ty::Bool
171                    | ty::Char
172                    | ty::Int(_)
173                    | ty::Uint(_)
174                    | ty::Float(_)
175                    | ty::Foreign(_)
176                    | ty::Str
177                    | ty::Array(_, _)
178                    | ty::Slice(_)
179                    | ty::RawPtr(_, _)
180                    | ty::Ref(_, _, _)
181                    | ty::FnDef(_, _)
182                    | ty::FnPtr(..)
183                    | ty::Dynamic(_, _, _)
184                    | ty::Closure(_, _)
185                    | ty::CoroutineClosure(_, _)
186                    | ty::Coroutine(_, _)
187                    | ty::CoroutineWitness(..)
188                    | ty::UnsafeBinder(_)
189                    | ty::Never
190                    | ty::Tuple(_)
191                    | ty::Error(_) => ConstValue::from_target_usize(0u64, &tcx),
192                };
193                let val = self.const_val_to_op(val, dest.layout.ty, Some(dest.layout))?;
194                self.copy_op(&val, dest)?;
195            }
196
197            sym::caller_location => {
198                let span = self.find_closest_untracked_caller_location();
199                let val = self.tcx.span_as_caller_location(span);
200                let val =
201                    self.const_val_to_op(val, self.tcx.caller_location_ty(), Some(dest.layout))?;
202                self.copy_op(&val, dest)?;
203            }
204
205            sym::align_of_val | sym::size_of_val => {
206                // Avoid `deref_pointer` -- this is not a deref, the ptr does not have to be
207                // dereferenceable!
208                let place = self.ref_to_mplace(&self.read_immediate(&args[0])?)?;
209                let (size, align) = self
210                    .size_and_align_of_val(&place)?
211                    .ok_or_else(|| err_unsup_format!("`extern type` does not have known layout"))?;
212
213                let result = match intrinsic_name {
214                    sym::align_of_val => align.bytes(),
215                    sym::size_of_val => size.bytes(),
216                    _ => bug!(),
217                };
218
219                self.write_scalar(Scalar::from_target_usize(result, self), dest)?;
220            }
221
222            sym::fadd_algebraic
223            | sym::fsub_algebraic
224            | sym::fmul_algebraic
225            | sym::fdiv_algebraic
226            | sym::frem_algebraic => {
227                let a = self.read_immediate(&args[0])?;
228                let b = self.read_immediate(&args[1])?;
229
230                let op = match intrinsic_name {
231                    sym::fadd_algebraic => BinOp::Add,
232                    sym::fsub_algebraic => BinOp::Sub,
233                    sym::fmul_algebraic => BinOp::Mul,
234                    sym::fdiv_algebraic => BinOp::Div,
235                    sym::frem_algebraic => BinOp::Rem,
236
237                    _ => bug!(),
238                };
239
240                let res = self.binary_op(op, &a, &b)?;
241                // `binary_op` already called `generate_nan` if needed.
242                let res = M::apply_float_nondet(self, res)?;
243                self.write_immediate(*res, dest)?;
244            }
245
246            sym::ctpop
247            | sym::cttz
248            | sym::cttz_nonzero
249            | sym::ctlz
250            | sym::ctlz_nonzero
251            | sym::bswap
252            | sym::bitreverse => {
253                let ty = instance_args.type_at(0);
254                let layout = self.layout_of(ty)?;
255                let val = self.read_scalar(&args[0])?;
256
257                let out_val = self.numeric_intrinsic(intrinsic_name, val, layout, dest.layout)?;
258                self.write_scalar(out_val, dest)?;
259            }
260            sym::saturating_add | sym::saturating_sub => {
261                let l = self.read_immediate(&args[0])?;
262                let r = self.read_immediate(&args[1])?;
263                let val = self.saturating_arith(
264                    if intrinsic_name == sym::saturating_add { BinOp::Add } else { BinOp::Sub },
265                    &l,
266                    &r,
267                )?;
268                self.write_scalar(val, dest)?;
269            }
270            sym::discriminant_value => {
271                let place = self.deref_pointer(&args[0])?;
272                let variant = self.read_discriminant(&place)?;
273                let discr = self.discriminant_for_variant(place.layout.ty, variant)?;
274                self.write_immediate(*discr, dest)?;
275            }
276            sym::exact_div => {
277                let l = self.read_immediate(&args[0])?;
278                let r = self.read_immediate(&args[1])?;
279                self.exact_div(&l, &r, dest)?;
280            }
281            sym::rotate_left | sym::rotate_right => {
282                // rotate_left: (X << (S % BW)) | (X >> ((BW - S) % BW))
283                // rotate_right: (X << ((BW - S) % BW)) | (X >> (S % BW))
284                let layout_val = self.layout_of(instance_args.type_at(0))?;
285                let val = self.read_scalar(&args[0])?;
286                let val_bits = val.to_bits(layout_val.size)?; // sign is ignored here
287
288                let layout_raw_shift = self.layout_of(self.tcx.types.u32)?;
289                let raw_shift = self.read_scalar(&args[1])?;
290                let raw_shift_bits = raw_shift.to_bits(layout_raw_shift.size)?;
291
292                let width_bits = u128::from(layout_val.size.bits());
293                let shift_bits = raw_shift_bits % width_bits;
294                let inv_shift_bits = (width_bits - shift_bits) % width_bits;
295                let result_bits = if intrinsic_name == sym::rotate_left {
296                    (val_bits << shift_bits) | (val_bits >> inv_shift_bits)
297                } else {
298                    (val_bits >> shift_bits) | (val_bits << inv_shift_bits)
299                };
300                let truncated_bits = layout_val.size.truncate(result_bits);
301                let result = Scalar::from_uint(truncated_bits, layout_val.size);
302                self.write_scalar(result, dest)?;
303            }
304            sym::copy => {
305                self.copy_intrinsic(&args[0], &args[1], &args[2], /*nonoverlapping*/ false)?;
306            }
307            sym::write_bytes => {
308                self.write_bytes_intrinsic(&args[0], &args[1], &args[2], "write_bytes")?;
309            }
310            sym::compare_bytes => {
311                let result = self.compare_bytes_intrinsic(&args[0], &args[1], &args[2])?;
312                self.write_scalar(result, dest)?;
313            }
314            sym::arith_offset => {
315                let ptr = self.read_pointer(&args[0])?;
316                let offset_count = self.read_target_isize(&args[1])?;
317                let pointee_ty = instance_args.type_at(0);
318
319                let pointee_size = i64::try_from(self.layout_of(pointee_ty)?.size.bytes()).unwrap();
320                let offset_bytes = offset_count.wrapping_mul(pointee_size);
321                let offset_ptr = ptr.wrapping_signed_offset(offset_bytes, self);
322                self.write_pointer(offset_ptr, dest)?;
323            }
324            sym::ptr_offset_from | sym::ptr_offset_from_unsigned => {
325                let a = self.read_pointer(&args[0])?;
326                let b = self.read_pointer(&args[1])?;
327
328                let usize_layout = self.layout_of(self.tcx.types.usize)?;
329                let isize_layout = self.layout_of(self.tcx.types.isize)?;
330
331                // Get offsets for both that are at least relative to the same base.
332                // With `OFFSET_IS_ADDR` this is trivial; without it we need either
333                // two integers or two pointers into the same allocation.
334                let (a_offset, b_offset, is_addr) = if M::Provenance::OFFSET_IS_ADDR {
335                    (a.addr().bytes(), b.addr().bytes(), /*is_addr*/ true)
336                } else {
337                    match (self.ptr_try_get_alloc_id(a, 0), self.ptr_try_get_alloc_id(b, 0)) {
338                        (Err(a), Err(b)) => {
339                            // Neither pointer points to an allocation, so they are both absolute.
340                            (a, b, /*is_addr*/ true)
341                        }
342                        (Ok((a_alloc_id, a_offset, _)), Ok((b_alloc_id, b_offset, _)))
343                            if a_alloc_id == b_alloc_id =>
344                        {
345                            // Found allocation for both, and it's the same.
346                            // Use these offsets for distance calculation.
347                            (a_offset.bytes(), b_offset.bytes(), /*is_addr*/ false)
348                        }
349                        _ => {
350                            // Not into the same allocation -- this is UB.
351                            throw_ub_custom!(
352                                fluent::const_eval_offset_from_different_allocations,
353                                name = intrinsic_name,
354                            );
355                        }
356                    }
357                };
358
359                // Compute distance: a - b.
360                let dist = {
361                    // Addresses are unsigned, so this is a `usize` computation. We have to do the
362                    // overflow check separately anyway.
363                    let (val, overflowed) = {
364                        let a_offset = ImmTy::from_uint(a_offset, usize_layout);
365                        let b_offset = ImmTy::from_uint(b_offset, usize_layout);
366                        self.binary_op(BinOp::SubWithOverflow, &a_offset, &b_offset)?
367                            .to_scalar_pair()
368                    };
369                    if overflowed.to_bool()? {
370                        // a < b
371                        if intrinsic_name == sym::ptr_offset_from_unsigned {
372                            throw_ub_custom!(
373                                fluent::const_eval_offset_from_unsigned_overflow,
374                                a_offset = a_offset,
375                                b_offset = b_offset,
376                                is_addr = is_addr,
377                            );
378                        }
379                        // The signed form of the intrinsic allows this. If we interpret the
380                        // difference as isize, we'll get the proper signed difference. If that
381                        // seems *positive* or equal to isize::MIN, they were more than isize::MAX apart.
382                        let dist = val.to_target_isize(self)?;
383                        if dist >= 0 || i128::from(dist) == self.pointer_size().signed_int_min() {
384                            throw_ub_custom!(
385                                fluent::const_eval_offset_from_underflow,
386                                name = intrinsic_name,
387                            );
388                        }
389                        dist
390                    } else {
391                        // b >= a
392                        let dist = val.to_target_isize(self)?;
393                        // If converting to isize produced a *negative* result, we had an overflow
394                        // because they were more than isize::MAX apart.
395                        if dist < 0 {
396                            throw_ub_custom!(
397                                fluent::const_eval_offset_from_overflow,
398                                name = intrinsic_name,
399                            );
400                        }
401                        dist
402                    }
403                };
404
405                // Check that the memory between them is dereferenceable at all, starting from the
406                // origin pointer: `dist` is `a - b`, so it is based on `b`.
407                self.check_ptr_access_signed(b, dist, CheckInAllocMsg::Dereferenceable)
408                    .map_err_kind(|_| {
409                        // This could mean they point to different allocations, or they point to the same allocation
410                        // but not the entire range between the pointers is in-bounds.
411                        if let Ok((a_alloc_id, ..)) = self.ptr_try_get_alloc_id(a, 0)
412                            && let Ok((b_alloc_id, ..)) = self.ptr_try_get_alloc_id(b, 0)
413                            && a_alloc_id == b_alloc_id
414                        {
415                            err_ub_custom!(
416                                fluent::const_eval_offset_from_out_of_bounds,
417                                name = intrinsic_name,
418                            )
419                        } else {
420                            err_ub_custom!(
421                                fluent::const_eval_offset_from_different_allocations,
422                                name = intrinsic_name,
423                            )
424                        }
425                    })?;
426                // Then check that this is also dereferenceable from `a`. This ensures that they are
427                // derived from the same allocation.
428                self.check_ptr_access_signed(
429                    a,
430                    dist.checked_neg().unwrap(), // i64::MIN is impossible as no allocation can be that large
431                    CheckInAllocMsg::Dereferenceable,
432                )
433                .map_err_kind(|_| {
434                    // Make the error more specific.
435                    err_ub_custom!(
436                        fluent::const_eval_offset_from_different_allocations,
437                        name = intrinsic_name,
438                    )
439                })?;
440
441                // Perform division by size to compute return value.
442                let ret_layout = if intrinsic_name == sym::ptr_offset_from_unsigned {
443                    assert!(0 <= dist && dist <= self.target_isize_max());
444                    usize_layout
445                } else {
446                    assert!(self.target_isize_min() <= dist && dist <= self.target_isize_max());
447                    isize_layout
448                };
449                let pointee_layout = self.layout_of(instance_args.type_at(0))?;
450                // If ret_layout is unsigned, we checked that so is the distance, so we are good.
451                let val = ImmTy::from_int(dist, ret_layout);
452                let size = ImmTy::from_int(pointee_layout.size.bytes(), ret_layout);
453                self.exact_div(&val, &size, dest)?;
454            }
455
456            sym::simd_insert => {
457                let index = u64::from(self.read_scalar(&args[1])?.to_u32()?);
458                let elem = &args[2];
459                let (input, input_len) = self.project_to_simd(&args[0])?;
460                let (dest, dest_len) = self.project_to_simd(dest)?;
461                assert_eq!(input_len, dest_len, "Return vector length must match input length");
462                // Bounds are not checked by typeck so we have to do it ourselves.
463                if index >= input_len {
464                    throw_ub_format!(
465                        "`simd_insert` index {index} is out-of-bounds of vector with length {input_len}"
466                    );
467                }
468
469                for i in 0..dest_len {
470                    let place = self.project_index(&dest, i)?;
471                    let value =
472                        if i == index { elem.clone() } else { self.project_index(&input, i)? };
473                    self.copy_op(&value, &place)?;
474                }
475            }
476            sym::simd_extract => {
477                let index = u64::from(self.read_scalar(&args[1])?.to_u32()?);
478                let (input, input_len) = self.project_to_simd(&args[0])?;
479                // Bounds are not checked by typeck so we have to do it ourselves.
480                if index >= input_len {
481                    throw_ub_format!(
482                        "`simd_extract` index {index} is out-of-bounds of vector with length {input_len}"
483                    );
484                }
485                self.copy_op(&self.project_index(&input, index)?, dest)?;
486            }
487            sym::black_box => {
488                // These just return their argument
489                self.copy_op(&args[0], dest)?;
490            }
491            sym::raw_eq => {
492                let result = self.raw_eq_intrinsic(&args[0], &args[1])?;
493                self.write_scalar(result, dest)?;
494            }
495            sym::typed_swap_nonoverlapping => {
496                self.typed_swap_nonoverlapping_intrinsic(&args[0], &args[1])?;
497            }
498
499            sym::vtable_size => {
500                let ptr = self.read_pointer(&args[0])?;
501                // `None` because we don't know which trait to expect here; any vtable is okay.
502                let (size, _align) = self.get_vtable_size_and_align(ptr, None)?;
503                self.write_scalar(Scalar::from_target_usize(size.bytes(), self), dest)?;
504            }
505            sym::vtable_align => {
506                let ptr = self.read_pointer(&args[0])?;
507                // `None` because we don't know which trait to expect here; any vtable is okay.
508                let (_size, align) = self.get_vtable_size_and_align(ptr, None)?;
509                self.write_scalar(Scalar::from_target_usize(align.bytes(), self), dest)?;
510            }
511
512            sym::minnumf16 => self.float_min_intrinsic::<Half>(args, dest)?,
513            sym::minnumf32 => self.float_min_intrinsic::<Single>(args, dest)?,
514            sym::minnumf64 => self.float_min_intrinsic::<Double>(args, dest)?,
515            sym::minnumf128 => self.float_min_intrinsic::<Quad>(args, dest)?,
516
517            sym::minimumf16 => self.float_minimum_intrinsic::<Half>(args, dest)?,
518            sym::minimumf32 => self.float_minimum_intrinsic::<Single>(args, dest)?,
519            sym::minimumf64 => self.float_minimum_intrinsic::<Double>(args, dest)?,
520            sym::minimumf128 => self.float_minimum_intrinsic::<Quad>(args, dest)?,
521
522            sym::maxnumf16 => self.float_max_intrinsic::<Half>(args, dest)?,
523            sym::maxnumf32 => self.float_max_intrinsic::<Single>(args, dest)?,
524            sym::maxnumf64 => self.float_max_intrinsic::<Double>(args, dest)?,
525            sym::maxnumf128 => self.float_max_intrinsic::<Quad>(args, dest)?,
526
527            sym::maximumf16 => self.float_maximum_intrinsic::<Half>(args, dest)?,
528            sym::maximumf32 => self.float_maximum_intrinsic::<Single>(args, dest)?,
529            sym::maximumf64 => self.float_maximum_intrinsic::<Double>(args, dest)?,
530            sym::maximumf128 => self.float_maximum_intrinsic::<Quad>(args, dest)?,
531
532            sym::copysignf16 => self.float_copysign_intrinsic::<Half>(args, dest)?,
533            sym::copysignf32 => self.float_copysign_intrinsic::<Single>(args, dest)?,
534            sym::copysignf64 => self.float_copysign_intrinsic::<Double>(args, dest)?,
535            sym::copysignf128 => self.float_copysign_intrinsic::<Quad>(args, dest)?,
536
537            sym::fabsf16 => self.float_abs_intrinsic::<Half>(args, dest)?,
538            sym::fabsf32 => self.float_abs_intrinsic::<Single>(args, dest)?,
539            sym::fabsf64 => self.float_abs_intrinsic::<Double>(args, dest)?,
540            sym::fabsf128 => self.float_abs_intrinsic::<Quad>(args, dest)?,
541
542            sym::floorf16 => self.float_round_intrinsic::<Half>(
543                args,
544                dest,
545                rustc_apfloat::Round::TowardNegative,
546            )?,
547            sym::floorf32 => self.float_round_intrinsic::<Single>(
548                args,
549                dest,
550                rustc_apfloat::Round::TowardNegative,
551            )?,
552            sym::floorf64 => self.float_round_intrinsic::<Double>(
553                args,
554                dest,
555                rustc_apfloat::Round::TowardNegative,
556            )?,
557            sym::floorf128 => self.float_round_intrinsic::<Quad>(
558                args,
559                dest,
560                rustc_apfloat::Round::TowardNegative,
561            )?,
562
563            sym::ceilf16 => self.float_round_intrinsic::<Half>(
564                args,
565                dest,
566                rustc_apfloat::Round::TowardPositive,
567            )?,
568            sym::ceilf32 => self.float_round_intrinsic::<Single>(
569                args,
570                dest,
571                rustc_apfloat::Round::TowardPositive,
572            )?,
573            sym::ceilf64 => self.float_round_intrinsic::<Double>(
574                args,
575                dest,
576                rustc_apfloat::Round::TowardPositive,
577            )?,
578            sym::ceilf128 => self.float_round_intrinsic::<Quad>(
579                args,
580                dest,
581                rustc_apfloat::Round::TowardPositive,
582            )?,
583
584            sym::truncf16 => {
585                self.float_round_intrinsic::<Half>(args, dest, rustc_apfloat::Round::TowardZero)?
586            }
587            sym::truncf32 => {
588                self.float_round_intrinsic::<Single>(args, dest, rustc_apfloat::Round::TowardZero)?
589            }
590            sym::truncf64 => {
591                self.float_round_intrinsic::<Double>(args, dest, rustc_apfloat::Round::TowardZero)?
592            }
593            sym::truncf128 => {
594                self.float_round_intrinsic::<Quad>(args, dest, rustc_apfloat::Round::TowardZero)?
595            }
596
597            sym::roundf16 => self.float_round_intrinsic::<Half>(
598                args,
599                dest,
600                rustc_apfloat::Round::NearestTiesToAway,
601            )?,
602            sym::roundf32 => self.float_round_intrinsic::<Single>(
603                args,
604                dest,
605                rustc_apfloat::Round::NearestTiesToAway,
606            )?,
607            sym::roundf64 => self.float_round_intrinsic::<Double>(
608                args,
609                dest,
610                rustc_apfloat::Round::NearestTiesToAway,
611            )?,
612            sym::roundf128 => self.float_round_intrinsic::<Quad>(
613                args,
614                dest,
615                rustc_apfloat::Round::NearestTiesToAway,
616            )?,
617
618            sym::round_ties_even_f16 => self.float_round_intrinsic::<Half>(
619                args,
620                dest,
621                rustc_apfloat::Round::NearestTiesToEven,
622            )?,
623            sym::round_ties_even_f32 => self.float_round_intrinsic::<Single>(
624                args,
625                dest,
626                rustc_apfloat::Round::NearestTiesToEven,
627            )?,
628            sym::round_ties_even_f64 => self.float_round_intrinsic::<Double>(
629                args,
630                dest,
631                rustc_apfloat::Round::NearestTiesToEven,
632            )?,
633            sym::round_ties_even_f128 => self.float_round_intrinsic::<Quad>(
634                args,
635                dest,
636                rustc_apfloat::Round::NearestTiesToEven,
637            )?,
638
639            // Unsupported intrinsic: skip the return_to_block below.
640            _ => return interp_ok(false),
641        }
642
643        trace!("{:?}", self.dump_place(&dest.clone().into()));
644        self.return_to_block(ret)?;
645        interp_ok(true)
646    }
647
648    pub(super) fn eval_nondiverging_intrinsic(
649        &mut self,
650        intrinsic: &NonDivergingIntrinsic<'tcx>,
651    ) -> InterpResult<'tcx> {
652        match intrinsic {
653            NonDivergingIntrinsic::Assume(op) => {
654                let op = self.eval_operand(op, None)?;
655                let cond = self.read_scalar(&op)?.to_bool()?;
656                if !cond {
657                    throw_ub_custom!(fluent::const_eval_assume_false);
658                }
659                interp_ok(())
660            }
661            NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping {
662                count,
663                src,
664                dst,
665            }) => {
666                let src = self.eval_operand(src, None)?;
667                let dst = self.eval_operand(dst, None)?;
668                let count = self.eval_operand(count, None)?;
669                self.copy_intrinsic(&src, &dst, &count, /* nonoverlapping */ true)
670            }
671        }
672    }
673
674    pub fn numeric_intrinsic(
675        &self,
676        name: Symbol,
677        val: Scalar<M::Provenance>,
678        layout: TyAndLayout<'tcx>,
679        ret_layout: TyAndLayout<'tcx>,
680    ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
681        assert!(layout.ty.is_integral(), "invalid type for numeric intrinsic: {}", layout.ty);
682        let bits = val.to_bits(layout.size)?; // these operations all ignore the sign
683        let extra = 128 - u128::from(layout.size.bits());
684        let bits_out = match name {
685            sym::ctpop => u128::from(bits.count_ones()),
686            sym::ctlz_nonzero | sym::cttz_nonzero if bits == 0 => {
687                throw_ub_custom!(fluent::const_eval_call_nonzero_intrinsic, name = name,);
688            }
689            sym::ctlz | sym::ctlz_nonzero => u128::from(bits.leading_zeros()) - extra,
690            sym::cttz | sym::cttz_nonzero => u128::from((bits << extra).trailing_zeros()) - extra,
691            sym::bswap => {
692                assert_eq!(layout, ret_layout);
693                (bits << extra).swap_bytes()
694            }
695            sym::bitreverse => {
696                assert_eq!(layout, ret_layout);
697                (bits << extra).reverse_bits()
698            }
699            _ => bug!("not a numeric intrinsic: {}", name),
700        };
701        interp_ok(Scalar::from_uint(bits_out, ret_layout.size))
702    }
703
704    pub fn exact_div(
705        &mut self,
706        a: &ImmTy<'tcx, M::Provenance>,
707        b: &ImmTy<'tcx, M::Provenance>,
708        dest: &PlaceTy<'tcx, M::Provenance>,
709    ) -> InterpResult<'tcx> {
710        assert_eq!(a.layout.ty, b.layout.ty);
711        assert_matches!(a.layout.ty.kind(), ty::Int(..) | ty::Uint(..));
712
713        // Performs an exact division, resulting in undefined behavior where
714        // `x % y != 0` or `y == 0` or `x == T::MIN && y == -1`.
715        // First, check x % y != 0 (or if that computation overflows).
716        let rem = self.binary_op(BinOp::Rem, a, b)?;
717        // sign does not matter for 0 test, so `to_bits` is fine
718        if rem.to_scalar().to_bits(a.layout.size)? != 0 {
719            throw_ub_custom!(
720                fluent::const_eval_exact_div_has_remainder,
721                a = format!("{a}"),
722                b = format!("{b}")
723            )
724        }
725        // `Rem` says this is all right, so we can let `Div` do its job.
726        let res = self.binary_op(BinOp::Div, a, b)?;
727        self.write_immediate(*res, dest)
728    }
729
730    pub fn saturating_arith(
731        &self,
732        mir_op: BinOp,
733        l: &ImmTy<'tcx, M::Provenance>,
734        r: &ImmTy<'tcx, M::Provenance>,
735    ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
736        assert_eq!(l.layout.ty, r.layout.ty);
737        assert_matches!(l.layout.ty.kind(), ty::Int(..) | ty::Uint(..));
738        assert_matches!(mir_op, BinOp::Add | BinOp::Sub);
739
740        let (val, overflowed) =
741            self.binary_op(mir_op.wrapping_to_overflowing().unwrap(), l, r)?.to_scalar_pair();
742        interp_ok(if overflowed.to_bool()? {
743            let size = l.layout.size;
744            if l.layout.backend_repr.is_signed() {
745                // For signed ints the saturated value depends on the sign of the first
746                // term since the sign of the second term can be inferred from this and
747                // the fact that the operation has overflowed (if either is 0 no
748                // overflow can occur)
749                let first_term: i128 = l.to_scalar().to_int(l.layout.size)?;
750                if first_term >= 0 {
751                    // Negative overflow not possible since the positive first term
752                    // can only increase an (in range) negative term for addition
753                    // or corresponding negated positive term for subtraction.
754                    Scalar::from_int(size.signed_int_max(), size)
755                } else {
756                    // Positive overflow not possible for similar reason.
757                    Scalar::from_int(size.signed_int_min(), size)
758                }
759            } else {
760                // unsigned
761                if matches!(mir_op, BinOp::Add) {
762                    // max unsigned
763                    Scalar::from_uint(size.unsigned_int_max(), size)
764                } else {
765                    // underflow to 0
766                    Scalar::from_uint(0u128, size)
767                }
768            }
769        } else {
770            val
771        })
772    }
773
774    /// Offsets a pointer by some multiple of its type, returning an error if the pointer leaves its
775    /// allocation.
776    pub fn ptr_offset_inbounds(
777        &self,
778        ptr: Pointer<Option<M::Provenance>>,
779        offset_bytes: i64,
780    ) -> InterpResult<'tcx, Pointer<Option<M::Provenance>>> {
781        // The offset must be in bounds starting from `ptr`.
782        self.check_ptr_access_signed(
783            ptr,
784            offset_bytes,
785            CheckInAllocMsg::InboundsPointerArithmetic,
786        )?;
787        // This also implies that there is no overflow, so we are done.
788        interp_ok(ptr.wrapping_signed_offset(offset_bytes, self))
789    }
790
791    /// Copy `count*size_of::<T>()` many bytes from `*src` to `*dst`.
792    pub(crate) fn copy_intrinsic(
793        &mut self,
794        src: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
795        dst: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
796        count: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
797        nonoverlapping: bool,
798    ) -> InterpResult<'tcx> {
799        let count = self.read_target_usize(count)?;
800        let layout = self.layout_of(src.layout.ty.builtin_deref(true).unwrap())?;
801        let (size, align) = (layout.size, layout.align.abi);
802
803        let size = self.compute_size_in_bytes(size, count).ok_or_else(|| {
804            err_ub_custom!(
805                fluent::const_eval_size_overflow,
806                name = if nonoverlapping { "copy_nonoverlapping" } else { "copy" }
807            )
808        })?;
809
810        let src = self.read_pointer(src)?;
811        let dst = self.read_pointer(dst)?;
812
813        self.check_ptr_align(src, align)?;
814        self.check_ptr_align(dst, align)?;
815
816        self.mem_copy(src, dst, size, nonoverlapping)
817    }
818
819    /// Does a *typed* swap of `*left` and `*right`.
820    fn typed_swap_nonoverlapping_intrinsic(
821        &mut self,
822        left: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
823        right: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
824    ) -> InterpResult<'tcx> {
825        let left = self.deref_pointer(left)?;
826        let right = self.deref_pointer(right)?;
827        assert_eq!(left.layout, right.layout);
828        assert!(left.layout.is_sized());
829        let kind = MemoryKind::Stack;
830        let temp = self.allocate(left.layout, kind)?;
831        self.copy_op(&left, &temp)?; // checks alignment of `left`
832
833        // We want to always enforce non-overlapping, even if this is a scalar type.
834        // Therefore we directly use the underlying `mem_copy` here.
835        self.mem_copy(right.ptr(), left.ptr(), left.layout.size, /*nonoverlapping*/ true)?;
836        // This means we also need to do the validation of the value that used to be in `right`
837        // ourselves. This value is now in `left.` The one that started out in `left` already got
838        // validated by the copy above.
839        if M::enforce_validity(self, left.layout) {
840            self.validate_operand(
841                &left.clone().into(),
842                M::enforce_validity_recursively(self, left.layout),
843                /*reset_provenance_and_padding*/ true,
844            )?;
845        }
846
847        self.copy_op(&temp, &right)?; // checks alignment of `right`
848
849        self.deallocate_ptr(temp.ptr(), None, kind)?;
850        interp_ok(())
851    }
852
853    pub fn write_bytes_intrinsic(
854        &mut self,
855        dst: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
856        byte: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
857        count: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
858        name: &'static str,
859    ) -> InterpResult<'tcx> {
860        let layout = self.layout_of(dst.layout.ty.builtin_deref(true).unwrap())?;
861
862        let dst = self.read_pointer(dst)?;
863        let byte = self.read_scalar(byte)?.to_u8()?;
864        let count = self.read_target_usize(count)?;
865
866        // `checked_mul` enforces a too small bound (the correct one would probably be target_isize_max),
867        // but no actual allocation can be big enough for the difference to be noticeable.
868        let len = self
869            .compute_size_in_bytes(layout.size, count)
870            .ok_or_else(|| err_ub_custom!(fluent::const_eval_size_overflow, name = name))?;
871
872        let bytes = std::iter::repeat(byte).take(len.bytes_usize());
873        self.write_bytes_ptr(dst, bytes)
874    }
875
876    pub(crate) fn compare_bytes_intrinsic(
877        &mut self,
878        left: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
879        right: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
880        byte_count: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
881    ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
882        let left = self.read_pointer(left)?;
883        let right = self.read_pointer(right)?;
884        let n = Size::from_bytes(self.read_target_usize(byte_count)?);
885
886        let left_bytes = self.read_bytes_ptr_strip_provenance(left, n)?;
887        let right_bytes = self.read_bytes_ptr_strip_provenance(right, n)?;
888
889        // `Ordering`'s discriminants are -1/0/+1, so casting does the right thing.
890        let result = Ord::cmp(left_bytes, right_bytes) as i32;
891        interp_ok(Scalar::from_i32(result))
892    }
893
894    pub(crate) fn raw_eq_intrinsic(
895        &mut self,
896        lhs: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
897        rhs: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
898    ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
899        let layout = self.layout_of(lhs.layout.ty.builtin_deref(true).unwrap())?;
900        assert!(layout.is_sized());
901
902        let get_bytes = |this: &InterpCx<'tcx, M>,
903                         op: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>|
904         -> InterpResult<'tcx, &[u8]> {
905            let ptr = this.read_pointer(op)?;
906            this.check_ptr_align(ptr, layout.align.abi)?;
907            let Some(alloc_ref) = self.get_ptr_alloc(ptr, layout.size)? else {
908                // zero-sized access
909                return interp_ok(&[]);
910            };
911            alloc_ref.get_bytes_strip_provenance()
912        };
913
914        let lhs_bytes = get_bytes(self, lhs)?;
915        let rhs_bytes = get_bytes(self, rhs)?;
916        interp_ok(Scalar::from_bool(lhs_bytes == rhs_bytes))
917    }
918
919    fn float_min_intrinsic<F>(
920        &mut self,
921        args: &[OpTy<'tcx, M::Provenance>],
922        dest: &PlaceTy<'tcx, M::Provenance>,
923    ) -> InterpResult<'tcx, ()>
924    where
925        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
926    {
927        let a: F = self.read_scalar(&args[0])?.to_float()?;
928        let b: F = self.read_scalar(&args[1])?.to_float()?;
929        let res = if a == b {
930            // They are definitely not NaN (those are never equal), but they could be `+0` and `-0`.
931            // Let the machine decide which one to return.
932            M::equal_float_min_max(self, a, b)
933        } else {
934            self.adjust_nan(a.min(b), &[a, b])
935        };
936        self.write_scalar(res, dest)?;
937        interp_ok(())
938    }
939
940    fn float_max_intrinsic<F>(
941        &mut self,
942        args: &[OpTy<'tcx, M::Provenance>],
943        dest: &PlaceTy<'tcx, M::Provenance>,
944    ) -> InterpResult<'tcx, ()>
945    where
946        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
947    {
948        let a: F = self.read_scalar(&args[0])?.to_float()?;
949        let b: F = self.read_scalar(&args[1])?.to_float()?;
950        let res = if a == b {
951            // They are definitely not NaN (those are never equal), but they could be `+0` and `-0`.
952            // Let the machine decide which one to return.
953            M::equal_float_min_max(self, a, b)
954        } else {
955            self.adjust_nan(a.max(b), &[a, b])
956        };
957        self.write_scalar(res, dest)?;
958        interp_ok(())
959    }
960
961    fn float_minimum_intrinsic<F>(
962        &mut self,
963        args: &[OpTy<'tcx, M::Provenance>],
964        dest: &PlaceTy<'tcx, M::Provenance>,
965    ) -> InterpResult<'tcx, ()>
966    where
967        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
968    {
969        let a: F = self.read_scalar(&args[0])?.to_float()?;
970        let b: F = self.read_scalar(&args[1])?.to_float()?;
971        let res = a.minimum(b);
972        let res = self.adjust_nan(res, &[a, b]);
973        self.write_scalar(res, dest)?;
974        interp_ok(())
975    }
976
977    fn float_maximum_intrinsic<F>(
978        &mut self,
979        args: &[OpTy<'tcx, M::Provenance>],
980        dest: &PlaceTy<'tcx, M::Provenance>,
981    ) -> InterpResult<'tcx, ()>
982    where
983        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
984    {
985        let a: F = self.read_scalar(&args[0])?.to_float()?;
986        let b: F = self.read_scalar(&args[1])?.to_float()?;
987        let res = a.maximum(b);
988        let res = self.adjust_nan(res, &[a, b]);
989        self.write_scalar(res, dest)?;
990        interp_ok(())
991    }
992
993    fn float_copysign_intrinsic<F>(
994        &mut self,
995        args: &[OpTy<'tcx, M::Provenance>],
996        dest: &PlaceTy<'tcx, M::Provenance>,
997    ) -> InterpResult<'tcx, ()>
998    where
999        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1000    {
1001        let a: F = self.read_scalar(&args[0])?.to_float()?;
1002        let b: F = self.read_scalar(&args[1])?.to_float()?;
1003        // bitwise, no NaN adjustments
1004        self.write_scalar(a.copy_sign(b), dest)?;
1005        interp_ok(())
1006    }
1007
1008    fn float_abs_intrinsic<F>(
1009        &mut self,
1010        args: &[OpTy<'tcx, M::Provenance>],
1011        dest: &PlaceTy<'tcx, M::Provenance>,
1012    ) -> InterpResult<'tcx, ()>
1013    where
1014        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1015    {
1016        let x: F = self.read_scalar(&args[0])?.to_float()?;
1017        // bitwise, no NaN adjustments
1018        self.write_scalar(x.abs(), dest)?;
1019        interp_ok(())
1020    }
1021
1022    fn float_round_intrinsic<F>(
1023        &mut self,
1024        args: &[OpTy<'tcx, M::Provenance>],
1025        dest: &PlaceTy<'tcx, M::Provenance>,
1026        mode: rustc_apfloat::Round,
1027    ) -> InterpResult<'tcx, ()>
1028    where
1029        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1030    {
1031        let x: F = self.read_scalar(&args[0])?.to_float()?;
1032        let res = x.round_to_integral(mode).value;
1033        let res = self.adjust_nan(res, &[x]);
1034        self.write_scalar(res, dest)?;
1035        interp_ok(())
1036    }
1037}