1use std::assert_matches::assert_matches;
6
7use rustc_abi::{FieldIdx, HasDataLayout, Size};
8use rustc_apfloat::ieee::{Double, Half, Quad, Single};
9use rustc_middle::mir::interpret::{read_target_uint, write_target_uint};
10use rustc_middle::mir::{self, BinOp, ConstValue, NonDivergingIntrinsic};
11use rustc_middle::ty::layout::TyAndLayout;
12use rustc_middle::ty::{Ty, TyCtxt};
13use rustc_middle::{bug, ty};
14use rustc_span::{Symbol, sym};
15use tracing::trace;
16
17use super::memory::MemoryKind;
18use super::util::ensure_monomorphic_enough;
19use super::{
20 Allocation, CheckInAllocMsg, ConstAllocation, ImmTy, InterpCx, InterpResult, Machine, OpTy,
21 PlaceTy, Pointer, PointerArithmetic, Provenance, Scalar, err_ub_custom, err_unsup_format,
22 interp_ok, throw_inval, throw_ub_custom, throw_ub_format,
23};
24use crate::fluent_generated as fluent;
25
26pub(crate) fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ConstAllocation<'tcx> {
28 let path = crate::util::type_name(tcx, ty);
29 let alloc = Allocation::from_bytes_byte_aligned_immutable(path.into_bytes(), ());
30 tcx.mk_const_alloc(alloc)
31}
32impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
33 fn write_type_id(
35 &mut self,
36 ty: Ty<'tcx>,
37 dest: &PlaceTy<'tcx, M::Provenance>,
38 ) -> InterpResult<'tcx, ()> {
39 let tcx = self.tcx;
40 let type_id_hash = tcx.type_id_hash(ty).as_u128();
41 let op = self.const_val_to_op(
42 ConstValue::Scalar(Scalar::from_u128(type_id_hash)),
43 tcx.types.u128,
44 None,
45 )?;
46 self.copy_op_allow_transmute(&op, dest)?;
47
48 let alloc_id = tcx.reserve_and_set_type_id_alloc(ty);
52 let arr = self.project_field(dest, FieldIdx::ZERO)?;
53 let mut elem_iter = self.project_array_fields(&arr)?;
54 while let Some((_, elem)) = elem_iter.next(self)? {
55 let hash_fragment = self.read_scalar(&elem)?.to_target_usize(&tcx)?;
57 let ptr = Pointer::new(alloc_id.into(), Size::from_bytes(hash_fragment));
58 let ptr = self.global_root_pointer(ptr)?;
59 let val = Scalar::from_pointer(ptr, &tcx);
60 self.write_scalar(val, &elem)?;
61 }
62 interp_ok(())
63 }
64
65 pub(crate) fn read_type_id(
67 &self,
68 op: &OpTy<'tcx, M::Provenance>,
69 ) -> InterpResult<'tcx, Ty<'tcx>> {
70 let ptr_size = self.pointer_size().bytes_usize();
73 let arr = self.project_field(op, FieldIdx::ZERO)?;
74
75 let mut ty_and_hash = None;
76 let mut elem_iter = self.project_array_fields(&arr)?;
77 while let Some((idx, elem)) = elem_iter.next(self)? {
78 let elem = self.read_pointer(&elem)?;
79 let (elem_ty, elem_hash) = self.get_ptr_type_id(elem)?;
80 let full_hash = match ty_and_hash {
83 None => {
84 let hash = self.tcx.type_id_hash(elem_ty).as_u128();
85 let mut hash_bytes = [0u8; 16];
86 write_target_uint(self.data_layout().endian, &mut hash_bytes, hash).unwrap();
87 ty_and_hash = Some((elem_ty, hash_bytes));
88 hash_bytes
89 }
90 Some((ty, hash_bytes)) => {
91 if ty != elem_ty {
92 throw_ub_format!(
93 "invalid `TypeId` value: not all bytes carry the same type id metadata"
94 );
95 }
96 hash_bytes
97 }
98 };
99 let hash_frag = &full_hash[(idx as usize) * ptr_size..][..ptr_size];
101 if read_target_uint(self.data_layout().endian, hash_frag).unwrap() != elem_hash.into() {
102 throw_ub_format!(
103 "invalid `TypeId` value: the hash does not match the type id metadata"
104 );
105 }
106 }
107
108 interp_ok(ty_and_hash.unwrap().0)
109 }
110
111 pub fn eval_intrinsic(
115 &mut self,
116 instance: ty::Instance<'tcx>,
117 args: &[OpTy<'tcx, M::Provenance>],
118 dest: &PlaceTy<'tcx, M::Provenance>,
119 ret: Option<mir::BasicBlock>,
120 ) -> InterpResult<'tcx, bool> {
121 let instance_args = instance.args;
122 let intrinsic_name = self.tcx.item_name(instance.def_id());
123 let tcx = self.tcx.tcx;
124
125 match intrinsic_name {
126 sym::type_name => {
127 let tp_ty = instance.args.type_at(0);
128 ensure_monomorphic_enough(tcx, tp_ty)?;
129 let alloc = alloc_type_name(tcx, tp_ty);
130 let val = ConstValue::Slice { data: alloc, meta: alloc.inner().size().bytes() };
131 let val = self.const_val_to_op(val, dest.layout.ty, Some(dest.layout))?;
132 self.copy_op(&val, dest)?;
133 }
134 sym::needs_drop => {
135 let tp_ty = instance.args.type_at(0);
136 ensure_monomorphic_enough(tcx, tp_ty)?;
137 let val = ConstValue::from_bool(tp_ty.needs_drop(tcx, self.typing_env));
138 let val = self.const_val_to_op(val, tcx.types.bool, Some(dest.layout))?;
139 self.copy_op(&val, dest)?;
140 }
141 sym::type_id => {
142 let tp_ty = instance.args.type_at(0);
143 ensure_monomorphic_enough(tcx, tp_ty)?;
144 self.write_type_id(tp_ty, dest)?;
145 }
146 sym::type_id_eq => {
147 let a_ty = self.read_type_id(&args[0])?;
148 let b_ty = self.read_type_id(&args[1])?;
149 self.write_scalar(Scalar::from_bool(a_ty == b_ty), dest)?;
150 }
151 sym::variant_count => {
152 let tp_ty = instance.args.type_at(0);
153 let ty = match tp_ty.kind() {
154 ty::Pat(base, _) => *base,
158 _ => tp_ty,
159 };
160 let val = match ty.kind() {
161 ty::Adt(adt, _) => {
163 ConstValue::from_target_usize(adt.variants().len() as u64, &tcx)
164 }
165 ty::Alias(..) | ty::Param(_) | ty::Placeholder(_) | ty::Infer(_) => {
166 throw_inval!(TooGeneric)
167 }
168 ty::Pat(..) => unreachable!(),
169 ty::Bound(_, _) => bug!("bound ty during ctfe"),
170 ty::Bool
171 | ty::Char
172 | ty::Int(_)
173 | ty::Uint(_)
174 | ty::Float(_)
175 | ty::Foreign(_)
176 | ty::Str
177 | ty::Array(_, _)
178 | ty::Slice(_)
179 | ty::RawPtr(_, _)
180 | ty::Ref(_, _, _)
181 | ty::FnDef(_, _)
182 | ty::FnPtr(..)
183 | ty::Dynamic(_, _, _)
184 | ty::Closure(_, _)
185 | ty::CoroutineClosure(_, _)
186 | ty::Coroutine(_, _)
187 | ty::CoroutineWitness(..)
188 | ty::UnsafeBinder(_)
189 | ty::Never
190 | ty::Tuple(_)
191 | ty::Error(_) => ConstValue::from_target_usize(0u64, &tcx),
192 };
193 let val = self.const_val_to_op(val, dest.layout.ty, Some(dest.layout))?;
194 self.copy_op(&val, dest)?;
195 }
196
197 sym::caller_location => {
198 let span = self.find_closest_untracked_caller_location();
199 let val = self.tcx.span_as_caller_location(span);
200 let val =
201 self.const_val_to_op(val, self.tcx.caller_location_ty(), Some(dest.layout))?;
202 self.copy_op(&val, dest)?;
203 }
204
205 sym::align_of_val | sym::size_of_val => {
206 let place = self.ref_to_mplace(&self.read_immediate(&args[0])?)?;
209 let (size, align) = self
210 .size_and_align_of_val(&place)?
211 .ok_or_else(|| err_unsup_format!("`extern type` does not have known layout"))?;
212
213 let result = match intrinsic_name {
214 sym::align_of_val => align.bytes(),
215 sym::size_of_val => size.bytes(),
216 _ => bug!(),
217 };
218
219 self.write_scalar(Scalar::from_target_usize(result, self), dest)?;
220 }
221
222 sym::fadd_algebraic
223 | sym::fsub_algebraic
224 | sym::fmul_algebraic
225 | sym::fdiv_algebraic
226 | sym::frem_algebraic => {
227 let a = self.read_immediate(&args[0])?;
228 let b = self.read_immediate(&args[1])?;
229
230 let op = match intrinsic_name {
231 sym::fadd_algebraic => BinOp::Add,
232 sym::fsub_algebraic => BinOp::Sub,
233 sym::fmul_algebraic => BinOp::Mul,
234 sym::fdiv_algebraic => BinOp::Div,
235 sym::frem_algebraic => BinOp::Rem,
236
237 _ => bug!(),
238 };
239
240 let res = self.binary_op(op, &a, &b)?;
241 let res = M::apply_float_nondet(self, res)?;
243 self.write_immediate(*res, dest)?;
244 }
245
246 sym::ctpop
247 | sym::cttz
248 | sym::cttz_nonzero
249 | sym::ctlz
250 | sym::ctlz_nonzero
251 | sym::bswap
252 | sym::bitreverse => {
253 let ty = instance_args.type_at(0);
254 let layout = self.layout_of(ty)?;
255 let val = self.read_scalar(&args[0])?;
256
257 let out_val = self.numeric_intrinsic(intrinsic_name, val, layout, dest.layout)?;
258 self.write_scalar(out_val, dest)?;
259 }
260 sym::saturating_add | sym::saturating_sub => {
261 let l = self.read_immediate(&args[0])?;
262 let r = self.read_immediate(&args[1])?;
263 let val = self.saturating_arith(
264 if intrinsic_name == sym::saturating_add { BinOp::Add } else { BinOp::Sub },
265 &l,
266 &r,
267 )?;
268 self.write_scalar(val, dest)?;
269 }
270 sym::discriminant_value => {
271 let place = self.deref_pointer(&args[0])?;
272 let variant = self.read_discriminant(&place)?;
273 let discr = self.discriminant_for_variant(place.layout.ty, variant)?;
274 self.write_immediate(*discr, dest)?;
275 }
276 sym::exact_div => {
277 let l = self.read_immediate(&args[0])?;
278 let r = self.read_immediate(&args[1])?;
279 self.exact_div(&l, &r, dest)?;
280 }
281 sym::rotate_left | sym::rotate_right => {
282 let layout_val = self.layout_of(instance_args.type_at(0))?;
285 let val = self.read_scalar(&args[0])?;
286 let val_bits = val.to_bits(layout_val.size)?; let layout_raw_shift = self.layout_of(self.tcx.types.u32)?;
289 let raw_shift = self.read_scalar(&args[1])?;
290 let raw_shift_bits = raw_shift.to_bits(layout_raw_shift.size)?;
291
292 let width_bits = u128::from(layout_val.size.bits());
293 let shift_bits = raw_shift_bits % width_bits;
294 let inv_shift_bits = (width_bits - shift_bits) % width_bits;
295 let result_bits = if intrinsic_name == sym::rotate_left {
296 (val_bits << shift_bits) | (val_bits >> inv_shift_bits)
297 } else {
298 (val_bits >> shift_bits) | (val_bits << inv_shift_bits)
299 };
300 let truncated_bits = layout_val.size.truncate(result_bits);
301 let result = Scalar::from_uint(truncated_bits, layout_val.size);
302 self.write_scalar(result, dest)?;
303 }
304 sym::copy => {
305 self.copy_intrinsic(&args[0], &args[1], &args[2], false)?;
306 }
307 sym::write_bytes => {
308 self.write_bytes_intrinsic(&args[0], &args[1], &args[2], "write_bytes")?;
309 }
310 sym::compare_bytes => {
311 let result = self.compare_bytes_intrinsic(&args[0], &args[1], &args[2])?;
312 self.write_scalar(result, dest)?;
313 }
314 sym::arith_offset => {
315 let ptr = self.read_pointer(&args[0])?;
316 let offset_count = self.read_target_isize(&args[1])?;
317 let pointee_ty = instance_args.type_at(0);
318
319 let pointee_size = i64::try_from(self.layout_of(pointee_ty)?.size.bytes()).unwrap();
320 let offset_bytes = offset_count.wrapping_mul(pointee_size);
321 let offset_ptr = ptr.wrapping_signed_offset(offset_bytes, self);
322 self.write_pointer(offset_ptr, dest)?;
323 }
324 sym::ptr_offset_from | sym::ptr_offset_from_unsigned => {
325 let a = self.read_pointer(&args[0])?;
326 let b = self.read_pointer(&args[1])?;
327
328 let usize_layout = self.layout_of(self.tcx.types.usize)?;
329 let isize_layout = self.layout_of(self.tcx.types.isize)?;
330
331 let (a_offset, b_offset, is_addr) = if M::Provenance::OFFSET_IS_ADDR {
335 (a.addr().bytes(), b.addr().bytes(), true)
336 } else {
337 match (self.ptr_try_get_alloc_id(a, 0), self.ptr_try_get_alloc_id(b, 0)) {
338 (Err(a), Err(b)) => {
339 (a, b, true)
341 }
342 (Ok((a_alloc_id, a_offset, _)), Ok((b_alloc_id, b_offset, _)))
343 if a_alloc_id == b_alloc_id =>
344 {
345 (a_offset.bytes(), b_offset.bytes(), false)
348 }
349 _ => {
350 throw_ub_custom!(
352 fluent::const_eval_offset_from_different_allocations,
353 name = intrinsic_name,
354 );
355 }
356 }
357 };
358
359 let dist = {
361 let (val, overflowed) = {
364 let a_offset = ImmTy::from_uint(a_offset, usize_layout);
365 let b_offset = ImmTy::from_uint(b_offset, usize_layout);
366 self.binary_op(BinOp::SubWithOverflow, &a_offset, &b_offset)?
367 .to_scalar_pair()
368 };
369 if overflowed.to_bool()? {
370 if intrinsic_name == sym::ptr_offset_from_unsigned {
372 throw_ub_custom!(
373 fluent::const_eval_offset_from_unsigned_overflow,
374 a_offset = a_offset,
375 b_offset = b_offset,
376 is_addr = is_addr,
377 );
378 }
379 let dist = val.to_target_isize(self)?;
383 if dist >= 0 || i128::from(dist) == self.pointer_size().signed_int_min() {
384 throw_ub_custom!(
385 fluent::const_eval_offset_from_underflow,
386 name = intrinsic_name,
387 );
388 }
389 dist
390 } else {
391 let dist = val.to_target_isize(self)?;
393 if dist < 0 {
396 throw_ub_custom!(
397 fluent::const_eval_offset_from_overflow,
398 name = intrinsic_name,
399 );
400 }
401 dist
402 }
403 };
404
405 self.check_ptr_access_signed(b, dist, CheckInAllocMsg::Dereferenceable)
408 .map_err_kind(|_| {
409 if let Ok((a_alloc_id, ..)) = self.ptr_try_get_alloc_id(a, 0)
412 && let Ok((b_alloc_id, ..)) = self.ptr_try_get_alloc_id(b, 0)
413 && a_alloc_id == b_alloc_id
414 {
415 err_ub_custom!(
416 fluent::const_eval_offset_from_out_of_bounds,
417 name = intrinsic_name,
418 )
419 } else {
420 err_ub_custom!(
421 fluent::const_eval_offset_from_different_allocations,
422 name = intrinsic_name,
423 )
424 }
425 })?;
426 self.check_ptr_access_signed(
429 a,
430 dist.checked_neg().unwrap(), CheckInAllocMsg::Dereferenceable,
432 )
433 .map_err_kind(|_| {
434 err_ub_custom!(
436 fluent::const_eval_offset_from_different_allocations,
437 name = intrinsic_name,
438 )
439 })?;
440
441 let ret_layout = if intrinsic_name == sym::ptr_offset_from_unsigned {
443 assert!(0 <= dist && dist <= self.target_isize_max());
444 usize_layout
445 } else {
446 assert!(self.target_isize_min() <= dist && dist <= self.target_isize_max());
447 isize_layout
448 };
449 let pointee_layout = self.layout_of(instance_args.type_at(0))?;
450 let val = ImmTy::from_int(dist, ret_layout);
452 let size = ImmTy::from_int(pointee_layout.size.bytes(), ret_layout);
453 self.exact_div(&val, &size, dest)?;
454 }
455
456 sym::simd_insert => {
457 let index = u64::from(self.read_scalar(&args[1])?.to_u32()?);
458 let elem = &args[2];
459 let (input, input_len) = self.project_to_simd(&args[0])?;
460 let (dest, dest_len) = self.project_to_simd(dest)?;
461 assert_eq!(input_len, dest_len, "Return vector length must match input length");
462 if index >= input_len {
464 throw_ub_format!(
465 "`simd_insert` index {index} is out-of-bounds of vector with length {input_len}"
466 );
467 }
468
469 for i in 0..dest_len {
470 let place = self.project_index(&dest, i)?;
471 let value =
472 if i == index { elem.clone() } else { self.project_index(&input, i)? };
473 self.copy_op(&value, &place)?;
474 }
475 }
476 sym::simd_extract => {
477 let index = u64::from(self.read_scalar(&args[1])?.to_u32()?);
478 let (input, input_len) = self.project_to_simd(&args[0])?;
479 if index >= input_len {
481 throw_ub_format!(
482 "`simd_extract` index {index} is out-of-bounds of vector with length {input_len}"
483 );
484 }
485 self.copy_op(&self.project_index(&input, index)?, dest)?;
486 }
487 sym::black_box => {
488 self.copy_op(&args[0], dest)?;
490 }
491 sym::raw_eq => {
492 let result = self.raw_eq_intrinsic(&args[0], &args[1])?;
493 self.write_scalar(result, dest)?;
494 }
495 sym::typed_swap_nonoverlapping => {
496 self.typed_swap_nonoverlapping_intrinsic(&args[0], &args[1])?;
497 }
498
499 sym::vtable_size => {
500 let ptr = self.read_pointer(&args[0])?;
501 let (size, _align) = self.get_vtable_size_and_align(ptr, None)?;
503 self.write_scalar(Scalar::from_target_usize(size.bytes(), self), dest)?;
504 }
505 sym::vtable_align => {
506 let ptr = self.read_pointer(&args[0])?;
507 let (_size, align) = self.get_vtable_size_and_align(ptr, None)?;
509 self.write_scalar(Scalar::from_target_usize(align.bytes(), self), dest)?;
510 }
511
512 sym::minnumf16 => self.float_min_intrinsic::<Half>(args, dest)?,
513 sym::minnumf32 => self.float_min_intrinsic::<Single>(args, dest)?,
514 sym::minnumf64 => self.float_min_intrinsic::<Double>(args, dest)?,
515 sym::minnumf128 => self.float_min_intrinsic::<Quad>(args, dest)?,
516
517 sym::minimumf16 => self.float_minimum_intrinsic::<Half>(args, dest)?,
518 sym::minimumf32 => self.float_minimum_intrinsic::<Single>(args, dest)?,
519 sym::minimumf64 => self.float_minimum_intrinsic::<Double>(args, dest)?,
520 sym::minimumf128 => self.float_minimum_intrinsic::<Quad>(args, dest)?,
521
522 sym::maxnumf16 => self.float_max_intrinsic::<Half>(args, dest)?,
523 sym::maxnumf32 => self.float_max_intrinsic::<Single>(args, dest)?,
524 sym::maxnumf64 => self.float_max_intrinsic::<Double>(args, dest)?,
525 sym::maxnumf128 => self.float_max_intrinsic::<Quad>(args, dest)?,
526
527 sym::maximumf16 => self.float_maximum_intrinsic::<Half>(args, dest)?,
528 sym::maximumf32 => self.float_maximum_intrinsic::<Single>(args, dest)?,
529 sym::maximumf64 => self.float_maximum_intrinsic::<Double>(args, dest)?,
530 sym::maximumf128 => self.float_maximum_intrinsic::<Quad>(args, dest)?,
531
532 sym::copysignf16 => self.float_copysign_intrinsic::<Half>(args, dest)?,
533 sym::copysignf32 => self.float_copysign_intrinsic::<Single>(args, dest)?,
534 sym::copysignf64 => self.float_copysign_intrinsic::<Double>(args, dest)?,
535 sym::copysignf128 => self.float_copysign_intrinsic::<Quad>(args, dest)?,
536
537 sym::fabsf16 => self.float_abs_intrinsic::<Half>(args, dest)?,
538 sym::fabsf32 => self.float_abs_intrinsic::<Single>(args, dest)?,
539 sym::fabsf64 => self.float_abs_intrinsic::<Double>(args, dest)?,
540 sym::fabsf128 => self.float_abs_intrinsic::<Quad>(args, dest)?,
541
542 sym::floorf16 => self.float_round_intrinsic::<Half>(
543 args,
544 dest,
545 rustc_apfloat::Round::TowardNegative,
546 )?,
547 sym::floorf32 => self.float_round_intrinsic::<Single>(
548 args,
549 dest,
550 rustc_apfloat::Round::TowardNegative,
551 )?,
552 sym::floorf64 => self.float_round_intrinsic::<Double>(
553 args,
554 dest,
555 rustc_apfloat::Round::TowardNegative,
556 )?,
557 sym::floorf128 => self.float_round_intrinsic::<Quad>(
558 args,
559 dest,
560 rustc_apfloat::Round::TowardNegative,
561 )?,
562
563 sym::ceilf16 => self.float_round_intrinsic::<Half>(
564 args,
565 dest,
566 rustc_apfloat::Round::TowardPositive,
567 )?,
568 sym::ceilf32 => self.float_round_intrinsic::<Single>(
569 args,
570 dest,
571 rustc_apfloat::Round::TowardPositive,
572 )?,
573 sym::ceilf64 => self.float_round_intrinsic::<Double>(
574 args,
575 dest,
576 rustc_apfloat::Round::TowardPositive,
577 )?,
578 sym::ceilf128 => self.float_round_intrinsic::<Quad>(
579 args,
580 dest,
581 rustc_apfloat::Round::TowardPositive,
582 )?,
583
584 sym::truncf16 => {
585 self.float_round_intrinsic::<Half>(args, dest, rustc_apfloat::Round::TowardZero)?
586 }
587 sym::truncf32 => {
588 self.float_round_intrinsic::<Single>(args, dest, rustc_apfloat::Round::TowardZero)?
589 }
590 sym::truncf64 => {
591 self.float_round_intrinsic::<Double>(args, dest, rustc_apfloat::Round::TowardZero)?
592 }
593 sym::truncf128 => {
594 self.float_round_intrinsic::<Quad>(args, dest, rustc_apfloat::Round::TowardZero)?
595 }
596
597 sym::roundf16 => self.float_round_intrinsic::<Half>(
598 args,
599 dest,
600 rustc_apfloat::Round::NearestTiesToAway,
601 )?,
602 sym::roundf32 => self.float_round_intrinsic::<Single>(
603 args,
604 dest,
605 rustc_apfloat::Round::NearestTiesToAway,
606 )?,
607 sym::roundf64 => self.float_round_intrinsic::<Double>(
608 args,
609 dest,
610 rustc_apfloat::Round::NearestTiesToAway,
611 )?,
612 sym::roundf128 => self.float_round_intrinsic::<Quad>(
613 args,
614 dest,
615 rustc_apfloat::Round::NearestTiesToAway,
616 )?,
617
618 sym::round_ties_even_f16 => self.float_round_intrinsic::<Half>(
619 args,
620 dest,
621 rustc_apfloat::Round::NearestTiesToEven,
622 )?,
623 sym::round_ties_even_f32 => self.float_round_intrinsic::<Single>(
624 args,
625 dest,
626 rustc_apfloat::Round::NearestTiesToEven,
627 )?,
628 sym::round_ties_even_f64 => self.float_round_intrinsic::<Double>(
629 args,
630 dest,
631 rustc_apfloat::Round::NearestTiesToEven,
632 )?,
633 sym::round_ties_even_f128 => self.float_round_intrinsic::<Quad>(
634 args,
635 dest,
636 rustc_apfloat::Round::NearestTiesToEven,
637 )?,
638
639 _ => return interp_ok(false),
641 }
642
643 trace!("{:?}", self.dump_place(&dest.clone().into()));
644 self.return_to_block(ret)?;
645 interp_ok(true)
646 }
647
648 pub(super) fn eval_nondiverging_intrinsic(
649 &mut self,
650 intrinsic: &NonDivergingIntrinsic<'tcx>,
651 ) -> InterpResult<'tcx> {
652 match intrinsic {
653 NonDivergingIntrinsic::Assume(op) => {
654 let op = self.eval_operand(op, None)?;
655 let cond = self.read_scalar(&op)?.to_bool()?;
656 if !cond {
657 throw_ub_custom!(fluent::const_eval_assume_false);
658 }
659 interp_ok(())
660 }
661 NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping {
662 count,
663 src,
664 dst,
665 }) => {
666 let src = self.eval_operand(src, None)?;
667 let dst = self.eval_operand(dst, None)?;
668 let count = self.eval_operand(count, None)?;
669 self.copy_intrinsic(&src, &dst, &count, true)
670 }
671 }
672 }
673
674 pub fn numeric_intrinsic(
675 &self,
676 name: Symbol,
677 val: Scalar<M::Provenance>,
678 layout: TyAndLayout<'tcx>,
679 ret_layout: TyAndLayout<'tcx>,
680 ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
681 assert!(layout.ty.is_integral(), "invalid type for numeric intrinsic: {}", layout.ty);
682 let bits = val.to_bits(layout.size)?; let extra = 128 - u128::from(layout.size.bits());
684 let bits_out = match name {
685 sym::ctpop => u128::from(bits.count_ones()),
686 sym::ctlz_nonzero | sym::cttz_nonzero if bits == 0 => {
687 throw_ub_custom!(fluent::const_eval_call_nonzero_intrinsic, name = name,);
688 }
689 sym::ctlz | sym::ctlz_nonzero => u128::from(bits.leading_zeros()) - extra,
690 sym::cttz | sym::cttz_nonzero => u128::from((bits << extra).trailing_zeros()) - extra,
691 sym::bswap => {
692 assert_eq!(layout, ret_layout);
693 (bits << extra).swap_bytes()
694 }
695 sym::bitreverse => {
696 assert_eq!(layout, ret_layout);
697 (bits << extra).reverse_bits()
698 }
699 _ => bug!("not a numeric intrinsic: {}", name),
700 };
701 interp_ok(Scalar::from_uint(bits_out, ret_layout.size))
702 }
703
704 pub fn exact_div(
705 &mut self,
706 a: &ImmTy<'tcx, M::Provenance>,
707 b: &ImmTy<'tcx, M::Provenance>,
708 dest: &PlaceTy<'tcx, M::Provenance>,
709 ) -> InterpResult<'tcx> {
710 assert_eq!(a.layout.ty, b.layout.ty);
711 assert_matches!(a.layout.ty.kind(), ty::Int(..) | ty::Uint(..));
712
713 let rem = self.binary_op(BinOp::Rem, a, b)?;
717 if rem.to_scalar().to_bits(a.layout.size)? != 0 {
719 throw_ub_custom!(
720 fluent::const_eval_exact_div_has_remainder,
721 a = format!("{a}"),
722 b = format!("{b}")
723 )
724 }
725 let res = self.binary_op(BinOp::Div, a, b)?;
727 self.write_immediate(*res, dest)
728 }
729
730 pub fn saturating_arith(
731 &self,
732 mir_op: BinOp,
733 l: &ImmTy<'tcx, M::Provenance>,
734 r: &ImmTy<'tcx, M::Provenance>,
735 ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
736 assert_eq!(l.layout.ty, r.layout.ty);
737 assert_matches!(l.layout.ty.kind(), ty::Int(..) | ty::Uint(..));
738 assert_matches!(mir_op, BinOp::Add | BinOp::Sub);
739
740 let (val, overflowed) =
741 self.binary_op(mir_op.wrapping_to_overflowing().unwrap(), l, r)?.to_scalar_pair();
742 interp_ok(if overflowed.to_bool()? {
743 let size = l.layout.size;
744 if l.layout.backend_repr.is_signed() {
745 let first_term: i128 = l.to_scalar().to_int(l.layout.size)?;
750 if first_term >= 0 {
751 Scalar::from_int(size.signed_int_max(), size)
755 } else {
756 Scalar::from_int(size.signed_int_min(), size)
758 }
759 } else {
760 if matches!(mir_op, BinOp::Add) {
762 Scalar::from_uint(size.unsigned_int_max(), size)
764 } else {
765 Scalar::from_uint(0u128, size)
767 }
768 }
769 } else {
770 val
771 })
772 }
773
774 pub fn ptr_offset_inbounds(
777 &self,
778 ptr: Pointer<Option<M::Provenance>>,
779 offset_bytes: i64,
780 ) -> InterpResult<'tcx, Pointer<Option<M::Provenance>>> {
781 self.check_ptr_access_signed(
783 ptr,
784 offset_bytes,
785 CheckInAllocMsg::InboundsPointerArithmetic,
786 )?;
787 interp_ok(ptr.wrapping_signed_offset(offset_bytes, self))
789 }
790
791 pub(crate) fn copy_intrinsic(
793 &mut self,
794 src: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
795 dst: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
796 count: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
797 nonoverlapping: bool,
798 ) -> InterpResult<'tcx> {
799 let count = self.read_target_usize(count)?;
800 let layout = self.layout_of(src.layout.ty.builtin_deref(true).unwrap())?;
801 let (size, align) = (layout.size, layout.align.abi);
802
803 let size = self.compute_size_in_bytes(size, count).ok_or_else(|| {
804 err_ub_custom!(
805 fluent::const_eval_size_overflow,
806 name = if nonoverlapping { "copy_nonoverlapping" } else { "copy" }
807 )
808 })?;
809
810 let src = self.read_pointer(src)?;
811 let dst = self.read_pointer(dst)?;
812
813 self.check_ptr_align(src, align)?;
814 self.check_ptr_align(dst, align)?;
815
816 self.mem_copy(src, dst, size, nonoverlapping)
817 }
818
819 fn typed_swap_nonoverlapping_intrinsic(
821 &mut self,
822 left: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
823 right: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
824 ) -> InterpResult<'tcx> {
825 let left = self.deref_pointer(left)?;
826 let right = self.deref_pointer(right)?;
827 assert_eq!(left.layout, right.layout);
828 assert!(left.layout.is_sized());
829 let kind = MemoryKind::Stack;
830 let temp = self.allocate(left.layout, kind)?;
831 self.copy_op(&left, &temp)?; self.mem_copy(right.ptr(), left.ptr(), left.layout.size, true)?;
836 if M::enforce_validity(self, left.layout) {
840 self.validate_operand(
841 &left.clone().into(),
842 M::enforce_validity_recursively(self, left.layout),
843 true,
844 )?;
845 }
846
847 self.copy_op(&temp, &right)?; self.deallocate_ptr(temp.ptr(), None, kind)?;
850 interp_ok(())
851 }
852
853 pub fn write_bytes_intrinsic(
854 &mut self,
855 dst: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
856 byte: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
857 count: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
858 name: &'static str,
859 ) -> InterpResult<'tcx> {
860 let layout = self.layout_of(dst.layout.ty.builtin_deref(true).unwrap())?;
861
862 let dst = self.read_pointer(dst)?;
863 let byte = self.read_scalar(byte)?.to_u8()?;
864 let count = self.read_target_usize(count)?;
865
866 let len = self
869 .compute_size_in_bytes(layout.size, count)
870 .ok_or_else(|| err_ub_custom!(fluent::const_eval_size_overflow, name = name))?;
871
872 let bytes = std::iter::repeat(byte).take(len.bytes_usize());
873 self.write_bytes_ptr(dst, bytes)
874 }
875
876 pub(crate) fn compare_bytes_intrinsic(
877 &mut self,
878 left: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
879 right: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
880 byte_count: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
881 ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
882 let left = self.read_pointer(left)?;
883 let right = self.read_pointer(right)?;
884 let n = Size::from_bytes(self.read_target_usize(byte_count)?);
885
886 let left_bytes = self.read_bytes_ptr_strip_provenance(left, n)?;
887 let right_bytes = self.read_bytes_ptr_strip_provenance(right, n)?;
888
889 let result = Ord::cmp(left_bytes, right_bytes) as i32;
891 interp_ok(Scalar::from_i32(result))
892 }
893
894 pub(crate) fn raw_eq_intrinsic(
895 &mut self,
896 lhs: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
897 rhs: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
898 ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
899 let layout = self.layout_of(lhs.layout.ty.builtin_deref(true).unwrap())?;
900 assert!(layout.is_sized());
901
902 let get_bytes = |this: &InterpCx<'tcx, M>,
903 op: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>|
904 -> InterpResult<'tcx, &[u8]> {
905 let ptr = this.read_pointer(op)?;
906 this.check_ptr_align(ptr, layout.align.abi)?;
907 let Some(alloc_ref) = self.get_ptr_alloc(ptr, layout.size)? else {
908 return interp_ok(&[]);
910 };
911 alloc_ref.get_bytes_strip_provenance()
912 };
913
914 let lhs_bytes = get_bytes(self, lhs)?;
915 let rhs_bytes = get_bytes(self, rhs)?;
916 interp_ok(Scalar::from_bool(lhs_bytes == rhs_bytes))
917 }
918
919 fn float_min_intrinsic<F>(
920 &mut self,
921 args: &[OpTy<'tcx, M::Provenance>],
922 dest: &PlaceTy<'tcx, M::Provenance>,
923 ) -> InterpResult<'tcx, ()>
924 where
925 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
926 {
927 let a: F = self.read_scalar(&args[0])?.to_float()?;
928 let b: F = self.read_scalar(&args[1])?.to_float()?;
929 let res = if a == b {
930 M::equal_float_min_max(self, a, b)
933 } else {
934 self.adjust_nan(a.min(b), &[a, b])
935 };
936 self.write_scalar(res, dest)?;
937 interp_ok(())
938 }
939
940 fn float_max_intrinsic<F>(
941 &mut self,
942 args: &[OpTy<'tcx, M::Provenance>],
943 dest: &PlaceTy<'tcx, M::Provenance>,
944 ) -> InterpResult<'tcx, ()>
945 where
946 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
947 {
948 let a: F = self.read_scalar(&args[0])?.to_float()?;
949 let b: F = self.read_scalar(&args[1])?.to_float()?;
950 let res = if a == b {
951 M::equal_float_min_max(self, a, b)
954 } else {
955 self.adjust_nan(a.max(b), &[a, b])
956 };
957 self.write_scalar(res, dest)?;
958 interp_ok(())
959 }
960
961 fn float_minimum_intrinsic<F>(
962 &mut self,
963 args: &[OpTy<'tcx, M::Provenance>],
964 dest: &PlaceTy<'tcx, M::Provenance>,
965 ) -> InterpResult<'tcx, ()>
966 where
967 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
968 {
969 let a: F = self.read_scalar(&args[0])?.to_float()?;
970 let b: F = self.read_scalar(&args[1])?.to_float()?;
971 let res = a.minimum(b);
972 let res = self.adjust_nan(res, &[a, b]);
973 self.write_scalar(res, dest)?;
974 interp_ok(())
975 }
976
977 fn float_maximum_intrinsic<F>(
978 &mut self,
979 args: &[OpTy<'tcx, M::Provenance>],
980 dest: &PlaceTy<'tcx, M::Provenance>,
981 ) -> InterpResult<'tcx, ()>
982 where
983 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
984 {
985 let a: F = self.read_scalar(&args[0])?.to_float()?;
986 let b: F = self.read_scalar(&args[1])?.to_float()?;
987 let res = a.maximum(b);
988 let res = self.adjust_nan(res, &[a, b]);
989 self.write_scalar(res, dest)?;
990 interp_ok(())
991 }
992
993 fn float_copysign_intrinsic<F>(
994 &mut self,
995 args: &[OpTy<'tcx, M::Provenance>],
996 dest: &PlaceTy<'tcx, M::Provenance>,
997 ) -> InterpResult<'tcx, ()>
998 where
999 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1000 {
1001 let a: F = self.read_scalar(&args[0])?.to_float()?;
1002 let b: F = self.read_scalar(&args[1])?.to_float()?;
1003 self.write_scalar(a.copy_sign(b), dest)?;
1005 interp_ok(())
1006 }
1007
1008 fn float_abs_intrinsic<F>(
1009 &mut self,
1010 args: &[OpTy<'tcx, M::Provenance>],
1011 dest: &PlaceTy<'tcx, M::Provenance>,
1012 ) -> InterpResult<'tcx, ()>
1013 where
1014 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1015 {
1016 let x: F = self.read_scalar(&args[0])?.to_float()?;
1017 self.write_scalar(x.abs(), dest)?;
1019 interp_ok(())
1020 }
1021
1022 fn float_round_intrinsic<F>(
1023 &mut self,
1024 args: &[OpTy<'tcx, M::Provenance>],
1025 dest: &PlaceTy<'tcx, M::Provenance>,
1026 mode: rustc_apfloat::Round,
1027 ) -> InterpResult<'tcx, ()>
1028 where
1029 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1030 {
1031 let x: F = self.read_scalar(&args[0])?.to_float()?;
1032 let res = x.round_to_integral(mode).value;
1033 let res = self.adjust_nan(res, &[x]);
1034 self.write_scalar(res, dest)?;
1035 interp_ok(())
1036 }
1037}