1use std::assert_matches::assert_matches;
6
7use rustc_abi::{FieldIdx, HasDataLayout, Size};
8use rustc_apfloat::ieee::{Double, Half, Quad, Single};
9use rustc_middle::mir::interpret::{CTFE_ALLOC_SALT, read_target_uint, write_target_uint};
10use rustc_middle::mir::{self, BinOp, ConstValue, NonDivergingIntrinsic};
11use rustc_middle::ty::layout::TyAndLayout;
12use rustc_middle::ty::{Ty, TyCtxt};
13use rustc_middle::{bug, ty};
14use rustc_span::{Symbol, sym};
15use tracing::trace;
16
17use super::memory::MemoryKind;
18use super::util::ensure_monomorphic_enough;
19use super::{
20 AllocId, CheckInAllocMsg, ImmTy, InterpCx, InterpResult, Machine, OpTy, PlaceTy, Pointer,
21 PointerArithmetic, Provenance, Scalar, err_ub_custom, err_unsup_format, interp_ok, throw_inval,
22 throw_ub_custom, throw_ub_format,
23};
24use crate::fluent_generated as fluent;
25
26pub(crate) fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> (AllocId, u64) {
28 let path = crate::util::type_name(tcx, ty);
29 let bytes = path.into_bytes();
30 let len = bytes.len().try_into().unwrap();
31 (tcx.allocate_bytes_dedup(bytes, CTFE_ALLOC_SALT), len)
32}
33impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
34 fn write_type_id(
36 &mut self,
37 ty: Ty<'tcx>,
38 dest: &PlaceTy<'tcx, M::Provenance>,
39 ) -> InterpResult<'tcx, ()> {
40 let tcx = self.tcx;
41 let type_id_hash = tcx.type_id_hash(ty).as_u128();
42 let op = self.const_val_to_op(
43 ConstValue::Scalar(Scalar::from_u128(type_id_hash)),
44 tcx.types.u128,
45 None,
46 )?;
47 self.copy_op_allow_transmute(&op, dest)?;
48
49 let alloc_id = tcx.reserve_and_set_type_id_alloc(ty);
53 let arr = self.project_field(dest, FieldIdx::ZERO)?;
54 let mut elem_iter = self.project_array_fields(&arr)?;
55 while let Some((_, elem)) = elem_iter.next(self)? {
56 let hash_fragment = self.read_scalar(&elem)?.to_target_usize(&tcx)?;
58 let ptr = Pointer::new(alloc_id.into(), Size::from_bytes(hash_fragment));
59 let ptr = self.global_root_pointer(ptr)?;
60 let val = Scalar::from_pointer(ptr, &tcx);
61 self.write_scalar(val, &elem)?;
62 }
63 interp_ok(())
64 }
65
66 pub(crate) fn read_type_id(
68 &self,
69 op: &OpTy<'tcx, M::Provenance>,
70 ) -> InterpResult<'tcx, Ty<'tcx>> {
71 let ptr_size = self.pointer_size().bytes_usize();
74 let arr = self.project_field(op, FieldIdx::ZERO)?;
75
76 let mut ty_and_hash = None;
77 let mut elem_iter = self.project_array_fields(&arr)?;
78 while let Some((idx, elem)) = elem_iter.next(self)? {
79 let elem = self.read_pointer(&elem)?;
80 let (elem_ty, elem_hash) = self.get_ptr_type_id(elem)?;
81 let full_hash = match ty_and_hash {
84 None => {
85 let hash = self.tcx.type_id_hash(elem_ty).as_u128();
86 let mut hash_bytes = [0u8; 16];
87 write_target_uint(self.data_layout().endian, &mut hash_bytes, hash).unwrap();
88 ty_and_hash = Some((elem_ty, hash_bytes));
89 hash_bytes
90 }
91 Some((ty, hash_bytes)) => {
92 if ty != elem_ty {
93 throw_ub_format!(
94 "invalid `TypeId` value: not all bytes carry the same type id metadata"
95 );
96 }
97 hash_bytes
98 }
99 };
100 let hash_frag = &full_hash[(idx as usize) * ptr_size..][..ptr_size];
102 if read_target_uint(self.data_layout().endian, hash_frag).unwrap() != elem_hash.into() {
103 throw_ub_format!(
104 "invalid `TypeId` value: the hash does not match the type id metadata"
105 );
106 }
107 }
108
109 interp_ok(ty_and_hash.unwrap().0)
110 }
111
112 pub fn eval_intrinsic(
116 &mut self,
117 instance: ty::Instance<'tcx>,
118 args: &[OpTy<'tcx, M::Provenance>],
119 dest: &PlaceTy<'tcx, M::Provenance>,
120 ret: Option<mir::BasicBlock>,
121 ) -> InterpResult<'tcx, bool> {
122 let instance_args = instance.args;
123 let intrinsic_name = self.tcx.item_name(instance.def_id());
124 let tcx = self.tcx.tcx;
125
126 match intrinsic_name {
127 sym::type_name => {
128 let tp_ty = instance.args.type_at(0);
129 ensure_monomorphic_enough(tcx, tp_ty)?;
130 let (alloc_id, meta) = alloc_type_name(tcx, tp_ty);
131 let val = ConstValue::Slice { alloc_id, meta };
132 let val = self.const_val_to_op(val, dest.layout.ty, Some(dest.layout))?;
133 self.copy_op(&val, dest)?;
134 }
135 sym::needs_drop => {
136 let tp_ty = instance.args.type_at(0);
137 ensure_monomorphic_enough(tcx, tp_ty)?;
138 let val = ConstValue::from_bool(tp_ty.needs_drop(tcx, self.typing_env));
139 let val = self.const_val_to_op(val, tcx.types.bool, Some(dest.layout))?;
140 self.copy_op(&val, dest)?;
141 }
142 sym::type_id => {
143 let tp_ty = instance.args.type_at(0);
144 ensure_monomorphic_enough(tcx, tp_ty)?;
145 self.write_type_id(tp_ty, dest)?;
146 }
147 sym::type_id_eq => {
148 let a_ty = self.read_type_id(&args[0])?;
149 let b_ty = self.read_type_id(&args[1])?;
150 self.write_scalar(Scalar::from_bool(a_ty == b_ty), dest)?;
151 }
152 sym::variant_count => {
153 let tp_ty = instance.args.type_at(0);
154 let ty = match tp_ty.kind() {
155 ty::Pat(base, _) => *base,
159 _ => tp_ty,
160 };
161 let val = match ty.kind() {
162 ty::Adt(adt, _) => {
164 ConstValue::from_target_usize(adt.variants().len() as u64, &tcx)
165 }
166 ty::Alias(..) | ty::Param(_) | ty::Placeholder(_) | ty::Infer(_) => {
167 throw_inval!(TooGeneric)
168 }
169 ty::Pat(..) => unreachable!(),
170 ty::Bound(_, _) => bug!("bound ty during ctfe"),
171 ty::Bool
172 | ty::Char
173 | ty::Int(_)
174 | ty::Uint(_)
175 | ty::Float(_)
176 | ty::Foreign(_)
177 | ty::Str
178 | ty::Array(_, _)
179 | ty::Slice(_)
180 | ty::RawPtr(_, _)
181 | ty::Ref(_, _, _)
182 | ty::FnDef(_, _)
183 | ty::FnPtr(..)
184 | ty::Dynamic(_, _)
185 | ty::Closure(_, _)
186 | ty::CoroutineClosure(_, _)
187 | ty::Coroutine(_, _)
188 | ty::CoroutineWitness(..)
189 | ty::UnsafeBinder(_)
190 | ty::Never
191 | ty::Tuple(_)
192 | ty::Error(_) => ConstValue::from_target_usize(0u64, &tcx),
193 };
194 let val = self.const_val_to_op(val, dest.layout.ty, Some(dest.layout))?;
195 self.copy_op(&val, dest)?;
196 }
197
198 sym::caller_location => {
199 let span = self.find_closest_untracked_caller_location();
200 let val = self.tcx.span_as_caller_location(span);
201 let val =
202 self.const_val_to_op(val, self.tcx.caller_location_ty(), Some(dest.layout))?;
203 self.copy_op(&val, dest)?;
204 }
205
206 sym::align_of_val | sym::size_of_val => {
207 let place = self.ref_to_mplace(&self.read_immediate(&args[0])?)?;
210 let (size, align) = self
211 .size_and_align_of_val(&place)?
212 .ok_or_else(|| err_unsup_format!("`extern type` does not have known layout"))?;
213
214 let result = match intrinsic_name {
215 sym::align_of_val => align.bytes(),
216 sym::size_of_val => size.bytes(),
217 _ => bug!(),
218 };
219
220 self.write_scalar(Scalar::from_target_usize(result, self), dest)?;
221 }
222
223 sym::fadd_algebraic
224 | sym::fsub_algebraic
225 | sym::fmul_algebraic
226 | sym::fdiv_algebraic
227 | sym::frem_algebraic => {
228 let a = self.read_immediate(&args[0])?;
229 let b = self.read_immediate(&args[1])?;
230
231 let op = match intrinsic_name {
232 sym::fadd_algebraic => BinOp::Add,
233 sym::fsub_algebraic => BinOp::Sub,
234 sym::fmul_algebraic => BinOp::Mul,
235 sym::fdiv_algebraic => BinOp::Div,
236 sym::frem_algebraic => BinOp::Rem,
237
238 _ => bug!(),
239 };
240
241 let res = self.binary_op(op, &a, &b)?;
242 let res = M::apply_float_nondet(self, res)?;
244 self.write_immediate(*res, dest)?;
245 }
246
247 sym::ctpop
248 | sym::cttz
249 | sym::cttz_nonzero
250 | sym::ctlz
251 | sym::ctlz_nonzero
252 | sym::bswap
253 | sym::bitreverse => {
254 let ty = instance_args.type_at(0);
255 let layout = self.layout_of(ty)?;
256 let val = self.read_scalar(&args[0])?;
257
258 let out_val = self.numeric_intrinsic(intrinsic_name, val, layout, dest.layout)?;
259 self.write_scalar(out_val, dest)?;
260 }
261 sym::saturating_add | sym::saturating_sub => {
262 let l = self.read_immediate(&args[0])?;
263 let r = self.read_immediate(&args[1])?;
264 let val = self.saturating_arith(
265 if intrinsic_name == sym::saturating_add { BinOp::Add } else { BinOp::Sub },
266 &l,
267 &r,
268 )?;
269 self.write_scalar(val, dest)?;
270 }
271 sym::discriminant_value => {
272 let place = self.deref_pointer(&args[0])?;
273 let variant = self.read_discriminant(&place)?;
274 let discr = self.discriminant_for_variant(place.layout.ty, variant)?;
275 self.write_immediate(*discr, dest)?;
276 }
277 sym::exact_div => {
278 let l = self.read_immediate(&args[0])?;
279 let r = self.read_immediate(&args[1])?;
280 self.exact_div(&l, &r, dest)?;
281 }
282 sym::rotate_left | sym::rotate_right => {
283 let layout_val = self.layout_of(instance_args.type_at(0))?;
286 let val = self.read_scalar(&args[0])?;
287 let val_bits = val.to_bits(layout_val.size)?; let layout_raw_shift = self.layout_of(self.tcx.types.u32)?;
290 let raw_shift = self.read_scalar(&args[1])?;
291 let raw_shift_bits = raw_shift.to_bits(layout_raw_shift.size)?;
292
293 let width_bits = u128::from(layout_val.size.bits());
294 let shift_bits = raw_shift_bits % width_bits;
295 let inv_shift_bits = (width_bits - shift_bits) % width_bits;
296 let result_bits = if intrinsic_name == sym::rotate_left {
297 (val_bits << shift_bits) | (val_bits >> inv_shift_bits)
298 } else {
299 (val_bits >> shift_bits) | (val_bits << inv_shift_bits)
300 };
301 let truncated_bits = layout_val.size.truncate(result_bits);
302 let result = Scalar::from_uint(truncated_bits, layout_val.size);
303 self.write_scalar(result, dest)?;
304 }
305 sym::copy => {
306 self.copy_intrinsic(&args[0], &args[1], &args[2], false)?;
307 }
308 sym::write_bytes => {
309 self.write_bytes_intrinsic(&args[0], &args[1], &args[2], "write_bytes")?;
310 }
311 sym::compare_bytes => {
312 let result = self.compare_bytes_intrinsic(&args[0], &args[1], &args[2])?;
313 self.write_scalar(result, dest)?;
314 }
315 sym::arith_offset => {
316 let ptr = self.read_pointer(&args[0])?;
317 let offset_count = self.read_target_isize(&args[1])?;
318 let pointee_ty = instance_args.type_at(0);
319
320 let pointee_size = i64::try_from(self.layout_of(pointee_ty)?.size.bytes()).unwrap();
321 let offset_bytes = offset_count.wrapping_mul(pointee_size);
322 let offset_ptr = ptr.wrapping_signed_offset(offset_bytes, self);
323 self.write_pointer(offset_ptr, dest)?;
324 }
325 sym::ptr_offset_from | sym::ptr_offset_from_unsigned => {
326 let a = self.read_pointer(&args[0])?;
327 let b = self.read_pointer(&args[1])?;
328
329 let usize_layout = self.layout_of(self.tcx.types.usize)?;
330 let isize_layout = self.layout_of(self.tcx.types.isize)?;
331
332 let (a_offset, b_offset, is_addr) = if M::Provenance::OFFSET_IS_ADDR {
336 (a.addr().bytes(), b.addr().bytes(), true)
337 } else {
338 match (self.ptr_try_get_alloc_id(a, 0), self.ptr_try_get_alloc_id(b, 0)) {
339 (Err(a), Err(b)) => {
340 (a, b, true)
342 }
343 (Ok((a_alloc_id, a_offset, _)), Ok((b_alloc_id, b_offset, _)))
344 if a_alloc_id == b_alloc_id =>
345 {
346 (a_offset.bytes(), b_offset.bytes(), false)
349 }
350 _ => {
351 throw_ub_custom!(
353 fluent::const_eval_offset_from_different_allocations,
354 name = intrinsic_name,
355 );
356 }
357 }
358 };
359
360 let dist = {
362 let (val, overflowed) = {
365 let a_offset = ImmTy::from_uint(a_offset, usize_layout);
366 let b_offset = ImmTy::from_uint(b_offset, usize_layout);
367 self.binary_op(BinOp::SubWithOverflow, &a_offset, &b_offset)?
368 .to_scalar_pair()
369 };
370 if overflowed.to_bool()? {
371 if intrinsic_name == sym::ptr_offset_from_unsigned {
373 throw_ub_custom!(
374 fluent::const_eval_offset_from_unsigned_overflow,
375 a_offset = a_offset,
376 b_offset = b_offset,
377 is_addr = is_addr,
378 );
379 }
380 let dist = val.to_target_isize(self)?;
384 if dist >= 0 || i128::from(dist) == self.pointer_size().signed_int_min() {
385 throw_ub_custom!(
386 fluent::const_eval_offset_from_underflow,
387 name = intrinsic_name,
388 );
389 }
390 dist
391 } else {
392 let dist = val.to_target_isize(self)?;
394 if dist < 0 {
397 throw_ub_custom!(
398 fluent::const_eval_offset_from_overflow,
399 name = intrinsic_name,
400 );
401 }
402 dist
403 }
404 };
405
406 self.check_ptr_access_signed(b, dist, CheckInAllocMsg::Dereferenceable)
409 .map_err_kind(|_| {
410 if let Ok((a_alloc_id, ..)) = self.ptr_try_get_alloc_id(a, 0)
413 && let Ok((b_alloc_id, ..)) = self.ptr_try_get_alloc_id(b, 0)
414 && a_alloc_id == b_alloc_id
415 {
416 err_ub_custom!(
417 fluent::const_eval_offset_from_out_of_bounds,
418 name = intrinsic_name,
419 )
420 } else {
421 err_ub_custom!(
422 fluent::const_eval_offset_from_different_allocations,
423 name = intrinsic_name,
424 )
425 }
426 })?;
427 self.check_ptr_access_signed(
430 a,
431 dist.checked_neg().unwrap(), CheckInAllocMsg::Dereferenceable,
433 )
434 .map_err_kind(|_| {
435 err_ub_custom!(
437 fluent::const_eval_offset_from_different_allocations,
438 name = intrinsic_name,
439 )
440 })?;
441
442 let ret_layout = if intrinsic_name == sym::ptr_offset_from_unsigned {
444 assert!(0 <= dist && dist <= self.target_isize_max());
445 usize_layout
446 } else {
447 assert!(self.target_isize_min() <= dist && dist <= self.target_isize_max());
448 isize_layout
449 };
450 let pointee_layout = self.layout_of(instance_args.type_at(0))?;
451 let val = ImmTy::from_int(dist, ret_layout);
453 let size = ImmTy::from_int(pointee_layout.size.bytes(), ret_layout);
454 self.exact_div(&val, &size, dest)?;
455 }
456
457 sym::simd_insert => {
458 let index = u64::from(self.read_scalar(&args[1])?.to_u32()?);
459 let elem = &args[2];
460 let (input, input_len) = self.project_to_simd(&args[0])?;
461 let (dest, dest_len) = self.project_to_simd(dest)?;
462 assert_eq!(input_len, dest_len, "Return vector length must match input length");
463 if index >= input_len {
465 throw_ub_format!(
466 "`simd_insert` index {index} is out-of-bounds of vector with length {input_len}"
467 );
468 }
469
470 for i in 0..dest_len {
471 let place = self.project_index(&dest, i)?;
472 let value =
473 if i == index { elem.clone() } else { self.project_index(&input, i)? };
474 self.copy_op(&value, &place)?;
475 }
476 }
477 sym::simd_extract => {
478 let index = u64::from(self.read_scalar(&args[1])?.to_u32()?);
479 let (input, input_len) = self.project_to_simd(&args[0])?;
480 if index >= input_len {
482 throw_ub_format!(
483 "`simd_extract` index {index} is out-of-bounds of vector with length {input_len}"
484 );
485 }
486 self.copy_op(&self.project_index(&input, index)?, dest)?;
487 }
488 sym::black_box => {
489 self.copy_op(&args[0], dest)?;
491 }
492 sym::raw_eq => {
493 let result = self.raw_eq_intrinsic(&args[0], &args[1])?;
494 self.write_scalar(result, dest)?;
495 }
496 sym::typed_swap_nonoverlapping => {
497 self.typed_swap_nonoverlapping_intrinsic(&args[0], &args[1])?;
498 }
499
500 sym::vtable_size => {
501 let ptr = self.read_pointer(&args[0])?;
502 let (size, _align) = self.get_vtable_size_and_align(ptr, None)?;
504 self.write_scalar(Scalar::from_target_usize(size.bytes(), self), dest)?;
505 }
506 sym::vtable_align => {
507 let ptr = self.read_pointer(&args[0])?;
508 let (_size, align) = self.get_vtable_size_and_align(ptr, None)?;
510 self.write_scalar(Scalar::from_target_usize(align.bytes(), self), dest)?;
511 }
512
513 sym::minnumf16 => self.float_min_intrinsic::<Half>(args, dest)?,
514 sym::minnumf32 => self.float_min_intrinsic::<Single>(args, dest)?,
515 sym::minnumf64 => self.float_min_intrinsic::<Double>(args, dest)?,
516 sym::minnumf128 => self.float_min_intrinsic::<Quad>(args, dest)?,
517
518 sym::minimumf16 => self.float_minimum_intrinsic::<Half>(args, dest)?,
519 sym::minimumf32 => self.float_minimum_intrinsic::<Single>(args, dest)?,
520 sym::minimumf64 => self.float_minimum_intrinsic::<Double>(args, dest)?,
521 sym::minimumf128 => self.float_minimum_intrinsic::<Quad>(args, dest)?,
522
523 sym::maxnumf16 => self.float_max_intrinsic::<Half>(args, dest)?,
524 sym::maxnumf32 => self.float_max_intrinsic::<Single>(args, dest)?,
525 sym::maxnumf64 => self.float_max_intrinsic::<Double>(args, dest)?,
526 sym::maxnumf128 => self.float_max_intrinsic::<Quad>(args, dest)?,
527
528 sym::maximumf16 => self.float_maximum_intrinsic::<Half>(args, dest)?,
529 sym::maximumf32 => self.float_maximum_intrinsic::<Single>(args, dest)?,
530 sym::maximumf64 => self.float_maximum_intrinsic::<Double>(args, dest)?,
531 sym::maximumf128 => self.float_maximum_intrinsic::<Quad>(args, dest)?,
532
533 sym::copysignf16 => self.float_copysign_intrinsic::<Half>(args, dest)?,
534 sym::copysignf32 => self.float_copysign_intrinsic::<Single>(args, dest)?,
535 sym::copysignf64 => self.float_copysign_intrinsic::<Double>(args, dest)?,
536 sym::copysignf128 => self.float_copysign_intrinsic::<Quad>(args, dest)?,
537
538 sym::fabsf16 => self.float_abs_intrinsic::<Half>(args, dest)?,
539 sym::fabsf32 => self.float_abs_intrinsic::<Single>(args, dest)?,
540 sym::fabsf64 => self.float_abs_intrinsic::<Double>(args, dest)?,
541 sym::fabsf128 => self.float_abs_intrinsic::<Quad>(args, dest)?,
542
543 sym::floorf16 => self.float_round_intrinsic::<Half>(
544 args,
545 dest,
546 rustc_apfloat::Round::TowardNegative,
547 )?,
548 sym::floorf32 => self.float_round_intrinsic::<Single>(
549 args,
550 dest,
551 rustc_apfloat::Round::TowardNegative,
552 )?,
553 sym::floorf64 => self.float_round_intrinsic::<Double>(
554 args,
555 dest,
556 rustc_apfloat::Round::TowardNegative,
557 )?,
558 sym::floorf128 => self.float_round_intrinsic::<Quad>(
559 args,
560 dest,
561 rustc_apfloat::Round::TowardNegative,
562 )?,
563
564 sym::ceilf16 => self.float_round_intrinsic::<Half>(
565 args,
566 dest,
567 rustc_apfloat::Round::TowardPositive,
568 )?,
569 sym::ceilf32 => self.float_round_intrinsic::<Single>(
570 args,
571 dest,
572 rustc_apfloat::Round::TowardPositive,
573 )?,
574 sym::ceilf64 => self.float_round_intrinsic::<Double>(
575 args,
576 dest,
577 rustc_apfloat::Round::TowardPositive,
578 )?,
579 sym::ceilf128 => self.float_round_intrinsic::<Quad>(
580 args,
581 dest,
582 rustc_apfloat::Round::TowardPositive,
583 )?,
584
585 sym::truncf16 => {
586 self.float_round_intrinsic::<Half>(args, dest, rustc_apfloat::Round::TowardZero)?
587 }
588 sym::truncf32 => {
589 self.float_round_intrinsic::<Single>(args, dest, rustc_apfloat::Round::TowardZero)?
590 }
591 sym::truncf64 => {
592 self.float_round_intrinsic::<Double>(args, dest, rustc_apfloat::Round::TowardZero)?
593 }
594 sym::truncf128 => {
595 self.float_round_intrinsic::<Quad>(args, dest, rustc_apfloat::Round::TowardZero)?
596 }
597
598 sym::roundf16 => self.float_round_intrinsic::<Half>(
599 args,
600 dest,
601 rustc_apfloat::Round::NearestTiesToAway,
602 )?,
603 sym::roundf32 => self.float_round_intrinsic::<Single>(
604 args,
605 dest,
606 rustc_apfloat::Round::NearestTiesToAway,
607 )?,
608 sym::roundf64 => self.float_round_intrinsic::<Double>(
609 args,
610 dest,
611 rustc_apfloat::Round::NearestTiesToAway,
612 )?,
613 sym::roundf128 => self.float_round_intrinsic::<Quad>(
614 args,
615 dest,
616 rustc_apfloat::Round::NearestTiesToAway,
617 )?,
618
619 sym::round_ties_even_f16 => self.float_round_intrinsic::<Half>(
620 args,
621 dest,
622 rustc_apfloat::Round::NearestTiesToEven,
623 )?,
624 sym::round_ties_even_f32 => self.float_round_intrinsic::<Single>(
625 args,
626 dest,
627 rustc_apfloat::Round::NearestTiesToEven,
628 )?,
629 sym::round_ties_even_f64 => self.float_round_intrinsic::<Double>(
630 args,
631 dest,
632 rustc_apfloat::Round::NearestTiesToEven,
633 )?,
634 sym::round_ties_even_f128 => self.float_round_intrinsic::<Quad>(
635 args,
636 dest,
637 rustc_apfloat::Round::NearestTiesToEven,
638 )?,
639 sym::fmaf16 => self.fma_intrinsic::<Half>(args, dest)?,
640 sym::fmaf32 => self.fma_intrinsic::<Single>(args, dest)?,
641 sym::fmaf64 => self.fma_intrinsic::<Double>(args, dest)?,
642 sym::fmaf128 => self.fma_intrinsic::<Quad>(args, dest)?,
643 sym::fmuladdf16 => self.float_muladd_intrinsic::<Half>(args, dest)?,
644 sym::fmuladdf32 => self.float_muladd_intrinsic::<Single>(args, dest)?,
645 sym::fmuladdf64 => self.float_muladd_intrinsic::<Double>(args, dest)?,
646 sym::fmuladdf128 => self.float_muladd_intrinsic::<Quad>(args, dest)?,
647
648 _ => return interp_ok(false),
650 }
651
652 trace!("{:?}", self.dump_place(&dest.clone().into()));
653 self.return_to_block(ret)?;
654 interp_ok(true)
655 }
656
657 pub(super) fn eval_nondiverging_intrinsic(
658 &mut self,
659 intrinsic: &NonDivergingIntrinsic<'tcx>,
660 ) -> InterpResult<'tcx> {
661 match intrinsic {
662 NonDivergingIntrinsic::Assume(op) => {
663 let op = self.eval_operand(op, None)?;
664 let cond = self.read_scalar(&op)?.to_bool()?;
665 if !cond {
666 throw_ub_custom!(fluent::const_eval_assume_false);
667 }
668 interp_ok(())
669 }
670 NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping {
671 count,
672 src,
673 dst,
674 }) => {
675 let src = self.eval_operand(src, None)?;
676 let dst = self.eval_operand(dst, None)?;
677 let count = self.eval_operand(count, None)?;
678 self.copy_intrinsic(&src, &dst, &count, true)
679 }
680 }
681 }
682
683 pub fn numeric_intrinsic(
684 &self,
685 name: Symbol,
686 val: Scalar<M::Provenance>,
687 layout: TyAndLayout<'tcx>,
688 ret_layout: TyAndLayout<'tcx>,
689 ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
690 assert!(layout.ty.is_integral(), "invalid type for numeric intrinsic: {}", layout.ty);
691 let bits = val.to_bits(layout.size)?; let extra = 128 - u128::from(layout.size.bits());
693 let bits_out = match name {
694 sym::ctpop => u128::from(bits.count_ones()),
695 sym::ctlz_nonzero | sym::cttz_nonzero if bits == 0 => {
696 throw_ub_custom!(fluent::const_eval_call_nonzero_intrinsic, name = name,);
697 }
698 sym::ctlz | sym::ctlz_nonzero => u128::from(bits.leading_zeros()) - extra,
699 sym::cttz | sym::cttz_nonzero => u128::from((bits << extra).trailing_zeros()) - extra,
700 sym::bswap => {
701 assert_eq!(layout, ret_layout);
702 (bits << extra).swap_bytes()
703 }
704 sym::bitreverse => {
705 assert_eq!(layout, ret_layout);
706 (bits << extra).reverse_bits()
707 }
708 _ => bug!("not a numeric intrinsic: {}", name),
709 };
710 interp_ok(Scalar::from_uint(bits_out, ret_layout.size))
711 }
712
713 pub fn exact_div(
714 &mut self,
715 a: &ImmTy<'tcx, M::Provenance>,
716 b: &ImmTy<'tcx, M::Provenance>,
717 dest: &PlaceTy<'tcx, M::Provenance>,
718 ) -> InterpResult<'tcx> {
719 assert_eq!(a.layout.ty, b.layout.ty);
720 assert_matches!(a.layout.ty.kind(), ty::Int(..) | ty::Uint(..));
721
722 let rem = self.binary_op(BinOp::Rem, a, b)?;
726 if rem.to_scalar().to_bits(a.layout.size)? != 0 {
728 throw_ub_custom!(
729 fluent::const_eval_exact_div_has_remainder,
730 a = format!("{a}"),
731 b = format!("{b}")
732 )
733 }
734 let res = self.binary_op(BinOp::Div, a, b)?;
736 self.write_immediate(*res, dest)
737 }
738
739 pub fn saturating_arith(
740 &self,
741 mir_op: BinOp,
742 l: &ImmTy<'tcx, M::Provenance>,
743 r: &ImmTy<'tcx, M::Provenance>,
744 ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
745 assert_eq!(l.layout.ty, r.layout.ty);
746 assert_matches!(l.layout.ty.kind(), ty::Int(..) | ty::Uint(..));
747 assert_matches!(mir_op, BinOp::Add | BinOp::Sub);
748
749 let (val, overflowed) =
750 self.binary_op(mir_op.wrapping_to_overflowing().unwrap(), l, r)?.to_scalar_pair();
751 interp_ok(if overflowed.to_bool()? {
752 let size = l.layout.size;
753 if l.layout.backend_repr.is_signed() {
754 let first_term: i128 = l.to_scalar().to_int(l.layout.size)?;
759 if first_term >= 0 {
760 Scalar::from_int(size.signed_int_max(), size)
764 } else {
765 Scalar::from_int(size.signed_int_min(), size)
767 }
768 } else {
769 if matches!(mir_op, BinOp::Add) {
771 Scalar::from_uint(size.unsigned_int_max(), size)
773 } else {
774 Scalar::from_uint(0u128, size)
776 }
777 }
778 } else {
779 val
780 })
781 }
782
783 pub fn ptr_offset_inbounds(
786 &self,
787 ptr: Pointer<Option<M::Provenance>>,
788 offset_bytes: i64,
789 ) -> InterpResult<'tcx, Pointer<Option<M::Provenance>>> {
790 self.check_ptr_access_signed(
792 ptr,
793 offset_bytes,
794 CheckInAllocMsg::InboundsPointerArithmetic,
795 )?;
796 interp_ok(ptr.wrapping_signed_offset(offset_bytes, self))
798 }
799
800 pub(crate) fn copy_intrinsic(
802 &mut self,
803 src: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
804 dst: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
805 count: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
806 nonoverlapping: bool,
807 ) -> InterpResult<'tcx> {
808 let count = self.read_target_usize(count)?;
809 let layout = self.layout_of(src.layout.ty.builtin_deref(true).unwrap())?;
810 let (size, align) = (layout.size, layout.align.abi);
811
812 let size = self.compute_size_in_bytes(size, count).ok_or_else(|| {
813 err_ub_custom!(
814 fluent::const_eval_size_overflow,
815 name = if nonoverlapping { "copy_nonoverlapping" } else { "copy" }
816 )
817 })?;
818
819 let src = self.read_pointer(src)?;
820 let dst = self.read_pointer(dst)?;
821
822 self.check_ptr_align(src, align)?;
823 self.check_ptr_align(dst, align)?;
824
825 self.mem_copy(src, dst, size, nonoverlapping)
826 }
827
828 fn typed_swap_nonoverlapping_intrinsic(
830 &mut self,
831 left: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
832 right: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
833 ) -> InterpResult<'tcx> {
834 let left = self.deref_pointer(left)?;
835 let right = self.deref_pointer(right)?;
836 assert_eq!(left.layout, right.layout);
837 assert!(left.layout.is_sized());
838 let kind = MemoryKind::Stack;
839 let temp = self.allocate(left.layout, kind)?;
840 self.copy_op(&left, &temp)?; self.mem_copy(right.ptr(), left.ptr(), left.layout.size, true)?;
845 if M::enforce_validity(self, left.layout) {
849 self.validate_operand(
850 &left.clone().into(),
851 M::enforce_validity_recursively(self, left.layout),
852 true,
853 )?;
854 }
855
856 self.copy_op(&temp, &right)?; self.deallocate_ptr(temp.ptr(), None, kind)?;
859 interp_ok(())
860 }
861
862 pub fn write_bytes_intrinsic(
863 &mut self,
864 dst: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
865 byte: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
866 count: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
867 name: &'static str,
868 ) -> InterpResult<'tcx> {
869 let layout = self.layout_of(dst.layout.ty.builtin_deref(true).unwrap())?;
870
871 let dst = self.read_pointer(dst)?;
872 let byte = self.read_scalar(byte)?.to_u8()?;
873 let count = self.read_target_usize(count)?;
874
875 let len = self
878 .compute_size_in_bytes(layout.size, count)
879 .ok_or_else(|| err_ub_custom!(fluent::const_eval_size_overflow, name = name))?;
880
881 let bytes = std::iter::repeat(byte).take(len.bytes_usize());
882 self.write_bytes_ptr(dst, bytes)
883 }
884
885 pub(crate) fn compare_bytes_intrinsic(
886 &mut self,
887 left: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
888 right: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
889 byte_count: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
890 ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
891 let left = self.read_pointer(left)?;
892 let right = self.read_pointer(right)?;
893 let n = Size::from_bytes(self.read_target_usize(byte_count)?);
894
895 let left_bytes = self.read_bytes_ptr_strip_provenance(left, n)?;
896 let right_bytes = self.read_bytes_ptr_strip_provenance(right, n)?;
897
898 let result = Ord::cmp(left_bytes, right_bytes) as i32;
900 interp_ok(Scalar::from_i32(result))
901 }
902
903 pub(crate) fn raw_eq_intrinsic(
904 &mut self,
905 lhs: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
906 rhs: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
907 ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
908 let layout = self.layout_of(lhs.layout.ty.builtin_deref(true).unwrap())?;
909 assert!(layout.is_sized());
910
911 let get_bytes = |this: &InterpCx<'tcx, M>,
912 op: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>|
913 -> InterpResult<'tcx, &[u8]> {
914 let ptr = this.read_pointer(op)?;
915 this.check_ptr_align(ptr, layout.align.abi)?;
916 let Some(alloc_ref) = self.get_ptr_alloc(ptr, layout.size)? else {
917 return interp_ok(&[]);
919 };
920 alloc_ref.get_bytes_strip_provenance()
921 };
922
923 let lhs_bytes = get_bytes(self, lhs)?;
924 let rhs_bytes = get_bytes(self, rhs)?;
925 interp_ok(Scalar::from_bool(lhs_bytes == rhs_bytes))
926 }
927
928 fn float_min_intrinsic<F>(
929 &mut self,
930 args: &[OpTy<'tcx, M::Provenance>],
931 dest: &PlaceTy<'tcx, M::Provenance>,
932 ) -> InterpResult<'tcx, ()>
933 where
934 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
935 {
936 let a: F = self.read_scalar(&args[0])?.to_float()?;
937 let b: F = self.read_scalar(&args[1])?.to_float()?;
938 let res = if a == b {
939 M::equal_float_min_max(self, a, b)
942 } else {
943 self.adjust_nan(a.min(b), &[a, b])
944 };
945 self.write_scalar(res, dest)?;
946 interp_ok(())
947 }
948
949 fn float_max_intrinsic<F>(
950 &mut self,
951 args: &[OpTy<'tcx, M::Provenance>],
952 dest: &PlaceTy<'tcx, M::Provenance>,
953 ) -> InterpResult<'tcx, ()>
954 where
955 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
956 {
957 let a: F = self.read_scalar(&args[0])?.to_float()?;
958 let b: F = self.read_scalar(&args[1])?.to_float()?;
959 let res = if a == b {
960 M::equal_float_min_max(self, a, b)
963 } else {
964 self.adjust_nan(a.max(b), &[a, b])
965 };
966 self.write_scalar(res, dest)?;
967 interp_ok(())
968 }
969
970 fn float_minimum_intrinsic<F>(
971 &mut self,
972 args: &[OpTy<'tcx, M::Provenance>],
973 dest: &PlaceTy<'tcx, M::Provenance>,
974 ) -> InterpResult<'tcx, ()>
975 where
976 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
977 {
978 let a: F = self.read_scalar(&args[0])?.to_float()?;
979 let b: F = self.read_scalar(&args[1])?.to_float()?;
980 let res = a.minimum(b);
981 let res = self.adjust_nan(res, &[a, b]);
982 self.write_scalar(res, dest)?;
983 interp_ok(())
984 }
985
986 fn float_maximum_intrinsic<F>(
987 &mut self,
988 args: &[OpTy<'tcx, M::Provenance>],
989 dest: &PlaceTy<'tcx, M::Provenance>,
990 ) -> InterpResult<'tcx, ()>
991 where
992 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
993 {
994 let a: F = self.read_scalar(&args[0])?.to_float()?;
995 let b: F = self.read_scalar(&args[1])?.to_float()?;
996 let res = a.maximum(b);
997 let res = self.adjust_nan(res, &[a, b]);
998 self.write_scalar(res, dest)?;
999 interp_ok(())
1000 }
1001
1002 fn float_copysign_intrinsic<F>(
1003 &mut self,
1004 args: &[OpTy<'tcx, M::Provenance>],
1005 dest: &PlaceTy<'tcx, M::Provenance>,
1006 ) -> InterpResult<'tcx, ()>
1007 where
1008 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1009 {
1010 let a: F = self.read_scalar(&args[0])?.to_float()?;
1011 let b: F = self.read_scalar(&args[1])?.to_float()?;
1012 self.write_scalar(a.copy_sign(b), dest)?;
1014 interp_ok(())
1015 }
1016
1017 fn float_abs_intrinsic<F>(
1018 &mut self,
1019 args: &[OpTy<'tcx, M::Provenance>],
1020 dest: &PlaceTy<'tcx, M::Provenance>,
1021 ) -> InterpResult<'tcx, ()>
1022 where
1023 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1024 {
1025 let x: F = self.read_scalar(&args[0])?.to_float()?;
1026 self.write_scalar(x.abs(), dest)?;
1028 interp_ok(())
1029 }
1030
1031 fn float_round_intrinsic<F>(
1032 &mut self,
1033 args: &[OpTy<'tcx, M::Provenance>],
1034 dest: &PlaceTy<'tcx, M::Provenance>,
1035 mode: rustc_apfloat::Round,
1036 ) -> InterpResult<'tcx, ()>
1037 where
1038 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1039 {
1040 let x: F = self.read_scalar(&args[0])?.to_float()?;
1041 let res = x.round_to_integral(mode).value;
1042 let res = self.adjust_nan(res, &[x]);
1043 self.write_scalar(res, dest)?;
1044 interp_ok(())
1045 }
1046
1047 fn fma_intrinsic<F>(
1048 &mut self,
1049 args: &[OpTy<'tcx, M::Provenance>],
1050 dest: &PlaceTy<'tcx, M::Provenance>,
1051 ) -> InterpResult<'tcx, ()>
1052 where
1053 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1054 {
1055 let a: F = self.read_scalar(&args[0])?.to_float()?;
1056 let b: F = self.read_scalar(&args[1])?.to_float()?;
1057 let c: F = self.read_scalar(&args[2])?.to_float()?;
1058
1059 let res = a.mul_add(b, c).value;
1060 let res = self.adjust_nan(res, &[a, b, c]);
1061 self.write_scalar(res, dest)?;
1062 interp_ok(())
1063 }
1064
1065 fn float_muladd_intrinsic<F>(
1066 &mut self,
1067 args: &[OpTy<'tcx, M::Provenance>],
1068 dest: &PlaceTy<'tcx, M::Provenance>,
1069 ) -> InterpResult<'tcx, ()>
1070 where
1071 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1072 {
1073 let a: F = self.read_scalar(&args[0])?.to_float()?;
1074 let b: F = self.read_scalar(&args[1])?.to_float()?;
1075 let c: F = self.read_scalar(&args[2])?.to_float()?;
1076
1077 let fuse = M::float_fuse_mul_add(self);
1078
1079 let res = if fuse { a.mul_add(b, c).value } else { ((a * b).value + c).value };
1080 let res = self.adjust_nan(res, &[a, b, c]);
1081 self.write_scalar(res, dest)?;
1082 interp_ok(())
1083 }
1084}