1mod simd;
6
7use std::assert_matches::assert_matches;
8
9use rustc_abi::{FieldIdx, HasDataLayout, Size};
10use rustc_apfloat::ieee::{Double, Half, Quad, Single};
11use rustc_middle::mir::interpret::{CTFE_ALLOC_SALT, read_target_uint, write_target_uint};
12use rustc_middle::mir::{self, BinOp, ConstValue, NonDivergingIntrinsic};
13use rustc_middle::ty::layout::TyAndLayout;
14use rustc_middle::ty::{FloatTy, Ty, TyCtxt};
15use rustc_middle::{bug, span_bug, ty};
16use rustc_span::{Symbol, sym};
17use tracing::trace;
18
19use super::memory::MemoryKind;
20use super::util::ensure_monomorphic_enough;
21use super::{
22 AllocId, CheckInAllocMsg, ImmTy, InterpCx, InterpResult, Machine, OpTy, PlaceTy, Pointer,
23 PointerArithmetic, Provenance, Scalar, err_ub_custom, err_unsup_format, interp_ok, throw_inval,
24 throw_ub_custom, throw_ub_format,
25};
26use crate::fluent_generated as fluent;
27
28pub(crate) fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> (AllocId, u64) {
30 let path = crate::util::type_name(tcx, ty);
31 let bytes = path.into_bytes();
32 let len = bytes.len().try_into().unwrap();
33 (tcx.allocate_bytes_dedup(bytes, CTFE_ALLOC_SALT), len)
34}
35impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
36 fn write_type_id(
38 &mut self,
39 ty: Ty<'tcx>,
40 dest: &PlaceTy<'tcx, M::Provenance>,
41 ) -> InterpResult<'tcx, ()> {
42 let tcx = self.tcx;
43 let type_id_hash = tcx.type_id_hash(ty).as_u128();
44 let op = self.const_val_to_op(
45 ConstValue::Scalar(Scalar::from_u128(type_id_hash)),
46 tcx.types.u128,
47 None,
48 )?;
49 self.copy_op_allow_transmute(&op, dest)?;
50
51 let alloc_id = tcx.reserve_and_set_type_id_alloc(ty);
55 let arr = self.project_field(dest, FieldIdx::ZERO)?;
56 let mut elem_iter = self.project_array_fields(&arr)?;
57 while let Some((_, elem)) = elem_iter.next(self)? {
58 let hash_fragment = self.read_scalar(&elem)?.to_target_usize(&tcx)?;
60 let ptr = Pointer::new(alloc_id.into(), Size::from_bytes(hash_fragment));
61 let ptr = self.global_root_pointer(ptr)?;
62 let val = Scalar::from_pointer(ptr, &tcx);
63 self.write_scalar(val, &elem)?;
64 }
65 interp_ok(())
66 }
67
68 pub(crate) fn read_type_id(
70 &self,
71 op: &OpTy<'tcx, M::Provenance>,
72 ) -> InterpResult<'tcx, Ty<'tcx>> {
73 let ptr_size = self.pointer_size().bytes_usize();
76 let arr = self.project_field(op, FieldIdx::ZERO)?;
77
78 let mut ty_and_hash = None;
79 let mut elem_iter = self.project_array_fields(&arr)?;
80 while let Some((idx, elem)) = elem_iter.next(self)? {
81 let elem = self.read_pointer(&elem)?;
82 let (elem_ty, elem_hash) = self.get_ptr_type_id(elem)?;
83 let full_hash = match ty_and_hash {
86 None => {
87 let hash = self.tcx.type_id_hash(elem_ty).as_u128();
88 let mut hash_bytes = [0u8; 16];
89 write_target_uint(self.data_layout().endian, &mut hash_bytes, hash).unwrap();
90 ty_and_hash = Some((elem_ty, hash_bytes));
91 hash_bytes
92 }
93 Some((ty, hash_bytes)) => {
94 if ty != elem_ty {
95 throw_ub_format!(
96 "invalid `TypeId` value: not all bytes carry the same type id metadata"
97 );
98 }
99 hash_bytes
100 }
101 };
102 let hash_frag = &full_hash[(idx as usize) * ptr_size..][..ptr_size];
104 if read_target_uint(self.data_layout().endian, hash_frag).unwrap() != elem_hash.into() {
105 throw_ub_format!(
106 "invalid `TypeId` value: the hash does not match the type id metadata"
107 );
108 }
109 }
110
111 interp_ok(ty_and_hash.unwrap().0)
112 }
113
114 pub fn eval_intrinsic(
118 &mut self,
119 instance: ty::Instance<'tcx>,
120 args: &[OpTy<'tcx, M::Provenance>],
121 dest: &PlaceTy<'tcx, M::Provenance>,
122 ret: Option<mir::BasicBlock>,
123 ) -> InterpResult<'tcx, bool> {
124 let instance_args = instance.args;
125 let intrinsic_name = self.tcx.item_name(instance.def_id());
126
127 if intrinsic_name.as_str().starts_with("simd_") {
128 return self.eval_simd_intrinsic(intrinsic_name, instance_args, args, dest, ret);
129 }
130
131 let tcx = self.tcx.tcx;
132
133 match intrinsic_name {
134 sym::type_name => {
135 let tp_ty = instance.args.type_at(0);
136 ensure_monomorphic_enough(tcx, tp_ty)?;
137 let (alloc_id, meta) = alloc_type_name(tcx, tp_ty);
138 let val = ConstValue::Slice { alloc_id, meta };
139 let val = self.const_val_to_op(val, dest.layout.ty, Some(dest.layout))?;
140 self.copy_op(&val, dest)?;
141 }
142 sym::needs_drop => {
143 let tp_ty = instance.args.type_at(0);
144 ensure_monomorphic_enough(tcx, tp_ty)?;
145 let val = ConstValue::from_bool(tp_ty.needs_drop(tcx, self.typing_env));
146 let val = self.const_val_to_op(val, tcx.types.bool, Some(dest.layout))?;
147 self.copy_op(&val, dest)?;
148 }
149 sym::type_id => {
150 let tp_ty = instance.args.type_at(0);
151 ensure_monomorphic_enough(tcx, tp_ty)?;
152 self.write_type_id(tp_ty, dest)?;
153 }
154 sym::type_id_eq => {
155 let a_ty = self.read_type_id(&args[0])?;
156 let b_ty = self.read_type_id(&args[1])?;
157 self.write_scalar(Scalar::from_bool(a_ty == b_ty), dest)?;
158 }
159 sym::size_of => {
160 let tp_ty = instance.args.type_at(0);
161 let layout = self.layout_of(tp_ty)?;
162 if !layout.is_sized() {
163 span_bug!(self.cur_span(), "unsized type for `size_of`");
164 }
165 let val = layout.size.bytes();
166 self.write_scalar(Scalar::from_target_usize(val, self), dest)?;
167 }
168 sym::align_of => {
169 let tp_ty = instance.args.type_at(0);
170 let layout = self.layout_of(tp_ty)?;
171 if !layout.is_sized() {
172 span_bug!(self.cur_span(), "unsized type for `align_of`");
173 }
174 let val = layout.align.bytes();
175 self.write_scalar(Scalar::from_target_usize(val, self), dest)?;
176 }
177 sym::variant_count => {
178 let tp_ty = instance.args.type_at(0);
179 let ty = match tp_ty.kind() {
180 ty::Pat(base, _) => *base,
184 _ => tp_ty,
185 };
186 let val = match ty.kind() {
187 ty::Adt(adt, _) => {
189 ConstValue::from_target_usize(adt.variants().len() as u64, &tcx)
190 }
191 ty::Alias(..) | ty::Param(_) | ty::Placeholder(_) | ty::Infer(_) => {
192 throw_inval!(TooGeneric)
193 }
194 ty::Pat(..) => unreachable!(),
195 ty::Bound(_, _) => bug!("bound ty during ctfe"),
196 ty::Bool
197 | ty::Char
198 | ty::Int(_)
199 | ty::Uint(_)
200 | ty::Float(_)
201 | ty::Foreign(_)
202 | ty::Str
203 | ty::Array(_, _)
204 | ty::Slice(_)
205 | ty::RawPtr(_, _)
206 | ty::Ref(_, _, _)
207 | ty::FnDef(_, _)
208 | ty::FnPtr(..)
209 | ty::Dynamic(_, _)
210 | ty::Closure(_, _)
211 | ty::CoroutineClosure(_, _)
212 | ty::Coroutine(_, _)
213 | ty::CoroutineWitness(..)
214 | ty::UnsafeBinder(_)
215 | ty::Never
216 | ty::Tuple(_)
217 | ty::Error(_) => ConstValue::from_target_usize(0u64, &tcx),
218 };
219 let val = self.const_val_to_op(val, dest.layout.ty, Some(dest.layout))?;
220 self.copy_op(&val, dest)?;
221 }
222
223 sym::caller_location => {
224 let span = self.find_closest_untracked_caller_location();
225 let val = self.tcx.span_as_caller_location(span);
226 let val =
227 self.const_val_to_op(val, self.tcx.caller_location_ty(), Some(dest.layout))?;
228 self.copy_op(&val, dest)?;
229 }
230
231 sym::align_of_val | sym::size_of_val => {
232 let place = self.ref_to_mplace(&self.read_immediate(&args[0])?)?;
235 let (size, align) = self
236 .size_and_align_of_val(&place)?
237 .ok_or_else(|| err_unsup_format!("`extern type` does not have known layout"))?;
238
239 let result = match intrinsic_name {
240 sym::align_of_val => align.bytes(),
241 sym::size_of_val => size.bytes(),
242 _ => bug!(),
243 };
244
245 self.write_scalar(Scalar::from_target_usize(result, self), dest)?;
246 }
247
248 sym::fadd_algebraic
249 | sym::fsub_algebraic
250 | sym::fmul_algebraic
251 | sym::fdiv_algebraic
252 | sym::frem_algebraic => {
253 let a = self.read_immediate(&args[0])?;
254 let b = self.read_immediate(&args[1])?;
255
256 let op = match intrinsic_name {
257 sym::fadd_algebraic => BinOp::Add,
258 sym::fsub_algebraic => BinOp::Sub,
259 sym::fmul_algebraic => BinOp::Mul,
260 sym::fdiv_algebraic => BinOp::Div,
261 sym::frem_algebraic => BinOp::Rem,
262
263 _ => bug!(),
264 };
265
266 let res = self.binary_op(op, &a, &b)?;
267 let res = M::apply_float_nondet(self, res)?;
269 self.write_immediate(*res, dest)?;
270 }
271
272 sym::ctpop
273 | sym::cttz
274 | sym::cttz_nonzero
275 | sym::ctlz
276 | sym::ctlz_nonzero
277 | sym::bswap
278 | sym::bitreverse => {
279 let ty = instance_args.type_at(0);
280 let layout = self.layout_of(ty)?;
281 let val = self.read_scalar(&args[0])?;
282
283 let out_val = self.numeric_intrinsic(intrinsic_name, val, layout, dest.layout)?;
284 self.write_scalar(out_val, dest)?;
285 }
286 sym::saturating_add | sym::saturating_sub => {
287 let l = self.read_immediate(&args[0])?;
288 let r = self.read_immediate(&args[1])?;
289 let val = self.saturating_arith(
290 if intrinsic_name == sym::saturating_add { BinOp::Add } else { BinOp::Sub },
291 &l,
292 &r,
293 )?;
294 self.write_scalar(val, dest)?;
295 }
296 sym::discriminant_value => {
297 let place = self.deref_pointer(&args[0])?;
298 let variant = self.read_discriminant(&place)?;
299 let discr = self.discriminant_for_variant(place.layout.ty, variant)?;
300 self.write_immediate(*discr, dest)?;
301 }
302 sym::exact_div => {
303 let l = self.read_immediate(&args[0])?;
304 let r = self.read_immediate(&args[1])?;
305 self.exact_div(&l, &r, dest)?;
306 }
307 sym::rotate_left | sym::rotate_right => {
308 let layout_val = self.layout_of(instance_args.type_at(0))?;
311 let val = self.read_scalar(&args[0])?;
312 let val_bits = val.to_bits(layout_val.size)?; let layout_raw_shift = self.layout_of(self.tcx.types.u32)?;
315 let raw_shift = self.read_scalar(&args[1])?;
316 let raw_shift_bits = raw_shift.to_bits(layout_raw_shift.size)?;
317
318 let width_bits = u128::from(layout_val.size.bits());
319 let shift_bits = raw_shift_bits % width_bits;
320 let inv_shift_bits = (width_bits - shift_bits) % width_bits;
321 let result_bits = if intrinsic_name == sym::rotate_left {
322 (val_bits << shift_bits) | (val_bits >> inv_shift_bits)
323 } else {
324 (val_bits >> shift_bits) | (val_bits << inv_shift_bits)
325 };
326 let truncated_bits = layout_val.size.truncate(result_bits);
327 let result = Scalar::from_uint(truncated_bits, layout_val.size);
328 self.write_scalar(result, dest)?;
329 }
330 sym::copy => {
331 self.copy_intrinsic(&args[0], &args[1], &args[2], false)?;
332 }
333 sym::write_bytes => {
334 self.write_bytes_intrinsic(&args[0], &args[1], &args[2], "write_bytes")?;
335 }
336 sym::compare_bytes => {
337 let result = self.compare_bytes_intrinsic(&args[0], &args[1], &args[2])?;
338 self.write_scalar(result, dest)?;
339 }
340 sym::arith_offset => {
341 let ptr = self.read_pointer(&args[0])?;
342 let offset_count = self.read_target_isize(&args[1])?;
343 let pointee_ty = instance_args.type_at(0);
344
345 let pointee_size = i64::try_from(self.layout_of(pointee_ty)?.size.bytes()).unwrap();
346 let offset_bytes = offset_count.wrapping_mul(pointee_size);
347 let offset_ptr = ptr.wrapping_signed_offset(offset_bytes, self);
348 self.write_pointer(offset_ptr, dest)?;
349 }
350 sym::ptr_offset_from | sym::ptr_offset_from_unsigned => {
351 let a = self.read_pointer(&args[0])?;
352 let b = self.read_pointer(&args[1])?;
353
354 let usize_layout = self.layout_of(self.tcx.types.usize)?;
355 let isize_layout = self.layout_of(self.tcx.types.isize)?;
356
357 let (a_offset, b_offset, is_addr) = if M::Provenance::OFFSET_IS_ADDR {
361 (a.addr().bytes(), b.addr().bytes(), true)
362 } else {
363 match (self.ptr_try_get_alloc_id(a, 0), self.ptr_try_get_alloc_id(b, 0)) {
364 (Err(a), Err(b)) => {
365 (a, b, true)
367 }
368 (Ok((a_alloc_id, a_offset, _)), Ok((b_alloc_id, b_offset, _)))
369 if a_alloc_id == b_alloc_id =>
370 {
371 (a_offset.bytes(), b_offset.bytes(), false)
374 }
375 _ => {
376 throw_ub_custom!(
378 fluent::const_eval_offset_from_different_allocations,
379 name = intrinsic_name,
380 );
381 }
382 }
383 };
384
385 let dist = {
387 let (val, overflowed) = {
390 let a_offset = ImmTy::from_uint(a_offset, usize_layout);
391 let b_offset = ImmTy::from_uint(b_offset, usize_layout);
392 self.binary_op(BinOp::SubWithOverflow, &a_offset, &b_offset)?
393 .to_scalar_pair()
394 };
395 if overflowed.to_bool()? {
396 if intrinsic_name == sym::ptr_offset_from_unsigned {
398 throw_ub_custom!(
399 fluent::const_eval_offset_from_unsigned_overflow,
400 a_offset = a_offset,
401 b_offset = b_offset,
402 is_addr = is_addr,
403 );
404 }
405 let dist = val.to_target_isize(self)?;
409 if dist >= 0 || i128::from(dist) == self.pointer_size().signed_int_min() {
410 throw_ub_custom!(
411 fluent::const_eval_offset_from_underflow,
412 name = intrinsic_name,
413 );
414 }
415 dist
416 } else {
417 let dist = val.to_target_isize(self)?;
419 if dist < 0 {
422 throw_ub_custom!(
423 fluent::const_eval_offset_from_overflow,
424 name = intrinsic_name,
425 );
426 }
427 dist
428 }
429 };
430
431 self.check_ptr_access_signed(b, dist, CheckInAllocMsg::Dereferenceable)
434 .map_err_kind(|_| {
435 if let Ok((a_alloc_id, ..)) = self.ptr_try_get_alloc_id(a, 0)
438 && let Ok((b_alloc_id, ..)) = self.ptr_try_get_alloc_id(b, 0)
439 && a_alloc_id == b_alloc_id
440 {
441 err_ub_custom!(
442 fluent::const_eval_offset_from_out_of_bounds,
443 name = intrinsic_name,
444 )
445 } else {
446 err_ub_custom!(
447 fluent::const_eval_offset_from_different_allocations,
448 name = intrinsic_name,
449 )
450 }
451 })?;
452 self.check_ptr_access_signed(
455 a,
456 dist.checked_neg().unwrap(), CheckInAllocMsg::Dereferenceable,
458 )
459 .map_err_kind(|_| {
460 err_ub_custom!(
462 fluent::const_eval_offset_from_different_allocations,
463 name = intrinsic_name,
464 )
465 })?;
466
467 let ret_layout = if intrinsic_name == sym::ptr_offset_from_unsigned {
469 assert!(0 <= dist && dist <= self.target_isize_max());
470 usize_layout
471 } else {
472 assert!(self.target_isize_min() <= dist && dist <= self.target_isize_max());
473 isize_layout
474 };
475 let pointee_layout = self.layout_of(instance_args.type_at(0))?;
476 let val = ImmTy::from_int(dist, ret_layout);
478 let size = ImmTy::from_int(pointee_layout.size.bytes(), ret_layout);
479 self.exact_div(&val, &size, dest)?;
480 }
481
482 sym::black_box => {
483 self.copy_op(&args[0], dest)?;
485 }
486 sym::raw_eq => {
487 let result = self.raw_eq_intrinsic(&args[0], &args[1])?;
488 self.write_scalar(result, dest)?;
489 }
490 sym::typed_swap_nonoverlapping => {
491 self.typed_swap_nonoverlapping_intrinsic(&args[0], &args[1])?;
492 }
493
494 sym::vtable_size => {
495 let ptr = self.read_pointer(&args[0])?;
496 let (size, _align) = self.get_vtable_size_and_align(ptr, None)?;
498 self.write_scalar(Scalar::from_target_usize(size.bytes(), self), dest)?;
499 }
500 sym::vtable_align => {
501 let ptr = self.read_pointer(&args[0])?;
502 let (_size, align) = self.get_vtable_size_and_align(ptr, None)?;
504 self.write_scalar(Scalar::from_target_usize(align.bytes(), self), dest)?;
505 }
506
507 sym::minnumf16 => self.float_min_intrinsic::<Half>(args, dest)?,
508 sym::minnumf32 => self.float_min_intrinsic::<Single>(args, dest)?,
509 sym::minnumf64 => self.float_min_intrinsic::<Double>(args, dest)?,
510 sym::minnumf128 => self.float_min_intrinsic::<Quad>(args, dest)?,
511
512 sym::minimumf16 => self.float_minimum_intrinsic::<Half>(args, dest)?,
513 sym::minimumf32 => self.float_minimum_intrinsic::<Single>(args, dest)?,
514 sym::minimumf64 => self.float_minimum_intrinsic::<Double>(args, dest)?,
515 sym::minimumf128 => self.float_minimum_intrinsic::<Quad>(args, dest)?,
516
517 sym::maxnumf16 => self.float_max_intrinsic::<Half>(args, dest)?,
518 sym::maxnumf32 => self.float_max_intrinsic::<Single>(args, dest)?,
519 sym::maxnumf64 => self.float_max_intrinsic::<Double>(args, dest)?,
520 sym::maxnumf128 => self.float_max_intrinsic::<Quad>(args, dest)?,
521
522 sym::maximumf16 => self.float_maximum_intrinsic::<Half>(args, dest)?,
523 sym::maximumf32 => self.float_maximum_intrinsic::<Single>(args, dest)?,
524 sym::maximumf64 => self.float_maximum_intrinsic::<Double>(args, dest)?,
525 sym::maximumf128 => self.float_maximum_intrinsic::<Quad>(args, dest)?,
526
527 sym::copysignf16 => self.float_copysign_intrinsic::<Half>(args, dest)?,
528 sym::copysignf32 => self.float_copysign_intrinsic::<Single>(args, dest)?,
529 sym::copysignf64 => self.float_copysign_intrinsic::<Double>(args, dest)?,
530 sym::copysignf128 => self.float_copysign_intrinsic::<Quad>(args, dest)?,
531
532 sym::fabsf16 => self.float_abs_intrinsic::<Half>(args, dest)?,
533 sym::fabsf32 => self.float_abs_intrinsic::<Single>(args, dest)?,
534 sym::fabsf64 => self.float_abs_intrinsic::<Double>(args, dest)?,
535 sym::fabsf128 => self.float_abs_intrinsic::<Quad>(args, dest)?,
536
537 sym::floorf16 => self.float_round_intrinsic::<Half>(
538 args,
539 dest,
540 rustc_apfloat::Round::TowardNegative,
541 )?,
542 sym::floorf32 => self.float_round_intrinsic::<Single>(
543 args,
544 dest,
545 rustc_apfloat::Round::TowardNegative,
546 )?,
547 sym::floorf64 => self.float_round_intrinsic::<Double>(
548 args,
549 dest,
550 rustc_apfloat::Round::TowardNegative,
551 )?,
552 sym::floorf128 => self.float_round_intrinsic::<Quad>(
553 args,
554 dest,
555 rustc_apfloat::Round::TowardNegative,
556 )?,
557
558 sym::ceilf16 => self.float_round_intrinsic::<Half>(
559 args,
560 dest,
561 rustc_apfloat::Round::TowardPositive,
562 )?,
563 sym::ceilf32 => self.float_round_intrinsic::<Single>(
564 args,
565 dest,
566 rustc_apfloat::Round::TowardPositive,
567 )?,
568 sym::ceilf64 => self.float_round_intrinsic::<Double>(
569 args,
570 dest,
571 rustc_apfloat::Round::TowardPositive,
572 )?,
573 sym::ceilf128 => self.float_round_intrinsic::<Quad>(
574 args,
575 dest,
576 rustc_apfloat::Round::TowardPositive,
577 )?,
578
579 sym::truncf16 => {
580 self.float_round_intrinsic::<Half>(args, dest, rustc_apfloat::Round::TowardZero)?
581 }
582 sym::truncf32 => {
583 self.float_round_intrinsic::<Single>(args, dest, rustc_apfloat::Round::TowardZero)?
584 }
585 sym::truncf64 => {
586 self.float_round_intrinsic::<Double>(args, dest, rustc_apfloat::Round::TowardZero)?
587 }
588 sym::truncf128 => {
589 self.float_round_intrinsic::<Quad>(args, dest, rustc_apfloat::Round::TowardZero)?
590 }
591
592 sym::roundf16 => self.float_round_intrinsic::<Half>(
593 args,
594 dest,
595 rustc_apfloat::Round::NearestTiesToAway,
596 )?,
597 sym::roundf32 => self.float_round_intrinsic::<Single>(
598 args,
599 dest,
600 rustc_apfloat::Round::NearestTiesToAway,
601 )?,
602 sym::roundf64 => self.float_round_intrinsic::<Double>(
603 args,
604 dest,
605 rustc_apfloat::Round::NearestTiesToAway,
606 )?,
607 sym::roundf128 => self.float_round_intrinsic::<Quad>(
608 args,
609 dest,
610 rustc_apfloat::Round::NearestTiesToAway,
611 )?,
612
613 sym::round_ties_even_f16 => self.float_round_intrinsic::<Half>(
614 args,
615 dest,
616 rustc_apfloat::Round::NearestTiesToEven,
617 )?,
618 sym::round_ties_even_f32 => self.float_round_intrinsic::<Single>(
619 args,
620 dest,
621 rustc_apfloat::Round::NearestTiesToEven,
622 )?,
623 sym::round_ties_even_f64 => self.float_round_intrinsic::<Double>(
624 args,
625 dest,
626 rustc_apfloat::Round::NearestTiesToEven,
627 )?,
628 sym::round_ties_even_f128 => self.float_round_intrinsic::<Quad>(
629 args,
630 dest,
631 rustc_apfloat::Round::NearestTiesToEven,
632 )?,
633 sym::fmaf16 => self.fma_intrinsic::<Half>(args, dest)?,
634 sym::fmaf32 => self.fma_intrinsic::<Single>(args, dest)?,
635 sym::fmaf64 => self.fma_intrinsic::<Double>(args, dest)?,
636 sym::fmaf128 => self.fma_intrinsic::<Quad>(args, dest)?,
637 sym::fmuladdf16 => self.float_muladd_intrinsic::<Half>(args, dest)?,
638 sym::fmuladdf32 => self.float_muladd_intrinsic::<Single>(args, dest)?,
639 sym::fmuladdf64 => self.float_muladd_intrinsic::<Double>(args, dest)?,
640 sym::fmuladdf128 => self.float_muladd_intrinsic::<Quad>(args, dest)?,
641
642 _ => return interp_ok(false),
644 }
645
646 trace!("{:?}", self.dump_place(&dest.clone().into()));
647 self.return_to_block(ret)?;
648 interp_ok(true)
649 }
650
651 pub(super) fn eval_nondiverging_intrinsic(
652 &mut self,
653 intrinsic: &NonDivergingIntrinsic<'tcx>,
654 ) -> InterpResult<'tcx> {
655 match intrinsic {
656 NonDivergingIntrinsic::Assume(op) => {
657 let op = self.eval_operand(op, None)?;
658 let cond = self.read_scalar(&op)?.to_bool()?;
659 if !cond {
660 throw_ub_custom!(fluent::const_eval_assume_false);
661 }
662 interp_ok(())
663 }
664 NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping {
665 count,
666 src,
667 dst,
668 }) => {
669 let src = self.eval_operand(src, None)?;
670 let dst = self.eval_operand(dst, None)?;
671 let count = self.eval_operand(count, None)?;
672 self.copy_intrinsic(&src, &dst, &count, true)
673 }
674 }
675 }
676
677 pub fn numeric_intrinsic(
678 &self,
679 name: Symbol,
680 val: Scalar<M::Provenance>,
681 layout: TyAndLayout<'tcx>,
682 ret_layout: TyAndLayout<'tcx>,
683 ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
684 assert!(layout.ty.is_integral(), "invalid type for numeric intrinsic: {}", layout.ty);
685 let bits = val.to_bits(layout.size)?; let extra = 128 - u128::from(layout.size.bits());
687 let bits_out = match name {
688 sym::ctpop => u128::from(bits.count_ones()),
689 sym::ctlz_nonzero | sym::cttz_nonzero if bits == 0 => {
690 throw_ub_custom!(fluent::const_eval_call_nonzero_intrinsic, name = name,);
691 }
692 sym::ctlz | sym::ctlz_nonzero => u128::from(bits.leading_zeros()) - extra,
693 sym::cttz | sym::cttz_nonzero => u128::from((bits << extra).trailing_zeros()) - extra,
694 sym::bswap => {
695 assert_eq!(layout, ret_layout);
696 (bits << extra).swap_bytes()
697 }
698 sym::bitreverse => {
699 assert_eq!(layout, ret_layout);
700 (bits << extra).reverse_bits()
701 }
702 _ => bug!("not a numeric intrinsic: {}", name),
703 };
704 interp_ok(Scalar::from_uint(bits_out, ret_layout.size))
705 }
706
707 pub fn exact_div(
708 &mut self,
709 a: &ImmTy<'tcx, M::Provenance>,
710 b: &ImmTy<'tcx, M::Provenance>,
711 dest: &PlaceTy<'tcx, M::Provenance>,
712 ) -> InterpResult<'tcx> {
713 assert_eq!(a.layout.ty, b.layout.ty);
714 assert_matches!(a.layout.ty.kind(), ty::Int(..) | ty::Uint(..));
715
716 let rem = self.binary_op(BinOp::Rem, a, b)?;
720 if rem.to_scalar().to_bits(a.layout.size)? != 0 {
722 throw_ub_custom!(
723 fluent::const_eval_exact_div_has_remainder,
724 a = format!("{a}"),
725 b = format!("{b}")
726 )
727 }
728 let res = self.binary_op(BinOp::Div, a, b)?;
730 self.write_immediate(*res, dest)
731 }
732
733 pub fn saturating_arith(
734 &self,
735 mir_op: BinOp,
736 l: &ImmTy<'tcx, M::Provenance>,
737 r: &ImmTy<'tcx, M::Provenance>,
738 ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
739 assert_eq!(l.layout.ty, r.layout.ty);
740 assert_matches!(l.layout.ty.kind(), ty::Int(..) | ty::Uint(..));
741 assert_matches!(mir_op, BinOp::Add | BinOp::Sub);
742
743 let (val, overflowed) =
744 self.binary_op(mir_op.wrapping_to_overflowing().unwrap(), l, r)?.to_scalar_pair();
745 interp_ok(if overflowed.to_bool()? {
746 let size = l.layout.size;
747 if l.layout.backend_repr.is_signed() {
748 let first_term: i128 = l.to_scalar().to_int(l.layout.size)?;
753 if first_term >= 0 {
754 Scalar::from_int(size.signed_int_max(), size)
758 } else {
759 Scalar::from_int(size.signed_int_min(), size)
761 }
762 } else {
763 if matches!(mir_op, BinOp::Add) {
765 Scalar::from_uint(size.unsigned_int_max(), size)
767 } else {
768 Scalar::from_uint(0u128, size)
770 }
771 }
772 } else {
773 val
774 })
775 }
776
777 pub fn ptr_offset_inbounds(
780 &self,
781 ptr: Pointer<Option<M::Provenance>>,
782 offset_bytes: i64,
783 ) -> InterpResult<'tcx, Pointer<Option<M::Provenance>>> {
784 self.check_ptr_access_signed(
786 ptr,
787 offset_bytes,
788 CheckInAllocMsg::InboundsPointerArithmetic,
789 )?;
790 interp_ok(ptr.wrapping_signed_offset(offset_bytes, self))
792 }
793
794 pub(crate) fn copy_intrinsic(
796 &mut self,
797 src: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
798 dst: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
799 count: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
800 nonoverlapping: bool,
801 ) -> InterpResult<'tcx> {
802 let count = self.read_target_usize(count)?;
803 let layout = self.layout_of(src.layout.ty.builtin_deref(true).unwrap())?;
804 let (size, align) = (layout.size, layout.align.abi);
805
806 let size = self.compute_size_in_bytes(size, count).ok_or_else(|| {
807 err_ub_custom!(
808 fluent::const_eval_size_overflow,
809 name = if nonoverlapping { "copy_nonoverlapping" } else { "copy" }
810 )
811 })?;
812
813 let src = self.read_pointer(src)?;
814 let dst = self.read_pointer(dst)?;
815
816 self.check_ptr_align(src, align)?;
817 self.check_ptr_align(dst, align)?;
818
819 self.mem_copy(src, dst, size, nonoverlapping)
820 }
821
822 fn typed_swap_nonoverlapping_intrinsic(
824 &mut self,
825 left: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
826 right: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
827 ) -> InterpResult<'tcx> {
828 let left = self.deref_pointer(left)?;
829 let right = self.deref_pointer(right)?;
830 assert_eq!(left.layout, right.layout);
831 assert!(left.layout.is_sized());
832 let kind = MemoryKind::Stack;
833 let temp = self.allocate(left.layout, kind)?;
834 self.copy_op(&left, &temp)?; self.mem_copy(right.ptr(), left.ptr(), left.layout.size, true)?;
839 if M::enforce_validity(self, left.layout) {
843 self.validate_operand(
844 &left.clone().into(),
845 M::enforce_validity_recursively(self, left.layout),
846 true,
847 )?;
848 }
849
850 self.copy_op(&temp, &right)?; self.deallocate_ptr(temp.ptr(), None, kind)?;
853 interp_ok(())
854 }
855
856 pub fn write_bytes_intrinsic(
857 &mut self,
858 dst: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
859 byte: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
860 count: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
861 name: &'static str,
862 ) -> InterpResult<'tcx> {
863 let layout = self.layout_of(dst.layout.ty.builtin_deref(true).unwrap())?;
864
865 let dst = self.read_pointer(dst)?;
866 let byte = self.read_scalar(byte)?.to_u8()?;
867 let count = self.read_target_usize(count)?;
868
869 let len = self
872 .compute_size_in_bytes(layout.size, count)
873 .ok_or_else(|| err_ub_custom!(fluent::const_eval_size_overflow, name = name))?;
874
875 let bytes = std::iter::repeat_n(byte, len.bytes_usize());
876 self.write_bytes_ptr(dst, bytes)
877 }
878
879 pub(crate) fn compare_bytes_intrinsic(
880 &mut self,
881 left: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
882 right: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
883 byte_count: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
884 ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
885 let left = self.read_pointer(left)?;
886 let right = self.read_pointer(right)?;
887 let n = Size::from_bytes(self.read_target_usize(byte_count)?);
888
889 let left_bytes = self.read_bytes_ptr_strip_provenance(left, n)?;
890 let right_bytes = self.read_bytes_ptr_strip_provenance(right, n)?;
891
892 let result = Ord::cmp(left_bytes, right_bytes) as i32;
894 interp_ok(Scalar::from_i32(result))
895 }
896
897 pub(crate) fn raw_eq_intrinsic(
898 &mut self,
899 lhs: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
900 rhs: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
901 ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
902 let layout = self.layout_of(lhs.layout.ty.builtin_deref(true).unwrap())?;
903 assert!(layout.is_sized());
904
905 let get_bytes = |this: &InterpCx<'tcx, M>,
906 op: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>|
907 -> InterpResult<'tcx, &[u8]> {
908 let ptr = this.read_pointer(op)?;
909 this.check_ptr_align(ptr, layout.align.abi)?;
910 let Some(alloc_ref) = self.get_ptr_alloc(ptr, layout.size)? else {
911 return interp_ok(&[]);
913 };
914 alloc_ref.get_bytes_strip_provenance()
915 };
916
917 let lhs_bytes = get_bytes(self, lhs)?;
918 let rhs_bytes = get_bytes(self, rhs)?;
919 interp_ok(Scalar::from_bool(lhs_bytes == rhs_bytes))
920 }
921
922 fn float_min_intrinsic<F>(
923 &mut self,
924 args: &[OpTy<'tcx, M::Provenance>],
925 dest: &PlaceTy<'tcx, M::Provenance>,
926 ) -> InterpResult<'tcx, ()>
927 where
928 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
929 {
930 let a: F = self.read_scalar(&args[0])?.to_float()?;
931 let b: F = self.read_scalar(&args[1])?.to_float()?;
932 let res = if a == b {
933 M::equal_float_min_max(self, a, b)
936 } else {
937 self.adjust_nan(a.min(b), &[a, b])
938 };
939 self.write_scalar(res, dest)?;
940 interp_ok(())
941 }
942
943 fn float_max_intrinsic<F>(
944 &mut self,
945 args: &[OpTy<'tcx, M::Provenance>],
946 dest: &PlaceTy<'tcx, M::Provenance>,
947 ) -> InterpResult<'tcx, ()>
948 where
949 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
950 {
951 let a: F = self.read_scalar(&args[0])?.to_float()?;
952 let b: F = self.read_scalar(&args[1])?.to_float()?;
953 let res = if a == b {
954 M::equal_float_min_max(self, a, b)
957 } else {
958 self.adjust_nan(a.max(b), &[a, b])
959 };
960 self.write_scalar(res, dest)?;
961 interp_ok(())
962 }
963
964 fn float_minimum_intrinsic<F>(
965 &mut self,
966 args: &[OpTy<'tcx, M::Provenance>],
967 dest: &PlaceTy<'tcx, M::Provenance>,
968 ) -> InterpResult<'tcx, ()>
969 where
970 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
971 {
972 let a: F = self.read_scalar(&args[0])?.to_float()?;
973 let b: F = self.read_scalar(&args[1])?.to_float()?;
974 let res = a.minimum(b);
975 let res = self.adjust_nan(res, &[a, b]);
976 self.write_scalar(res, dest)?;
977 interp_ok(())
978 }
979
980 fn float_maximum_intrinsic<F>(
981 &mut self,
982 args: &[OpTy<'tcx, M::Provenance>],
983 dest: &PlaceTy<'tcx, M::Provenance>,
984 ) -> InterpResult<'tcx, ()>
985 where
986 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
987 {
988 let a: F = self.read_scalar(&args[0])?.to_float()?;
989 let b: F = self.read_scalar(&args[1])?.to_float()?;
990 let res = a.maximum(b);
991 let res = self.adjust_nan(res, &[a, b]);
992 self.write_scalar(res, dest)?;
993 interp_ok(())
994 }
995
996 fn float_copysign_intrinsic<F>(
997 &mut self,
998 args: &[OpTy<'tcx, M::Provenance>],
999 dest: &PlaceTy<'tcx, M::Provenance>,
1000 ) -> InterpResult<'tcx, ()>
1001 where
1002 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1003 {
1004 let a: F = self.read_scalar(&args[0])?.to_float()?;
1005 let b: F = self.read_scalar(&args[1])?.to_float()?;
1006 self.write_scalar(a.copy_sign(b), dest)?;
1008 interp_ok(())
1009 }
1010
1011 fn float_abs_intrinsic<F>(
1012 &mut self,
1013 args: &[OpTy<'tcx, M::Provenance>],
1014 dest: &PlaceTy<'tcx, M::Provenance>,
1015 ) -> InterpResult<'tcx, ()>
1016 where
1017 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1018 {
1019 let x: F = self.read_scalar(&args[0])?.to_float()?;
1020 self.write_scalar(x.abs(), dest)?;
1022 interp_ok(())
1023 }
1024
1025 fn float_round_intrinsic<F>(
1026 &mut self,
1027 args: &[OpTy<'tcx, M::Provenance>],
1028 dest: &PlaceTy<'tcx, M::Provenance>,
1029 mode: rustc_apfloat::Round,
1030 ) -> InterpResult<'tcx, ()>
1031 where
1032 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1033 {
1034 let x: F = self.read_scalar(&args[0])?.to_float()?;
1035 let res = x.round_to_integral(mode).value;
1036 let res = self.adjust_nan(res, &[x]);
1037 self.write_scalar(res, dest)?;
1038 interp_ok(())
1039 }
1040
1041 fn fma_intrinsic<F>(
1042 &mut self,
1043 args: &[OpTy<'tcx, M::Provenance>],
1044 dest: &PlaceTy<'tcx, M::Provenance>,
1045 ) -> InterpResult<'tcx, ()>
1046 where
1047 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1048 {
1049 let a: F = self.read_scalar(&args[0])?.to_float()?;
1050 let b: F = self.read_scalar(&args[1])?.to_float()?;
1051 let c: F = self.read_scalar(&args[2])?.to_float()?;
1052
1053 let res = a.mul_add(b, c).value;
1054 let res = self.adjust_nan(res, &[a, b, c]);
1055 self.write_scalar(res, dest)?;
1056 interp_ok(())
1057 }
1058
1059 fn float_muladd_intrinsic<F>(
1060 &mut self,
1061 args: &[OpTy<'tcx, M::Provenance>],
1062 dest: &PlaceTy<'tcx, M::Provenance>,
1063 ) -> InterpResult<'tcx, ()>
1064 where
1065 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
1066 {
1067 let a: F = self.read_scalar(&args[0])?.to_float()?;
1068 let b: F = self.read_scalar(&args[1])?.to_float()?;
1069 let c: F = self.read_scalar(&args[2])?.to_float()?;
1070
1071 let fuse = M::float_fuse_mul_add(self);
1072
1073 let res = if fuse { a.mul_add(b, c).value } else { ((a * b).value + c).value };
1074 let res = self.adjust_nan(res, &[a, b, c]);
1075 self.write_scalar(res, dest)?;
1076 interp_ok(())
1077 }
1078
1079 pub fn float_to_int_checked(
1083 &self,
1084 src: &ImmTy<'tcx, M::Provenance>,
1085 cast_to: TyAndLayout<'tcx>,
1086 round: rustc_apfloat::Round,
1087 ) -> InterpResult<'tcx, Option<ImmTy<'tcx, M::Provenance>>> {
1088 fn float_to_int_inner<'tcx, F: rustc_apfloat::Float, M: Machine<'tcx>>(
1089 ecx: &InterpCx<'tcx, M>,
1090 src: F,
1091 cast_to: TyAndLayout<'tcx>,
1092 round: rustc_apfloat::Round,
1093 ) -> (Scalar<M::Provenance>, rustc_apfloat::Status) {
1094 let int_size = cast_to.layout.size;
1095 match cast_to.ty.kind() {
1096 ty::Uint(_) => {
1098 let res = src.to_u128_r(int_size.bits_usize(), round, &mut false);
1099 (Scalar::from_uint(res.value, int_size), res.status)
1100 }
1101 ty::Int(_) => {
1103 let res = src.to_i128_r(int_size.bits_usize(), round, &mut false);
1104 (Scalar::from_int(res.value, int_size), res.status)
1105 }
1106 _ => span_bug!(
1108 ecx.cur_span(),
1109 "attempted float-to-int conversion with non-int output type {}",
1110 cast_to.ty,
1111 ),
1112 }
1113 }
1114
1115 let ty::Float(fty) = src.layout.ty.kind() else {
1116 bug!("float_to_int_checked: non-float input type {}", src.layout.ty)
1117 };
1118
1119 let (val, status) = match fty {
1120 FloatTy::F16 => float_to_int_inner(self, src.to_scalar().to_f16()?, cast_to, round),
1121 FloatTy::F32 => float_to_int_inner(self, src.to_scalar().to_f32()?, cast_to, round),
1122 FloatTy::F64 => float_to_int_inner(self, src.to_scalar().to_f64()?, cast_to, round),
1123 FloatTy::F128 => float_to_int_inner(self, src.to_scalar().to_f128()?, cast_to, round),
1124 };
1125
1126 if status.intersects(
1127 rustc_apfloat::Status::INVALID_OP
1128 | rustc_apfloat::Status::OVERFLOW
1129 | rustc_apfloat::Status::UNDERFLOW,
1130 ) {
1131 interp_ok(None)
1134 } else {
1135 interp_ok(Some(ImmTy::from_scalar(val, cast_to)))
1138 }
1139 }
1140}