1use std::assert_matches::assert_matches;
6
7use either::{Either, Left, Right};
8use rustc_abi::{BackendRepr, HasDataLayout, Size};
9use rustc_middle::ty::Ty;
10use rustc_middle::ty::layout::TyAndLayout;
11use rustc_middle::{bug, mir, span_bug};
12use tracing::field::Empty;
13use tracing::{instrument, trace};
14
15use super::{
16 AllocInit, AllocRef, AllocRefMut, CheckAlignMsg, CtfeProvenance, ImmTy, Immediate, InterpCx,
17 InterpResult, Machine, MemoryKind, Misalignment, OffsetMode, OpTy, Operand, Pointer,
18 Projectable, Provenance, Scalar, alloc_range, interp_ok, mir_assign_valid_types,
19};
20use crate::enter_trace_span;
21
22#[derive(Copy, Clone, Hash, PartialEq, Eq, Debug)]
23pub enum MemPlaceMeta<Prov: Provenance = CtfeProvenance> {
25 Meta(Scalar<Prov>),
27 None,
29}
30
31impl<Prov: Provenance> MemPlaceMeta<Prov> {
32 #[cfg_attr(debug_assertions, track_caller)] pub fn unwrap_meta(self) -> Scalar<Prov> {
34 match self {
35 Self::Meta(s) => s,
36 Self::None => {
37 bug!("expected wide pointer extra data (e.g. slice length or trait object vtable)")
38 }
39 }
40 }
41
42 #[inline(always)]
43 pub fn has_meta(self) -> bool {
44 match self {
45 Self::Meta(_) => true,
46 Self::None => false,
47 }
48 }
49}
50
51#[derive(Copy, Clone, Hash, PartialEq, Eq, Debug)]
52pub(super) struct MemPlace<Prov: Provenance = CtfeProvenance> {
53 pub ptr: Pointer<Option<Prov>>,
55 pub meta: MemPlaceMeta<Prov>,
59 misaligned: Option<Misalignment>,
61}
62
63impl<Prov: Provenance> MemPlace<Prov> {
64 fn map_provenance(self, f: impl FnOnce(Prov) -> Prov) -> Self {
66 MemPlace { ptr: self.ptr.map_provenance(|p| p.map(f)), ..self }
67 }
68
69 #[inline]
71 fn to_ref(self, cx: &impl HasDataLayout) -> Immediate<Prov> {
72 Immediate::new_pointer_with_meta(self.ptr, self.meta, cx)
73 }
74
75 #[inline]
76 fn offset_with_meta_<'tcx, M: Machine<'tcx, Provenance = Prov>>(
78 self,
79 offset: Size,
80 mode: OffsetMode,
81 meta: MemPlaceMeta<Prov>,
82 ecx: &InterpCx<'tcx, M>,
83 ) -> InterpResult<'tcx, Self> {
84 debug_assert!(
85 !meta.has_meta() || self.meta.has_meta(),
86 "cannot use `offset_with_meta` to add metadata to a place"
87 );
88 let ptr = match mode {
89 OffsetMode::Inbounds => {
90 ecx.ptr_offset_inbounds(self.ptr, offset.bytes().try_into().unwrap())?
91 }
92 OffsetMode::Wrapping => self.ptr.wrapping_offset(offset, ecx),
93 };
94 interp_ok(MemPlace { ptr, meta, misaligned: self.misaligned })
95 }
96}
97
98#[derive(Clone, Hash, Eq, PartialEq)]
100pub struct MPlaceTy<'tcx, Prov: Provenance = CtfeProvenance> {
101 mplace: MemPlace<Prov>,
102 pub layout: TyAndLayout<'tcx>,
103}
104
105impl<Prov: Provenance> std::fmt::Debug for MPlaceTy<'_, Prov> {
106 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
107 f.debug_struct("MPlaceTy")
109 .field("mplace", &self.mplace)
110 .field("ty", &format_args!("{}", self.layout.ty))
111 .finish()
112 }
113}
114
115impl<'tcx, Prov: Provenance> MPlaceTy<'tcx, Prov> {
116 #[inline]
120 pub fn fake_alloc_zst(layout: TyAndLayout<'tcx>) -> Self {
121 assert!(layout.is_zst());
122 let align = layout.align.abi;
123 let ptr = Pointer::without_provenance(align.bytes()); MPlaceTy { mplace: MemPlace { ptr, meta: MemPlaceMeta::None, misaligned: None }, layout }
125 }
126
127 pub fn map_provenance(self, f: impl FnOnce(Prov) -> Prov) -> Self {
129 MPlaceTy { mplace: self.mplace.map_provenance(f), ..self }
130 }
131
132 #[inline(always)]
133 pub(super) fn mplace(&self) -> &MemPlace<Prov> {
134 &self.mplace
135 }
136
137 #[inline(always)]
138 pub fn ptr(&self) -> Pointer<Option<Prov>> {
139 self.mplace.ptr
140 }
141
142 #[inline(always)]
143 pub fn to_ref(&self, cx: &impl HasDataLayout) -> Immediate<Prov> {
144 self.mplace.to_ref(cx)
145 }
146}
147
148impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for MPlaceTy<'tcx, Prov> {
149 #[inline(always)]
150 fn layout(&self) -> TyAndLayout<'tcx> {
151 self.layout
152 }
153
154 #[inline(always)]
155 fn meta(&self) -> MemPlaceMeta<Prov> {
156 self.mplace.meta
157 }
158
159 fn offset_with_meta<M: Machine<'tcx, Provenance = Prov>>(
160 &self,
161 offset: Size,
162 mode: OffsetMode,
163 meta: MemPlaceMeta<Prov>,
164 layout: TyAndLayout<'tcx>,
165 ecx: &InterpCx<'tcx, M>,
166 ) -> InterpResult<'tcx, Self> {
167 interp_ok(MPlaceTy {
168 mplace: self.mplace.offset_with_meta_(offset, mode, meta, ecx)?,
169 layout,
170 })
171 }
172
173 #[inline(always)]
174 fn to_op<M: Machine<'tcx, Provenance = Prov>>(
175 &self,
176 _ecx: &InterpCx<'tcx, M>,
177 ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
178 interp_ok(self.clone().into())
179 }
180}
181
182#[derive(Copy, Clone, Debug)]
183pub(super) enum Place<Prov: Provenance = CtfeProvenance> {
184 Ptr(MemPlace<Prov>),
186
187 Local { local: mir::Local, offset: Option<Size>, locals_addr: usize },
199}
200
201#[derive(Clone)]
208pub struct PlaceTy<'tcx, Prov: Provenance = CtfeProvenance> {
209 place: Place<Prov>, pub layout: TyAndLayout<'tcx>,
211}
212
213impl<Prov: Provenance> std::fmt::Debug for PlaceTy<'_, Prov> {
214 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
215 f.debug_struct("PlaceTy")
217 .field("place", &self.place)
218 .field("ty", &format_args!("{}", self.layout.ty))
219 .finish()
220 }
221}
222
223impl<'tcx, Prov: Provenance> From<MPlaceTy<'tcx, Prov>> for PlaceTy<'tcx, Prov> {
224 #[inline(always)]
225 fn from(mplace: MPlaceTy<'tcx, Prov>) -> Self {
226 PlaceTy { place: Place::Ptr(mplace.mplace), layout: mplace.layout }
227 }
228}
229
230impl<'tcx, Prov: Provenance> PlaceTy<'tcx, Prov> {
231 #[inline(always)]
232 pub(super) fn place(&self) -> &Place<Prov> {
233 &self.place
234 }
235
236 #[inline(always)]
238 pub fn as_mplace_or_local(
239 &self,
240 ) -> Either<MPlaceTy<'tcx, Prov>, (mir::Local, Option<Size>, usize, TyAndLayout<'tcx>)> {
241 match self.place {
242 Place::Ptr(mplace) => Left(MPlaceTy { mplace, layout: self.layout }),
243 Place::Local { local, offset, locals_addr } => {
244 Right((local, offset, locals_addr, self.layout))
245 }
246 }
247 }
248
249 #[inline(always)]
250 #[cfg_attr(debug_assertions, track_caller)] pub fn assert_mem_place(&self) -> MPlaceTy<'tcx, Prov> {
252 self.as_mplace_or_local().left().unwrap_or_else(|| {
253 bug!(
254 "PlaceTy of type {} was a local when it was expected to be an MPlace",
255 self.layout.ty
256 )
257 })
258 }
259}
260
261impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for PlaceTy<'tcx, Prov> {
262 #[inline(always)]
263 fn layout(&self) -> TyAndLayout<'tcx> {
264 self.layout
265 }
266
267 #[inline]
268 fn meta(&self) -> MemPlaceMeta<Prov> {
269 match self.as_mplace_or_local() {
270 Left(mplace) => mplace.meta(),
271 Right(_) => {
272 debug_assert!(self.layout.is_sized(), "unsized locals should live in memory");
273 MemPlaceMeta::None
274 }
275 }
276 }
277
278 fn offset_with_meta<M: Machine<'tcx, Provenance = Prov>>(
279 &self,
280 offset: Size,
281 mode: OffsetMode,
282 meta: MemPlaceMeta<Prov>,
283 layout: TyAndLayout<'tcx>,
284 ecx: &InterpCx<'tcx, M>,
285 ) -> InterpResult<'tcx, Self> {
286 interp_ok(match self.as_mplace_or_local() {
287 Left(mplace) => mplace.offset_with_meta(offset, mode, meta, layout, ecx)?.into(),
288 Right((local, old_offset, locals_addr, _)) => {
289 debug_assert!(layout.is_sized(), "unsized locals should live in memory");
290 assert_matches!(meta, MemPlaceMeta::None); assert!(offset + layout.size <= self.layout.size);
295
296 let new_offset = old_offset.unwrap_or(Size::ZERO) + offset;
298
299 PlaceTy {
300 place: Place::Local { local, offset: Some(new_offset), locals_addr },
301 layout,
302 }
303 }
304 })
305 }
306
307 #[inline(always)]
308 fn to_op<M: Machine<'tcx, Provenance = Prov>>(
309 &self,
310 ecx: &InterpCx<'tcx, M>,
311 ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
312 ecx.place_to_op(self)
313 }
314}
315
316impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> {
318 #[inline(always)]
319 pub fn as_mplace_or_imm(&self) -> Either<MPlaceTy<'tcx, Prov>, ImmTy<'tcx, Prov>> {
320 match self.op() {
321 Operand::Indirect(mplace) => Left(MPlaceTy { mplace: *mplace, layout: self.layout }),
322 Operand::Immediate(imm) => Right(ImmTy::from_immediate(*imm, self.layout)),
323 }
324 }
325
326 #[inline(always)]
327 #[cfg_attr(debug_assertions, track_caller)] pub fn assert_mem_place(&self) -> MPlaceTy<'tcx, Prov> {
329 self.as_mplace_or_imm().left().unwrap_or_else(|| {
330 bug!(
331 "OpTy of type {} was immediate when it was expected to be an MPlace",
332 self.layout.ty
333 )
334 })
335 }
336}
337
338pub trait Writeable<'tcx, Prov: Provenance>: Projectable<'tcx, Prov> {
340 fn to_place(&self) -> PlaceTy<'tcx, Prov>;
341
342 fn force_mplace<M: Machine<'tcx, Provenance = Prov>>(
343 &self,
344 ecx: &mut InterpCx<'tcx, M>,
345 ) -> InterpResult<'tcx, MPlaceTy<'tcx, Prov>>;
346}
347
348impl<'tcx, Prov: Provenance> Writeable<'tcx, Prov> for PlaceTy<'tcx, Prov> {
349 #[inline(always)]
350 fn to_place(&self) -> PlaceTy<'tcx, Prov> {
351 self.clone()
352 }
353
354 #[inline(always)]
355 fn force_mplace<M: Machine<'tcx, Provenance = Prov>>(
356 &self,
357 ecx: &mut InterpCx<'tcx, M>,
358 ) -> InterpResult<'tcx, MPlaceTy<'tcx, Prov>> {
359 ecx.force_allocation(self)
360 }
361}
362
363impl<'tcx, Prov: Provenance> Writeable<'tcx, Prov> for MPlaceTy<'tcx, Prov> {
364 #[inline(always)]
365 fn to_place(&self) -> PlaceTy<'tcx, Prov> {
366 self.clone().into()
367 }
368
369 #[inline(always)]
370 fn force_mplace<M: Machine<'tcx, Provenance = Prov>>(
371 &self,
372 _ecx: &mut InterpCx<'tcx, M>,
373 ) -> InterpResult<'tcx, MPlaceTy<'tcx, Prov>> {
374 interp_ok(self.clone())
375 }
376}
377
378impl<'tcx, Prov, M> InterpCx<'tcx, M>
380where
381 Prov: Provenance,
382 M: Machine<'tcx, Provenance = Prov>,
383{
384 fn ptr_with_meta_to_mplace(
385 &self,
386 ptr: Pointer<Option<M::Provenance>>,
387 meta: MemPlaceMeta<M::Provenance>,
388 layout: TyAndLayout<'tcx>,
389 unaligned: bool,
390 ) -> MPlaceTy<'tcx, M::Provenance> {
391 let misaligned =
392 if unaligned { None } else { self.is_ptr_misaligned(ptr, layout.align.abi) };
393 MPlaceTy { mplace: MemPlace { ptr, meta, misaligned }, layout }
394 }
395
396 pub fn ptr_to_mplace(
397 &self,
398 ptr: Pointer<Option<M::Provenance>>,
399 layout: TyAndLayout<'tcx>,
400 ) -> MPlaceTy<'tcx, M::Provenance> {
401 assert!(layout.is_sized());
402 self.ptr_with_meta_to_mplace(ptr, MemPlaceMeta::None, layout, false)
403 }
404
405 pub fn ptr_to_mplace_unaligned(
406 &self,
407 ptr: Pointer<Option<M::Provenance>>,
408 layout: TyAndLayout<'tcx>,
409 ) -> MPlaceTy<'tcx, M::Provenance> {
410 assert!(layout.is_sized());
411 self.ptr_with_meta_to_mplace(ptr, MemPlaceMeta::None, layout, true)
412 }
413
414 pub fn ref_to_mplace(
421 &self,
422 val: &ImmTy<'tcx, M::Provenance>,
423 ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> {
424 let pointee_type =
425 val.layout.ty.builtin_deref(true).expect("`ref_to_mplace` called on non-ptr type");
426 let layout = self.layout_of(pointee_type)?;
427 let (ptr, meta) = val.to_scalar_and_meta();
428
429 let ptr = ptr.to_pointer(self)?;
432 interp_ok(self.ptr_with_meta_to_mplace(ptr, meta, layout, false))
433 }
434
435 pub fn mplace_to_ref(
439 &self,
440 mplace: &MPlaceTy<'tcx, M::Provenance>,
441 ) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> {
442 let imm = mplace.mplace.to_ref(self);
443 let layout = self.layout_of(Ty::new_mut_ptr(self.tcx.tcx, mplace.layout.ty))?;
444 interp_ok(ImmTy::from_immediate(imm, layout))
445 }
446
447 #[instrument(skip(self), level = "trace")]
450 pub fn deref_pointer(
451 &self,
452 src: &impl Projectable<'tcx, M::Provenance>,
453 ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> {
454 if src.layout().ty.is_box() {
455 bug!("dereferencing {}", src.layout().ty);
459 }
460
461 let val = self.read_immediate(src)?;
462 trace!("deref to {} on {:?}", val.layout.ty, *val);
463
464 let mplace = self.ref_to_mplace(&val)?;
465 interp_ok(mplace)
466 }
467
468 #[inline]
469 pub(super) fn get_place_alloc(
470 &self,
471 mplace: &MPlaceTy<'tcx, M::Provenance>,
472 ) -> InterpResult<'tcx, Option<AllocRef<'_, 'tcx, M::Provenance, M::AllocExtra, M::Bytes>>>
473 {
474 let (size, _align) = self
475 .size_and_align_of_val(mplace)?
476 .unwrap_or((mplace.layout.size, mplace.layout.align.abi));
477 let a = self.get_ptr_alloc(mplace.ptr(), size)?;
480 self.check_misalign(mplace.mplace.misaligned, CheckAlignMsg::BasedOn)?;
481 interp_ok(a)
482 }
483
484 #[inline]
485 pub(super) fn get_place_alloc_mut(
486 &mut self,
487 mplace: &MPlaceTy<'tcx, M::Provenance>,
488 ) -> InterpResult<'tcx, Option<AllocRefMut<'_, 'tcx, M::Provenance, M::AllocExtra, M::Bytes>>>
489 {
490 let (size, _align) = self
491 .size_and_align_of_val(mplace)?
492 .unwrap_or((mplace.layout.size, mplace.layout.align.abi));
493 let misalign_res = self.check_misalign(mplace.mplace.misaligned, CheckAlignMsg::BasedOn);
497 let (a, ()) = self.get_ptr_alloc_mut(mplace.ptr(), size).and(misalign_res)?;
499 interp_ok(a)
500 }
501
502 pub fn local_to_place(
504 &self,
505 local: mir::Local,
506 ) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> {
507 let frame = self.frame();
508 let layout = self.layout_of_local(frame, local, None)?;
509 let place = if layout.is_sized() {
510 Place::Local { local, offset: None, locals_addr: frame.locals_addr() }
512 } else {
513 match frame.locals[local].access()? {
515 Operand::Immediate(_) => bug!(),
516 Operand::Indirect(mplace) => Place::Ptr(*mplace),
517 }
518 };
519 interp_ok(PlaceTy { place, layout })
520 }
521
522 #[instrument(skip(self), level = "trace")]
525 pub fn eval_place(
526 &self,
527 mir_place: mir::Place<'tcx>,
528 ) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> {
529 let _span =
530 enter_trace_span!(M, step::eval_place, ?mir_place, tracing_separate_thread = Empty);
531
532 let mut place = self.local_to_place(mir_place.local)?;
533 for elem in mir_place.projection.iter() {
535 place = self.project(&place, elem)?
536 }
537
538 trace!("{:?}", self.dump_place(&place));
539 if cfg!(debug_assertions) {
541 let normalized_place_ty = self
542 .instantiate_from_current_frame_and_normalize_erasing_regions(
543 mir_place.ty(&self.frame().body.local_decls, *self.tcx).ty,
544 )?;
545 if !mir_assign_valid_types(
546 *self.tcx,
547 self.typing_env,
548 self.layout_of(normalized_place_ty)?,
549 place.layout,
550 ) {
551 span_bug!(
552 self.cur_span(),
553 "eval_place of a MIR place with type {} produced an interpreter place with type {}",
554 normalized_place_ty,
555 place.layout.ty,
556 )
557 }
558 }
559 interp_ok(place)
560 }
561
562 #[inline(always)]
565 fn as_mplace_or_mutable_local(
566 &mut self,
567 place: &PlaceTy<'tcx, M::Provenance>,
568 ) -> InterpResult<
569 'tcx,
570 Either<
571 MPlaceTy<'tcx, M::Provenance>,
572 (&mut Immediate<M::Provenance>, TyAndLayout<'tcx>, mir::Local),
573 >,
574 > {
575 interp_ok(match place.to_place().as_mplace_or_local() {
576 Left(mplace) => Left(mplace),
577 Right((local, offset, locals_addr, layout)) => {
578 if offset.is_some() {
579 Left(place.force_mplace(self)?)
582 } else {
583 debug_assert_eq!(locals_addr, self.frame().locals_addr());
584 debug_assert_eq!(self.layout_of_local(self.frame(), local, None)?, layout);
585 match self.frame_mut().locals[local].access_mut()? {
586 Operand::Indirect(mplace) => {
587 Left(MPlaceTy { mplace: *mplace, layout })
589 }
590 Operand::Immediate(local_val) => {
591 Right((local_val, layout, local))
593 }
594 }
595 }
596 }
597 })
598 }
599
600 #[inline(always)]
602 #[instrument(skip(self), level = "trace")]
603 pub fn write_immediate(
604 &mut self,
605 src: Immediate<M::Provenance>,
606 dest: &impl Writeable<'tcx, M::Provenance>,
607 ) -> InterpResult<'tcx> {
608 self.write_immediate_no_validate(src, dest)?;
609
610 if M::enforce_validity(self, dest.layout()) {
611 self.validate_operand(
614 &dest.to_place(),
615 M::enforce_validity_recursively(self, dest.layout()),
616 true,
617 )?;
618 }
619
620 interp_ok(())
621 }
622
623 #[inline(always)]
625 pub fn write_scalar(
626 &mut self,
627 val: impl Into<Scalar<M::Provenance>>,
628 dest: &impl Writeable<'tcx, M::Provenance>,
629 ) -> InterpResult<'tcx> {
630 self.write_immediate(Immediate::Scalar(val.into()), dest)
631 }
632
633 #[inline(always)]
635 pub fn write_pointer(
636 &mut self,
637 ptr: impl Into<Pointer<Option<M::Provenance>>>,
638 dest: &impl Writeable<'tcx, M::Provenance>,
639 ) -> InterpResult<'tcx> {
640 self.write_scalar(Scalar::from_maybe_pointer(ptr.into(), self), dest)
641 }
642
643 pub(super) fn write_immediate_no_validate(
647 &mut self,
648 src: Immediate<M::Provenance>,
649 dest: &impl Writeable<'tcx, M::Provenance>,
650 ) -> InterpResult<'tcx> {
651 assert!(dest.layout().is_sized(), "Cannot write unsized immediate data");
652
653 match self.as_mplace_or_mutable_local(&dest.to_place())? {
654 Right((local_val, local_layout, local)) => {
655 *local_val = src;
657 if !self.validation_in_progress() {
659 M::after_local_write(self, local, false)?;
660 }
661 if cfg!(debug_assertions) {
665 src.assert_matches_abi(
666 local_layout.backend_repr,
667 "invalid immediate for given destination place",
668 self,
669 );
670 }
671 }
672 Left(mplace) => {
673 self.write_immediate_to_mplace_no_validate(src, mplace.layout, mplace.mplace)?;
674 }
675 }
676 interp_ok(())
677 }
678
679 fn write_immediate_to_mplace_no_validate(
683 &mut self,
684 value: Immediate<M::Provenance>,
685 layout: TyAndLayout<'tcx>,
686 dest: MemPlace<M::Provenance>,
687 ) -> InterpResult<'tcx> {
688 value.assert_matches_abi(
691 layout.backend_repr,
692 "invalid immediate for given destination place",
693 self,
694 );
695 let tcx = *self.tcx;
701 let Some(mut alloc) = self.get_place_alloc_mut(&MPlaceTy { mplace: dest, layout })? else {
702 return interp_ok(());
704 };
705
706 match value {
707 Immediate::Scalar(scalar) => {
708 alloc.write_scalar(alloc_range(Size::ZERO, scalar.size()), scalar)
709 }
710 Immediate::ScalarPair(a_val, b_val) => {
711 let BackendRepr::ScalarPair(a, b) = layout.backend_repr else {
712 span_bug!(
713 self.cur_span(),
714 "write_immediate_to_mplace: invalid ScalarPair layout: {:#?}",
715 layout
716 )
717 };
718 let b_offset = a.size(&tcx).align_to(b.align(&tcx).abi);
719 assert!(b_offset.bytes() > 0); alloc.write_scalar(alloc_range(Size::ZERO, a_val.size()), a_val)?;
726 alloc.write_scalar(alloc_range(b_offset, b_val.size()), b_val)?;
727 interp_ok(())
729 }
730 Immediate::Uninit => alloc.write_uninit_full(),
731 }
732 }
733
734 pub fn write_uninit(
735 &mut self,
736 dest: &impl Writeable<'tcx, M::Provenance>,
737 ) -> InterpResult<'tcx> {
738 match self.as_mplace_or_mutable_local(&dest.to_place())? {
739 Right((local_val, _local_layout, local)) => {
740 *local_val = Immediate::Uninit;
741 if !self.validation_in_progress() {
743 M::after_local_write(self, local, false)?;
744 }
745 }
746 Left(mplace) => {
747 let Some(mut alloc) = self.get_place_alloc_mut(&mplace)? else {
748 return interp_ok(());
750 };
751 alloc.write_uninit_full()?;
752 }
753 }
754 interp_ok(())
755 }
756
757 pub fn clear_provenance(
759 &mut self,
760 dest: &impl Writeable<'tcx, M::Provenance>,
761 ) -> InterpResult<'tcx> {
762 match self.as_mplace_or_mutable_local(&dest.to_place())? {
763 Right((local_val, _local_layout, local)) => {
764 local_val.clear_provenance()?;
765 if !self.validation_in_progress() {
767 M::after_local_write(self, local, false)?;
768 }
769 }
770 Left(mplace) => {
771 let Some(mut alloc) = self.get_place_alloc_mut(&mplace)? else {
772 return interp_ok(());
774 };
775 alloc.clear_provenance()?;
776 }
777 }
778 interp_ok(())
779 }
780
781 #[inline(always)]
784 pub fn copy_op_allow_transmute(
785 &mut self,
786 src: &impl Projectable<'tcx, M::Provenance>,
787 dest: &impl Writeable<'tcx, M::Provenance>,
788 ) -> InterpResult<'tcx> {
789 self.copy_op_inner(src, dest, true)
790 }
791
792 #[inline(always)]
795 pub fn copy_op(
796 &mut self,
797 src: &impl Projectable<'tcx, M::Provenance>,
798 dest: &impl Writeable<'tcx, M::Provenance>,
799 ) -> InterpResult<'tcx> {
800 self.copy_op_inner(src, dest, false)
801 }
802
803 #[inline(always)]
806 #[instrument(skip(self), level = "trace")]
807 fn copy_op_inner(
808 &mut self,
809 src: &impl Projectable<'tcx, M::Provenance>,
810 dest: &impl Writeable<'tcx, M::Provenance>,
811 allow_transmute: bool,
812 ) -> InterpResult<'tcx> {
813 self.copy_op_no_validate(src, dest, allow_transmute)?;
820
821 if M::enforce_validity(self, dest.layout()) {
822 let dest = dest.to_place();
823 if src.layout().ty != dest.layout().ty {
827 self.validate_operand(
828 &dest.transmute(src.layout(), self)?,
829 M::enforce_validity_recursively(self, src.layout()),
830 true,
831 )?;
832 }
833 self.validate_operand(
834 &dest,
835 M::enforce_validity_recursively(self, dest.layout()),
836 true,
837 )?;
838 }
839
840 interp_ok(())
841 }
842
843 #[instrument(skip(self), level = "trace")]
848 fn copy_op_no_validate(
849 &mut self,
850 src: &impl Projectable<'tcx, M::Provenance>,
851 dest: &impl Writeable<'tcx, M::Provenance>,
852 allow_transmute: bool,
853 ) -> InterpResult<'tcx> {
854 let layout_compat =
857 mir_assign_valid_types(*self.tcx, self.typing_env, src.layout(), dest.layout());
858 if !allow_transmute && !layout_compat {
859 span_bug!(
860 self.cur_span(),
861 "type mismatch when copying!\nsrc: {},\ndest: {}",
862 src.layout().ty,
863 dest.layout().ty,
864 );
865 }
866
867 let src = match self.read_immediate_raw(src)? {
870 Right(src_val) => {
871 assert!(!src.layout().is_unsized());
872 assert!(!dest.layout().is_unsized());
873 assert_eq!(src.layout().size, dest.layout().size);
874 return if layout_compat {
876 self.write_immediate_no_validate(*src_val, dest)
877 } else {
878 let dest_mem = dest.force_mplace(self)?;
883 self.write_immediate_to_mplace_no_validate(
884 *src_val,
885 src.layout(),
886 dest_mem.mplace,
887 )
888 };
889 }
890 Left(mplace) => mplace,
891 };
892 trace!("copy_op: {:?} <- {:?}: {}", *dest, src, dest.layout().ty);
894
895 let dest = dest.force_mplace(self)?;
896 let Some((dest_size, _)) = self.size_and_align_of_val(&dest)? else {
897 span_bug!(self.cur_span(), "copy_op needs (dynamically) sized values")
898 };
899 if cfg!(debug_assertions) {
900 let src_size = self.size_and_align_of_val(&src)?.unwrap().0;
901 assert_eq!(src_size, dest_size, "Cannot copy differently-sized data");
902 } else {
903 assert_eq!(src.layout.size, dest.layout.size);
905 }
906
907 self.mem_copy(src.ptr(), dest.ptr(), dest_size, true)?;
915 self.check_misalign(src.mplace.misaligned, CheckAlignMsg::BasedOn)?;
916 self.check_misalign(dest.mplace.misaligned, CheckAlignMsg::BasedOn)?;
917 interp_ok(())
918 }
919
920 #[instrument(skip(self), level = "trace")]
925 pub fn force_allocation(
926 &mut self,
927 place: &PlaceTy<'tcx, M::Provenance>,
928 ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> {
929 let mplace = match place.place {
930 Place::Local { local, offset, locals_addr } => {
931 debug_assert_eq!(locals_addr, self.frame().locals_addr());
932 let whole_local = match self.frame_mut().locals[local].access_mut()? {
933 &mut Operand::Immediate(local_val) => {
934 let local_layout = self.layout_of_local(&self.frame(), local, None)?;
940 assert!(local_layout.is_sized(), "unsized locals cannot be immediate");
941 let mplace = self.allocate(local_layout, MemoryKind::Stack)?;
942 if !matches!(local_val, Immediate::Uninit) {
944 self.write_immediate_to_mplace_no_validate(
948 local_val,
949 local_layout,
950 mplace.mplace,
951 )?;
952 }
953 M::after_local_moved_to_memory(self, local, &mplace)?;
954 *self.frame_mut().locals[local].access_mut().unwrap() =
958 Operand::Indirect(mplace.mplace);
959 mplace.mplace
960 }
961 &mut Operand::Indirect(mplace) => mplace, };
963 if let Some(offset) = offset {
964 whole_local.offset_with_meta_(
966 offset,
967 OffsetMode::Wrapping,
968 MemPlaceMeta::None,
969 self,
970 )?
971 } else {
972 whole_local
974 }
975 }
976 Place::Ptr(mplace) => mplace,
977 };
978 interp_ok(MPlaceTy { mplace, layout: place.layout })
980 }
981
982 pub fn allocate_dyn(
983 &mut self,
984 layout: TyAndLayout<'tcx>,
985 kind: MemoryKind<M::MemoryKind>,
986 meta: MemPlaceMeta<M::Provenance>,
987 ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> {
988 let Some((size, align)) = self.size_and_align_from_meta(&meta, &layout)? else {
989 span_bug!(self.cur_span(), "cannot allocate space for `extern` type, size is not known")
990 };
991 let ptr = self.allocate_ptr(size, align, kind, AllocInit::Uninit)?;
992 interp_ok(self.ptr_with_meta_to_mplace(ptr.into(), meta, layout, false))
993 }
994
995 pub fn allocate(
996 &mut self,
997 layout: TyAndLayout<'tcx>,
998 kind: MemoryKind<M::MemoryKind>,
999 ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> {
1000 assert!(layout.is_sized());
1001 self.allocate_dyn(layout, kind, MemPlaceMeta::None)
1002 }
1003
1004 pub fn allocate_bytes_dedup(
1007 &mut self,
1008 bytes: &[u8],
1009 ) -> InterpResult<'tcx, Pointer<M::Provenance>> {
1010 let salt = M::get_global_alloc_salt(self, None);
1011 let id = self.tcx.allocate_bytes_dedup(bytes, salt);
1012
1013 M::adjust_alloc_root_pointer(
1015 &self,
1016 Pointer::from(id),
1017 M::GLOBAL_KIND.map(MemoryKind::Machine),
1018 )
1019 }
1020
1021 pub fn allocate_str_dedup(
1024 &mut self,
1025 s: &str,
1026 ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> {
1027 let bytes = s.as_bytes();
1028 let ptr = self.allocate_bytes_dedup(bytes)?;
1029
1030 let meta = Scalar::from_target_usize(u64::try_from(bytes.len()).unwrap(), self);
1032
1033 let layout = self.layout_of(self.tcx.types.str_).unwrap();
1035
1036 interp_ok(self.ptr_with_meta_to_mplace(
1038 ptr.into(),
1039 MemPlaceMeta::Meta(meta),
1040 layout,
1041 false,
1042 ))
1043 }
1044
1045 pub fn raw_const_to_mplace(
1046 &self,
1047 raw: mir::ConstAlloc<'tcx>,
1048 ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> {
1049 let _ = self.tcx.global_alloc(raw.alloc_id);
1051 let ptr = self.global_root_pointer(Pointer::from(raw.alloc_id))?;
1052 let layout = self.layout_of(raw.ty)?;
1053 interp_ok(self.ptr_to_mplace(ptr.into(), layout))
1054 }
1055}
1056
1057#[cfg(target_pointer_width = "64")]
1059mod size_asserts {
1060 use rustc_data_structures::static_assert_size;
1061
1062 use super::*;
1063 static_assert_size!(MPlaceTy<'_>, 64);
1065 static_assert_size!(MemPlace, 48);
1066 static_assert_size!(MemPlaceMeta, 24);
1067 static_assert_size!(Place, 48);
1068 static_assert_size!(PlaceTy<'_>, 64);
1069 }