rustc_const_eval/interpret/
place.rs

1//! Computations on places -- field projections, going from mir::Place, and writing
2//! into a place.
3//! All high-level functions to write to memory work on places as destinations.
4
5use std::assert_matches::assert_matches;
6
7use either::{Either, Left, Right};
8use rustc_abi::{BackendRepr, HasDataLayout, Size};
9use rustc_middle::ty::Ty;
10use rustc_middle::ty::layout::TyAndLayout;
11use rustc_middle::{bug, mir, span_bug};
12use tracing::field::Empty;
13use tracing::{instrument, trace};
14
15use super::{
16    AllocInit, AllocRef, AllocRefMut, CheckAlignMsg, CtfeProvenance, ImmTy, Immediate, InterpCx,
17    InterpResult, Machine, MemoryKind, Misalignment, OffsetMode, OpTy, Operand, Pointer,
18    Projectable, Provenance, Scalar, alloc_range, interp_ok, mir_assign_valid_types,
19};
20use crate::enter_trace_span;
21
22#[derive(Copy, Clone, Hash, PartialEq, Eq, Debug)]
23/// Information required for the sound usage of a `MemPlace`.
24pub enum MemPlaceMeta<Prov: Provenance = CtfeProvenance> {
25    /// The unsized payload (e.g. length for slices or vtable pointer for trait objects).
26    Meta(Scalar<Prov>),
27    /// `Sized` types or unsized `extern type`
28    None,
29}
30
31impl<Prov: Provenance> MemPlaceMeta<Prov> {
32    #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
33    pub fn unwrap_meta(self) -> Scalar<Prov> {
34        match self {
35            Self::Meta(s) => s,
36            Self::None => {
37                bug!("expected wide pointer extra data (e.g. slice length or trait object vtable)")
38            }
39        }
40    }
41
42    #[inline(always)]
43    pub fn has_meta(self) -> bool {
44        match self {
45            Self::Meta(_) => true,
46            Self::None => false,
47        }
48    }
49}
50
51#[derive(Copy, Clone, Hash, PartialEq, Eq, Debug)]
52pub(super) struct MemPlace<Prov: Provenance = CtfeProvenance> {
53    /// The pointer can be a pure integer, with the `None` provenance.
54    pub ptr: Pointer<Option<Prov>>,
55    /// Metadata for unsized places. Interpretation is up to the type.
56    /// Must not be present for sized types, but can be missing for unsized types
57    /// (e.g., `extern type`).
58    pub meta: MemPlaceMeta<Prov>,
59    /// Stores whether this place was created based on a sufficiently aligned pointer.
60    misaligned: Option<Misalignment>,
61}
62
63impl<Prov: Provenance> MemPlace<Prov> {
64    /// Adjust the provenance of the main pointer (metadata is unaffected).
65    fn map_provenance(self, f: impl FnOnce(Prov) -> Prov) -> Self {
66        MemPlace { ptr: self.ptr.map_provenance(|p| p.map(f)), ..self }
67    }
68
69    /// Turn a mplace into a (thin or wide) pointer, as a reference, pointing to the same space.
70    #[inline]
71    fn to_ref(self, cx: &impl HasDataLayout) -> Immediate<Prov> {
72        Immediate::new_pointer_with_meta(self.ptr, self.meta, cx)
73    }
74
75    #[inline]
76    // Not called `offset_with_meta` to avoid confusion with the trait method.
77    fn offset_with_meta_<'tcx, M: Machine<'tcx, Provenance = Prov>>(
78        self,
79        offset: Size,
80        mode: OffsetMode,
81        meta: MemPlaceMeta<Prov>,
82        ecx: &InterpCx<'tcx, M>,
83    ) -> InterpResult<'tcx, Self> {
84        debug_assert!(
85            !meta.has_meta() || self.meta.has_meta(),
86            "cannot use `offset_with_meta` to add metadata to a place"
87        );
88        let ptr = match mode {
89            OffsetMode::Inbounds => {
90                ecx.ptr_offset_inbounds(self.ptr, offset.bytes().try_into().unwrap())?
91            }
92            OffsetMode::Wrapping => self.ptr.wrapping_offset(offset, ecx),
93        };
94        interp_ok(MemPlace { ptr, meta, misaligned: self.misaligned })
95    }
96}
97
98/// A MemPlace with its layout. Constructing it is only possible in this module.
99#[derive(Clone, Hash, Eq, PartialEq)]
100pub struct MPlaceTy<'tcx, Prov: Provenance = CtfeProvenance> {
101    mplace: MemPlace<Prov>,
102    pub layout: TyAndLayout<'tcx>,
103}
104
105impl<Prov: Provenance> std::fmt::Debug for MPlaceTy<'_, Prov> {
106    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
107        // Printing `layout` results in too much noise; just print a nice version of the type.
108        f.debug_struct("MPlaceTy")
109            .field("mplace", &self.mplace)
110            .field("ty", &format_args!("{}", self.layout.ty))
111            .finish()
112    }
113}
114
115impl<'tcx, Prov: Provenance> MPlaceTy<'tcx, Prov> {
116    /// Produces a MemPlace that works for ZST but nothing else.
117    /// Conceptually this is a new allocation, but it doesn't actually create an allocation so you
118    /// don't need to worry about memory leaks.
119    #[inline]
120    pub fn fake_alloc_zst(layout: TyAndLayout<'tcx>) -> Self {
121        assert!(layout.is_zst());
122        let align = layout.align.abi;
123        let ptr = Pointer::without_provenance(align.bytes()); // no provenance, absolute address
124        MPlaceTy { mplace: MemPlace { ptr, meta: MemPlaceMeta::None, misaligned: None }, layout }
125    }
126
127    /// Adjust the provenance of the main pointer (metadata is unaffected).
128    pub fn map_provenance(self, f: impl FnOnce(Prov) -> Prov) -> Self {
129        MPlaceTy { mplace: self.mplace.map_provenance(f), ..self }
130    }
131
132    #[inline(always)]
133    pub(super) fn mplace(&self) -> &MemPlace<Prov> {
134        &self.mplace
135    }
136
137    #[inline(always)]
138    pub fn ptr(&self) -> Pointer<Option<Prov>> {
139        self.mplace.ptr
140    }
141
142    #[inline(always)]
143    pub fn to_ref(&self, cx: &impl HasDataLayout) -> Immediate<Prov> {
144        self.mplace.to_ref(cx)
145    }
146}
147
148impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for MPlaceTy<'tcx, Prov> {
149    #[inline(always)]
150    fn layout(&self) -> TyAndLayout<'tcx> {
151        self.layout
152    }
153
154    #[inline(always)]
155    fn meta(&self) -> MemPlaceMeta<Prov> {
156        self.mplace.meta
157    }
158
159    fn offset_with_meta<M: Machine<'tcx, Provenance = Prov>>(
160        &self,
161        offset: Size,
162        mode: OffsetMode,
163        meta: MemPlaceMeta<Prov>,
164        layout: TyAndLayout<'tcx>,
165        ecx: &InterpCx<'tcx, M>,
166    ) -> InterpResult<'tcx, Self> {
167        interp_ok(MPlaceTy {
168            mplace: self.mplace.offset_with_meta_(offset, mode, meta, ecx)?,
169            layout,
170        })
171    }
172
173    #[inline(always)]
174    fn to_op<M: Machine<'tcx, Provenance = Prov>>(
175        &self,
176        _ecx: &InterpCx<'tcx, M>,
177    ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
178        interp_ok(self.clone().into())
179    }
180}
181
182#[derive(Copy, Clone, Debug)]
183pub(super) enum Place<Prov: Provenance = CtfeProvenance> {
184    /// A place referring to a value allocated in the `Memory` system.
185    Ptr(MemPlace<Prov>),
186
187    /// To support alloc-free locals, we are able to write directly to a local. The offset indicates
188    /// where in the local this place is located; if it is `None`, no projection has been applied
189    /// and the type of the place is exactly the type of the local.
190    /// Such projections are meaningful even if the offset is 0, since they can change layouts.
191    /// (Without that optimization, we'd just always be a `MemPlace`.)
192    /// `Local` places always refer to the current stack frame, so they are unstable under
193    /// function calls/returns and switching betweens stacks of different threads!
194    /// We carry around the address of the `locals` buffer of the correct stack frame as a sanity
195    /// check to be able to catch some cases of using a dangling `Place`.
196    ///
197    /// This variant shall not be used for unsized types -- those must always live in memory.
198    Local { local: mir::Local, offset: Option<Size>, locals_addr: usize },
199}
200
201/// An evaluated place, together with its type.
202///
203/// This may reference a stack frame by its index, so `PlaceTy` should generally not be kept around
204/// for longer than a single operation. Popping and then pushing a stack frame can make `PlaceTy`
205/// point to the wrong destination. If the interpreter has multiple stacks, stack switching will
206/// also invalidate a `PlaceTy`.
207#[derive(Clone)]
208pub struct PlaceTy<'tcx, Prov: Provenance = CtfeProvenance> {
209    place: Place<Prov>, // Keep this private; it helps enforce invariants.
210    pub layout: TyAndLayout<'tcx>,
211}
212
213impl<Prov: Provenance> std::fmt::Debug for PlaceTy<'_, Prov> {
214    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
215        // Printing `layout` results in too much noise; just print a nice version of the type.
216        f.debug_struct("PlaceTy")
217            .field("place", &self.place)
218            .field("ty", &format_args!("{}", self.layout.ty))
219            .finish()
220    }
221}
222
223impl<'tcx, Prov: Provenance> From<MPlaceTy<'tcx, Prov>> for PlaceTy<'tcx, Prov> {
224    #[inline(always)]
225    fn from(mplace: MPlaceTy<'tcx, Prov>) -> Self {
226        PlaceTy { place: Place::Ptr(mplace.mplace), layout: mplace.layout }
227    }
228}
229
230impl<'tcx, Prov: Provenance> PlaceTy<'tcx, Prov> {
231    #[inline(always)]
232    pub(super) fn place(&self) -> &Place<Prov> {
233        &self.place
234    }
235
236    /// A place is either an mplace or some local.
237    #[inline(always)]
238    pub fn as_mplace_or_local(
239        &self,
240    ) -> Either<MPlaceTy<'tcx, Prov>, (mir::Local, Option<Size>, usize, TyAndLayout<'tcx>)> {
241        match self.place {
242            Place::Ptr(mplace) => Left(MPlaceTy { mplace, layout: self.layout }),
243            Place::Local { local, offset, locals_addr } => {
244                Right((local, offset, locals_addr, self.layout))
245            }
246        }
247    }
248
249    #[inline(always)]
250    #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
251    pub fn assert_mem_place(&self) -> MPlaceTy<'tcx, Prov> {
252        self.as_mplace_or_local().left().unwrap_or_else(|| {
253            bug!(
254                "PlaceTy of type {} was a local when it was expected to be an MPlace",
255                self.layout.ty
256            )
257        })
258    }
259}
260
261impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for PlaceTy<'tcx, Prov> {
262    #[inline(always)]
263    fn layout(&self) -> TyAndLayout<'tcx> {
264        self.layout
265    }
266
267    #[inline]
268    fn meta(&self) -> MemPlaceMeta<Prov> {
269        match self.as_mplace_or_local() {
270            Left(mplace) => mplace.meta(),
271            Right(_) => {
272                debug_assert!(self.layout.is_sized(), "unsized locals should live in memory");
273                MemPlaceMeta::None
274            }
275        }
276    }
277
278    fn offset_with_meta<M: Machine<'tcx, Provenance = Prov>>(
279        &self,
280        offset: Size,
281        mode: OffsetMode,
282        meta: MemPlaceMeta<Prov>,
283        layout: TyAndLayout<'tcx>,
284        ecx: &InterpCx<'tcx, M>,
285    ) -> InterpResult<'tcx, Self> {
286        interp_ok(match self.as_mplace_or_local() {
287            Left(mplace) => mplace.offset_with_meta(offset, mode, meta, layout, ecx)?.into(),
288            Right((local, old_offset, locals_addr, _)) => {
289                debug_assert!(layout.is_sized(), "unsized locals should live in memory");
290                assert_matches!(meta, MemPlaceMeta::None); // we couldn't store it anyway...
291                // `Place::Local` are always in-bounds of their surrounding local, so we can just
292                // check directly if this remains in-bounds. This cannot actually be violated since
293                // projections are type-checked and bounds-checked.
294                assert!(offset + layout.size <= self.layout.size);
295
296                // Size `+`, ensures no overflow.
297                let new_offset = old_offset.unwrap_or(Size::ZERO) + offset;
298
299                PlaceTy {
300                    place: Place::Local { local, offset: Some(new_offset), locals_addr },
301                    layout,
302                }
303            }
304        })
305    }
306
307    #[inline(always)]
308    fn to_op<M: Machine<'tcx, Provenance = Prov>>(
309        &self,
310        ecx: &InterpCx<'tcx, M>,
311    ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
312        ecx.place_to_op(self)
313    }
314}
315
316// These are defined here because they produce a place.
317impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> {
318    #[inline(always)]
319    pub fn as_mplace_or_imm(&self) -> Either<MPlaceTy<'tcx, Prov>, ImmTy<'tcx, Prov>> {
320        match self.op() {
321            Operand::Indirect(mplace) => Left(MPlaceTy { mplace: *mplace, layout: self.layout }),
322            Operand::Immediate(imm) => Right(ImmTy::from_immediate(*imm, self.layout)),
323        }
324    }
325
326    #[inline(always)]
327    #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
328    pub fn assert_mem_place(&self) -> MPlaceTy<'tcx, Prov> {
329        self.as_mplace_or_imm().left().unwrap_or_else(|| {
330            bug!(
331                "OpTy of type {} was immediate when it was expected to be an MPlace",
332                self.layout.ty
333            )
334        })
335    }
336}
337
338/// The `Weiteable` trait describes interpreter values that can be written to.
339pub trait Writeable<'tcx, Prov: Provenance>: Projectable<'tcx, Prov> {
340    fn to_place(&self) -> PlaceTy<'tcx, Prov>;
341
342    fn force_mplace<M: Machine<'tcx, Provenance = Prov>>(
343        &self,
344        ecx: &mut InterpCx<'tcx, M>,
345    ) -> InterpResult<'tcx, MPlaceTy<'tcx, Prov>>;
346}
347
348impl<'tcx, Prov: Provenance> Writeable<'tcx, Prov> for PlaceTy<'tcx, Prov> {
349    #[inline(always)]
350    fn to_place(&self) -> PlaceTy<'tcx, Prov> {
351        self.clone()
352    }
353
354    #[inline(always)]
355    fn force_mplace<M: Machine<'tcx, Provenance = Prov>>(
356        &self,
357        ecx: &mut InterpCx<'tcx, M>,
358    ) -> InterpResult<'tcx, MPlaceTy<'tcx, Prov>> {
359        ecx.force_allocation(self)
360    }
361}
362
363impl<'tcx, Prov: Provenance> Writeable<'tcx, Prov> for MPlaceTy<'tcx, Prov> {
364    #[inline(always)]
365    fn to_place(&self) -> PlaceTy<'tcx, Prov> {
366        self.clone().into()
367    }
368
369    #[inline(always)]
370    fn force_mplace<M: Machine<'tcx, Provenance = Prov>>(
371        &self,
372        _ecx: &mut InterpCx<'tcx, M>,
373    ) -> InterpResult<'tcx, MPlaceTy<'tcx, Prov>> {
374        interp_ok(self.clone())
375    }
376}
377
378// FIXME: Working around https://github.com/rust-lang/rust/issues/54385
379impl<'tcx, Prov, M> InterpCx<'tcx, M>
380where
381    Prov: Provenance,
382    M: Machine<'tcx, Provenance = Prov>,
383{
384    fn ptr_with_meta_to_mplace(
385        &self,
386        ptr: Pointer<Option<M::Provenance>>,
387        meta: MemPlaceMeta<M::Provenance>,
388        layout: TyAndLayout<'tcx>,
389        unaligned: bool,
390    ) -> MPlaceTy<'tcx, M::Provenance> {
391        let misaligned =
392            if unaligned { None } else { self.is_ptr_misaligned(ptr, layout.align.abi) };
393        MPlaceTy { mplace: MemPlace { ptr, meta, misaligned }, layout }
394    }
395
396    pub fn ptr_to_mplace(
397        &self,
398        ptr: Pointer<Option<M::Provenance>>,
399        layout: TyAndLayout<'tcx>,
400    ) -> MPlaceTy<'tcx, M::Provenance> {
401        assert!(layout.is_sized());
402        self.ptr_with_meta_to_mplace(ptr, MemPlaceMeta::None, layout, /*unaligned*/ false)
403    }
404
405    pub fn ptr_to_mplace_unaligned(
406        &self,
407        ptr: Pointer<Option<M::Provenance>>,
408        layout: TyAndLayout<'tcx>,
409    ) -> MPlaceTy<'tcx, M::Provenance> {
410        assert!(layout.is_sized());
411        self.ptr_with_meta_to_mplace(ptr, MemPlaceMeta::None, layout, /*unaligned*/ true)
412    }
413
414    /// Take a value, which represents a (thin or wide) reference, and make it a place.
415    /// Alignment is just based on the type. This is the inverse of `mplace_to_ref()`.
416    ///
417    /// Only call this if you are sure the place is "valid" (aligned and inbounds), or do not
418    /// want to ever use the place for memory access!
419    /// Generally prefer `deref_pointer`.
420    pub fn ref_to_mplace(
421        &self,
422        val: &ImmTy<'tcx, M::Provenance>,
423    ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> {
424        let pointee_type =
425            val.layout.ty.builtin_deref(true).expect("`ref_to_mplace` called on non-ptr type");
426        let layout = self.layout_of(pointee_type)?;
427        let (ptr, meta) = val.to_scalar_and_meta();
428
429        // `ref_to_mplace` is called on raw pointers even if they don't actually get dereferenced;
430        // we hence can't call `size_and_align_of` since that asserts more validity than we want.
431        let ptr = ptr.to_pointer(self)?;
432        interp_ok(self.ptr_with_meta_to_mplace(ptr, meta, layout, /*unaligned*/ false))
433    }
434
435    /// Turn a mplace into a (thin or wide) mutable raw pointer, pointing to the same space.
436    /// `align` information is lost!
437    /// This is the inverse of `ref_to_mplace`.
438    pub fn mplace_to_ref(
439        &self,
440        mplace: &MPlaceTy<'tcx, M::Provenance>,
441    ) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> {
442        let imm = mplace.mplace.to_ref(self);
443        let layout = self.layout_of(Ty::new_mut_ptr(self.tcx.tcx, mplace.layout.ty))?;
444        interp_ok(ImmTy::from_immediate(imm, layout))
445    }
446
447    /// Take an operand, representing a pointer, and dereference it to a place.
448    /// Corresponds to the `*` operator in Rust.
449    #[instrument(skip(self), level = "trace")]
450    pub fn deref_pointer(
451        &self,
452        src: &impl Projectable<'tcx, M::Provenance>,
453    ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> {
454        if src.layout().ty.is_box() {
455            // Derefer should have removed all Box derefs.
456            // Some `Box` are not immediates (if they have a custom allocator)
457            // so the code below would fail.
458            bug!("dereferencing {}", src.layout().ty);
459        }
460
461        let val = self.read_immediate(src)?;
462        trace!("deref to {} on {:?}", val.layout.ty, *val);
463
464        let mplace = self.ref_to_mplace(&val)?;
465        interp_ok(mplace)
466    }
467
468    #[inline]
469    pub(super) fn get_place_alloc(
470        &self,
471        mplace: &MPlaceTy<'tcx, M::Provenance>,
472    ) -> InterpResult<'tcx, Option<AllocRef<'_, 'tcx, M::Provenance, M::AllocExtra, M::Bytes>>>
473    {
474        let (size, _align) = self
475            .size_and_align_of_val(mplace)?
476            .unwrap_or((mplace.layout.size, mplace.layout.align.abi));
477        // We check alignment separately, and *after* checking everything else.
478        // If an access is both OOB and misaligned, we want to see the bounds error.
479        let a = self.get_ptr_alloc(mplace.ptr(), size)?;
480        self.check_misalign(mplace.mplace.misaligned, CheckAlignMsg::BasedOn)?;
481        interp_ok(a)
482    }
483
484    #[inline]
485    pub(super) fn get_place_alloc_mut(
486        &mut self,
487        mplace: &MPlaceTy<'tcx, M::Provenance>,
488    ) -> InterpResult<'tcx, Option<AllocRefMut<'_, 'tcx, M::Provenance, M::AllocExtra, M::Bytes>>>
489    {
490        let (size, _align) = self
491            .size_and_align_of_val(mplace)?
492            .unwrap_or((mplace.layout.size, mplace.layout.align.abi));
493        // We check alignment separately, and raise that error *after* checking everything else.
494        // If an access is both OOB and misaligned, we want to see the bounds error.
495        // However we have to call `check_misalign` first to make the borrow checker happy.
496        let misalign_res = self.check_misalign(mplace.mplace.misaligned, CheckAlignMsg::BasedOn);
497        // An error from get_ptr_alloc_mut takes precedence.
498        let (a, ()) = self.get_ptr_alloc_mut(mplace.ptr(), size).and(misalign_res)?;
499        interp_ok(a)
500    }
501
502    /// Turn a local in the current frame into a place.
503    pub fn local_to_place(
504        &self,
505        local: mir::Local,
506    ) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> {
507        let frame = self.frame();
508        let layout = self.layout_of_local(frame, local, None)?;
509        let place = if layout.is_sized() {
510            // We can just always use the `Local` for sized values.
511            Place::Local { local, offset: None, locals_addr: frame.locals_addr() }
512        } else {
513            // Other parts of the system rely on `Place::Local` never being unsized.
514            match frame.locals[local].access()? {
515                Operand::Immediate(_) => bug!(),
516                Operand::Indirect(mplace) => Place::Ptr(*mplace),
517            }
518        };
519        interp_ok(PlaceTy { place, layout })
520    }
521
522    /// Computes a place. You should only use this if you intend to write into this
523    /// place; for reading, a more efficient alternative is `eval_place_to_op`.
524    #[instrument(skip(self), level = "trace")]
525    pub fn eval_place(
526        &self,
527        mir_place: mir::Place<'tcx>,
528    ) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> {
529        let _span =
530            enter_trace_span!(M, step::eval_place, ?mir_place, tracing_separate_thread = Empty);
531
532        let mut place = self.local_to_place(mir_place.local)?;
533        // Using `try_fold` turned out to be bad for performance, hence the loop.
534        for elem in mir_place.projection.iter() {
535            place = self.project(&place, elem)?
536        }
537
538        trace!("{:?}", self.dump_place(&place));
539        // Sanity-check the type we ended up with.
540        if cfg!(debug_assertions) {
541            let normalized_place_ty = self
542                .instantiate_from_current_frame_and_normalize_erasing_regions(
543                    mir_place.ty(&self.frame().body.local_decls, *self.tcx).ty,
544                )?;
545            if !mir_assign_valid_types(
546                *self.tcx,
547                self.typing_env,
548                self.layout_of(normalized_place_ty)?,
549                place.layout,
550            ) {
551                span_bug!(
552                    self.cur_span(),
553                    "eval_place of a MIR place with type {} produced an interpreter place with type {}",
554                    normalized_place_ty,
555                    place.layout.ty,
556                )
557            }
558        }
559        interp_ok(place)
560    }
561
562    /// Given a place, returns either the underlying mplace or a reference to where the value of
563    /// this place is stored.
564    #[inline(always)]
565    fn as_mplace_or_mutable_local(
566        &mut self,
567        place: &PlaceTy<'tcx, M::Provenance>,
568    ) -> InterpResult<
569        'tcx,
570        Either<
571            MPlaceTy<'tcx, M::Provenance>,
572            (&mut Immediate<M::Provenance>, TyAndLayout<'tcx>, mir::Local),
573        >,
574    > {
575        interp_ok(match place.to_place().as_mplace_or_local() {
576            Left(mplace) => Left(mplace),
577            Right((local, offset, locals_addr, layout)) => {
578                if offset.is_some() {
579                    // This has been projected to a part of this local, or had the type changed.
580                    // FIXME: there are cases where we could still avoid allocating an mplace.
581                    Left(place.force_mplace(self)?)
582                } else {
583                    debug_assert_eq!(locals_addr, self.frame().locals_addr());
584                    debug_assert_eq!(self.layout_of_local(self.frame(), local, None)?, layout);
585                    match self.frame_mut().locals[local].access_mut()? {
586                        Operand::Indirect(mplace) => {
587                            // The local is in memory.
588                            Left(MPlaceTy { mplace: *mplace, layout })
589                        }
590                        Operand::Immediate(local_val) => {
591                            // The local still has the optimized representation.
592                            Right((local_val, layout, local))
593                        }
594                    }
595                }
596            }
597        })
598    }
599
600    /// Write an immediate to a place
601    #[inline(always)]
602    #[instrument(skip(self), level = "trace")]
603    pub fn write_immediate(
604        &mut self,
605        src: Immediate<M::Provenance>,
606        dest: &impl Writeable<'tcx, M::Provenance>,
607    ) -> InterpResult<'tcx> {
608        self.write_immediate_no_validate(src, dest)?;
609
610        if M::enforce_validity(self, dest.layout()) {
611            // Data got changed, better make sure it matches the type!
612            // Also needed to reset padding.
613            self.validate_operand(
614                &dest.to_place(),
615                M::enforce_validity_recursively(self, dest.layout()),
616                /*reset_provenance_and_padding*/ true,
617            )?;
618        }
619
620        interp_ok(())
621    }
622
623    /// Write a scalar to a place
624    #[inline(always)]
625    pub fn write_scalar(
626        &mut self,
627        val: impl Into<Scalar<M::Provenance>>,
628        dest: &impl Writeable<'tcx, M::Provenance>,
629    ) -> InterpResult<'tcx> {
630        self.write_immediate(Immediate::Scalar(val.into()), dest)
631    }
632
633    /// Write a pointer to a place
634    #[inline(always)]
635    pub fn write_pointer(
636        &mut self,
637        ptr: impl Into<Pointer<Option<M::Provenance>>>,
638        dest: &impl Writeable<'tcx, M::Provenance>,
639    ) -> InterpResult<'tcx> {
640        self.write_scalar(Scalar::from_maybe_pointer(ptr.into(), self), dest)
641    }
642
643    /// Write an immediate to a place.
644    /// If you use this you are responsible for validating that things got copied at the
645    /// right type.
646    pub(super) fn write_immediate_no_validate(
647        &mut self,
648        src: Immediate<M::Provenance>,
649        dest: &impl Writeable<'tcx, M::Provenance>,
650    ) -> InterpResult<'tcx> {
651        assert!(dest.layout().is_sized(), "Cannot write unsized immediate data");
652
653        match self.as_mplace_or_mutable_local(&dest.to_place())? {
654            Right((local_val, local_layout, local)) => {
655                // Local can be updated in-place.
656                *local_val = src;
657                // Call the machine hook (the data race detector needs to know about this write).
658                if !self.validation_in_progress() {
659                    M::after_local_write(self, local, /*storage_live*/ false)?;
660                }
661                // Double-check that the value we are storing and the local fit to each other.
662                // Things can ge wrong in quite weird ways when this is violated.
663                // Unfortunately this is too expensive to do in release builds.
664                if cfg!(debug_assertions) {
665                    src.assert_matches_abi(
666                        local_layout.backend_repr,
667                        "invalid immediate for given destination place",
668                        self,
669                    );
670                }
671            }
672            Left(mplace) => {
673                self.write_immediate_to_mplace_no_validate(src, mplace.layout, mplace.mplace)?;
674            }
675        }
676        interp_ok(())
677    }
678
679    /// Write an immediate to memory.
680    /// If you use this you are responsible for validating that things got copied at the
681    /// right layout.
682    fn write_immediate_to_mplace_no_validate(
683        &mut self,
684        value: Immediate<M::Provenance>,
685        layout: TyAndLayout<'tcx>,
686        dest: MemPlace<M::Provenance>,
687    ) -> InterpResult<'tcx> {
688        // We use the sizes from `value` below.
689        // Ensure that matches the type of the place it is written to.
690        value.assert_matches_abi(
691            layout.backend_repr,
692            "invalid immediate for given destination place",
693            self,
694        );
695        // Note that it is really important that the type here is the right one, and matches the
696        // type things are read at. In case `value` is a `ScalarPair`, we don't do any magic here
697        // to handle padding properly, which is only correct if we never look at this data with the
698        // wrong type.
699
700        let tcx = *self.tcx;
701        let Some(mut alloc) = self.get_place_alloc_mut(&MPlaceTy { mplace: dest, layout })? else {
702            // zero-sized access
703            return interp_ok(());
704        };
705
706        match value {
707            Immediate::Scalar(scalar) => {
708                alloc.write_scalar(alloc_range(Size::ZERO, scalar.size()), scalar)
709            }
710            Immediate::ScalarPair(a_val, b_val) => {
711                let BackendRepr::ScalarPair(a, b) = layout.backend_repr else {
712                    span_bug!(
713                        self.cur_span(),
714                        "write_immediate_to_mplace: invalid ScalarPair layout: {:#?}",
715                        layout
716                    )
717                };
718                let b_offset = a.size(&tcx).align_to(b.align(&tcx).abi);
719                assert!(b_offset.bytes() > 0); // in `operand_field` we use the offset to tell apart the fields
720
721                // It is tempting to verify `b_offset` against `layout.fields.offset(1)`,
722                // but that does not work: We could be a newtype around a pair, then the
723                // fields do not match the `ScalarPair` components.
724
725                alloc.write_scalar(alloc_range(Size::ZERO, a_val.size()), a_val)?;
726                alloc.write_scalar(alloc_range(b_offset, b_val.size()), b_val)?;
727                // We don't have to reset padding here, `write_immediate` will anyway do a validation run.
728                interp_ok(())
729            }
730            Immediate::Uninit => alloc.write_uninit_full(),
731        }
732    }
733
734    pub fn write_uninit(
735        &mut self,
736        dest: &impl Writeable<'tcx, M::Provenance>,
737    ) -> InterpResult<'tcx> {
738        match self.as_mplace_or_mutable_local(&dest.to_place())? {
739            Right((local_val, _local_layout, local)) => {
740                *local_val = Immediate::Uninit;
741                // Call the machine hook (the data race detector needs to know about this write).
742                if !self.validation_in_progress() {
743                    M::after_local_write(self, local, /*storage_live*/ false)?;
744                }
745            }
746            Left(mplace) => {
747                let Some(mut alloc) = self.get_place_alloc_mut(&mplace)? else {
748                    // Zero-sized access
749                    return interp_ok(());
750                };
751                alloc.write_uninit_full()?;
752            }
753        }
754        interp_ok(())
755    }
756
757    /// Remove all provenance in the given place.
758    pub fn clear_provenance(
759        &mut self,
760        dest: &impl Writeable<'tcx, M::Provenance>,
761    ) -> InterpResult<'tcx> {
762        match self.as_mplace_or_mutable_local(&dest.to_place())? {
763            Right((local_val, _local_layout, local)) => {
764                local_val.clear_provenance()?;
765                // Call the machine hook (the data race detector needs to know about this write).
766                if !self.validation_in_progress() {
767                    M::after_local_write(self, local, /*storage_live*/ false)?;
768                }
769            }
770            Left(mplace) => {
771                let Some(mut alloc) = self.get_place_alloc_mut(&mplace)? else {
772                    // Zero-sized access
773                    return interp_ok(());
774                };
775                alloc.clear_provenance()?;
776            }
777        }
778        interp_ok(())
779    }
780
781    /// Copies the data from an operand to a place.
782    /// The layouts of the `src` and `dest` may disagree.
783    #[inline(always)]
784    pub fn copy_op_allow_transmute(
785        &mut self,
786        src: &impl Projectable<'tcx, M::Provenance>,
787        dest: &impl Writeable<'tcx, M::Provenance>,
788    ) -> InterpResult<'tcx> {
789        self.copy_op_inner(src, dest, /* allow_transmute */ true)
790    }
791
792    /// Copies the data from an operand to a place.
793    /// `src` and `dest` must have the same layout and the copied value will be validated.
794    #[inline(always)]
795    pub fn copy_op(
796        &mut self,
797        src: &impl Projectable<'tcx, M::Provenance>,
798        dest: &impl Writeable<'tcx, M::Provenance>,
799    ) -> InterpResult<'tcx> {
800        self.copy_op_inner(src, dest, /* allow_transmute */ false)
801    }
802
803    /// Copies the data from an operand to a place.
804    /// `allow_transmute` indicates whether the layouts may disagree.
805    #[inline(always)]
806    #[instrument(skip(self), level = "trace")]
807    fn copy_op_inner(
808        &mut self,
809        src: &impl Projectable<'tcx, M::Provenance>,
810        dest: &impl Writeable<'tcx, M::Provenance>,
811        allow_transmute: bool,
812    ) -> InterpResult<'tcx> {
813        // These are technically *two* typed copies: `src` is a not-yet-loaded value,
814        // so we're doing a typed copy at `src` type from there to some intermediate storage.
815        // And then we're doing a second typed copy from that intermediate storage to `dest`.
816        // But as an optimization, we only make a single direct copy here.
817
818        // Do the actual copy.
819        self.copy_op_no_validate(src, dest, allow_transmute)?;
820
821        if M::enforce_validity(self, dest.layout()) {
822            let dest = dest.to_place();
823            // Given that there were two typed copies, we have to ensure this is valid at both types,
824            // and we have to ensure this loses provenance and padding according to both types.
825            // But if the types are identical, we only do one pass.
826            if src.layout().ty != dest.layout().ty {
827                self.validate_operand(
828                    &dest.transmute(src.layout(), self)?,
829                    M::enforce_validity_recursively(self, src.layout()),
830                    /*reset_provenance_and_padding*/ true,
831                )?;
832            }
833            self.validate_operand(
834                &dest,
835                M::enforce_validity_recursively(self, dest.layout()),
836                /*reset_provenance_and_padding*/ true,
837            )?;
838        }
839
840        interp_ok(())
841    }
842
843    /// Copies the data from an operand to a place.
844    /// `allow_transmute` indicates whether the layouts may disagree.
845    /// Also, if you use this you are responsible for validating that things get copied at the
846    /// right type.
847    #[instrument(skip(self), level = "trace")]
848    fn copy_op_no_validate(
849        &mut self,
850        src: &impl Projectable<'tcx, M::Provenance>,
851        dest: &impl Writeable<'tcx, M::Provenance>,
852        allow_transmute: bool,
853    ) -> InterpResult<'tcx> {
854        // We do NOT compare the types for equality, because well-typed code can
855        // actually "transmute" `&mut T` to `&T` in an assignment without a cast.
856        let layout_compat =
857            mir_assign_valid_types(*self.tcx, self.typing_env, src.layout(), dest.layout());
858        if !allow_transmute && !layout_compat {
859            span_bug!(
860                self.cur_span(),
861                "type mismatch when copying!\nsrc: {},\ndest: {}",
862                src.layout().ty,
863                dest.layout().ty,
864            );
865        }
866
867        // Let us see if the layout is simple so we take a shortcut,
868        // avoid force_allocation.
869        let src = match self.read_immediate_raw(src)? {
870            Right(src_val) => {
871                assert!(!src.layout().is_unsized());
872                assert!(!dest.layout().is_unsized());
873                assert_eq!(src.layout().size, dest.layout().size);
874                // Yay, we got a value that we can write directly.
875                return if layout_compat {
876                    self.write_immediate_no_validate(*src_val, dest)
877                } else {
878                    // This is tricky. The problematic case is `ScalarPair`: the `src_val` was
879                    // loaded using the offsets defined by `src.layout`. When we put this back into
880                    // the destination, we have to use the same offsets! So (a) we make sure we
881                    // write back to memory, and (b) we use `dest` *with the source layout*.
882                    let dest_mem = dest.force_mplace(self)?;
883                    self.write_immediate_to_mplace_no_validate(
884                        *src_val,
885                        src.layout(),
886                        dest_mem.mplace,
887                    )
888                };
889            }
890            Left(mplace) => mplace,
891        };
892        // Slow path, this does not fit into an immediate. Just memcpy.
893        trace!("copy_op: {:?} <- {:?}: {}", *dest, src, dest.layout().ty);
894
895        let dest = dest.force_mplace(self)?;
896        let Some((dest_size, _)) = self.size_and_align_of_val(&dest)? else {
897            span_bug!(self.cur_span(), "copy_op needs (dynamically) sized values")
898        };
899        if cfg!(debug_assertions) {
900            let src_size = self.size_and_align_of_val(&src)?.unwrap().0;
901            assert_eq!(src_size, dest_size, "Cannot copy differently-sized data");
902        } else {
903            // As a cheap approximation, we compare the fixed parts of the size.
904            assert_eq!(src.layout.size, dest.layout.size);
905        }
906
907        // Setting `nonoverlapping` here only has an effect when we don't hit the fast-path above,
908        // but that should at least match what LLVM does where `memcpy` is also only used when the
909        // type does not have Scalar/ScalarPair layout.
910        // (Or as the `Assign` docs put it, assignments "not producing primitives" must be
911        // non-overlapping.)
912        // We check alignment separately, and *after* checking everything else.
913        // If an access is both OOB and misaligned, we want to see the bounds error.
914        self.mem_copy(src.ptr(), dest.ptr(), dest_size, /*nonoverlapping*/ true)?;
915        self.check_misalign(src.mplace.misaligned, CheckAlignMsg::BasedOn)?;
916        self.check_misalign(dest.mplace.misaligned, CheckAlignMsg::BasedOn)?;
917        interp_ok(())
918    }
919
920    /// Ensures that a place is in memory, and returns where it is.
921    /// If the place currently refers to a local that doesn't yet have a matching allocation,
922    /// create such an allocation.
923    /// This is essentially `force_to_memplace`.
924    #[instrument(skip(self), level = "trace")]
925    pub fn force_allocation(
926        &mut self,
927        place: &PlaceTy<'tcx, M::Provenance>,
928    ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> {
929        let mplace = match place.place {
930            Place::Local { local, offset, locals_addr } => {
931                debug_assert_eq!(locals_addr, self.frame().locals_addr());
932                let whole_local = match self.frame_mut().locals[local].access_mut()? {
933                    &mut Operand::Immediate(local_val) => {
934                        // We need to make an allocation.
935
936                        // We need the layout of the local. We can NOT use the layout we got,
937                        // that might e.g., be an inner field of a struct with `Scalar` layout,
938                        // that has different alignment than the outer field.
939                        let local_layout = self.layout_of_local(&self.frame(), local, None)?;
940                        assert!(local_layout.is_sized(), "unsized locals cannot be immediate");
941                        let mplace = self.allocate(local_layout, MemoryKind::Stack)?;
942                        // Preserve old value. (As an optimization, we can skip this if it was uninit.)
943                        if !matches!(local_val, Immediate::Uninit) {
944                            // We don't have to validate as we can assume the local was already
945                            // valid for its type. We must not use any part of `place` here, that
946                            // could be a projection to a part of the local!
947                            self.write_immediate_to_mplace_no_validate(
948                                local_val,
949                                local_layout,
950                                mplace.mplace,
951                            )?;
952                        }
953                        M::after_local_moved_to_memory(self, local, &mplace)?;
954                        // Now we can call `access_mut` again, asserting it goes well, and actually
955                        // overwrite things. This points to the entire allocation, not just the part
956                        // the place refers to, i.e. we do this before we apply `offset`.
957                        *self.frame_mut().locals[local].access_mut().unwrap() =
958                            Operand::Indirect(mplace.mplace);
959                        mplace.mplace
960                    }
961                    &mut Operand::Indirect(mplace) => mplace, // this already was an indirect local
962                };
963                if let Some(offset) = offset {
964                    // This offset is always inbounds, no need to check it again.
965                    whole_local.offset_with_meta_(
966                        offset,
967                        OffsetMode::Wrapping,
968                        MemPlaceMeta::None,
969                        self,
970                    )?
971                } else {
972                    // Preserve wide place metadata, do not call `offset`.
973                    whole_local
974                }
975            }
976            Place::Ptr(mplace) => mplace,
977        };
978        // Return with the original layout and align, so that the caller can go on
979        interp_ok(MPlaceTy { mplace, layout: place.layout })
980    }
981
982    pub fn allocate_dyn(
983        &mut self,
984        layout: TyAndLayout<'tcx>,
985        kind: MemoryKind<M::MemoryKind>,
986        meta: MemPlaceMeta<M::Provenance>,
987    ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> {
988        let Some((size, align)) = self.size_and_align_from_meta(&meta, &layout)? else {
989            span_bug!(self.cur_span(), "cannot allocate space for `extern` type, size is not known")
990        };
991        let ptr = self.allocate_ptr(size, align, kind, AllocInit::Uninit)?;
992        interp_ok(self.ptr_with_meta_to_mplace(ptr.into(), meta, layout, /*unaligned*/ false))
993    }
994
995    pub fn allocate(
996        &mut self,
997        layout: TyAndLayout<'tcx>,
998        kind: MemoryKind<M::MemoryKind>,
999    ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> {
1000        assert!(layout.is_sized());
1001        self.allocate_dyn(layout, kind, MemPlaceMeta::None)
1002    }
1003
1004    /// Allocates a sequence of bytes in the interpreter's memory with alignment 1.
1005    /// This is allocated in immutable global memory and deduplicated.
1006    pub fn allocate_bytes_dedup(
1007        &mut self,
1008        bytes: &[u8],
1009    ) -> InterpResult<'tcx, Pointer<M::Provenance>> {
1010        let salt = M::get_global_alloc_salt(self, None);
1011        let id = self.tcx.allocate_bytes_dedup(bytes, salt);
1012
1013        // Turn untagged "global" pointers (obtained via `tcx`) into the machine pointer to the allocation.
1014        M::adjust_alloc_root_pointer(
1015            &self,
1016            Pointer::from(id),
1017            M::GLOBAL_KIND.map(MemoryKind::Machine),
1018        )
1019    }
1020
1021    /// Allocates a string in the interpreter's memory, returning it as a (wide) place.
1022    /// This is allocated in immutable global memory and deduplicated.
1023    pub fn allocate_str_dedup(
1024        &mut self,
1025        s: &str,
1026    ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> {
1027        let bytes = s.as_bytes();
1028        let ptr = self.allocate_bytes_dedup(bytes)?;
1029
1030        // Create length metadata for the string.
1031        let meta = Scalar::from_target_usize(u64::try_from(bytes.len()).unwrap(), self);
1032
1033        // Get layout for Rust's str type.
1034        let layout = self.layout_of(self.tcx.types.str_).unwrap();
1035
1036        // Combine pointer and metadata into a wide pointer.
1037        interp_ok(self.ptr_with_meta_to_mplace(
1038            ptr.into(),
1039            MemPlaceMeta::Meta(meta),
1040            layout,
1041            /*unaligned*/ false,
1042        ))
1043    }
1044
1045    pub fn raw_const_to_mplace(
1046        &self,
1047        raw: mir::ConstAlloc<'tcx>,
1048    ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> {
1049        // This must be an allocation in `tcx`
1050        let _ = self.tcx.global_alloc(raw.alloc_id);
1051        let ptr = self.global_root_pointer(Pointer::from(raw.alloc_id))?;
1052        let layout = self.layout_of(raw.ty)?;
1053        interp_ok(self.ptr_to_mplace(ptr.into(), layout))
1054    }
1055}
1056
1057// Some nodes are used a lot. Make sure they don't unintentionally get bigger.
1058#[cfg(target_pointer_width = "64")]
1059mod size_asserts {
1060    use rustc_data_structures::static_assert_size;
1061
1062    use super::*;
1063    // tidy-alphabetical-start
1064    static_assert_size!(MPlaceTy<'_>, 64);
1065    static_assert_size!(MemPlace, 48);
1066    static_assert_size!(MemPlaceMeta, 24);
1067    static_assert_size!(Place, 48);
1068    static_assert_size!(PlaceTy<'_>, 64);
1069    // tidy-alphabetical-end
1070}