rustc_const_eval/interpret/
operand.rs

1//! Functions concerning immediate values and operands, and reading from operands.
2//! All high-level functions to read from memory work on operands as sources.
3
4use std::assert_matches::assert_matches;
5
6use either::{Either, Left, Right};
7use rustc_abi as abi;
8use rustc_abi::{BackendRepr, HasDataLayout, Size};
9use rustc_hir::def::Namespace;
10use rustc_middle::mir::interpret::ScalarSizeMismatch;
11use rustc_middle::ty::layout::{HasTyCtxt, HasTypingEnv, TyAndLayout};
12use rustc_middle::ty::print::{FmtPrinter, PrettyPrinter};
13use rustc_middle::ty::{ConstInt, ScalarInt, Ty, TyCtxt};
14use rustc_middle::{bug, mir, span_bug, ty};
15use rustc_span::DUMMY_SP;
16use tracing::field::Empty;
17use tracing::trace;
18
19use super::{
20    CtfeProvenance, Frame, InterpCx, InterpResult, MPlaceTy, Machine, MemPlace, MemPlaceMeta,
21    OffsetMode, PlaceTy, Pointer, Projectable, Provenance, Scalar, alloc_range, err_ub,
22    from_known_layout, interp_ok, mir_assign_valid_types, throw_ub,
23};
24use crate::enter_trace_span;
25
26/// An `Immediate` represents a single immediate self-contained Rust value.
27///
28/// For optimization of a few very common cases, there is also a representation for a pair of
29/// primitive values (`ScalarPair`). It allows Miri to avoid making allocations for checked binary
30/// operations and wide pointers. This idea was taken from rustc's codegen.
31/// In particular, thanks to `ScalarPair`, arithmetic operations and casts can be entirely
32/// defined on `Immediate`, and do not have to work with a `Place`.
33#[derive(Copy, Clone, Debug)]
34pub enum Immediate<Prov: Provenance = CtfeProvenance> {
35    /// A single scalar value (must have *initialized* `Scalar` ABI).
36    Scalar(Scalar<Prov>),
37    /// A pair of two scalar value (must have `ScalarPair` ABI where both fields are
38    /// `Scalar::Initialized`).
39    ScalarPair(Scalar<Prov>, Scalar<Prov>),
40    /// A value of fully uninitialized memory. Can have arbitrary size and layout, but must be sized.
41    Uninit,
42}
43
44impl<Prov: Provenance> From<Scalar<Prov>> for Immediate<Prov> {
45    #[inline(always)]
46    fn from(val: Scalar<Prov>) -> Self {
47        Immediate::Scalar(val)
48    }
49}
50
51impl<Prov: Provenance> Immediate<Prov> {
52    pub fn new_pointer_with_meta(
53        ptr: Pointer<Option<Prov>>,
54        meta: MemPlaceMeta<Prov>,
55        cx: &impl HasDataLayout,
56    ) -> Self {
57        let ptr = Scalar::from_maybe_pointer(ptr, cx);
58        match meta {
59            MemPlaceMeta::None => Immediate::from(ptr),
60            MemPlaceMeta::Meta(meta) => Immediate::ScalarPair(ptr, meta),
61        }
62    }
63
64    pub fn new_slice(ptr: Pointer<Option<Prov>>, len: u64, cx: &impl HasDataLayout) -> Self {
65        Immediate::ScalarPair(
66            Scalar::from_maybe_pointer(ptr, cx),
67            Scalar::from_target_usize(len, cx),
68        )
69    }
70
71    pub fn new_dyn_trait(
72        val: Pointer<Option<Prov>>,
73        vtable: Pointer<Option<Prov>>,
74        cx: &impl HasDataLayout,
75    ) -> Self {
76        Immediate::ScalarPair(
77            Scalar::from_maybe_pointer(val, cx),
78            Scalar::from_maybe_pointer(vtable, cx),
79        )
80    }
81
82    #[inline]
83    #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
84    pub fn to_scalar(self) -> Scalar<Prov> {
85        match self {
86            Immediate::Scalar(val) => val,
87            Immediate::ScalarPair(..) => bug!("Got a scalar pair where a scalar was expected"),
88            Immediate::Uninit => bug!("Got uninit where a scalar was expected"),
89        }
90    }
91
92    #[inline]
93    #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
94    pub fn to_scalar_int(self) -> ScalarInt {
95        self.to_scalar().try_to_scalar_int().unwrap()
96    }
97
98    #[inline]
99    #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
100    pub fn to_scalar_pair(self) -> (Scalar<Prov>, Scalar<Prov>) {
101        match self {
102            Immediate::ScalarPair(val1, val2) => (val1, val2),
103            Immediate::Scalar(..) => bug!("Got a scalar where a scalar pair was expected"),
104            Immediate::Uninit => bug!("Got uninit where a scalar pair was expected"),
105        }
106    }
107
108    /// Returns the scalar from the first component and optionally the 2nd component as metadata.
109    #[inline]
110    #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
111    pub fn to_scalar_and_meta(self) -> (Scalar<Prov>, MemPlaceMeta<Prov>) {
112        match self {
113            Immediate::ScalarPair(val1, val2) => (val1, MemPlaceMeta::Meta(val2)),
114            Immediate::Scalar(val) => (val, MemPlaceMeta::None),
115            Immediate::Uninit => bug!("Got uninit where a scalar or scalar pair was expected"),
116        }
117    }
118
119    /// Assert that this immediate is a valid value for the given ABI.
120    pub fn assert_matches_abi(self, abi: BackendRepr, msg: &str, cx: &impl HasDataLayout) {
121        match (self, abi) {
122            (Immediate::Scalar(scalar), BackendRepr::Scalar(s)) => {
123                assert_eq!(scalar.size(), s.size(cx), "{msg}: scalar value has wrong size");
124                if !matches!(s.primitive(), abi::Primitive::Pointer(..)) {
125                    // This is not a pointer, it should not carry provenance.
126                    assert!(
127                        matches!(scalar, Scalar::Int(..)),
128                        "{msg}: scalar value should be an integer, but has provenance"
129                    );
130                }
131            }
132            (Immediate::ScalarPair(a_val, b_val), BackendRepr::ScalarPair(a, b)) => {
133                assert_eq!(
134                    a_val.size(),
135                    a.size(cx),
136                    "{msg}: first component of scalar pair has wrong size"
137                );
138                if !matches!(a.primitive(), abi::Primitive::Pointer(..)) {
139                    assert!(
140                        matches!(a_val, Scalar::Int(..)),
141                        "{msg}: first component of scalar pair should be an integer, but has provenance"
142                    );
143                }
144                assert_eq!(
145                    b_val.size(),
146                    b.size(cx),
147                    "{msg}: second component of scalar pair has wrong size"
148                );
149                if !matches!(b.primitive(), abi::Primitive::Pointer(..)) {
150                    assert!(
151                        matches!(b_val, Scalar::Int(..)),
152                        "{msg}: second component of scalar pair should be an integer, but has provenance"
153                    );
154                }
155            }
156            (Immediate::Uninit, _) => {
157                assert!(abi.is_sized(), "{msg}: unsized immediates are not a thing");
158            }
159            _ => {
160                bug!("{msg}: value {self:?} does not match ABI {abi:?})",)
161            }
162        }
163    }
164
165    pub fn clear_provenance<'tcx>(&mut self) -> InterpResult<'tcx> {
166        match self {
167            Immediate::Scalar(s) => {
168                s.clear_provenance()?;
169            }
170            Immediate::ScalarPair(a, b) => {
171                a.clear_provenance()?;
172                b.clear_provenance()?;
173            }
174            Immediate::Uninit => {}
175        }
176        interp_ok(())
177    }
178}
179
180// ScalarPair needs a type to interpret, so we often have an immediate and a type together
181// as input for binary and cast operations.
182#[derive(Clone)]
183pub struct ImmTy<'tcx, Prov: Provenance = CtfeProvenance> {
184    imm: Immediate<Prov>,
185    pub layout: TyAndLayout<'tcx>,
186}
187
188impl<Prov: Provenance> std::fmt::Display for ImmTy<'_, Prov> {
189    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
190        /// Helper function for printing a scalar to a FmtPrinter
191        fn p<'a, 'tcx, Prov: Provenance>(
192            cx: &mut FmtPrinter<'a, 'tcx>,
193            s: Scalar<Prov>,
194            ty: Ty<'tcx>,
195        ) -> Result<(), std::fmt::Error> {
196            match s {
197                Scalar::Int(int) => cx.pretty_print_const_scalar_int(int, ty, true),
198                Scalar::Ptr(ptr, _sz) => {
199                    // Just print the ptr value. `pretty_print_const_scalar_ptr` would also try to
200                    // print what is points to, which would fail since it has no access to the local
201                    // memory.
202                    cx.pretty_print_const_pointer(ptr, ty)
203                }
204            }
205        }
206        ty::tls::with(|tcx| {
207            match self.imm {
208                Immediate::Scalar(s) => {
209                    if let Some(ty) = tcx.lift(self.layout.ty) {
210                        let s =
211                            FmtPrinter::print_string(tcx, Namespace::ValueNS, |cx| p(cx, s, ty))?;
212                        f.write_str(&s)?;
213                        return Ok(());
214                    }
215                    write!(f, "{:x}: {}", s, self.layout.ty)
216                }
217                Immediate::ScalarPair(a, b) => {
218                    // FIXME(oli-obk): at least print tuples and slices nicely
219                    write!(f, "({:x}, {:x}): {}", a, b, self.layout.ty)
220                }
221                Immediate::Uninit => {
222                    write!(f, "uninit: {}", self.layout.ty)
223                }
224            }
225        })
226    }
227}
228
229impl<Prov: Provenance> std::fmt::Debug for ImmTy<'_, Prov> {
230    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
231        // Printing `layout` results in too much noise; just print a nice version of the type.
232        f.debug_struct("ImmTy")
233            .field("imm", &self.imm)
234            .field("ty", &format_args!("{}", self.layout.ty))
235            .finish()
236    }
237}
238
239impl<'tcx, Prov: Provenance> std::ops::Deref for ImmTy<'tcx, Prov> {
240    type Target = Immediate<Prov>;
241    #[inline(always)]
242    fn deref(&self) -> &Immediate<Prov> {
243        &self.imm
244    }
245}
246
247impl<'tcx, Prov: Provenance> ImmTy<'tcx, Prov> {
248    #[inline]
249    pub fn from_scalar(val: Scalar<Prov>, layout: TyAndLayout<'tcx>) -> Self {
250        debug_assert!(layout.backend_repr.is_scalar(), "`ImmTy::from_scalar` on non-scalar layout");
251        debug_assert_eq!(val.size(), layout.size);
252        ImmTy { imm: val.into(), layout }
253    }
254
255    #[inline]
256    pub fn from_scalar_pair(a: Scalar<Prov>, b: Scalar<Prov>, layout: TyAndLayout<'tcx>) -> Self {
257        debug_assert!(
258            matches!(layout.backend_repr, BackendRepr::ScalarPair(..)),
259            "`ImmTy::from_scalar_pair` on non-scalar-pair layout"
260        );
261        let imm = Immediate::ScalarPair(a, b);
262        ImmTy { imm, layout }
263    }
264
265    #[inline(always)]
266    pub fn from_immediate(imm: Immediate<Prov>, layout: TyAndLayout<'tcx>) -> Self {
267        // Without a `cx` we cannot call `assert_matches_abi`.
268        debug_assert!(
269            match (imm, layout.backend_repr) {
270                (Immediate::Scalar(..), BackendRepr::Scalar(..)) => true,
271                (Immediate::ScalarPair(..), BackendRepr::ScalarPair(..)) => true,
272                (Immediate::Uninit, _) if layout.is_sized() => true,
273                _ => false,
274            },
275            "immediate {imm:?} does not fit to layout {layout:?}",
276        );
277        ImmTy { imm, layout }
278    }
279
280    #[inline]
281    pub fn uninit(layout: TyAndLayout<'tcx>) -> Self {
282        debug_assert!(layout.is_sized(), "immediates must be sized");
283        ImmTy { imm: Immediate::Uninit, layout }
284    }
285
286    #[inline]
287    pub fn from_scalar_int(s: ScalarInt, layout: TyAndLayout<'tcx>) -> Self {
288        Self::from_scalar(Scalar::from(s), layout)
289    }
290
291    #[inline]
292    pub fn from_uint(i: impl Into<u128>, layout: TyAndLayout<'tcx>) -> Self {
293        Self::from_scalar(Scalar::from_uint(i, layout.size), layout)
294    }
295
296    #[inline]
297    pub fn from_int(i: impl Into<i128>, layout: TyAndLayout<'tcx>) -> Self {
298        Self::from_scalar(Scalar::from_int(i, layout.size), layout)
299    }
300
301    #[inline]
302    pub fn from_bool(b: bool, tcx: TyCtxt<'tcx>) -> Self {
303        // Can use any typing env, since `bool` is always monomorphic.
304        let layout = tcx
305            .layout_of(ty::TypingEnv::fully_monomorphized().as_query_input(tcx.types.bool))
306            .unwrap();
307        Self::from_scalar(Scalar::from_bool(b), layout)
308    }
309
310    #[inline]
311    pub fn from_ordering(c: std::cmp::Ordering, tcx: TyCtxt<'tcx>) -> Self {
312        // Can use any typing env, since `Ordering` is always monomorphic.
313        let ty = tcx.ty_ordering_enum(DUMMY_SP);
314        let layout =
315            tcx.layout_of(ty::TypingEnv::fully_monomorphized().as_query_input(ty)).unwrap();
316        Self::from_scalar(Scalar::Int(c.into()), layout)
317    }
318
319    pub fn from_pair(a: Self, b: Self, cx: &(impl HasTypingEnv<'tcx> + HasTyCtxt<'tcx>)) -> Self {
320        let layout = cx
321            .tcx()
322            .layout_of(
323                cx.typing_env().as_query_input(Ty::new_tup(cx.tcx(), &[a.layout.ty, b.layout.ty])),
324            )
325            .unwrap();
326        Self::from_scalar_pair(a.to_scalar(), b.to_scalar(), layout)
327    }
328
329    /// Return the immediate as a `ScalarInt`. Ensures that it has the size that the layout of the
330    /// immediate indicates.
331    #[inline]
332    pub fn to_scalar_int(&self) -> InterpResult<'tcx, ScalarInt> {
333        let s = self.to_scalar().to_scalar_int()?;
334        if s.size() != self.layout.size {
335            throw_ub!(ScalarSizeMismatch(ScalarSizeMismatch {
336                target_size: self.layout.size.bytes(),
337                data_size: s.size().bytes(),
338            }));
339        }
340        interp_ok(s)
341    }
342
343    #[inline]
344    pub fn to_const_int(self) -> ConstInt {
345        assert!(self.layout.ty.is_integral());
346        let int = self.imm.to_scalar_int();
347        assert_eq!(int.size(), self.layout.size);
348        ConstInt::new(int, self.layout.ty.is_signed(), self.layout.ty.is_ptr_sized_integral())
349    }
350
351    #[inline]
352    #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
353    pub fn to_pair(self, cx: &(impl HasTyCtxt<'tcx> + HasTypingEnv<'tcx>)) -> (Self, Self) {
354        let layout = self.layout;
355        let (val0, val1) = self.to_scalar_pair();
356        (
357            ImmTy::from_scalar(val0, layout.field(cx, 0)),
358            ImmTy::from_scalar(val1, layout.field(cx, 1)),
359        )
360    }
361
362    /// Compute the "sub-immediate" that is located within the `base` at the given offset with the
363    /// given layout.
364    // Not called `offset` to avoid confusion with the trait method.
365    fn offset_(&self, offset: Size, layout: TyAndLayout<'tcx>, cx: &impl HasDataLayout) -> Self {
366        // Verify that the input matches its type.
367        if cfg!(debug_assertions) {
368            self.assert_matches_abi(
369                self.layout.backend_repr,
370                "invalid input to Immediate::offset",
371                cx,
372            );
373        }
374        // `ImmTy` have already been checked to be in-bounds, so we can just check directly if this
375        // remains in-bounds. This cannot actually be violated since projections are type-checked
376        // and bounds-checked.
377        assert!(
378            offset + layout.size <= self.layout.size,
379            "attempting to project to field at offset {} with size {} into immediate with layout {:#?}",
380            offset.bytes(),
381            layout.size.bytes(),
382            self.layout,
383        );
384        // This makes several assumptions about what layouts we will encounter; we match what
385        // codegen does as good as we can (see `extract_field` in `rustc_codegen_ssa/src/mir/operand.rs`).
386        let inner_val: Immediate<_> = match (**self, self.layout.backend_repr) {
387            // If the entire value is uninit, then so is the field (can happen in ConstProp).
388            (Immediate::Uninit, _) => Immediate::Uninit,
389            // If the field is uninhabited, we can forget the data (can happen in ConstProp).
390            // `enum S { A(!), B, C }` is an example of an enum with Scalar layout that
391            // has an uninhabited variant, which means this case is possible.
392            _ if layout.is_uninhabited() => Immediate::Uninit,
393            // the field contains no information, can be left uninit
394            // (Scalar/ScalarPair can contain even aligned ZST, not just 1-ZST)
395            _ if layout.is_zst() => Immediate::Uninit,
396            // some fieldless enum variants can have non-zero size but still `Aggregate` ABI... try
397            // to detect those here and also give them no data
398            _ if matches!(layout.backend_repr, BackendRepr::Memory { .. })
399                && matches!(layout.variants, abi::Variants::Single { .. })
400                && matches!(&layout.fields, abi::FieldsShape::Arbitrary { offsets, .. } if offsets.len() == 0) =>
401            {
402                Immediate::Uninit
403            }
404            // the field covers the entire type
405            _ if layout.size == self.layout.size => {
406                assert_eq!(offset.bytes(), 0);
407                **self
408            }
409            // extract fields from types with `ScalarPair` ABI
410            (Immediate::ScalarPair(a_val, b_val), BackendRepr::ScalarPair(a, b)) => {
411                Immediate::from(if offset.bytes() == 0 {
412                    a_val
413                } else {
414                    assert_eq!(offset, a.size(cx).align_to(b.align(cx).abi));
415                    b_val
416                })
417            }
418            // everything else is a bug
419            _ => bug!(
420                "invalid field access on immediate {} at offset {}, original layout {:#?}",
421                self,
422                offset.bytes(),
423                self.layout
424            ),
425        };
426        // Ensure the new layout matches the new value.
427        inner_val.assert_matches_abi(
428            layout.backend_repr,
429            "invalid field type in Immediate::offset",
430            cx,
431        );
432
433        ImmTy::from_immediate(inner_val, layout)
434    }
435}
436
437impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for ImmTy<'tcx, Prov> {
438    #[inline(always)]
439    fn layout(&self) -> TyAndLayout<'tcx> {
440        self.layout
441    }
442
443    #[inline(always)]
444    fn meta(&self) -> MemPlaceMeta<Prov> {
445        debug_assert!(self.layout.is_sized()); // unsized ImmTy can only exist temporarily and should never reach this here
446        MemPlaceMeta::None
447    }
448
449    fn offset_with_meta<M: Machine<'tcx, Provenance = Prov>>(
450        &self,
451        offset: Size,
452        _mode: OffsetMode,
453        meta: MemPlaceMeta<Prov>,
454        layout: TyAndLayout<'tcx>,
455        ecx: &InterpCx<'tcx, M>,
456    ) -> InterpResult<'tcx, Self> {
457        assert_matches!(meta, MemPlaceMeta::None); // we can't store this anywhere anyway
458        interp_ok(self.offset_(offset, layout, ecx))
459    }
460
461    #[inline(always)]
462    fn to_op<M: Machine<'tcx, Provenance = Prov>>(
463        &self,
464        _ecx: &InterpCx<'tcx, M>,
465    ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
466        interp_ok(self.clone().into())
467    }
468}
469
470/// An `Operand` is the result of computing a `mir::Operand`. It can be immediate,
471/// or still in memory. The latter is an optimization, to delay reading that chunk of
472/// memory and to avoid having to store arbitrary-sized data here.
473#[derive(Copy, Clone, Debug)]
474pub(super) enum Operand<Prov: Provenance = CtfeProvenance> {
475    Immediate(Immediate<Prov>),
476    Indirect(MemPlace<Prov>),
477}
478
479#[derive(Clone)]
480pub struct OpTy<'tcx, Prov: Provenance = CtfeProvenance> {
481    op: Operand<Prov>, // Keep this private; it helps enforce invariants.
482    pub layout: TyAndLayout<'tcx>,
483}
484
485impl<Prov: Provenance> std::fmt::Debug for OpTy<'_, Prov> {
486    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
487        // Printing `layout` results in too much noise; just print a nice version of the type.
488        f.debug_struct("OpTy")
489            .field("op", &self.op)
490            .field("ty", &format_args!("{}", self.layout.ty))
491            .finish()
492    }
493}
494
495impl<'tcx, Prov: Provenance> From<ImmTy<'tcx, Prov>> for OpTy<'tcx, Prov> {
496    #[inline(always)]
497    fn from(val: ImmTy<'tcx, Prov>) -> Self {
498        OpTy { op: Operand::Immediate(val.imm), layout: val.layout }
499    }
500}
501
502impl<'tcx, Prov: Provenance> From<MPlaceTy<'tcx, Prov>> for OpTy<'tcx, Prov> {
503    #[inline(always)]
504    fn from(mplace: MPlaceTy<'tcx, Prov>) -> Self {
505        OpTy { op: Operand::Indirect(*mplace.mplace()), layout: mplace.layout }
506    }
507}
508
509impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> {
510    #[inline(always)]
511    pub(super) fn op(&self) -> &Operand<Prov> {
512        &self.op
513    }
514}
515
516impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for OpTy<'tcx, Prov> {
517    #[inline(always)]
518    fn layout(&self) -> TyAndLayout<'tcx> {
519        self.layout
520    }
521
522    #[inline]
523    fn meta(&self) -> MemPlaceMeta<Prov> {
524        match self.as_mplace_or_imm() {
525            Left(mplace) => mplace.meta(),
526            Right(_) => {
527                debug_assert!(self.layout.is_sized(), "unsized immediates are not a thing");
528                MemPlaceMeta::None
529            }
530        }
531    }
532
533    fn offset_with_meta<M: Machine<'tcx, Provenance = Prov>>(
534        &self,
535        offset: Size,
536        mode: OffsetMode,
537        meta: MemPlaceMeta<Prov>,
538        layout: TyAndLayout<'tcx>,
539        ecx: &InterpCx<'tcx, M>,
540    ) -> InterpResult<'tcx, Self> {
541        match self.as_mplace_or_imm() {
542            Left(mplace) => {
543                interp_ok(mplace.offset_with_meta(offset, mode, meta, layout, ecx)?.into())
544            }
545            Right(imm) => {
546                assert_matches!(meta, MemPlaceMeta::None); // no place to store metadata here
547                // Every part of an uninit is uninit.
548                interp_ok(imm.offset_(offset, layout, ecx).into())
549            }
550        }
551    }
552
553    #[inline(always)]
554    fn to_op<M: Machine<'tcx, Provenance = Prov>>(
555        &self,
556        _ecx: &InterpCx<'tcx, M>,
557    ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
558        interp_ok(self.clone())
559    }
560}
561
562impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
563    /// Try reading an immediate in memory; this is interesting particularly for `ScalarPair`.
564    /// Returns `None` if the layout does not permit loading this as a value.
565    ///
566    /// This is an internal function; call `read_immediate` instead.
567    fn read_immediate_from_mplace_raw(
568        &self,
569        mplace: &MPlaceTy<'tcx, M::Provenance>,
570    ) -> InterpResult<'tcx, Option<ImmTy<'tcx, M::Provenance>>> {
571        if mplace.layout.is_unsized() {
572            // Don't touch unsized
573            return interp_ok(None);
574        }
575
576        let Some(alloc) = self.get_place_alloc(mplace)? else {
577            // zero-sized type can be left uninit
578            return interp_ok(Some(ImmTy::uninit(mplace.layout)));
579        };
580
581        // It may seem like all types with `Scalar` or `ScalarPair` ABI are fair game at this point.
582        // However, `MaybeUninit<u64>` is considered a `Scalar` as far as its layout is concerned --
583        // and yet cannot be represented by an interpreter `Scalar`, since we have to handle the
584        // case where some of the bytes are initialized and others are not. So, we need an extra
585        // check that walks over the type of `mplace` to make sure it is truly correct to treat this
586        // like a `Scalar` (or `ScalarPair`).
587        interp_ok(match mplace.layout.backend_repr {
588            BackendRepr::Scalar(abi::Scalar::Initialized { value: s, .. }) => {
589                let size = s.size(self);
590                assert_eq!(size, mplace.layout.size, "abi::Scalar size does not match layout size");
591                let scalar = alloc.read_scalar(
592                    alloc_range(Size::ZERO, size),
593                    /*read_provenance*/ matches!(s, abi::Primitive::Pointer(_)),
594                )?;
595                Some(ImmTy::from_scalar(scalar, mplace.layout))
596            }
597            BackendRepr::ScalarPair(
598                abi::Scalar::Initialized { value: a, .. },
599                abi::Scalar::Initialized { value: b, .. },
600            ) => {
601                // We checked `ptr_align` above, so all fields will have the alignment they need.
602                // We would anyway check against `ptr_align.restrict_for_offset(b_offset)`,
603                // which `ptr.offset(b_offset)` cannot possibly fail to satisfy.
604                let (a_size, b_size) = (a.size(self), b.size(self));
605                let b_offset = a_size.align_to(b.align(self).abi);
606                assert!(b_offset.bytes() > 0); // in `operand_field` we use the offset to tell apart the fields
607                let a_val = alloc.read_scalar(
608                    alloc_range(Size::ZERO, a_size),
609                    /*read_provenance*/ matches!(a, abi::Primitive::Pointer(_)),
610                )?;
611                let b_val = alloc.read_scalar(
612                    alloc_range(b_offset, b_size),
613                    /*read_provenance*/ matches!(b, abi::Primitive::Pointer(_)),
614                )?;
615                Some(ImmTy::from_immediate(Immediate::ScalarPair(a_val, b_val), mplace.layout))
616            }
617            _ => {
618                // Neither a scalar nor scalar pair.
619                None
620            }
621        })
622    }
623
624    /// Try returning an immediate for the operand. If the layout does not permit loading this as an
625    /// immediate, return where in memory we can find the data.
626    /// Note that for a given layout, this operation will either always return Left or Right!
627    /// succeed!  Whether it returns Left depends on whether the layout can be represented
628    /// in an `Immediate`, not on which data is stored there currently.
629    ///
630    /// This is an internal function that should not usually be used; call `read_immediate` instead.
631    /// ConstProp needs it, though.
632    pub fn read_immediate_raw(
633        &self,
634        src: &impl Projectable<'tcx, M::Provenance>,
635    ) -> InterpResult<'tcx, Either<MPlaceTy<'tcx, M::Provenance>, ImmTy<'tcx, M::Provenance>>> {
636        interp_ok(match src.to_op(self)?.as_mplace_or_imm() {
637            Left(ref mplace) => {
638                if let Some(val) = self.read_immediate_from_mplace_raw(mplace)? {
639                    Right(val)
640                } else {
641                    Left(mplace.clone())
642                }
643            }
644            Right(val) => Right(val),
645        })
646    }
647
648    /// Read an immediate from a place, asserting that that is possible with the given layout.
649    ///
650    /// If this succeeds, the `ImmTy` is never `Uninit`.
651    #[inline(always)]
652    pub fn read_immediate(
653        &self,
654        op: &impl Projectable<'tcx, M::Provenance>,
655    ) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> {
656        if !matches!(
657            op.layout().backend_repr,
658            BackendRepr::Scalar(abi::Scalar::Initialized { .. })
659                | BackendRepr::ScalarPair(
660                    abi::Scalar::Initialized { .. },
661                    abi::Scalar::Initialized { .. }
662                )
663        ) {
664            span_bug!(self.cur_span(), "primitive read not possible for type: {}", op.layout().ty);
665        }
666        let imm = self.read_immediate_raw(op)?.right().unwrap();
667        if matches!(*imm, Immediate::Uninit) {
668            throw_ub!(InvalidUninitBytes(None));
669        }
670        interp_ok(imm)
671    }
672
673    /// Read a scalar from a place
674    pub fn read_scalar(
675        &self,
676        op: &impl Projectable<'tcx, M::Provenance>,
677    ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
678        interp_ok(self.read_immediate(op)?.to_scalar())
679    }
680
681    // Pointer-sized reads are fairly common and need target layout access, so we wrap them in
682    // convenience functions.
683
684    /// Read a pointer from a place.
685    pub fn read_pointer(
686        &self,
687        op: &impl Projectable<'tcx, M::Provenance>,
688    ) -> InterpResult<'tcx, Pointer<Option<M::Provenance>>> {
689        self.read_scalar(op)?.to_pointer(self)
690    }
691    /// Read a pointer-sized unsigned integer from a place.
692    pub fn read_target_usize(
693        &self,
694        op: &impl Projectable<'tcx, M::Provenance>,
695    ) -> InterpResult<'tcx, u64> {
696        self.read_scalar(op)?.to_target_usize(self)
697    }
698    /// Read a pointer-sized signed integer from a place.
699    pub fn read_target_isize(
700        &self,
701        op: &impl Projectable<'tcx, M::Provenance>,
702    ) -> InterpResult<'tcx, i64> {
703        self.read_scalar(op)?.to_target_isize(self)
704    }
705
706    /// Turn the wide MPlace into a string (must already be dereferenced!)
707    pub fn read_str(&self, mplace: &MPlaceTy<'tcx, M::Provenance>) -> InterpResult<'tcx, &str> {
708        let len = mplace.len(self)?;
709        let bytes = self.read_bytes_ptr_strip_provenance(mplace.ptr(), Size::from_bytes(len))?;
710        let s = std::str::from_utf8(bytes).map_err(|err| err_ub!(InvalidStr(err)))?;
711        interp_ok(s)
712    }
713
714    /// Read from a local of the current frame. Convenience method for [`InterpCx::local_at_frame_to_op`].
715    pub fn local_to_op(
716        &self,
717        local: mir::Local,
718        layout: Option<TyAndLayout<'tcx>>,
719    ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
720        self.local_at_frame_to_op(self.frame(), local, layout)
721    }
722
723    /// Read from a local of a given frame.
724    /// Will not access memory, instead an indirect `Operand` is returned.
725    ///
726    /// This is public because it is used by [Aquascope](https://github.com/cognitive-engineering-lab/aquascope/)
727    /// to get an OpTy from a local.
728    pub fn local_at_frame_to_op(
729        &self,
730        frame: &Frame<'tcx, M::Provenance, M::FrameExtra>,
731        local: mir::Local,
732        layout: Option<TyAndLayout<'tcx>>,
733    ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
734        let layout = self.layout_of_local(frame, local, layout)?;
735        let op = *frame.locals[local].access()?;
736        if matches!(op, Operand::Immediate(_)) {
737            assert!(!layout.is_unsized());
738        }
739        M::after_local_read(self, frame, local)?;
740        interp_ok(OpTy { op, layout })
741    }
742
743    /// Every place can be read from, so we can turn them into an operand.
744    /// This will definitely return `Indirect` if the place is a `Ptr`, i.e., this
745    /// will never actually read from memory.
746    pub fn place_to_op(
747        &self,
748        place: &PlaceTy<'tcx, M::Provenance>,
749    ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
750        match place.as_mplace_or_local() {
751            Left(mplace) => interp_ok(mplace.into()),
752            Right((local, offset, locals_addr, _)) => {
753                debug_assert!(place.layout.is_sized()); // only sized locals can ever be `Place::Local`.
754                debug_assert_eq!(locals_addr, self.frame().locals_addr());
755                let base = self.local_to_op(local, None)?;
756                interp_ok(match offset {
757                    Some(offset) => base.offset(offset, place.layout, self)?,
758                    None => {
759                        // In the common case this hasn't been projected.
760                        debug_assert_eq!(place.layout, base.layout);
761                        base
762                    }
763                })
764            }
765        }
766    }
767
768    /// Evaluate a place with the goal of reading from it. This lets us sometimes
769    /// avoid allocations.
770    pub fn eval_place_to_op(
771        &self,
772        mir_place: mir::Place<'tcx>,
773        layout: Option<TyAndLayout<'tcx>>,
774    ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
775        let _span = enter_trace_span!(
776            M,
777            step::eval_place_to_op,
778            ?mir_place,
779            tracing_separate_thread = Empty
780        );
781
782        // Do not use the layout passed in as argument if the base we are looking at
783        // here is not the entire place.
784        let layout = if mir_place.projection.is_empty() { layout } else { None };
785
786        let mut op = self.local_to_op(mir_place.local, layout)?;
787        // Using `try_fold` turned out to be bad for performance, hence the loop.
788        for elem in mir_place.projection.iter() {
789            op = self.project(&op, elem)?
790        }
791
792        trace!("eval_place_to_op: got {:?}", op);
793        // Sanity-check the type we ended up with.
794        if cfg!(debug_assertions) {
795            let normalized_place_ty = self
796                .instantiate_from_current_frame_and_normalize_erasing_regions(
797                    mir_place.ty(&self.frame().body.local_decls, *self.tcx).ty,
798                )?;
799            if !mir_assign_valid_types(
800                *self.tcx,
801                self.typing_env(),
802                self.layout_of(normalized_place_ty)?,
803                op.layout,
804            ) {
805                span_bug!(
806                    self.cur_span(),
807                    "eval_place of a MIR place with type {} produced an interpreter operand with type {}",
808                    normalized_place_ty,
809                    op.layout.ty,
810                )
811            }
812        }
813        interp_ok(op)
814    }
815
816    /// Evaluate the operand, returning a place where you can then find the data.
817    /// If you already know the layout, you can save two table lookups
818    /// by passing it in here.
819    #[inline]
820    pub fn eval_operand(
821        &self,
822        mir_op: &mir::Operand<'tcx>,
823        layout: Option<TyAndLayout<'tcx>>,
824    ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
825        let _span =
826            enter_trace_span!(M, step::eval_operand, ?mir_op, tracing_separate_thread = Empty);
827
828        use rustc_middle::mir::Operand::*;
829        let op = match mir_op {
830            // FIXME: do some more logic on `move` to invalidate the old location
831            &Copy(place) | &Move(place) => self.eval_place_to_op(place, layout)?,
832
833            Constant(constant) => {
834                let c = self.instantiate_from_current_frame_and_normalize_erasing_regions(
835                    constant.const_,
836                )?;
837
838                // This can still fail:
839                // * During ConstProp, with `TooGeneric` or since the `required_consts` were not all
840                //   checked yet.
841                // * During CTFE, since promoteds in `const`/`static` initializer bodies can fail.
842                self.eval_mir_constant(&c, constant.span, layout)?
843            }
844        };
845        trace!("{:?}: {:?}", mir_op, op);
846        interp_ok(op)
847    }
848
849    pub(crate) fn const_val_to_op(
850        &self,
851        val_val: mir::ConstValue,
852        ty: Ty<'tcx>,
853        layout: Option<TyAndLayout<'tcx>>,
854    ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
855        // Other cases need layout.
856        let adjust_scalar = |scalar| -> InterpResult<'tcx, _> {
857            interp_ok(match scalar {
858                Scalar::Ptr(ptr, size) => Scalar::Ptr(self.global_root_pointer(ptr)?, size),
859                Scalar::Int(int) => Scalar::Int(int),
860            })
861        };
862        let layout =
863            from_known_layout(self.tcx, self.typing_env(), layout, || self.layout_of(ty).into())?;
864        let imm = match val_val {
865            mir::ConstValue::Indirect { alloc_id, offset } => {
866                // This is const data, no mutation allowed.
867                let ptr = self.global_root_pointer(Pointer::new(
868                    CtfeProvenance::from(alloc_id).as_immutable(),
869                    offset,
870                ))?;
871                return interp_ok(self.ptr_to_mplace(ptr.into(), layout).into());
872            }
873            mir::ConstValue::Scalar(x) => adjust_scalar(x)?.into(),
874            mir::ConstValue::ZeroSized => Immediate::Uninit,
875            mir::ConstValue::Slice { alloc_id, meta } => {
876                // This is const data, no mutation allowed.
877                let ptr = Pointer::new(CtfeProvenance::from(alloc_id).as_immutable(), Size::ZERO);
878                Immediate::new_slice(self.global_root_pointer(ptr)?.into(), meta, self)
879            }
880        };
881        interp_ok(OpTy { op: Operand::Immediate(imm), layout })
882    }
883}
884
885// Some nodes are used a lot. Make sure they don't unintentionally get bigger.
886#[cfg(target_pointer_width = "64")]
887mod size_asserts {
888    use rustc_data_structures::static_assert_size;
889
890    use super::*;
891    // tidy-alphabetical-start
892    static_assert_size!(ImmTy<'_>, 64);
893    static_assert_size!(Immediate, 48);
894    static_assert_size!(OpTy<'_>, 72);
895    static_assert_size!(Operand, 56);
896    // tidy-alphabetical-end
897}