rustc_const_eval/interpret/
operand.rs

Help
1//! Functions concerning immediate values and operands, and reading from operands.
2//! All high-level functions to read from memory work on operands as sources.
3
4use std::assert_matches::assert_matches;
5
6use either::{Either, Left, Right};
7use rustc_abi as abi;
8use rustc_abi::{BackendRepr, HasDataLayout, Size};
9use rustc_hir::def::Namespace;
10use rustc_middle::mir::interpret::ScalarSizeMismatch;
11use rustc_middle::ty::layout::{HasTyCtxt, HasTypingEnv, LayoutOf, TyAndLayout};
12use rustc_middle::ty::print::{FmtPrinter, PrettyPrinter};
13use rustc_middle::ty::{ConstInt, ScalarInt, Ty, TyCtxt};
14use rustc_middle::{bug, mir, span_bug, ty};
15use tracing::trace;
16
17use super::{
18    CtfeProvenance, Frame, InterpCx, InterpResult, MPlaceTy, Machine, MemPlace, MemPlaceMeta,
19    OffsetMode, PlaceTy, Pointer, Projectable, Provenance, Scalar, alloc_range, err_ub,
20    from_known_layout, interp_ok, mir_assign_valid_types, throw_ub,
21};
22
23/// An `Immediate` represents a single immediate self-contained Rust value.
24///
25/// For optimization of a few very common cases, there is also a representation for a pair of
26/// primitive values (`ScalarPair`). It allows Miri to avoid making allocations for checked binary
27/// operations and wide pointers. This idea was taken from rustc's codegen.
28/// In particular, thanks to `ScalarPair`, arithmetic operations and casts can be entirely
29/// defined on `Immediate`, and do not have to work with a `Place`.
30#[derive(Copy, Clone, Debug)]
31pub enum Immediate<Prov: Provenance = CtfeProvenance> {
32    /// A single scalar value (must have *initialized* `Scalar` ABI).
33    Scalar(Scalar<Prov>),
34    /// A pair of two scalar value (must have `ScalarPair` ABI where both fields are
35    /// `Scalar::Initialized`).
36    ScalarPair(Scalar<Prov>, Scalar<Prov>),
37    /// A value of fully uninitialized memory. Can have arbitrary size and layout, but must be sized.
38    Uninit,
39}
40
41impl<Prov: Provenance> From<Scalar<Prov>> for Immediate<Prov> {
42    #[inline(always)]
43    fn from(val: Scalar<Prov>) -> Self {
44        Immediate::Scalar(val)
45    }
46}
47
48impl<Prov: Provenance> Immediate<Prov> {
49    pub fn new_pointer_with_meta(
50        ptr: Pointer<Option<Prov>>,
51        meta: MemPlaceMeta<Prov>,
52        cx: &impl HasDataLayout,
53    ) -> Self {
54        let ptr = Scalar::from_maybe_pointer(ptr, cx);
55        match meta {
56            MemPlaceMeta::None => Immediate::from(ptr),
57            MemPlaceMeta::Meta(meta) => Immediate::ScalarPair(ptr, meta),
58        }
59    }
60
61    pub fn new_slice(ptr: Pointer<Option<Prov>>, len: u64, cx: &impl HasDataLayout) -> Self {
62        Immediate::ScalarPair(
63            Scalar::from_maybe_pointer(ptr, cx),
64            Scalar::from_target_usize(len, cx),
65        )
66    }
67
68    pub fn new_dyn_trait(
69        val: Pointer<Option<Prov>>,
70        vtable: Pointer<Option<Prov>>,
71        cx: &impl HasDataLayout,
72    ) -> Self {
73        Immediate::ScalarPair(
74            Scalar::from_maybe_pointer(val, cx),
75            Scalar::from_maybe_pointer(vtable, cx),
76        )
77    }
78
79    #[inline]
80    #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
81    pub fn to_scalar(self) -> Scalar<Prov> {
82        match self {
83            Immediate::Scalar(val) => val,
84            Immediate::ScalarPair(..) => bug!("Got a scalar pair where a scalar was expected"),
85            Immediate::Uninit => bug!("Got uninit where a scalar was expected"),
86        }
87    }
88
89    #[inline]
90    #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
91    pub fn to_scalar_int(self) -> ScalarInt {
92        self.to_scalar().try_to_scalar_int().unwrap()
93    }
94
95    #[inline]
96    #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
97    pub fn to_scalar_pair(self) -> (Scalar<Prov>, Scalar<Prov>) {
98        match self {
99            Immediate::ScalarPair(val1, val2) => (val1, val2),
100            Immediate::Scalar(..) => bug!("Got a scalar where a scalar pair was expected"),
101            Immediate::Uninit => bug!("Got uninit where a scalar pair was expected"),
102        }
103    }
104
105    /// Returns the scalar from the first component and optionally the 2nd component as metadata.
106    #[inline]
107    #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
108    pub fn to_scalar_and_meta(self) -> (Scalar<Prov>, MemPlaceMeta<Prov>) {
109        match self {
110            Immediate::ScalarPair(val1, val2) => (val1, MemPlaceMeta::Meta(val2)),
111            Immediate::Scalar(val) => (val, MemPlaceMeta::None),
112            Immediate::Uninit => bug!("Got uninit where a scalar or scalar pair was expected"),
113        }
114    }
115
116    /// Assert that this immediate is a valid value for the given ABI.
117    pub fn assert_matches_abi(self, abi: BackendRepr, msg: &str, cx: &impl HasDataLayout) {
118        match (self, abi) {
119            (Immediate::Scalar(scalar), BackendRepr::Scalar(s)) => {
120                assert_eq!(scalar.size(), s.size(cx), "{msg}: scalar value has wrong size");
121                if !matches!(s.primitive(), abi::Primitive::Pointer(..)) {
122                    // This is not a pointer, it should not carry provenance.
123                    assert!(
124                        matches!(scalar, Scalar::Int(..)),
125                        "{msg}: scalar value should be an integer, but has provenance"
126                    );
127                }
128            }
129            (Immediate::ScalarPair(a_val, b_val), BackendRepr::ScalarPair(a, b)) => {
130                assert_eq!(
131                    a_val.size(),
132                    a.size(cx),
133                    "{msg}: first component of scalar pair has wrong size"
134                );
135                if !matches!(a.primitive(), abi::Primitive::Pointer(..)) {
136                    assert!(
137                        matches!(a_val, Scalar::Int(..)),
138                        "{msg}: first component of scalar pair should be an integer, but has provenance"
139                    );
140                }
141                assert_eq!(
142                    b_val.size(),
143                    b.size(cx),
144                    "{msg}: second component of scalar pair has wrong size"
145                );
146                if !matches!(b.primitive(), abi::Primitive::Pointer(..)) {
147                    assert!(
148                        matches!(b_val, Scalar::Int(..)),
149                        "{msg}: second component of scalar pair should be an integer, but has provenance"
150                    );
151                }
152            }
153            (Immediate::Uninit, _) => {
154                assert!(abi.is_sized(), "{msg}: unsized immediates are not a thing");
155            }
156            _ => {
157                bug!("{msg}: value {self:?} does not match ABI {abi:?})",)
158            }
159        }
160    }
161
162    pub fn clear_provenance<'tcx>(&mut self) -> InterpResult<'tcx> {
163        match self {
164            Immediate::Scalar(s) => {
165                s.clear_provenance()?;
166            }
167            Immediate::ScalarPair(a, b) => {
168                a.clear_provenance()?;
169                b.clear_provenance()?;
170            }
171            Immediate::Uninit => {}
172        }
173        interp_ok(())
174    }
175}
176
177// ScalarPair needs a type to interpret, so we often have an immediate and a type together
178// as input for binary and cast operations.
179#[derive(Clone)]
180pub struct ImmTy<'tcx, Prov: Provenance = CtfeProvenance> {
181    imm: Immediate<Prov>,
182    pub layout: TyAndLayout<'tcx>,
183}
184
185impl<Prov: Provenance> std::fmt::Display for ImmTy<'_, Prov> {
186    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
187        /// Helper function for printing a scalar to a FmtPrinter
188        fn p<'a, 'tcx, Prov: Provenance>(
189            cx: &mut FmtPrinter<'a, 'tcx>,
190            s: Scalar<Prov>,
191            ty: Ty<'tcx>,
192        ) -> Result<(), std::fmt::Error> {
193            match s {
194                Scalar::Int(int) => cx.pretty_print_const_scalar_int(int, ty, true),
195                Scalar::Ptr(ptr, _sz) => {
196                    // Just print the ptr value. `pretty_print_const_scalar_ptr` would also try to
197                    // print what is points to, which would fail since it has no access to the local
198                    // memory.
199                    cx.pretty_print_const_pointer(ptr, ty)
200                }
201            }
202        }
203        ty::tls::with(|tcx| {
204            match self.imm {
205                Immediate::Scalar(s) => {
206                    if let Some(ty) = tcx.lift(self.layout.ty) {
207                        let s =
208                            FmtPrinter::print_string(tcx, Namespace::ValueNS, |cx| p(cx, s, ty))?;
209                        f.write_str(&s)?;
210                        return Ok(());
211                    }
212                    write!(f, "{:x}: {}", s, self.layout.ty)
213                }
214                Immediate::ScalarPair(a, b) => {
215                    // FIXME(oli-obk): at least print tuples and slices nicely
216                    write!(f, "({:x}, {:x}): {}", a, b, self.layout.ty)
217                }
218                Immediate::Uninit => {
219                    write!(f, "uninit: {}", self.layout.ty)
220                }
221            }
222        })
223    }
224}
225
226impl<Prov: Provenance> std::fmt::Debug for ImmTy<'_, Prov> {
227    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
228        // Printing `layout` results in too much noise; just print a nice version of the type.
229        f.debug_struct("ImmTy")
230            .field("imm", &self.imm)
231            .field("ty", &format_args!("{}", self.layout.ty))
232            .finish()
233    }
234}
235
236impl<'tcx, Prov: Provenance> std::ops::Deref for ImmTy<'tcx, Prov> {
237    type Target = Immediate<Prov>;
238    #[inline(always)]
239    fn deref(&self) -> &Immediate<Prov> {
240        &self.imm
241    }
242}
243
244impl<'tcx, Prov: Provenance> ImmTy<'tcx, Prov> {
245    #[inline]
246    pub fn from_scalar(val: Scalar<Prov>, layout: TyAndLayout<'tcx>) -> Self {
247        debug_assert!(layout.backend_repr.is_scalar(), "`ImmTy::from_scalar` on non-scalar layout");
248        debug_assert_eq!(val.size(), layout.size);
249        ImmTy { imm: val.into(), layout }
250    }
251
252    #[inline]
253    pub fn from_scalar_pair(a: Scalar<Prov>, b: Scalar<Prov>, layout: TyAndLayout<'tcx>) -> Self {
254        debug_assert!(
255            matches!(layout.backend_repr, BackendRepr::ScalarPair(..)),
256            "`ImmTy::from_scalar_pair` on non-scalar-pair layout"
257        );
258        let imm = Immediate::ScalarPair(a, b);
259        ImmTy { imm, layout }
260    }
261
262    #[inline(always)]
263    pub fn from_immediate(imm: Immediate<Prov>, layout: TyAndLayout<'tcx>) -> Self {
264        // Without a `cx` we cannot call `assert_matches_abi`.
265        debug_assert!(
266            match (imm, layout.backend_repr) {
267                (Immediate::Scalar(..), BackendRepr::Scalar(..)) => true,
268                (Immediate::ScalarPair(..), BackendRepr::ScalarPair(..)) => true,
269                (Immediate::Uninit, _) if layout.is_sized() => true,
270                _ => false,
271            },
272            "immediate {imm:?} does not fit to layout {layout:?}",
273        );
274        ImmTy { imm, layout }
275    }
276
277    #[inline]
278    pub fn uninit(layout: TyAndLayout<'tcx>) -> Self {
279        debug_assert!(layout.is_sized(), "immediates must be sized");
280        ImmTy { imm: Immediate::Uninit, layout }
281    }
282
283    #[inline]
284    pub fn from_scalar_int(s: ScalarInt, layout: TyAndLayout<'tcx>) -> Self {
285        Self::from_scalar(Scalar::from(s), layout)
286    }
287
288    #[inline]
289    pub fn from_uint(i: impl Into<u128>, layout: TyAndLayout<'tcx>) -> Self {
290        Self::from_scalar(Scalar::from_uint(i, layout.size), layout)
291    }
292
293    #[inline]
294    pub fn from_int(i: impl Into<i128>, layout: TyAndLayout<'tcx>) -> Self {
295        Self::from_scalar(Scalar::from_int(i, layout.size), layout)
296    }
297
298    #[inline]
299    pub fn from_bool(b: bool, tcx: TyCtxt<'tcx>) -> Self {
300        // Can use any typing env, since `bool` is always monomorphic.
301        let layout = tcx
302            .layout_of(ty::TypingEnv::fully_monomorphized().as_query_input(tcx.types.bool))
303            .unwrap();
304        Self::from_scalar(Scalar::from_bool(b), layout)
305    }
306
307    #[inline]
308    pub fn from_ordering(c: std::cmp::Ordering, tcx: TyCtxt<'tcx>) -> Self {
309        // Can use any typing env, since `Ordering` is always monomorphic.
310        let ty = tcx.ty_ordering_enum(None);
311        let layout =
312            tcx.layout_of(ty::TypingEnv::fully_monomorphized().as_query_input(ty)).unwrap();
313        Self::from_scalar(Scalar::from_i8(c as i8), layout)
314    }
315
316    pub fn from_pair(a: Self, b: Self, cx: &(impl HasTypingEnv<'tcx> + HasTyCtxt<'tcx>)) -> Self {
317        let layout = cx
318            .tcx()
319            .layout_of(
320                cx.typing_env().as_query_input(Ty::new_tup(cx.tcx(), &[a.layout.ty, b.layout.ty])),
321            )
322            .unwrap();
323        Self::from_scalar_pair(a.to_scalar(), b.to_scalar(), layout)
324    }
325
326    /// Return the immediate as a `ScalarInt`. Ensures that it has the size that the layout of the
327    /// immediate indicates.
328    #[inline]
329    pub fn to_scalar_int(&self) -> InterpResult<'tcx, ScalarInt> {
330        let s = self.to_scalar().to_scalar_int()?;
331        if s.size() != self.layout.size {
332            throw_ub!(ScalarSizeMismatch(ScalarSizeMismatch {
333                target_size: self.layout.size.bytes(),
334                data_size: s.size().bytes(),
335            }));
336        }
337        interp_ok(s)
338    }
339
340    #[inline]
341    pub fn to_const_int(self) -> ConstInt {
342        assert!(self.layout.ty.is_integral());
343        let int = self.imm.to_scalar_int();
344        assert_eq!(int.size(), self.layout.size);
345        ConstInt::new(int, self.layout.ty.is_signed(), self.layout.ty.is_ptr_sized_integral())
346    }
347
348    #[inline]
349    #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
350    pub fn to_pair(self, cx: &(impl HasTyCtxt<'tcx> + HasTypingEnv<'tcx>)) -> (Self, Self) {
351        let layout = self.layout;
352        let (val0, val1) = self.to_scalar_pair();
353        (
354            ImmTy::from_scalar(val0, layout.field(cx, 0)),
355            ImmTy::from_scalar(val1, layout.field(cx, 1)),
356        )
357    }
358
359    /// Compute the "sub-immediate" that is located within the `base` at the given offset with the
360    /// given layout.
361    // Not called `offset` to avoid confusion with the trait method.
362    fn offset_(&self, offset: Size, layout: TyAndLayout<'tcx>, cx: &impl HasDataLayout) -> Self {
363        // Verify that the input matches its type.
364        if cfg!(debug_assertions) {
365            self.assert_matches_abi(
366                self.layout.backend_repr,
367                "invalid input to Immediate::offset",
368                cx,
369            );
370        }
371        // `ImmTy` have already been checked to be in-bounds, so we can just check directly if this
372        // remains in-bounds. This cannot actually be violated since projections are type-checked
373        // and bounds-checked.
374        assert!(
375            offset + layout.size <= self.layout.size,
376            "attempting to project to field at offset {} with size {} into immediate with layout {:#?}",
377            offset.bytes(),
378            layout.size.bytes(),
379            self.layout,
380        );
381        // This makes several assumptions about what layouts we will encounter; we match what
382        // codegen does as good as we can (see `extract_field` in `rustc_codegen_ssa/src/mir/operand.rs`).
383        let inner_val: Immediate<_> = match (**self, self.layout.backend_repr) {
384            // If the entire value is uninit, then so is the field (can happen in ConstProp).
385            (Immediate::Uninit, _) => Immediate::Uninit,
386            // If the field is uninhabited, we can forget the data (can happen in ConstProp).
387            // `enum S { A(!), B, C }` is an example of an enum with Scalar layout that
388            // has an `Uninhabited` variant, which means this case is possible.
389            _ if layout.is_uninhabited() => Immediate::Uninit,
390            // the field contains no information, can be left uninit
391            // (Scalar/ScalarPair can contain even aligned ZST, not just 1-ZST)
392            _ if layout.is_zst() => Immediate::Uninit,
393            // some fieldless enum variants can have non-zero size but still `Aggregate` ABI... try
394            // to detect those here and also give them no data
395            _ if matches!(layout.backend_repr, BackendRepr::Memory { .. })
396                && matches!(layout.variants, abi::Variants::Single { .. })
397                && matches!(&layout.fields, abi::FieldsShape::Arbitrary { offsets, .. } if offsets.len() == 0) =>
398            {
399                Immediate::Uninit
400            }
401            // the field covers the entire type
402            _ if layout.size == self.layout.size => {
403                assert_eq!(offset.bytes(), 0);
404                **self
405            }
406            // extract fields from types with `ScalarPair` ABI
407            (Immediate::ScalarPair(a_val, b_val), BackendRepr::ScalarPair(a, b)) => {
408                Immediate::from(if offset.bytes() == 0 {
409                    a_val
410                } else {
411                    assert_eq!(offset, a.size(cx).align_to(b.align(cx).abi));
412                    b_val
413                })
414            }
415            // everything else is a bug
416            _ => bug!(
417                "invalid field access on immediate {} at offset {}, original layout {:#?}",
418                self,
419                offset.bytes(),
420                self.layout
421            ),
422        };
423        // Ensure the new layout matches the new value.
424        inner_val.assert_matches_abi(
425            layout.backend_repr,
426            "invalid field type in Immediate::offset",
427            cx,
428        );
429
430        ImmTy::from_immediate(inner_val, layout)
431    }
432}
433
434impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for ImmTy<'tcx, Prov> {
435    #[inline(always)]
436    fn layout(&self) -> TyAndLayout<'tcx> {
437        self.layout
438    }
439
440    #[inline(always)]
441    fn meta(&self) -> MemPlaceMeta<Prov> {
442        debug_assert!(self.layout.is_sized()); // unsized ImmTy can only exist temporarily and should never reach this here
443        MemPlaceMeta::None
444    }
445
446    fn offset_with_meta<M: Machine<'tcx, Provenance = Prov>>(
447        &self,
448        offset: Size,
449        _mode: OffsetMode,
450        meta: MemPlaceMeta<Prov>,
451        layout: TyAndLayout<'tcx>,
452        ecx: &InterpCx<'tcx, M>,
453    ) -> InterpResult<'tcx, Self> {
454        assert_matches!(meta, MemPlaceMeta::None); // we can't store this anywhere anyway
455        interp_ok(self.offset_(offset, layout, ecx))
456    }
457
458    #[inline(always)]
459    fn to_op<M: Machine<'tcx, Provenance = Prov>>(
460        &self,
461        _ecx: &InterpCx<'tcx, M>,
462    ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
463        interp_ok(self.clone().into())
464    }
465}
466
467/// An `Operand` is the result of computing a `mir::Operand`. It can be immediate,
468/// or still in memory. The latter is an optimization, to delay reading that chunk of
469/// memory and to avoid having to store arbitrary-sized data here.
470#[derive(Copy, Clone, Debug)]
471pub(super) enum Operand<Prov: Provenance = CtfeProvenance> {
472    Immediate(Immediate<Prov>),
473    Indirect(MemPlace<Prov>),
474}
475
476#[derive(Clone)]
477pub struct OpTy<'tcx, Prov: Provenance = CtfeProvenance> {
478    op: Operand<Prov>, // Keep this private; it helps enforce invariants.
479    pub layout: TyAndLayout<'tcx>,
480}
481
482impl<Prov: Provenance> std::fmt::Debug for OpTy<'_, Prov> {
483    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
484        // Printing `layout` results in too much noise; just print a nice version of the type.
485        f.debug_struct("OpTy")
486            .field("op", &self.op)
487            .field("ty", &format_args!("{}", self.layout.ty))
488            .finish()
489    }
490}
491
492impl<'tcx, Prov: Provenance> From<ImmTy<'tcx, Prov>> for OpTy<'tcx, Prov> {
493    #[inline(always)]
494    fn from(val: ImmTy<'tcx, Prov>) -> Self {
495        OpTy { op: Operand::Immediate(val.imm), layout: val.layout }
496    }
497}
498
499impl<'tcx, Prov: Provenance> From<MPlaceTy<'tcx, Prov>> for OpTy<'tcx, Prov> {
500    #[inline(always)]
501    fn from(mplace: MPlaceTy<'tcx, Prov>) -> Self {
502        OpTy { op: Operand::Indirect(*mplace.mplace()), layout: mplace.layout }
503    }
504}
505
506impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> {
507    #[inline(always)]
508    pub(super) fn op(&self) -> &Operand<Prov> {
509        &self.op
510    }
511}
512
513impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for OpTy<'tcx, Prov> {
514    #[inline(always)]
515    fn layout(&self) -> TyAndLayout<'tcx> {
516        self.layout
517    }
518
519    #[inline]
520    fn meta(&self) -> MemPlaceMeta<Prov> {
521        match self.as_mplace_or_imm() {
522            Left(mplace) => mplace.meta(),
523            Right(_) => {
524                debug_assert!(self.layout.is_sized(), "unsized immediates are not a thing");
525                MemPlaceMeta::None
526            }
527        }
528    }
529
530    fn offset_with_meta<M: Machine<'tcx, Provenance = Prov>>(
531        &self,
532        offset: Size,
533        mode: OffsetMode,
534        meta: MemPlaceMeta<Prov>,
535        layout: TyAndLayout<'tcx>,
536        ecx: &InterpCx<'tcx, M>,
537    ) -> InterpResult<'tcx, Self> {
538        match self.as_mplace_or_imm() {
539            Left(mplace) => {
540                interp_ok(mplace.offset_with_meta(offset, mode, meta, layout, ecx)?.into())
541            }
542            Right(imm) => {
543                assert_matches!(meta, MemPlaceMeta::None); // no place to store metadata here
544                // Every part of an uninit is uninit.
545                interp_ok(imm.offset_(offset, layout, ecx).into())
546            }
547        }
548    }
549
550    #[inline(always)]
551    fn to_op<M: Machine<'tcx, Provenance = Prov>>(
552        &self,
553        _ecx: &InterpCx<'tcx, M>,
554    ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
555        interp_ok(self.clone())
556    }
557}
558
559impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
560    /// Try reading an immediate in memory; this is interesting particularly for `ScalarPair`.
561    /// Returns `None` if the layout does not permit loading this as a value.
562    ///
563    /// This is an internal function; call `read_immediate` instead.
564    fn read_immediate_from_mplace_raw(
565        &self,
566        mplace: &MPlaceTy<'tcx, M::Provenance>,
567    ) -> InterpResult<'tcx, Option<ImmTy<'tcx, M::Provenance>>> {
568        if mplace.layout.is_unsized() {
569            // Don't touch unsized
570            return interp_ok(None);
571        }
572
573        let Some(alloc) = self.get_place_alloc(mplace)? else {
574            // zero-sized type can be left uninit
575            return interp_ok(Some(ImmTy::uninit(mplace.layout)));
576        };
577
578        // It may seem like all types with `Scalar` or `ScalarPair` ABI are fair game at this point.
579        // However, `MaybeUninit<u64>` is considered a `Scalar` as far as its layout is concerned --
580        // and yet cannot be represented by an interpreter `Scalar`, since we have to handle the
581        // case where some of the bytes are initialized and others are not. So, we need an extra
582        // check that walks over the type of `mplace` to make sure it is truly correct to treat this
583        // like a `Scalar` (or `ScalarPair`).
584        interp_ok(match mplace.layout.backend_repr {
585            BackendRepr::Scalar(abi::Scalar::Initialized { value: s, .. }) => {
586                let size = s.size(self);
587                assert_eq!(size, mplace.layout.size, "abi::Scalar size does not match layout size");
588                let scalar = alloc.read_scalar(
589                    alloc_range(Size::ZERO, size),
590                    /*read_provenance*/ matches!(s, abi::Primitive::Pointer(_)),
591                )?;
592                Some(ImmTy::from_scalar(scalar, mplace.layout))
593            }
594            BackendRepr::ScalarPair(
595                abi::Scalar::Initialized { value: a, .. },
596                abi::Scalar::Initialized { value: b, .. },
597            ) => {
598                // We checked `ptr_align` above, so all fields will have the alignment they need.
599                // We would anyway check against `ptr_align.restrict_for_offset(b_offset)`,
600                // which `ptr.offset(b_offset)` cannot possibly fail to satisfy.
601                let (a_size, b_size) = (a.size(self), b.size(self));
602                let b_offset = a_size.align_to(b.align(self).abi);
603                assert!(b_offset.bytes() > 0); // in `operand_field` we use the offset to tell apart the fields
604                let a_val = alloc.read_scalar(
605                    alloc_range(Size::ZERO, a_size),
606                    /*read_provenance*/ matches!(a, abi::Primitive::Pointer(_)),
607                )?;
608                let b_val = alloc.read_scalar(
609                    alloc_range(b_offset, b_size),
610                    /*read_provenance*/ matches!(b, abi::Primitive::Pointer(_)),
611                )?;
612                Some(ImmTy::from_immediate(Immediate::ScalarPair(a_val, b_val), mplace.layout))
613            }
614            _ => {
615                // Neither a scalar nor scalar pair.
616                None
617            }
618        })
619    }
620
621    /// Try returning an immediate for the operand. If the layout does not permit loading this as an
622    /// immediate, return where in memory we can find the data.
623    /// Note that for a given layout, this operation will either always return Left or Right!
624    /// succeed!  Whether it returns Left depends on whether the layout can be represented
625    /// in an `Immediate`, not on which data is stored there currently.
626    ///
627    /// This is an internal function that should not usually be used; call `read_immediate` instead.
628    /// ConstProp needs it, though.
629    pub fn read_immediate_raw(
630        &self,
631        src: &impl Projectable<'tcx, M::Provenance>,
632    ) -> InterpResult<'tcx, Either<MPlaceTy<'tcx, M::Provenance>, ImmTy<'tcx, M::Provenance>>> {
633        interp_ok(match src.to_op(self)?.as_mplace_or_imm() {
634            Left(ref mplace) => {
635                if let Some(val) = self.read_immediate_from_mplace_raw(mplace)? {
636                    Right(val)
637                } else {
638                    Left(mplace.clone())
639                }
640            }
641            Right(val) => Right(val),
642        })
643    }
644
645    /// Read an immediate from a place, asserting that that is possible with the given layout.
646    ///
647    /// If this succeeds, the `ImmTy` is never `Uninit`.
648    #[inline(always)]
649    pub fn read_immediate(
650        &self,
651        op: &impl Projectable<'tcx, M::Provenance>,
652    ) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> {
653        if !matches!(
654            op.layout().backend_repr,
655            BackendRepr::Scalar(abi::Scalar::Initialized { .. })
656                | BackendRepr::ScalarPair(
657                    abi::Scalar::Initialized { .. },
658                    abi::Scalar::Initialized { .. }
659                )
660        ) {
661            span_bug!(self.cur_span(), "primitive read not possible for type: {}", op.layout().ty);
662        }
663        let imm = self.read_immediate_raw(op)?.right().unwrap();
664        if matches!(*imm, Immediate::Uninit) {
665            throw_ub!(InvalidUninitBytes(None));
666        }
667        interp_ok(imm)
668    }
669
670    /// Read a scalar from a place
671    pub fn read_scalar(
672        &self,
673        op: &impl Projectable<'tcx, M::Provenance>,
674    ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
675        interp_ok(self.read_immediate(op)?.to_scalar())
676    }
677
678    // Pointer-sized reads are fairly common and need target layout access, so we wrap them in
679    // convenience functions.
680
681    /// Read a pointer from a place.
682    pub fn read_pointer(
683        &self,
684        op: &impl Projectable<'tcx, M::Provenance>,
685    ) -> InterpResult<'tcx, Pointer<Option<M::Provenance>>> {
686        self.read_scalar(op)?.to_pointer(self)
687    }
688    /// Read a pointer-sized unsigned integer from a place.
689    pub fn read_target_usize(
690        &self,
691        op: &impl Projectable<'tcx, M::Provenance>,
692    ) -> InterpResult<'tcx, u64> {
693        self.read_scalar(op)?.to_target_usize(self)
694    }
695    /// Read a pointer-sized signed integer from a place.
696    pub fn read_target_isize(
697        &self,
698        op: &impl Projectable<'tcx, M::Provenance>,
699    ) -> InterpResult<'tcx, i64> {
700        self.read_scalar(op)?.to_target_isize(self)
701    }
702
703    /// Turn the wide MPlace into a string (must already be dereferenced!)
704    pub fn read_str(&self, mplace: &MPlaceTy<'tcx, M::Provenance>) -> InterpResult<'tcx, &str> {
705        let len = mplace.len(self)?;
706        let bytes = self.read_bytes_ptr_strip_provenance(mplace.ptr(), Size::from_bytes(len))?;
707        let s = std::str::from_utf8(bytes).map_err(|err| err_ub!(InvalidStr(err)))?;
708        interp_ok(s)
709    }
710
711    /// Read from a local of the current frame. Convenience method for [`InterpCx::local_at_frame_to_op`].
712    pub fn local_to_op(
713        &self,
714        local: mir::Local,
715        layout: Option<TyAndLayout<'tcx>>,
716    ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
717        self.local_at_frame_to_op(self.frame(), local, layout)
718    }
719
720    /// Read from a local of a given frame.
721    /// Will not access memory, instead an indirect `Operand` is returned.
722    ///
723    /// This is public because it is used by [Aquascope](https://github.com/cognitive-engineering-lab/aquascope/)
724    /// to get an OpTy from a local.
725    pub fn local_at_frame_to_op(
726        &self,
727        frame: &Frame<'tcx, M::Provenance, M::FrameExtra>,
728        local: mir::Local,
729        layout: Option<TyAndLayout<'tcx>>,
730    ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
731        let layout = self.layout_of_local(frame, local, layout)?;
732        let op = *frame.locals[local].access()?;
733        if matches!(op, Operand::Immediate(_)) {
734            assert!(!layout.is_unsized());
735        }
736        M::after_local_read(self, frame, local)?;
737        interp_ok(OpTy { op, layout })
738    }
739
740    /// Every place can be read from, so we can turn them into an operand.
741    /// This will definitely return `Indirect` if the place is a `Ptr`, i.e., this
742    /// will never actually read from memory.
743    pub fn place_to_op(
744        &self,
745        place: &PlaceTy<'tcx, M::Provenance>,
746    ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
747        match place.as_mplace_or_local() {
748            Left(mplace) => interp_ok(mplace.into()),
749            Right((local, offset, locals_addr, _)) => {
750                debug_assert!(place.layout.is_sized()); // only sized locals can ever be `Place::Local`.
751                debug_assert_eq!(locals_addr, self.frame().locals_addr());
752                let base = self.local_to_op(local, None)?;
753                interp_ok(match offset {
754                    Some(offset) => base.offset(offset, place.layout, self)?,
755                    None => {
756                        // In the common case this hasn't been projected.
757                        debug_assert_eq!(place.layout, base.layout);
758                        base
759                    }
760                })
761            }
762        }
763    }
764
765    /// Evaluate a place with the goal of reading from it. This lets us sometimes
766    /// avoid allocations.
767    pub fn eval_place_to_op(
768        &self,
769        mir_place: mir::Place<'tcx>,
770        layout: Option<TyAndLayout<'tcx>>,
771    ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
772        // Do not use the layout passed in as argument if the base we are looking at
773        // here is not the entire place.
774        let layout = if mir_place.projection.is_empty() { layout } else { None };
775
776        let mut op = self.local_to_op(mir_place.local, layout)?;
777        // Using `try_fold` turned out to be bad for performance, hence the loop.
778        for elem in mir_place.projection.iter() {
779            op = self.project(&op, elem)?
780        }
781
782        trace!("eval_place_to_op: got {:?}", op);
783        // Sanity-check the type we ended up with.
784        if cfg!(debug_assertions) {
785            let normalized_place_ty = self
786                .instantiate_from_current_frame_and_normalize_erasing_regions(
787                    mir_place.ty(&self.frame().body.local_decls, *self.tcx).ty,
788                )?;
789            if !mir_assign_valid_types(
790                *self.tcx,
791                self.typing_env(),
792                self.layout_of(normalized_place_ty)?,
793                op.layout,
794            ) {
795                span_bug!(
796                    self.cur_span(),
797                    "eval_place of a MIR place with type {} produced an interpreter operand with type {}",
798                    normalized_place_ty,
799                    op.layout.ty,
800                )
801            }
802        }
803        interp_ok(op)
804    }
805
806    /// Evaluate the operand, returning a place where you can then find the data.
807    /// If you already know the layout, you can save two table lookups
808    /// by passing it in here.
809    #[inline]
810    pub fn eval_operand(
811        &self,
812        mir_op: &mir::Operand<'tcx>,
813        layout: Option<TyAndLayout<'tcx>>,
814    ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
815        use rustc_middle::mir::Operand::*;
816        let op = match mir_op {
817            // FIXME: do some more logic on `move` to invalidate the old location
818            &Copy(place) | &Move(place) => self.eval_place_to_op(place, layout)?,
819
820            Constant(constant) => {
821                let c = self.instantiate_from_current_frame_and_normalize_erasing_regions(
822                    constant.const_,
823                )?;
824
825                // This can still fail:
826                // * During ConstProp, with `TooGeneric` or since the `required_consts` were not all
827                //   checked yet.
828                // * During CTFE, since promoteds in `const`/`static` initializer bodies can fail.
829                self.eval_mir_constant(&c, constant.span, layout)?
830            }
831        };
832        trace!("{:?}: {:?}", mir_op, op);
833        interp_ok(op)
834    }
835
836    pub(crate) fn const_val_to_op(
837        &self,
838        val_val: mir::ConstValue<'tcx>,
839        ty: Ty<'tcx>,
840        layout: Option<TyAndLayout<'tcx>>,
841    ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
842        // Other cases need layout.
843        let adjust_scalar = |scalar| -> InterpResult<'tcx, _> {
844            interp_ok(match scalar {
845                Scalar::Ptr(ptr, size) => Scalar::Ptr(self.global_root_pointer(ptr)?, size),
846                Scalar::Int(int) => Scalar::Int(int),
847            })
848        };
849        let layout =
850            from_known_layout(self.tcx, self.typing_env(), layout, || self.layout_of(ty).into())?;
851        let imm = match val_val {
852            mir::ConstValue::Indirect { alloc_id, offset } => {
853                // This is const data, no mutation allowed.
854                let ptr = self.global_root_pointer(Pointer::new(
855                    CtfeProvenance::from(alloc_id).as_immutable(),
856                    offset,
857                ))?;
858                return interp_ok(self.ptr_to_mplace(ptr.into(), layout).into());
859            }
860            mir::ConstValue::Scalar(x) => adjust_scalar(x)?.into(),
861            mir::ConstValue::ZeroSized => Immediate::Uninit,
862            mir::ConstValue::Slice { data, meta } => {
863                // This is const data, no mutation allowed.
864                let alloc_id = self.tcx.reserve_and_set_memory_alloc(data);
865                let ptr = Pointer::new(CtfeProvenance::from(alloc_id).as_immutable(), Size::ZERO);
866                Immediate::new_slice(self.global_root_pointer(ptr)?.into(), meta, self)
867            }
868        };
869        interp_ok(OpTy { op: Operand::Immediate(imm), layout })
870    }
871}
872
873// Some nodes are used a lot. Make sure they don't unintentionally get bigger.
874#[cfg(target_pointer_width = "64")]
875mod size_asserts {
876    use rustc_data_structures::static_assert_size;
877
878    use super::*;
879    // tidy-alphabetical-start
880    static_assert_size!(Immediate, 48);
881    static_assert_size!(ImmTy<'_>, 64);
882    static_assert_size!(Operand, 56);
883    static_assert_size!(OpTy<'_>, 72);
884    // tidy-alphabetical-end
885}
rustc_const_eval/interpret/operand.rs

rustc_const_eval/interpret/
operand.rs
