rustc_middle/mir/interpret/
allocation.rs

1//! The virtual memory representation of the MIR interpreter.
2
3mod init_mask;
4mod provenance_map;
5
6use std::borrow::Cow;
7use std::hash::Hash;
8use std::ops::{Deref, DerefMut, Range};
9use std::{fmt, hash, ptr};
10
11use either::{Left, Right};
12use init_mask::*;
13pub use init_mask::{InitChunk, InitChunkIter};
14use provenance_map::*;
15use rustc_abi::{Align, HasDataLayout, Size};
16use rustc_ast::Mutability;
17use rustc_data_structures::intern::Interned;
18use rustc_macros::HashStable;
19use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
20
21use super::{
22    AllocId, BadBytesAccess, CtfeProvenance, InterpErrorKind, InterpResult, Pointer,
23    PointerArithmetic, Provenance, ResourceExhaustionInfo, Scalar, ScalarSizeMismatch,
24    UndefinedBehaviorInfo, UnsupportedOpInfo, interp_ok, read_target_uint, write_target_uint,
25};
26use crate::ty;
27
28/// Functionality required for the bytes of an `Allocation`.
29pub trait AllocBytes: Clone + fmt::Debug + Deref<Target = [u8]> + DerefMut<Target = [u8]> {
30    /// Create an `AllocBytes` from a slice of `u8`.
31    fn from_bytes<'a>(slice: impl Into<Cow<'a, [u8]>>, _align: Align) -> Self;
32
33    /// Create a zeroed `AllocBytes` of the specified size and alignment.
34    /// Returns `None` if we ran out of memory on the host.
35    fn zeroed(size: Size, _align: Align) -> Option<Self>;
36
37    /// Gives direct access to the raw underlying storage.
38    ///
39    /// Crucially this pointer is compatible with:
40    /// - other pointers returned by this method, and
41    /// - references returned from `deref()`, as long as there was no write.
42    fn as_mut_ptr(&mut self) -> *mut u8;
43
44    /// Gives direct access to the raw underlying storage.
45    ///
46    /// Crucially this pointer is compatible with:
47    /// - other pointers returned by this method, and
48    /// - references returned from `deref()`, as long as there was no write.
49    fn as_ptr(&self) -> *const u8;
50}
51
52/// Default `bytes` for `Allocation` is a `Box<u8>`.
53impl AllocBytes for Box<[u8]> {
54    fn from_bytes<'a>(slice: impl Into<Cow<'a, [u8]>>, _align: Align) -> Self {
55        Box::<[u8]>::from(slice.into())
56    }
57
58    fn zeroed(size: Size, _align: Align) -> Option<Self> {
59        let bytes = Box::<[u8]>::try_new_zeroed_slice(size.bytes().try_into().ok()?).ok()?;
60        // SAFETY: the box was zero-allocated, which is a valid initial value for Box<[u8]>
61        let bytes = unsafe { bytes.assume_init() };
62        Some(bytes)
63    }
64
65    fn as_mut_ptr(&mut self) -> *mut u8 {
66        Box::as_mut_ptr(self).cast()
67    }
68
69    fn as_ptr(&self) -> *const u8 {
70        Box::as_ptr(self).cast()
71    }
72}
73
74/// This type represents an Allocation in the Miri/CTFE core engine.
75///
76/// Its public API is rather low-level, working directly with allocation offsets and a custom error
77/// type to account for the lack of an AllocId on this level. The Miri/CTFE core engine `memory`
78/// module provides higher-level access.
79// Note: for performance reasons when interning, some of the `Allocation` fields can be partially
80// hashed. (see the `Hash` impl below for more details), so the impl is not derived.
81#[derive(Clone, Eq, PartialEq)]
82#[derive(HashStable)]
83pub struct Allocation<Prov: Provenance = CtfeProvenance, Extra = (), Bytes = Box<[u8]>> {
84    /// The actual bytes of the allocation.
85    /// Note that the bytes of a pointer represent the offset of the pointer.
86    bytes: Bytes,
87    /// Maps from byte addresses to extra provenance data for each pointer.
88    /// Only the first byte of a pointer is inserted into the map; i.e.,
89    /// every entry in this map applies to `pointer_size` consecutive bytes starting
90    /// at the given offset.
91    provenance: ProvenanceMap<Prov>,
92    /// Denotes which part of this allocation is initialized.
93    init_mask: InitMask,
94    /// The alignment of the allocation to detect unaligned reads.
95    /// (`Align` guarantees that this is a power of two.)
96    pub align: Align,
97    /// `true` if the allocation is mutable.
98    /// Also used by codegen to determine if a static should be put into mutable memory,
99    /// which happens for `static mut` and `static` with interior mutability.
100    pub mutability: Mutability,
101    /// Extra state for the machine.
102    pub extra: Extra,
103}
104
105/// Helper struct that packs an alignment, mutability, and "all bytes are zero" flag together.
106///
107/// Alignment values always have 2 free high bits, and we check for this in our [`Encodable`] impl.
108struct AllocFlags {
109    align: Align,
110    mutability: Mutability,
111    all_zero: bool,
112}
113
114impl<E: Encoder> Encodable<E> for AllocFlags {
115    fn encode(&self, encoder: &mut E) {
116        // Make sure Align::MAX can be stored with the high 2 bits unset.
117        const {
118            let max_supported_align_repr = u8::MAX >> 2;
119            let max_supported_align = 1 << max_supported_align_repr;
120            assert!(Align::MAX.bytes() <= max_supported_align)
121        }
122
123        let mut flags = self.align.bytes().trailing_zeros() as u8;
124        flags |= match self.mutability {
125            Mutability::Not => 0,
126            Mutability::Mut => 1 << 6,
127        };
128        flags |= (self.all_zero as u8) << 7;
129        flags.encode(encoder);
130    }
131}
132
133impl<D: Decoder> Decodable<D> for AllocFlags {
134    fn decode(decoder: &mut D) -> Self {
135        let flags: u8 = Decodable::decode(decoder);
136        let align = flags & 0b0011_1111;
137        let mutability = flags & 0b0100_0000;
138        let all_zero = flags & 0b1000_0000;
139
140        let align = Align::from_bytes(1 << align).unwrap();
141        let mutability = match mutability {
142            0 => Mutability::Not,
143            _ => Mutability::Mut,
144        };
145        let all_zero = all_zero > 0;
146
147        AllocFlags { align, mutability, all_zero }
148    }
149}
150
151/// Efficiently detect whether a slice of `u8` is all zero.
152///
153/// This is used in encoding of [`Allocation`] to special-case all-zero allocations. It is only
154/// optimized a little, because for many allocations the encoding of the actual bytes does not
155/// dominate runtime.
156#[inline]
157fn all_zero(buf: &[u8]) -> bool {
158    // In the empty case we wouldn't encode any contents even without this system where we
159    // special-case allocations whose contents are all 0. We can return anything in the empty case.
160    if buf.is_empty() {
161        return true;
162    }
163    // Just fast-rejecting based on the first element significantly reduces the amount that we end
164    // up walking the whole array.
165    if buf[0] != 0 {
166        return false;
167    }
168
169    // This strategy of combining all slice elements with & or | is unbeatable for the large
170    // all-zero case because it is so well-understood by autovectorization.
171    buf.iter().fold(true, |acc, b| acc & (*b == 0))
172}
173
174/// Custom encoder for [`Allocation`] to more efficiently represent the case where all bytes are 0.
175impl<Prov: Provenance, Extra, Bytes, E: Encoder> Encodable<E> for Allocation<Prov, Extra, Bytes>
176where
177    Bytes: AllocBytes,
178    ProvenanceMap<Prov>: Encodable<E>,
179    Extra: Encodable<E>,
180{
181    fn encode(&self, encoder: &mut E) {
182        let all_zero = all_zero(&self.bytes);
183        AllocFlags { align: self.align, mutability: self.mutability, all_zero }.encode(encoder);
184
185        encoder.emit_usize(self.bytes.len());
186        if !all_zero {
187            encoder.emit_raw_bytes(&self.bytes);
188        }
189        self.provenance.encode(encoder);
190        self.init_mask.encode(encoder);
191        self.extra.encode(encoder);
192    }
193}
194
195impl<Prov: Provenance, Extra, Bytes, D: Decoder> Decodable<D> for Allocation<Prov, Extra, Bytes>
196where
197    Bytes: AllocBytes,
198    ProvenanceMap<Prov>: Decodable<D>,
199    Extra: Decodable<D>,
200{
201    fn decode(decoder: &mut D) -> Self {
202        let AllocFlags { align, mutability, all_zero } = Decodable::decode(decoder);
203
204        let len = decoder.read_usize();
205        let bytes = if all_zero { vec![0u8; len] } else { decoder.read_raw_bytes(len).to_vec() };
206        let bytes = Bytes::from_bytes(bytes, align);
207
208        let provenance = Decodable::decode(decoder);
209        let init_mask = Decodable::decode(decoder);
210        let extra = Decodable::decode(decoder);
211
212        Self { bytes, provenance, init_mask, align, mutability, extra }
213    }
214}
215
216/// This is the maximum size we will hash at a time, when interning an `Allocation` and its
217/// `InitMask`. Note, we hash that amount of bytes twice: at the start, and at the end of a buffer.
218/// Used when these two structures are large: we only partially hash the larger fields in that
219/// situation. See the comment at the top of their respective `Hash` impl for more details.
220const MAX_BYTES_TO_HASH: usize = 64;
221
222/// This is the maximum size (in bytes) for which a buffer will be fully hashed, when interning.
223/// Otherwise, it will be partially hashed in 2 slices, requiring at least 2 `MAX_BYTES_TO_HASH`
224/// bytes.
225const MAX_HASHED_BUFFER_LEN: usize = 2 * MAX_BYTES_TO_HASH;
226
227// Const allocations are only hashed for interning. However, they can be large, making the hashing
228// expensive especially since it uses `FxHash`: it's better suited to short keys, not potentially
229// big buffers like the actual bytes of allocation. We can partially hash some fields when they're
230// large.
231impl hash::Hash for Allocation {
232    fn hash<H: hash::Hasher>(&self, state: &mut H) {
233        let Self {
234            bytes,
235            provenance,
236            init_mask,
237            align,
238            mutability,
239            extra: (), // don't bother hashing ()
240        } = self;
241
242        // Partially hash the `bytes` buffer when it is large. To limit collisions with common
243        // prefixes and suffixes, we hash the length and some slices of the buffer.
244        let byte_count = bytes.len();
245        if byte_count > MAX_HASHED_BUFFER_LEN {
246            // Hash the buffer's length.
247            byte_count.hash(state);
248
249            // And its head and tail.
250            bytes[..MAX_BYTES_TO_HASH].hash(state);
251            bytes[byte_count - MAX_BYTES_TO_HASH..].hash(state);
252        } else {
253            bytes.hash(state);
254        }
255
256        // Hash the other fields as usual.
257        provenance.hash(state);
258        init_mask.hash(state);
259        align.hash(state);
260        mutability.hash(state);
261    }
262}
263
264/// Interned types generally have an `Outer` type and an `Inner` type, where
265/// `Outer` is a newtype around `Interned<Inner>`, and all the operations are
266/// done on `Outer`, because all occurrences are interned. E.g. `Ty` is an
267/// outer type and `TyKind` is its inner type.
268///
269/// Here things are different because only const allocations are interned. This
270/// means that both the inner type (`Allocation`) and the outer type
271/// (`ConstAllocation`) are used quite a bit.
272#[derive(Copy, Clone, PartialEq, Eq, Hash, HashStable)]
273#[rustc_pass_by_value]
274pub struct ConstAllocation<'tcx>(pub Interned<'tcx, Allocation>);
275
276impl<'tcx> fmt::Debug for ConstAllocation<'tcx> {
277    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
278        // The debug representation of this is very verbose and basically useless,
279        // so don't print it.
280        write!(f, "ConstAllocation {{ .. }}")
281    }
282}
283
284impl<'tcx> ConstAllocation<'tcx> {
285    pub fn inner(self) -> &'tcx Allocation {
286        self.0.0
287    }
288}
289
290/// We have our own error type that does not know about the `AllocId`; that information
291/// is added when converting to `InterpError`.
292#[derive(Debug)]
293pub enum AllocError {
294    /// A scalar had the wrong size.
295    ScalarSizeMismatch(ScalarSizeMismatch),
296    /// Encountered a pointer where we needed raw bytes.
297    ReadPointerAsInt(Option<BadBytesAccess>),
298    /// Partially overwriting a pointer.
299    OverwritePartialPointer(Size),
300    /// Partially copying a pointer.
301    ReadPartialPointer(Size),
302    /// Using uninitialized data where it is not allowed.
303    InvalidUninitBytes(Option<BadBytesAccess>),
304}
305pub type AllocResult<T = ()> = Result<T, AllocError>;
306
307impl From<ScalarSizeMismatch> for AllocError {
308    fn from(s: ScalarSizeMismatch) -> Self {
309        AllocError::ScalarSizeMismatch(s)
310    }
311}
312
313impl AllocError {
314    pub fn to_interp_error<'tcx>(self, alloc_id: AllocId) -> InterpErrorKind<'tcx> {
315        use AllocError::*;
316        match self {
317            ScalarSizeMismatch(s) => {
318                InterpErrorKind::UndefinedBehavior(UndefinedBehaviorInfo::ScalarSizeMismatch(s))
319            }
320            ReadPointerAsInt(info) => InterpErrorKind::Unsupported(
321                UnsupportedOpInfo::ReadPointerAsInt(info.map(|b| (alloc_id, b))),
322            ),
323            OverwritePartialPointer(offset) => InterpErrorKind::Unsupported(
324                UnsupportedOpInfo::OverwritePartialPointer(Pointer::new(alloc_id, offset)),
325            ),
326            ReadPartialPointer(offset) => InterpErrorKind::Unsupported(
327                UnsupportedOpInfo::ReadPartialPointer(Pointer::new(alloc_id, offset)),
328            ),
329            InvalidUninitBytes(info) => InterpErrorKind::UndefinedBehavior(
330                UndefinedBehaviorInfo::InvalidUninitBytes(info.map(|b| (alloc_id, b))),
331            ),
332        }
333    }
334}
335
336/// The information that makes up a memory access: offset and size.
337#[derive(Copy, Clone)]
338pub struct AllocRange {
339    pub start: Size,
340    pub size: Size,
341}
342
343impl fmt::Debug for AllocRange {
344    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
345        write!(f, "[{:#x}..{:#x}]", self.start.bytes(), self.end().bytes())
346    }
347}
348
349/// Free-starting constructor for less syntactic overhead.
350#[inline(always)]
351pub fn alloc_range(start: Size, size: Size) -> AllocRange {
352    AllocRange { start, size }
353}
354
355impl From<Range<Size>> for AllocRange {
356    #[inline]
357    fn from(r: Range<Size>) -> Self {
358        alloc_range(r.start, r.end - r.start) // `Size` subtraction (overflow-checked)
359    }
360}
361
362impl From<Range<usize>> for AllocRange {
363    #[inline]
364    fn from(r: Range<usize>) -> Self {
365        AllocRange::from(Size::from_bytes(r.start)..Size::from_bytes(r.end))
366    }
367}
368
369impl AllocRange {
370    #[inline(always)]
371    pub fn end(self) -> Size {
372        self.start + self.size // This does overflow checking.
373    }
374
375    /// Returns the `subrange` within this range; panics if it is not a subrange.
376    #[inline]
377    pub fn subrange(self, subrange: AllocRange) -> AllocRange {
378        let sub_start = self.start + subrange.start;
379        let range = alloc_range(sub_start, subrange.size);
380        assert!(range.end() <= self.end(), "access outside the bounds for given AllocRange");
381        range
382    }
383}
384
385/// Whether a new allocation should be initialized with zero-bytes.
386pub enum AllocInit {
387    Uninit,
388    Zero,
389}
390
391// The constructors are all without extra; the extra gets added by a machine hook later.
392impl<Prov: Provenance, Bytes: AllocBytes> Allocation<Prov, (), Bytes> {
393    /// Creates an allocation initialized by the given bytes
394    pub fn from_bytes<'a>(
395        slice: impl Into<Cow<'a, [u8]>>,
396        align: Align,
397        mutability: Mutability,
398    ) -> Self {
399        let bytes = Bytes::from_bytes(slice, align);
400        let size = Size::from_bytes(bytes.len());
401        Self {
402            bytes,
403            provenance: ProvenanceMap::new(),
404            init_mask: InitMask::new(size, true),
405            align,
406            mutability,
407            extra: (),
408        }
409    }
410
411    pub fn from_bytes_byte_aligned_immutable<'a>(slice: impl Into<Cow<'a, [u8]>>) -> Self {
412        Allocation::from_bytes(slice, Align::ONE, Mutability::Not)
413    }
414
415    fn new_inner<R>(
416        size: Size,
417        align: Align,
418        init: AllocInit,
419        fail: impl FnOnce() -> R,
420    ) -> Result<Self, R> {
421        // We raise an error if we cannot create the allocation on the host.
422        // This results in an error that can happen non-deterministically, since the memory
423        // available to the compiler can change between runs. Normally queries are always
424        // deterministic. However, we can be non-deterministic here because all uses of const
425        // evaluation (including ConstProp!) will make compilation fail (via hard error
426        // or ICE) upon encountering a `MemoryExhausted` error.
427        let bytes = Bytes::zeroed(size, align).ok_or_else(fail)?;
428
429        Ok(Allocation {
430            bytes,
431            provenance: ProvenanceMap::new(),
432            init_mask: InitMask::new(
433                size,
434                match init {
435                    AllocInit::Uninit => false,
436                    AllocInit::Zero => true,
437                },
438            ),
439            align,
440            mutability: Mutability::Mut,
441            extra: (),
442        })
443    }
444
445    /// Try to create an Allocation of `size` bytes, failing if there is not enough memory
446    /// available to the compiler to do so.
447    pub fn try_new<'tcx>(size: Size, align: Align, init: AllocInit) -> InterpResult<'tcx, Self> {
448        Self::new_inner(size, align, init, || {
449            ty::tls::with(|tcx| tcx.dcx().delayed_bug("exhausted memory during interpretation"));
450            InterpErrorKind::ResourceExhaustion(ResourceExhaustionInfo::MemoryExhausted)
451        })
452        .into()
453    }
454
455    /// Try to create an Allocation of `size` bytes, panics if there is not enough memory
456    /// available to the compiler to do so.
457    ///
458    /// Example use case: To obtain an Allocation filled with specific data,
459    /// first call this function and then call write_scalar to fill in the right data.
460    pub fn new(size: Size, align: Align, init: AllocInit) -> Self {
461        match Self::new_inner(size, align, init, || {
462            panic!(
463                "interpreter ran out of memory: cannot create allocation of {} bytes",
464                size.bytes()
465            );
466        }) {
467            Ok(x) => x,
468            Err(x) => x,
469        }
470    }
471
472    /// Add the extra.
473    pub fn with_extra<Extra>(self, extra: Extra) -> Allocation<Prov, Extra, Bytes> {
474        Allocation {
475            bytes: self.bytes,
476            provenance: self.provenance,
477            init_mask: self.init_mask,
478            align: self.align,
479            mutability: self.mutability,
480            extra,
481        }
482    }
483}
484
485impl Allocation {
486    /// Adjust allocation from the ones in `tcx` to a custom Machine instance
487    /// with a different `Provenance` and `Byte` type.
488    pub fn adjust_from_tcx<'tcx, Prov: Provenance, Bytes: AllocBytes>(
489        &self,
490        cx: &impl HasDataLayout,
491        mut alloc_bytes: impl FnMut(&[u8], Align) -> InterpResult<'tcx, Bytes>,
492        mut adjust_ptr: impl FnMut(Pointer<CtfeProvenance>) -> InterpResult<'tcx, Pointer<Prov>>,
493    ) -> InterpResult<'tcx, Allocation<Prov, (), Bytes>> {
494        // Copy the data.
495        let mut bytes = alloc_bytes(&*self.bytes, self.align)?;
496        // Adjust provenance of pointers stored in this allocation.
497        let mut new_provenance = Vec::with_capacity(self.provenance.ptrs().len());
498        let ptr_size = cx.data_layout().pointer_size.bytes_usize();
499        let endian = cx.data_layout().endian;
500        for &(offset, alloc_id) in self.provenance.ptrs().iter() {
501            let idx = offset.bytes_usize();
502            let ptr_bytes = &mut bytes[idx..idx + ptr_size];
503            let bits = read_target_uint(endian, ptr_bytes).unwrap();
504            let (ptr_prov, ptr_offset) =
505                adjust_ptr(Pointer::new(alloc_id, Size::from_bytes(bits)))?.into_parts();
506            write_target_uint(endian, ptr_bytes, ptr_offset.bytes().into()).unwrap();
507            new_provenance.push((offset, ptr_prov));
508        }
509        // Create allocation.
510        interp_ok(Allocation {
511            bytes,
512            provenance: ProvenanceMap::from_presorted_ptrs(new_provenance),
513            init_mask: self.init_mask.clone(),
514            align: self.align,
515            mutability: self.mutability,
516            extra: self.extra,
517        })
518    }
519}
520
521/// Raw accessors. Provide access to otherwise private bytes.
522impl<Prov: Provenance, Extra, Bytes: AllocBytes> Allocation<Prov, Extra, Bytes> {
523    pub fn len(&self) -> usize {
524        self.bytes.len()
525    }
526
527    pub fn size(&self) -> Size {
528        Size::from_bytes(self.len())
529    }
530
531    /// Looks at a slice which may contain uninitialized bytes or provenance. This differs
532    /// from `get_bytes_with_uninit_and_ptr` in that it does no provenance checks (even on the
533    /// edges) at all.
534    /// This must not be used for reads affecting the interpreter execution.
535    pub fn inspect_with_uninit_and_ptr_outside_interpreter(&self, range: Range<usize>) -> &[u8] {
536        &self.bytes[range]
537    }
538
539    /// Returns the mask indicating which bytes are initialized.
540    pub fn init_mask(&self) -> &InitMask {
541        &self.init_mask
542    }
543
544    /// Returns the provenance map.
545    pub fn provenance(&self) -> &ProvenanceMap<Prov> {
546        &self.provenance
547    }
548}
549
550/// Byte accessors.
551impl<Prov: Provenance, Extra, Bytes: AllocBytes> Allocation<Prov, Extra, Bytes> {
552    /// This is the entirely abstraction-violating way to just grab the raw bytes without
553    /// caring about provenance or initialization.
554    ///
555    /// This function also guarantees that the resulting pointer will remain stable
556    /// even when new allocations are pushed to the `HashMap`. `mem_copy_repeatedly` relies
557    /// on that.
558    #[inline]
559    pub fn get_bytes_unchecked(&self, range: AllocRange) -> &[u8] {
560        &self.bytes[range.start.bytes_usize()..range.end().bytes_usize()]
561    }
562
563    /// Checks that these bytes are initialized, and then strip provenance (if possible) and return
564    /// them.
565    ///
566    /// It is the caller's responsibility to check bounds and alignment beforehand.
567    /// Most likely, you want to use the `PlaceTy` and `OperandTy`-based methods
568    /// on `InterpCx` instead.
569    #[inline]
570    pub fn get_bytes_strip_provenance(
571        &self,
572        cx: &impl HasDataLayout,
573        range: AllocRange,
574    ) -> AllocResult<&[u8]> {
575        self.init_mask.is_range_initialized(range).map_err(|uninit_range| {
576            AllocError::InvalidUninitBytes(Some(BadBytesAccess {
577                access: range,
578                bad: uninit_range,
579            }))
580        })?;
581        if !Prov::OFFSET_IS_ADDR && !self.provenance.range_empty(range, cx) {
582            // Find the provenance.
583            let (offset, _prov) = self
584                .provenance
585                .range_ptrs_get(range, cx)
586                .first()
587                .copied()
588                .expect("there must be provenance somewhere here");
589            let start = offset.max(range.start); // the pointer might begin before `range`!
590            let end = (offset + cx.pointer_size()).min(range.end()); // the pointer might end after `range`!
591            return Err(AllocError::ReadPointerAsInt(Some(BadBytesAccess {
592                access: range,
593                bad: AllocRange::from(start..end),
594            })));
595        }
596        Ok(self.get_bytes_unchecked(range))
597    }
598
599    /// This is the entirely abstraction-violating way to just get mutable access to the raw bytes.
600    /// Just calling this already marks everything as defined and removes provenance, so be sure to
601    /// actually overwrite all the data there!
602    ///
603    /// It is the caller's responsibility to check bounds and alignment beforehand.
604    /// Most likely, you want to use the `PlaceTy` and `OperandTy`-based methods
605    /// on `InterpCx` instead.
606    pub fn get_bytes_unchecked_for_overwrite(
607        &mut self,
608        cx: &impl HasDataLayout,
609        range: AllocRange,
610    ) -> AllocResult<&mut [u8]> {
611        self.mark_init(range, true);
612        self.provenance.clear(range, cx)?;
613
614        Ok(&mut self.bytes[range.start.bytes_usize()..range.end().bytes_usize()])
615    }
616
617    /// A raw pointer variant of `get_bytes_unchecked_for_overwrite` that avoids invalidating existing immutable aliases
618    /// into this memory.
619    pub fn get_bytes_unchecked_for_overwrite_ptr(
620        &mut self,
621        cx: &impl HasDataLayout,
622        range: AllocRange,
623    ) -> AllocResult<*mut [u8]> {
624        self.mark_init(range, true);
625        self.provenance.clear(range, cx)?;
626
627        assert!(range.end().bytes_usize() <= self.bytes.len()); // need to do our own bounds-check
628        // Crucially, we go via `AllocBytes::as_mut_ptr`, not `AllocBytes::deref_mut`.
629        let begin_ptr = self.bytes.as_mut_ptr().wrapping_add(range.start.bytes_usize());
630        let len = range.end().bytes_usize() - range.start.bytes_usize();
631        Ok(ptr::slice_from_raw_parts_mut(begin_ptr, len))
632    }
633
634    /// This gives direct mutable access to the entire buffer, just exposing their internal state
635    /// without resetting anything. Directly exposes `AllocBytes::as_mut_ptr`. Only works if
636    /// `OFFSET_IS_ADDR` is true.
637    pub fn get_bytes_unchecked_raw_mut(&mut self) -> *mut u8 {
638        assert!(Prov::OFFSET_IS_ADDR);
639        self.bytes.as_mut_ptr()
640    }
641
642    /// This gives direct immutable access to the entire buffer, just exposing their internal state
643    /// without resetting anything. Directly exposes `AllocBytes::as_ptr`. Only works if
644    /// `OFFSET_IS_ADDR` is true.
645    pub fn get_bytes_unchecked_raw(&self) -> *const u8 {
646        assert!(Prov::OFFSET_IS_ADDR);
647        self.bytes.as_ptr()
648    }
649}
650
651/// Reading and writing.
652impl<Prov: Provenance, Extra, Bytes: AllocBytes> Allocation<Prov, Extra, Bytes> {
653    /// Sets the init bit for the given range.
654    fn mark_init(&mut self, range: AllocRange, is_init: bool) {
655        if range.size.bytes() == 0 {
656            return;
657        }
658        assert!(self.mutability == Mutability::Mut);
659        self.init_mask.set_range(range, is_init);
660    }
661
662    /// Reads a *non-ZST* scalar.
663    ///
664    /// If `read_provenance` is `true`, this will also read provenance; otherwise (if the machine
665    /// supports that) provenance is entirely ignored.
666    ///
667    /// ZSTs can't be read because in order to obtain a `Pointer`, we need to check
668    /// for ZSTness anyway due to integer pointers being valid for ZSTs.
669    ///
670    /// It is the caller's responsibility to check bounds and alignment beforehand.
671    /// Most likely, you want to call `InterpCx::read_scalar` instead of this method.
672    pub fn read_scalar(
673        &self,
674        cx: &impl HasDataLayout,
675        range: AllocRange,
676        read_provenance: bool,
677    ) -> AllocResult<Scalar<Prov>> {
678        // First and foremost, if anything is uninit, bail.
679        if self.init_mask.is_range_initialized(range).is_err() {
680            return Err(AllocError::InvalidUninitBytes(None));
681        }
682
683        // Get the integer part of the result. We HAVE TO check provenance before returning this!
684        let bytes = self.get_bytes_unchecked(range);
685        let bits = read_target_uint(cx.data_layout().endian, bytes).unwrap();
686
687        if read_provenance {
688            assert_eq!(range.size, cx.data_layout().pointer_size);
689
690            // When reading data with provenance, the easy case is finding provenance exactly where we
691            // are reading, then we can put data and provenance back together and return that.
692            if let Some(prov) = self.provenance.get_ptr(range.start) {
693                // Now we can return the bits, with their appropriate provenance.
694                let ptr = Pointer::new(prov, Size::from_bytes(bits));
695                return Ok(Scalar::from_pointer(ptr, cx));
696            }
697
698            // If we can work on pointers byte-wise, join the byte-wise provenances.
699            if Prov::OFFSET_IS_ADDR {
700                let mut prov = self.provenance.get(range.start, cx);
701                for offset in Size::from_bytes(1)..range.size {
702                    let this_prov = self.provenance.get(range.start + offset, cx);
703                    prov = Prov::join(prov, this_prov);
704                }
705                // Now use this provenance.
706                let ptr = Pointer::new(prov, Size::from_bytes(bits));
707                return Ok(Scalar::from_maybe_pointer(ptr, cx));
708            } else {
709                // Without OFFSET_IS_ADDR, the only remaining case we can handle is total absence of
710                // provenance.
711                if self.provenance.range_empty(range, cx) {
712                    return Ok(Scalar::from_uint(bits, range.size));
713                }
714                // Else we have mixed provenance, that doesn't work.
715                return Err(AllocError::ReadPartialPointer(range.start));
716            }
717        } else {
718            // We are *not* reading a pointer.
719            // If we can just ignore provenance or there is none, that's easy.
720            if Prov::OFFSET_IS_ADDR || self.provenance.range_empty(range, cx) {
721                // We just strip provenance.
722                return Ok(Scalar::from_uint(bits, range.size));
723            }
724            // There is some provenance and we don't have OFFSET_IS_ADDR. This doesn't work.
725            return Err(AllocError::ReadPointerAsInt(None));
726        }
727    }
728
729    /// Writes a *non-ZST* scalar.
730    ///
731    /// ZSTs can't be read because in order to obtain a `Pointer`, we need to check
732    /// for ZSTness anyway due to integer pointers being valid for ZSTs.
733    ///
734    /// It is the caller's responsibility to check bounds and alignment beforehand.
735    /// Most likely, you want to call `InterpCx::write_scalar` instead of this method.
736    pub fn write_scalar(
737        &mut self,
738        cx: &impl HasDataLayout,
739        range: AllocRange,
740        val: Scalar<Prov>,
741    ) -> AllocResult {
742        assert!(self.mutability == Mutability::Mut);
743
744        // `to_bits_or_ptr_internal` is the right method because we just want to store this data
745        // as-is into memory. This also double-checks that `val.size()` matches `range.size`.
746        let (bytes, provenance) = match val.to_bits_or_ptr_internal(range.size)? {
747            Right(ptr) => {
748                let (provenance, offset) = ptr.into_parts();
749                (u128::from(offset.bytes()), Some(provenance))
750            }
751            Left(data) => (data, None),
752        };
753
754        let endian = cx.data_layout().endian;
755        // Yes we do overwrite all the bytes in `dst`.
756        let dst = self.get_bytes_unchecked_for_overwrite(cx, range)?;
757        write_target_uint(endian, dst, bytes).unwrap();
758
759        // See if we have to also store some provenance.
760        if let Some(provenance) = provenance {
761            assert_eq!(range.size, cx.data_layout().pointer_size);
762            self.provenance.insert_ptr(range.start, provenance, cx);
763        }
764
765        Ok(())
766    }
767
768    /// Write "uninit" to the given memory range.
769    pub fn write_uninit(&mut self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult {
770        self.mark_init(range, false);
771        self.provenance.clear(range, cx)?;
772        Ok(())
773    }
774
775    /// Initialize all previously uninitialized bytes in the entire allocation, and set
776    /// provenance of everything to `Wildcard`. Before calling this, make sure all
777    /// provenance in this allocation is exposed!
778    pub fn prepare_for_native_write(&mut self) -> AllocResult {
779        let full_range = AllocRange { start: Size::ZERO, size: Size::from_bytes(self.len()) };
780        // Overwrite uninitialized bytes with 0, to ensure we don't leak whatever their value happens to be.
781        for chunk in self.init_mask.range_as_init_chunks(full_range) {
782            if !chunk.is_init() {
783                let uninit_bytes = &mut self.bytes
784                    [chunk.range().start.bytes_usize()..chunk.range().end.bytes_usize()];
785                uninit_bytes.fill(0);
786            }
787        }
788        // Mark everything as initialized now.
789        self.mark_init(full_range, true);
790
791        // Set provenance of all bytes to wildcard.
792        self.provenance.write_wildcards(self.len());
793
794        // Also expose the provenance of the interpreter-level allocation, so it can
795        // be written by FFI. The `black_box` is defensive programming as LLVM likes
796        // to (incorrectly) optimize away ptr2int casts whose result is unused.
797        std::hint::black_box(self.get_bytes_unchecked_raw_mut().expose_provenance());
798
799        Ok(())
800    }
801
802    /// Remove all provenance in the given memory range.
803    pub fn clear_provenance(&mut self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult {
804        self.provenance.clear(range, cx)?;
805        return Ok(());
806    }
807
808    /// Applies a previously prepared provenance copy.
809    /// The affected range, as defined in the parameters to `provenance().prepare_copy` is expected
810    /// to be clear of provenance.
811    ///
812    /// This is dangerous to use as it can violate internal `Allocation` invariants!
813    /// It only exists to support an efficient implementation of `mem_copy_repeatedly`.
814    pub fn provenance_apply_copy(&mut self, copy: ProvenanceCopy<Prov>) {
815        self.provenance.apply_copy(copy)
816    }
817
818    /// Applies a previously prepared copy of the init mask.
819    ///
820    /// This is dangerous to use as it can violate internal `Allocation` invariants!
821    /// It only exists to support an efficient implementation of `mem_copy_repeatedly`.
822    pub fn init_mask_apply_copy(&mut self, copy: InitCopy, range: AllocRange, repeat: u64) {
823        self.init_mask.apply_copy(copy, range, repeat)
824    }
825}