1use std::any::Any;
5use std::borrow::Cow;
6use std::cell::{Cell, RefCell};
7use std::collections::hash_map::Entry;
8use std::path::Path;
9use std::rc::Rc;
10use std::{fmt, process};
11
12use rand::rngs::StdRng;
13use rand::{Rng, SeedableRng};
14use rustc_abi::{Align, ExternAbi, Size};
15use rustc_apfloat::{Float, FloatConvert};
16use rustc_attr_data_structures::InlineAttr;
17use rustc_data_structures::fx::{FxHashMap, FxHashSet};
18#[allow(unused)]
19use rustc_data_structures::static_assert_size;
20use rustc_middle::mir;
21use rustc_middle::query::TyCtxtAt;
22use rustc_middle::ty::layout::{
23 HasTyCtxt, HasTypingEnv, LayoutCx, LayoutError, LayoutOf, TyAndLayout,
24};
25use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
26use rustc_session::config::InliningThreshold;
27use rustc_span::def_id::{CrateNum, DefId};
28use rustc_span::{Span, SpanData, Symbol};
29use rustc_target::callconv::FnAbi;
30
31use crate::alloc_addresses::EvalContextExt;
32use crate::concurrency::cpu_affinity::{self, CpuAffinityMask};
33use crate::concurrency::data_race::{self, NaReadType, NaWriteType};
34use crate::concurrency::{AllocDataRaceHandler, GenmcCtx, GlobalDataRaceHandler, weak_memory};
35use crate::*;
36
37pub const SIGRTMIN: i32 = 34;
41
42pub const SIGRTMAX: i32 = 42;
46
47const ADDRS_PER_ANON_GLOBAL: usize = 32;
51
52pub struct FrameExtra<'tcx> {
54 pub borrow_tracker: Option<borrow_tracker::FrameState>,
56
57 pub catch_unwind: Option<CatchUnwindData<'tcx>>,
61
62 pub timing: Option<measureme::DetachedTiming>,
66
67 pub is_user_relevant: bool,
72
73 salt: usize,
78
79 pub data_race: Option<data_race::FrameState>,
81}
82
83impl<'tcx> std::fmt::Debug for FrameExtra<'tcx> {
84 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
85 let FrameExtra {
87 borrow_tracker,
88 catch_unwind,
89 timing: _,
90 is_user_relevant,
91 salt,
92 data_race,
93 } = self;
94 f.debug_struct("FrameData")
95 .field("borrow_tracker", borrow_tracker)
96 .field("catch_unwind", catch_unwind)
97 .field("is_user_relevant", is_user_relevant)
98 .field("salt", salt)
99 .field("data_race", data_race)
100 .finish()
101 }
102}
103
104impl VisitProvenance for FrameExtra<'_> {
105 fn visit_provenance(&self, visit: &mut VisitWith<'_>) {
106 let FrameExtra {
107 catch_unwind,
108 borrow_tracker,
109 timing: _,
110 is_user_relevant: _,
111 salt: _,
112 data_race: _,
113 } = self;
114
115 catch_unwind.visit_provenance(visit);
116 borrow_tracker.visit_provenance(visit);
117 }
118}
119
120#[derive(Debug, Copy, Clone, PartialEq, Eq)]
122pub enum MiriMemoryKind {
123 Rust,
125 Miri,
127 C,
129 WinHeap,
131 WinLocal,
133 Machine,
136 Runtime,
139 Global,
142 ExternStatic,
145 Tls,
148 Mmap,
150}
151
152impl From<MiriMemoryKind> for MemoryKind {
153 #[inline(always)]
154 fn from(kind: MiriMemoryKind) -> MemoryKind {
155 MemoryKind::Machine(kind)
156 }
157}
158
159impl MayLeak for MiriMemoryKind {
160 #[inline(always)]
161 fn may_leak(self) -> bool {
162 use self::MiriMemoryKind::*;
163 match self {
164 Rust | Miri | C | WinHeap | WinLocal | Runtime => false,
165 Machine | Global | ExternStatic | Tls | Mmap => true,
166 }
167 }
168}
169
170impl MiriMemoryKind {
171 fn should_save_allocation_span(self) -> bool {
173 use self::MiriMemoryKind::*;
174 match self {
175 Rust | Miri | C | WinHeap | WinLocal | Mmap => true,
177 Machine | Global | ExternStatic | Tls | Runtime => false,
179 }
180 }
181}
182
183impl fmt::Display for MiriMemoryKind {
184 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
185 use self::MiriMemoryKind::*;
186 match self {
187 Rust => write!(f, "Rust heap"),
188 Miri => write!(f, "Miri bare-metal heap"),
189 C => write!(f, "C heap"),
190 WinHeap => write!(f, "Windows heap"),
191 WinLocal => write!(f, "Windows local memory"),
192 Machine => write!(f, "machine-managed memory"),
193 Runtime => write!(f, "language runtime memory"),
194 Global => write!(f, "global (static or const)"),
195 ExternStatic => write!(f, "extern static"),
196 Tls => write!(f, "thread-local static"),
197 Mmap => write!(f, "mmap"),
198 }
199 }
200}
201
202pub type MemoryKind = interpret::MemoryKind<MiriMemoryKind>;
203
204#[derive(Clone, Copy, PartialEq, Eq, Hash)]
210pub enum Provenance {
211 Concrete {
214 alloc_id: AllocId,
215 tag: BorTag,
217 },
218 Wildcard,
235}
236
237#[derive(Copy, Clone, PartialEq)]
239pub enum ProvenanceExtra {
240 Concrete(BorTag),
241 Wildcard,
242}
243
244#[cfg(target_pointer_width = "64")]
245static_assert_size!(StrictPointer, 24);
246#[cfg(target_pointer_width = "64")]
250static_assert_size!(Scalar, 32);
251
252impl fmt::Debug for Provenance {
253 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
254 match self {
255 Provenance::Concrete { alloc_id, tag } => {
256 if f.alternate() {
258 write!(f, "[{alloc_id:#?}]")?;
259 } else {
260 write!(f, "[{alloc_id:?}]")?;
261 }
262 write!(f, "{tag:?}")?;
264 }
265 Provenance::Wildcard => {
266 write!(f, "[wildcard]")?;
267 }
268 }
269 Ok(())
270 }
271}
272
273impl interpret::Provenance for Provenance {
274 const OFFSET_IS_ADDR: bool = true;
276
277 const WILDCARD: Option<Self> = Some(Provenance::Wildcard);
279
280 fn get_alloc_id(self) -> Option<AllocId> {
281 match self {
282 Provenance::Concrete { alloc_id, .. } => Some(alloc_id),
283 Provenance::Wildcard => None,
284 }
285 }
286
287 fn fmt(ptr: &interpret::Pointer<Self>, f: &mut fmt::Formatter<'_>) -> fmt::Result {
288 let (prov, addr) = ptr.into_parts(); write!(f, "{:#x}", addr.bytes())?;
290 if f.alternate() {
291 write!(f, "{prov:#?}")?;
292 } else {
293 write!(f, "{prov:?}")?;
294 }
295 Ok(())
296 }
297
298 fn join(left: Option<Self>, right: Option<Self>) -> Option<Self> {
299 match (left, right) {
300 (
302 Some(Provenance::Concrete { alloc_id: left_alloc, tag: left_tag }),
303 Some(Provenance::Concrete { alloc_id: right_alloc, tag: right_tag }),
304 ) if left_alloc == right_alloc && left_tag == right_tag => left,
305 (Some(Provenance::Wildcard), o) | (o, Some(Provenance::Wildcard)) => o,
308 _ => None,
310 }
311 }
312}
313
314impl fmt::Debug for ProvenanceExtra {
315 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
316 match self {
317 ProvenanceExtra::Concrete(pid) => write!(f, "{pid:?}"),
318 ProvenanceExtra::Wildcard => write!(f, "<wildcard>"),
319 }
320 }
321}
322
323impl ProvenanceExtra {
324 pub fn and_then<T>(self, f: impl FnOnce(BorTag) -> Option<T>) -> Option<T> {
325 match self {
326 ProvenanceExtra::Concrete(pid) => f(pid),
327 ProvenanceExtra::Wildcard => None,
328 }
329 }
330}
331
332#[derive(Debug)]
334pub struct AllocExtra<'tcx> {
335 pub borrow_tracker: Option<borrow_tracker::AllocState>,
337 pub data_race: AllocDataRaceHandler,
341 pub backtrace: Option<Vec<FrameInfo<'tcx>>>,
346 pub sync: FxHashMap<Size, Box<dyn Any>>,
351}
352
353impl<'tcx> Clone for AllocExtra<'tcx> {
356 fn clone(&self) -> Self {
357 panic!("our allocations should never be cloned");
358 }
359}
360
361impl VisitProvenance for AllocExtra<'_> {
362 fn visit_provenance(&self, visit: &mut VisitWith<'_>) {
363 let AllocExtra { borrow_tracker, data_race, backtrace: _, sync: _ } = self;
364
365 borrow_tracker.visit_provenance(visit);
366 data_race.visit_provenance(visit);
367 }
368}
369
370pub struct PrimitiveLayouts<'tcx> {
372 pub unit: TyAndLayout<'tcx>,
373 pub i8: TyAndLayout<'tcx>,
374 pub i16: TyAndLayout<'tcx>,
375 pub i32: TyAndLayout<'tcx>,
376 pub i64: TyAndLayout<'tcx>,
377 pub i128: TyAndLayout<'tcx>,
378 pub isize: TyAndLayout<'tcx>,
379 pub u8: TyAndLayout<'tcx>,
380 pub u16: TyAndLayout<'tcx>,
381 pub u32: TyAndLayout<'tcx>,
382 pub u64: TyAndLayout<'tcx>,
383 pub u128: TyAndLayout<'tcx>,
384 pub usize: TyAndLayout<'tcx>,
385 pub bool: TyAndLayout<'tcx>,
386 pub mut_raw_ptr: TyAndLayout<'tcx>, pub const_raw_ptr: TyAndLayout<'tcx>, }
389
390impl<'tcx> PrimitiveLayouts<'tcx> {
391 fn new(layout_cx: LayoutCx<'tcx>) -> Result<Self, &'tcx LayoutError<'tcx>> {
392 let tcx = layout_cx.tcx();
393 let mut_raw_ptr = Ty::new_mut_ptr(tcx, tcx.types.unit);
394 let const_raw_ptr = Ty::new_imm_ptr(tcx, tcx.types.unit);
395 Ok(Self {
396 unit: layout_cx.layout_of(tcx.types.unit)?,
397 i8: layout_cx.layout_of(tcx.types.i8)?,
398 i16: layout_cx.layout_of(tcx.types.i16)?,
399 i32: layout_cx.layout_of(tcx.types.i32)?,
400 i64: layout_cx.layout_of(tcx.types.i64)?,
401 i128: layout_cx.layout_of(tcx.types.i128)?,
402 isize: layout_cx.layout_of(tcx.types.isize)?,
403 u8: layout_cx.layout_of(tcx.types.u8)?,
404 u16: layout_cx.layout_of(tcx.types.u16)?,
405 u32: layout_cx.layout_of(tcx.types.u32)?,
406 u64: layout_cx.layout_of(tcx.types.u64)?,
407 u128: layout_cx.layout_of(tcx.types.u128)?,
408 usize: layout_cx.layout_of(tcx.types.usize)?,
409 bool: layout_cx.layout_of(tcx.types.bool)?,
410 mut_raw_ptr: layout_cx.layout_of(mut_raw_ptr)?,
411 const_raw_ptr: layout_cx.layout_of(const_raw_ptr)?,
412 })
413 }
414
415 pub fn uint(&self, size: Size) -> Option<TyAndLayout<'tcx>> {
416 match size.bits() {
417 8 => Some(self.u8),
418 16 => Some(self.u16),
419 32 => Some(self.u32),
420 64 => Some(self.u64),
421 128 => Some(self.u128),
422 _ => None,
423 }
424 }
425
426 pub fn int(&self, size: Size) -> Option<TyAndLayout<'tcx>> {
427 match size.bits() {
428 8 => Some(self.i8),
429 16 => Some(self.i16),
430 32 => Some(self.i32),
431 64 => Some(self.i64),
432 128 => Some(self.i128),
433 _ => None,
434 }
435 }
436}
437
438pub struct MiriMachine<'tcx> {
443 pub tcx: TyCtxt<'tcx>,
445
446 pub borrow_tracker: Option<borrow_tracker::GlobalState>,
448
449 pub data_race: GlobalDataRaceHandler,
455
456 pub alloc_addresses: alloc_addresses::GlobalState,
458
459 pub(crate) env_vars: EnvVars<'tcx>,
461
462 pub(crate) main_fn_ret_place: Option<MPlaceTy<'tcx>>,
464
465 pub(crate) argc: Option<Pointer>,
469 pub(crate) argv: Option<Pointer>,
470 pub(crate) cmd_line: Option<Pointer>,
471
472 pub(crate) tls: TlsData<'tcx>,
474
475 pub(crate) isolated_op: IsolatedOp,
479
480 pub(crate) validation: ValidationMode,
482
483 pub(crate) fds: shims::FdTable,
485 pub(crate) dirs: shims::DirTable,
487
488 pub(crate) epoll_interests: shims::EpollInterestTable,
490
491 pub(crate) monotonic_clock: MonotonicClock,
493
494 pub(crate) threads: ThreadManager<'tcx>,
496
497 pub(crate) thread_cpu_affinity: FxHashMap<ThreadId, CpuAffinityMask>,
501
502 pub(crate) sync: SynchronizationObjects,
504
505 pub(crate) layouts: PrimitiveLayouts<'tcx>,
507
508 pub(crate) static_roots: Vec<AllocId>,
510
511 profiler: Option<measureme::Profiler>,
514 string_cache: FxHashMap<String, measureme::StringId>,
517
518 pub(crate) exported_symbols_cache: FxHashMap<Symbol, Option<Instance<'tcx>>>,
521
522 pub(crate) backtrace_style: BacktraceStyle,
524
525 pub(crate) local_crates: Vec<CrateNum>,
527
528 extern_statics: FxHashMap<Symbol, StrictPointer>,
530
531 pub(crate) rng: RefCell<StdRng>,
534
535 #[cfg(target_os = "linux")]
537 pub(crate) allocator: Option<Rc<RefCell<crate::alloc::isolated_alloc::IsolatedAlloc>>>,
538
539 tracked_alloc_ids: FxHashSet<AllocId>,
542 track_alloc_accesses: bool,
544
545 pub(crate) check_alignment: AlignmentCheck,
547
548 pub(crate) cmpxchg_weak_failure_rate: f64,
550
551 pub(crate) preemption_rate: f64,
553
554 pub(crate) report_progress: Option<u32>,
556 pub(crate) basic_block_count: u64,
558
559 #[cfg(unix)]
561 pub native_lib: Vec<(libloading::Library, std::path::PathBuf)>,
562 #[cfg(not(unix))]
563 pub native_lib: Vec<!>,
564
565 pub(crate) gc_interval: u32,
567 pub(crate) since_gc: u32,
569
570 pub(crate) num_cpus: u32,
572
573 pub(crate) page_size: u64,
575 pub(crate) stack_addr: u64,
576 pub(crate) stack_size: u64,
577
578 pub(crate) collect_leak_backtraces: bool,
580
581 pub(crate) allocation_spans: RefCell<FxHashMap<AllocId, (Span, Option<Span>)>>,
584
585 const_cache: RefCell<FxHashMap<(mir::Const<'tcx>, usize), OpTy<'tcx>>>,
589
590 pub(crate) symbolic_alignment: RefCell<FxHashMap<AllocId, (Size, Align)>>,
597
598 union_data_ranges: FxHashMap<Ty<'tcx>, RangeSet>,
600
601 pub(crate) pthread_mutex_sanity: Cell<bool>,
603 pub(crate) pthread_rwlock_sanity: Cell<bool>,
604 pub(crate) pthread_condvar_sanity: Cell<bool>,
605
606 pub(crate) sb_extern_type_warned: Cell<bool>,
608 #[cfg(unix)]
610 pub(crate) native_call_mem_warned: Cell<bool>,
611 pub(crate) reject_in_isolation_warned: RefCell<FxHashSet<String>>,
613 pub(crate) int2ptr_warned: RefCell<FxHashSet<Span>>,
615
616 pub(crate) mangle_internal_symbol_cache: FxHashMap<&'static str, String>,
618
619 pub force_intrinsic_fallback: bool,
621}
622
623impl<'tcx> MiriMachine<'tcx> {
624 pub(crate) fn new(
625 config: &MiriConfig,
626 layout_cx: LayoutCx<'tcx>,
627 genmc_ctx: Option<Rc<GenmcCtx>>,
628 ) -> Self {
629 let tcx = layout_cx.tcx();
630 let local_crates = helpers::get_local_crates(tcx);
631 let layouts =
632 PrimitiveLayouts::new(layout_cx).expect("Couldn't get layouts of primitive types");
633 let profiler = config.measureme_out.as_ref().map(|out| {
634 let crate_name =
635 tcx.sess.opts.crate_name.clone().unwrap_or_else(|| "unknown-crate".to_string());
636 let pid = process::id();
637 let filename = format!("{crate_name}-{pid:07}");
642 let path = Path::new(out).join(filename);
643 measureme::Profiler::new(path).expect("Couldn't create `measureme` profiler")
644 });
645 let rng = StdRng::seed_from_u64(config.seed.unwrap_or(0));
646 let borrow_tracker = config.borrow_tracker.map(|bt| bt.instantiate_global_state(config));
647 let data_race = if config.genmc_mode {
648 GlobalDataRaceHandler::Genmc(genmc_ctx.unwrap())
650 } else if config.data_race_detector {
651 GlobalDataRaceHandler::Vclocks(Box::new(data_race::GlobalState::new(config)))
652 } else {
653 GlobalDataRaceHandler::None
654 };
655 let page_size = if let Some(page_size) = config.page_size {
659 page_size
660 } else {
661 let target = &tcx.sess.target;
662 match target.arch.as_ref() {
663 "wasm32" | "wasm64" => 64 * 1024, "aarch64" => {
665 if target.options.vendor.as_ref() == "apple" {
666 16 * 1024
670 } else {
671 4 * 1024
672 }
673 }
674 _ => 4 * 1024,
675 }
676 };
677 let stack_addr = if tcx.pointer_size().bits() < 32 { page_size } else { page_size * 32 };
679 let stack_size =
680 if tcx.pointer_size().bits() < 32 { page_size * 4 } else { page_size * 16 };
681 assert!(
682 usize::try_from(config.num_cpus).unwrap() <= cpu_affinity::MAX_CPUS,
683 "miri only supports up to {} CPUs, but {} were configured",
684 cpu_affinity::MAX_CPUS,
685 config.num_cpus
686 );
687 let threads = ThreadManager::new(config);
688 let mut thread_cpu_affinity = FxHashMap::default();
689 if matches!(&*tcx.sess.target.os, "linux" | "freebsd" | "android") {
690 thread_cpu_affinity
691 .insert(threads.active_thread(), CpuAffinityMask::new(&layout_cx, config.num_cpus));
692 }
693 MiriMachine {
694 tcx,
695 borrow_tracker,
696 data_race,
697 alloc_addresses: RefCell::new(alloc_addresses::GlobalStateInner::new(config, stack_addr)),
698 env_vars: EnvVars::default(),
700 main_fn_ret_place: None,
701 argc: None,
702 argv: None,
703 cmd_line: None,
704 tls: TlsData::default(),
705 isolated_op: config.isolated_op,
706 validation: config.validation,
707 fds: shims::FdTable::init(config.mute_stdout_stderr),
708 epoll_interests: shims::EpollInterestTable::new(),
709 dirs: Default::default(),
710 layouts,
711 threads,
712 thread_cpu_affinity,
713 sync: SynchronizationObjects::default(),
714 static_roots: Vec::new(),
715 profiler,
716 string_cache: Default::default(),
717 exported_symbols_cache: FxHashMap::default(),
718 backtrace_style: config.backtrace_style,
719 local_crates,
720 extern_statics: FxHashMap::default(),
721 rng: RefCell::new(rng),
722 #[cfg(target_os = "linux")]
723 allocator: if !config.native_lib.is_empty() {
724 Some(Rc::new(RefCell::new(crate::alloc::isolated_alloc::IsolatedAlloc::new())))
725 } else { None },
726 tracked_alloc_ids: config.tracked_alloc_ids.clone(),
727 track_alloc_accesses: config.track_alloc_accesses,
728 check_alignment: config.check_alignment,
729 cmpxchg_weak_failure_rate: config.cmpxchg_weak_failure_rate,
730 preemption_rate: config.preemption_rate,
731 report_progress: config.report_progress,
732 basic_block_count: 0,
733 monotonic_clock: MonotonicClock::new(config.isolated_op == IsolatedOp::Allow),
734 #[cfg(unix)]
735 native_lib: config.native_lib.iter().map(|lib_file_path| {
736 let host_triple = rustc_session::config::host_tuple();
737 let target_triple = tcx.sess.opts.target_triple.tuple();
738 if host_triple != target_triple {
740 panic!(
741 "calling native C functions in linked .so file requires host and target to be the same: \
742 host={host_triple}, target={target_triple}",
743 );
744 }
745 (
749 unsafe {
750 libloading::Library::new(lib_file_path)
751 .expect("failed to read specified extern shared object file")
752 },
753 lib_file_path.clone(),
754 )
755 }).collect(),
756 #[cfg(not(unix))]
757 native_lib: config.native_lib.iter().map(|_| {
758 panic!("calling functions from native libraries via FFI is only supported on Unix")
759 }).collect(),
760 gc_interval: config.gc_interval,
761 since_gc: 0,
762 num_cpus: config.num_cpus,
763 page_size,
764 stack_addr,
765 stack_size,
766 collect_leak_backtraces: config.collect_leak_backtraces,
767 allocation_spans: RefCell::new(FxHashMap::default()),
768 const_cache: RefCell::new(FxHashMap::default()),
769 symbolic_alignment: RefCell::new(FxHashMap::default()),
770 union_data_ranges: FxHashMap::default(),
771 pthread_mutex_sanity: Cell::new(false),
772 pthread_rwlock_sanity: Cell::new(false),
773 pthread_condvar_sanity: Cell::new(false),
774 sb_extern_type_warned: Cell::new(false),
775 #[cfg(unix)]
776 native_call_mem_warned: Cell::new(false),
777 reject_in_isolation_warned: Default::default(),
778 int2ptr_warned: Default::default(),
779 mangle_internal_symbol_cache: Default::default(),
780 force_intrinsic_fallback: config.force_intrinsic_fallback,
781 }
782 }
783
784 pub(crate) fn late_init(
785 ecx: &mut MiriInterpCx<'tcx>,
786 config: &MiriConfig,
787 on_main_stack_empty: StackEmptyCallback<'tcx>,
788 ) -> InterpResult<'tcx> {
789 EnvVars::init(ecx, config)?;
790 MiriMachine::init_extern_statics(ecx)?;
791 ThreadManager::init(ecx, on_main_stack_empty);
792 interp_ok(())
793 }
794
795 pub(crate) fn add_extern_static(ecx: &mut MiriInterpCx<'tcx>, name: &str, ptr: Pointer) {
796 let ptr = ptr.into_pointer_or_addr().unwrap();
798 ecx.machine.extern_statics.try_insert(Symbol::intern(name), ptr).unwrap();
799 }
800
801 pub(crate) fn communicate(&self) -> bool {
802 self.isolated_op == IsolatedOp::Allow
803 }
804
805 pub(crate) fn is_local(&self, frame: &FrameInfo<'_>) -> bool {
807 let def_id = frame.instance.def_id();
808 def_id.is_local() || self.local_crates.contains(&def_id.krate)
809 }
810
811 pub(crate) fn handle_abnormal_termination(&mut self) {
813 drop(self.profiler.take());
818 }
819
820 pub(crate) fn page_align(&self) -> Align {
821 Align::from_bytes(self.page_size).unwrap()
822 }
823
824 pub(crate) fn allocated_span(&self, alloc_id: AllocId) -> Option<SpanData> {
825 self.allocation_spans
826 .borrow()
827 .get(&alloc_id)
828 .map(|(allocated, _deallocated)| allocated.data())
829 }
830
831 pub(crate) fn deallocated_span(&self, alloc_id: AllocId) -> Option<SpanData> {
832 self.allocation_spans
833 .borrow()
834 .get(&alloc_id)
835 .and_then(|(_allocated, deallocated)| *deallocated)
836 .map(Span::data)
837 }
838
839 fn init_allocation(
840 ecx: &MiriInterpCx<'tcx>,
841 id: AllocId,
842 kind: MemoryKind,
843 size: Size,
844 align: Align,
845 ) -> InterpResult<'tcx, AllocExtra<'tcx>> {
846 if ecx.machine.tracked_alloc_ids.contains(&id) {
847 ecx.emit_diagnostic(NonHaltingDiagnostic::CreatedAlloc(id, size, align, kind));
848 }
849
850 let borrow_tracker = ecx
851 .machine
852 .borrow_tracker
853 .as_ref()
854 .map(|bt| bt.borrow_mut().new_allocation(id, size, kind, &ecx.machine));
855
856 let data_race = match &ecx.machine.data_race {
857 GlobalDataRaceHandler::None => AllocDataRaceHandler::None,
858 GlobalDataRaceHandler::Vclocks(data_race) =>
859 AllocDataRaceHandler::Vclocks(
860 data_race::AllocState::new_allocation(
861 data_race,
862 &ecx.machine.threads,
863 size,
864 kind,
865 ecx.machine.current_span(),
866 ),
867 data_race.weak_memory.then(weak_memory::AllocState::new_allocation),
868 ),
869 GlobalDataRaceHandler::Genmc(_genmc_ctx) => {
870 AllocDataRaceHandler::Genmc
873 }
874 };
875
876 let backtrace = if kind.may_leak() || !ecx.machine.collect_leak_backtraces {
880 None
881 } else {
882 Some(ecx.generate_stacktrace())
883 };
884
885 if matches!(kind, MemoryKind::Machine(kind) if kind.should_save_allocation_span()) {
886 ecx.machine
887 .allocation_spans
888 .borrow_mut()
889 .insert(id, (ecx.machine.current_span(), None));
890 }
891
892 interp_ok(AllocExtra { borrow_tracker, data_race, backtrace, sync: FxHashMap::default() })
893 }
894}
895
896impl VisitProvenance for MiriMachine<'_> {
897 fn visit_provenance(&self, visit: &mut VisitWith<'_>) {
898 #[rustfmt::skip]
899 let MiriMachine {
900 threads,
901 thread_cpu_affinity: _,
902 sync: _,
903 tls,
904 env_vars,
905 main_fn_ret_place,
906 argc,
907 argv,
908 cmd_line,
909 extern_statics,
910 dirs,
911 borrow_tracker,
912 data_race,
913 alloc_addresses,
914 fds,
915 epoll_interests:_,
916 tcx: _,
917 isolated_op: _,
918 validation: _,
919 monotonic_clock: _,
920 layouts: _,
921 static_roots: _,
922 profiler: _,
923 string_cache: _,
924 exported_symbols_cache: _,
925 backtrace_style: _,
926 local_crates: _,
927 rng: _,
928 #[cfg(target_os = "linux")]
929 allocator: _,
930 tracked_alloc_ids: _,
931 track_alloc_accesses: _,
932 check_alignment: _,
933 cmpxchg_weak_failure_rate: _,
934 preemption_rate: _,
935 report_progress: _,
936 basic_block_count: _,
937 native_lib: _,
938 gc_interval: _,
939 since_gc: _,
940 num_cpus: _,
941 page_size: _,
942 stack_addr: _,
943 stack_size: _,
944 collect_leak_backtraces: _,
945 allocation_spans: _,
946 const_cache: _,
947 symbolic_alignment: _,
948 union_data_ranges: _,
949 pthread_mutex_sanity: _,
950 pthread_rwlock_sanity: _,
951 pthread_condvar_sanity: _,
952 sb_extern_type_warned: _,
953 #[cfg(unix)]
954 native_call_mem_warned: _,
955 reject_in_isolation_warned: _,
956 int2ptr_warned: _,
957 mangle_internal_symbol_cache: _,
958 force_intrinsic_fallback: _,
959 } = self;
960
961 threads.visit_provenance(visit);
962 tls.visit_provenance(visit);
963 env_vars.visit_provenance(visit);
964 dirs.visit_provenance(visit);
965 fds.visit_provenance(visit);
966 data_race.visit_provenance(visit);
967 borrow_tracker.visit_provenance(visit);
968 alloc_addresses.visit_provenance(visit);
969 main_fn_ret_place.visit_provenance(visit);
970 argc.visit_provenance(visit);
971 argv.visit_provenance(visit);
972 cmd_line.visit_provenance(visit);
973 for ptr in extern_statics.values() {
974 ptr.visit_provenance(visit);
975 }
976 }
977}
978
979pub type MiriInterpCx<'tcx> = InterpCx<'tcx, MiriMachine<'tcx>>;
981
982pub trait MiriInterpCxExt<'tcx> {
984 fn eval_context_ref<'a>(&'a self) -> &'a MiriInterpCx<'tcx>;
985 fn eval_context_mut<'a>(&'a mut self) -> &'a mut MiriInterpCx<'tcx>;
986}
987impl<'tcx> MiriInterpCxExt<'tcx> for MiriInterpCx<'tcx> {
988 #[inline(always)]
989 fn eval_context_ref(&self) -> &MiriInterpCx<'tcx> {
990 self
991 }
992 #[inline(always)]
993 fn eval_context_mut(&mut self) -> &mut MiriInterpCx<'tcx> {
994 self
995 }
996}
997
998impl<'tcx> Machine<'tcx> for MiriMachine<'tcx> {
1000 type MemoryKind = MiriMemoryKind;
1001 type ExtraFnVal = DynSym;
1002
1003 type FrameExtra = FrameExtra<'tcx>;
1004 type AllocExtra = AllocExtra<'tcx>;
1005
1006 type Provenance = Provenance;
1007 type ProvenanceExtra = ProvenanceExtra;
1008 type Bytes = MiriAllocBytes;
1009
1010 type MemoryMap =
1011 MonoHashMap<AllocId, (MemoryKind, Allocation<Provenance, Self::AllocExtra, Self::Bytes>)>;
1012
1013 const GLOBAL_KIND: Option<MiriMemoryKind> = Some(MiriMemoryKind::Global);
1014
1015 const PANIC_ON_ALLOC_FAIL: bool = false;
1016
1017 #[inline(always)]
1018 fn enforce_alignment(ecx: &MiriInterpCx<'tcx>) -> bool {
1019 ecx.machine.check_alignment != AlignmentCheck::None
1020 }
1021
1022 #[inline(always)]
1023 fn alignment_check(
1024 ecx: &MiriInterpCx<'tcx>,
1025 alloc_id: AllocId,
1026 alloc_align: Align,
1027 alloc_kind: AllocKind,
1028 offset: Size,
1029 align: Align,
1030 ) -> Option<Misalignment> {
1031 if ecx.machine.check_alignment != AlignmentCheck::Symbolic {
1032 return None;
1034 }
1035 if alloc_kind != AllocKind::LiveData {
1036 return None;
1038 }
1039 let (promised_offset, promised_align) = ecx
1041 .machine
1042 .symbolic_alignment
1043 .borrow()
1044 .get(&alloc_id)
1045 .copied()
1046 .unwrap_or((Size::ZERO, alloc_align));
1047 if promised_align < align {
1048 Some(Misalignment { has: promised_align, required: align })
1050 } else {
1051 let distance = offset.bytes().wrapping_sub(promised_offset.bytes());
1053 if distance % align.bytes() == 0 {
1055 None
1057 } else {
1058 let distance_pow2 = 1 << distance.trailing_zeros();
1060 Some(Misalignment {
1061 has: Align::from_bytes(distance_pow2).unwrap(),
1062 required: align,
1063 })
1064 }
1065 }
1066 }
1067
1068 #[inline(always)]
1069 fn enforce_validity(ecx: &MiriInterpCx<'tcx>, _layout: TyAndLayout<'tcx>) -> bool {
1070 ecx.machine.validation != ValidationMode::No
1071 }
1072 #[inline(always)]
1073 fn enforce_validity_recursively(
1074 ecx: &InterpCx<'tcx, Self>,
1075 _layout: TyAndLayout<'tcx>,
1076 ) -> bool {
1077 ecx.machine.validation == ValidationMode::Deep
1078 }
1079
1080 #[inline(always)]
1081 fn ignore_optional_overflow_checks(ecx: &MiriInterpCx<'tcx>) -> bool {
1082 !ecx.tcx.sess.overflow_checks()
1083 }
1084
1085 fn check_fn_target_features(
1086 ecx: &MiriInterpCx<'tcx>,
1087 instance: ty::Instance<'tcx>,
1088 ) -> InterpResult<'tcx> {
1089 let attrs = ecx.tcx.codegen_fn_attrs(instance.def_id());
1090 if attrs
1091 .target_features
1092 .iter()
1093 .any(|feature| !ecx.tcx.sess.target_features.contains(&feature.name))
1094 {
1095 let unavailable = attrs
1096 .target_features
1097 .iter()
1098 .filter(|&feature| {
1099 !feature.implied && !ecx.tcx.sess.target_features.contains(&feature.name)
1100 })
1101 .fold(String::new(), |mut s, feature| {
1102 if !s.is_empty() {
1103 s.push_str(", ");
1104 }
1105 s.push_str(feature.name.as_str());
1106 s
1107 });
1108 let msg = format!(
1109 "calling a function that requires unavailable target features: {unavailable}"
1110 );
1111 if ecx.tcx.sess.target.is_like_wasm {
1114 throw_machine_stop!(TerminationInfo::Abort(msg));
1115 } else {
1116 throw_ub_format!("{msg}");
1117 }
1118 }
1119 interp_ok(())
1120 }
1121
1122 #[inline(always)]
1123 fn find_mir_or_eval_fn(
1124 ecx: &mut MiriInterpCx<'tcx>,
1125 instance: ty::Instance<'tcx>,
1126 abi: &FnAbi<'tcx, Ty<'tcx>>,
1127 args: &[FnArg<'tcx, Provenance>],
1128 dest: &PlaceTy<'tcx>,
1129 ret: Option<mir::BasicBlock>,
1130 unwind: mir::UnwindAction,
1131 ) -> InterpResult<'tcx, Option<(&'tcx mir::Body<'tcx>, ty::Instance<'tcx>)>> {
1132 if ecx.tcx.is_foreign_item(instance.def_id()) {
1134 let args = ecx.copy_fn_args(args); let link_name = Symbol::intern(ecx.tcx.symbol_name(instance).name);
1142 return ecx.emulate_foreign_item(link_name, abi, &args, dest, ret, unwind);
1143 }
1144
1145 interp_ok(Some((ecx.load_mir(instance.def, None)?, instance)))
1147 }
1148
1149 #[inline(always)]
1150 fn call_extra_fn(
1151 ecx: &mut MiriInterpCx<'tcx>,
1152 fn_val: DynSym,
1153 abi: &FnAbi<'tcx, Ty<'tcx>>,
1154 args: &[FnArg<'tcx, Provenance>],
1155 dest: &PlaceTy<'tcx>,
1156 ret: Option<mir::BasicBlock>,
1157 unwind: mir::UnwindAction,
1158 ) -> InterpResult<'tcx> {
1159 let args = ecx.copy_fn_args(args); ecx.emulate_dyn_sym(fn_val, abi, &args, dest, ret, unwind)
1161 }
1162
1163 #[inline(always)]
1164 fn call_intrinsic(
1165 ecx: &mut MiriInterpCx<'tcx>,
1166 instance: ty::Instance<'tcx>,
1167 args: &[OpTy<'tcx>],
1168 dest: &PlaceTy<'tcx>,
1169 ret: Option<mir::BasicBlock>,
1170 unwind: mir::UnwindAction,
1171 ) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>> {
1172 ecx.call_intrinsic(instance, args, dest, ret, unwind)
1173 }
1174
1175 #[inline(always)]
1176 fn assert_panic(
1177 ecx: &mut MiriInterpCx<'tcx>,
1178 msg: &mir::AssertMessage<'tcx>,
1179 unwind: mir::UnwindAction,
1180 ) -> InterpResult<'tcx> {
1181 ecx.assert_panic(msg, unwind)
1182 }
1183
1184 fn panic_nounwind(ecx: &mut InterpCx<'tcx, Self>, msg: &str) -> InterpResult<'tcx> {
1185 ecx.start_panic_nounwind(msg)
1186 }
1187
1188 fn unwind_terminate(
1189 ecx: &mut InterpCx<'tcx, Self>,
1190 reason: mir::UnwindTerminateReason,
1191 ) -> InterpResult<'tcx> {
1192 let panic = ecx.tcx.lang_items().get(reason.lang_item()).unwrap();
1194 let panic = ty::Instance::mono(ecx.tcx.tcx, panic);
1195 ecx.call_function(
1196 panic,
1197 ExternAbi::Rust,
1198 &[],
1199 None,
1200 StackPopCleanup::Goto { ret: None, unwind: mir::UnwindAction::Unreachable },
1201 )?;
1202 interp_ok(())
1203 }
1204
1205 #[inline(always)]
1206 fn binary_ptr_op(
1207 ecx: &MiriInterpCx<'tcx>,
1208 bin_op: mir::BinOp,
1209 left: &ImmTy<'tcx>,
1210 right: &ImmTy<'tcx>,
1211 ) -> InterpResult<'tcx, ImmTy<'tcx>> {
1212 ecx.binary_ptr_op(bin_op, left, right)
1213 }
1214
1215 #[inline(always)]
1216 fn generate_nan<F1: Float + FloatConvert<F2>, F2: Float>(
1217 ecx: &InterpCx<'tcx, Self>,
1218 inputs: &[F1],
1219 ) -> F2 {
1220 ecx.generate_nan(inputs)
1221 }
1222
1223 #[inline(always)]
1224 fn apply_float_nondet(
1225 ecx: &mut InterpCx<'tcx, Self>,
1226 val: ImmTy<'tcx>,
1227 ) -> InterpResult<'tcx, ImmTy<'tcx>> {
1228 crate::math::apply_random_float_error_to_imm(ecx, val, 2 )
1229 }
1230
1231 #[inline(always)]
1232 fn equal_float_min_max<F: Float>(ecx: &MiriInterpCx<'tcx>, a: F, b: F) -> F {
1233 ecx.equal_float_min_max(a, b)
1234 }
1235
1236 #[inline(always)]
1237 fn ub_checks(ecx: &InterpCx<'tcx, Self>) -> InterpResult<'tcx, bool> {
1238 interp_ok(ecx.tcx.sess.ub_checks())
1239 }
1240
1241 #[inline(always)]
1242 fn contract_checks(ecx: &InterpCx<'tcx, Self>) -> InterpResult<'tcx, bool> {
1243 interp_ok(ecx.tcx.sess.contract_checks())
1244 }
1245
1246 #[inline(always)]
1247 fn thread_local_static_pointer(
1248 ecx: &mut MiriInterpCx<'tcx>,
1249 def_id: DefId,
1250 ) -> InterpResult<'tcx, StrictPointer> {
1251 ecx.get_or_create_thread_local_alloc(def_id)
1252 }
1253
1254 fn extern_static_pointer(
1255 ecx: &MiriInterpCx<'tcx>,
1256 def_id: DefId,
1257 ) -> InterpResult<'tcx, StrictPointer> {
1258 let link_name = Symbol::intern(ecx.tcx.symbol_name(Instance::mono(*ecx.tcx, def_id)).name);
1259 if let Some(&ptr) = ecx.machine.extern_statics.get(&link_name) {
1260 let Provenance::Concrete { alloc_id, .. } = ptr.provenance else {
1264 panic!("extern_statics cannot contain wildcards")
1265 };
1266 let info = ecx.get_alloc_info(alloc_id);
1267 let def_ty = ecx.tcx.type_of(def_id).instantiate_identity();
1268 let extern_decl_layout =
1269 ecx.tcx.layout_of(ecx.typing_env().as_query_input(def_ty)).unwrap();
1270 if extern_decl_layout.size != info.size || extern_decl_layout.align.abi != info.align {
1271 throw_unsup_format!(
1272 "extern static `{link_name}` has been declared as `{krate}::{name}` \
1273 with a size of {decl_size} bytes and alignment of {decl_align} bytes, \
1274 but Miri emulates it via an extern static shim \
1275 with a size of {shim_size} bytes and alignment of {shim_align} bytes",
1276 name = ecx.tcx.def_path_str(def_id),
1277 krate = ecx.tcx.crate_name(def_id.krate),
1278 decl_size = extern_decl_layout.size.bytes(),
1279 decl_align = extern_decl_layout.align.abi.bytes(),
1280 shim_size = info.size.bytes(),
1281 shim_align = info.align.bytes(),
1282 )
1283 }
1284 interp_ok(ptr)
1285 } else {
1286 throw_unsup_format!("extern static `{link_name}` is not supported by Miri",)
1287 }
1288 }
1289
1290 fn init_local_allocation(
1291 ecx: &MiriInterpCx<'tcx>,
1292 id: AllocId,
1293 kind: MemoryKind,
1294 size: Size,
1295 align: Align,
1296 ) -> InterpResult<'tcx, Self::AllocExtra> {
1297 assert!(kind != MiriMemoryKind::Global.into());
1298 MiriMachine::init_allocation(ecx, id, kind, size, align)
1299 }
1300
1301 fn adjust_alloc_root_pointer(
1302 ecx: &MiriInterpCx<'tcx>,
1303 ptr: interpret::Pointer<CtfeProvenance>,
1304 kind: Option<MemoryKind>,
1305 ) -> InterpResult<'tcx, interpret::Pointer<Provenance>> {
1306 let kind = kind.expect("we set our GLOBAL_KIND so this cannot be None");
1307 let alloc_id = ptr.provenance.alloc_id();
1308 if cfg!(debug_assertions) {
1309 match ecx.tcx.try_get_global_alloc(alloc_id) {
1311 Some(GlobalAlloc::Static(def_id)) if ecx.tcx.is_thread_local_static(def_id) => {
1312 panic!("adjust_alloc_root_pointer called on thread-local static")
1313 }
1314 Some(GlobalAlloc::Static(def_id)) if ecx.tcx.is_foreign_item(def_id) => {
1315 panic!("adjust_alloc_root_pointer called on extern static")
1316 }
1317 _ => {}
1318 }
1319 }
1320 let tag = if let Some(borrow_tracker) = &ecx.machine.borrow_tracker {
1322 borrow_tracker.borrow_mut().root_ptr_tag(alloc_id, &ecx.machine)
1323 } else {
1324 BorTag::default()
1326 };
1327 ecx.adjust_alloc_root_pointer(ptr, tag, kind)
1328 }
1329
1330 #[inline(always)]
1332 fn ptr_from_addr_cast(ecx: &MiriInterpCx<'tcx>, addr: u64) -> InterpResult<'tcx, Pointer> {
1333 ecx.ptr_from_addr_cast(addr)
1334 }
1335
1336 #[inline(always)]
1340 fn expose_provenance(
1341 ecx: &InterpCx<'tcx, Self>,
1342 provenance: Self::Provenance,
1343 ) -> InterpResult<'tcx> {
1344 ecx.expose_provenance(provenance)
1345 }
1346
1347 fn ptr_get_alloc(
1359 ecx: &MiriInterpCx<'tcx>,
1360 ptr: StrictPointer,
1361 size: i64,
1362 ) -> Option<(AllocId, Size, Self::ProvenanceExtra)> {
1363 let rel = ecx.ptr_get_alloc(ptr, size);
1364
1365 rel.map(|(alloc_id, size)| {
1366 let tag = match ptr.provenance {
1367 Provenance::Concrete { tag, .. } => ProvenanceExtra::Concrete(tag),
1368 Provenance::Wildcard => ProvenanceExtra::Wildcard,
1369 };
1370 (alloc_id, size, tag)
1371 })
1372 }
1373
1374 fn adjust_global_allocation<'b>(
1383 ecx: &InterpCx<'tcx, Self>,
1384 id: AllocId,
1385 alloc: &'b Allocation,
1386 ) -> InterpResult<'tcx, Cow<'b, Allocation<Self::Provenance, Self::AllocExtra, Self::Bytes>>>
1387 {
1388 let alloc = alloc.adjust_from_tcx(
1389 &ecx.tcx,
1390 |bytes, align| ecx.get_global_alloc_bytes(id, bytes, align),
1391 |ptr| ecx.global_root_pointer(ptr),
1392 )?;
1393 let kind = MiriMemoryKind::Global.into();
1394 let extra = MiriMachine::init_allocation(ecx, id, kind, alloc.size(), alloc.align)?;
1395 interp_ok(Cow::Owned(alloc.with_extra(extra)))
1396 }
1397
1398 #[inline(always)]
1399 fn before_memory_read(
1400 _tcx: TyCtxtAt<'tcx>,
1401 machine: &Self,
1402 alloc_extra: &AllocExtra<'tcx>,
1403 ptr: Pointer,
1404 (alloc_id, prov_extra): (AllocId, Self::ProvenanceExtra),
1405 range: AllocRange,
1406 ) -> InterpResult<'tcx> {
1407 if machine.track_alloc_accesses && machine.tracked_alloc_ids.contains(&alloc_id) {
1408 machine
1409 .emit_diagnostic(NonHaltingDiagnostic::AccessedAlloc(alloc_id, AccessKind::Read));
1410 }
1411 match &machine.data_race {
1413 GlobalDataRaceHandler::None => {}
1414 GlobalDataRaceHandler::Genmc(genmc_ctx) =>
1415 genmc_ctx.memory_load(machine, ptr.addr(), range.size)?,
1416 GlobalDataRaceHandler::Vclocks(_data_race) => {
1417 let AllocDataRaceHandler::Vclocks(data_race, weak_memory) = &alloc_extra.data_race
1418 else {
1419 unreachable!();
1420 };
1421 data_race.read(alloc_id, range, NaReadType::Read, None, machine)?;
1422 if let Some(weak_memory) = weak_memory {
1423 weak_memory.memory_accessed(range, machine.data_race.as_vclocks_ref().unwrap());
1424 }
1425 }
1426 }
1427 if let Some(borrow_tracker) = &alloc_extra.borrow_tracker {
1428 borrow_tracker.before_memory_read(alloc_id, prov_extra, range, machine)?;
1429 }
1430 interp_ok(())
1431 }
1432
1433 #[inline(always)]
1434 fn before_memory_write(
1435 _tcx: TyCtxtAt<'tcx>,
1436 machine: &mut Self,
1437 alloc_extra: &mut AllocExtra<'tcx>,
1438 ptr: Pointer,
1439 (alloc_id, prov_extra): (AllocId, Self::ProvenanceExtra),
1440 range: AllocRange,
1441 ) -> InterpResult<'tcx> {
1442 if machine.track_alloc_accesses && machine.tracked_alloc_ids.contains(&alloc_id) {
1443 machine
1444 .emit_diagnostic(NonHaltingDiagnostic::AccessedAlloc(alloc_id, AccessKind::Write));
1445 }
1446 match &machine.data_race {
1447 GlobalDataRaceHandler::None => {}
1448 GlobalDataRaceHandler::Genmc(genmc_ctx) => {
1449 genmc_ctx.memory_store(machine, ptr.addr(), range.size)?;
1450 }
1451 GlobalDataRaceHandler::Vclocks(_global_state) => {
1452 let AllocDataRaceHandler::Vclocks(data_race, weak_memory) =
1453 &mut alloc_extra.data_race
1454 else {
1455 unreachable!()
1456 };
1457 data_race.write(alloc_id, range, NaWriteType::Write, None, machine)?;
1458 if let Some(weak_memory) = weak_memory {
1459 weak_memory.memory_accessed(range, machine.data_race.as_vclocks_ref().unwrap());
1460 }
1461 }
1462 }
1463 if let Some(borrow_tracker) = &mut alloc_extra.borrow_tracker {
1464 borrow_tracker.before_memory_write(alloc_id, prov_extra, range, machine)?;
1465 }
1466 interp_ok(())
1467 }
1468
1469 #[inline(always)]
1470 fn before_memory_deallocation(
1471 _tcx: TyCtxtAt<'tcx>,
1472 machine: &mut Self,
1473 alloc_extra: &mut AllocExtra<'tcx>,
1474 ptr: Pointer,
1475 (alloc_id, prove_extra): (AllocId, Self::ProvenanceExtra),
1476 size: Size,
1477 align: Align,
1478 kind: MemoryKind,
1479 ) -> InterpResult<'tcx> {
1480 if machine.tracked_alloc_ids.contains(&alloc_id) {
1481 machine.emit_diagnostic(NonHaltingDiagnostic::FreedAlloc(alloc_id));
1482 }
1483 match &machine.data_race {
1484 GlobalDataRaceHandler::None => {}
1485 GlobalDataRaceHandler::Genmc(genmc_ctx) =>
1486 genmc_ctx.handle_dealloc(machine, ptr.addr(), size, align, kind)?,
1487 GlobalDataRaceHandler::Vclocks(_global_state) => {
1488 let data_race = alloc_extra.data_race.as_vclocks_mut().unwrap();
1489 data_race.write(
1490 alloc_id,
1491 alloc_range(Size::ZERO, size),
1492 NaWriteType::Deallocate,
1493 None,
1494 machine,
1495 )?;
1496 }
1497 }
1498 if let Some(borrow_tracker) = &mut alloc_extra.borrow_tracker {
1499 borrow_tracker.before_memory_deallocation(alloc_id, prove_extra, size, machine)?;
1500 }
1501 if let Some((_, deallocated_at)) = machine.allocation_spans.borrow_mut().get_mut(&alloc_id)
1502 {
1503 *deallocated_at = Some(machine.current_span());
1504 }
1505 machine.free_alloc_id(alloc_id, size, align, kind);
1506 interp_ok(())
1507 }
1508
1509 #[inline(always)]
1510 fn retag_ptr_value(
1511 ecx: &mut InterpCx<'tcx, Self>,
1512 kind: mir::RetagKind,
1513 val: &ImmTy<'tcx>,
1514 ) -> InterpResult<'tcx, ImmTy<'tcx>> {
1515 if ecx.machine.borrow_tracker.is_some() {
1516 ecx.retag_ptr_value(kind, val)
1517 } else {
1518 interp_ok(val.clone())
1519 }
1520 }
1521
1522 #[inline(always)]
1523 fn retag_place_contents(
1524 ecx: &mut InterpCx<'tcx, Self>,
1525 kind: mir::RetagKind,
1526 place: &PlaceTy<'tcx>,
1527 ) -> InterpResult<'tcx> {
1528 if ecx.machine.borrow_tracker.is_some() {
1529 ecx.retag_place_contents(kind, place)?;
1530 }
1531 interp_ok(())
1532 }
1533
1534 fn protect_in_place_function_argument(
1535 ecx: &mut InterpCx<'tcx, Self>,
1536 place: &MPlaceTy<'tcx>,
1537 ) -> InterpResult<'tcx> {
1538 let protected_place = if ecx.machine.borrow_tracker.is_some() {
1541 ecx.protect_place(place)?
1542 } else {
1543 place.clone()
1545 };
1546 ecx.write_uninit(&protected_place)?;
1551 interp_ok(())
1553 }
1554
1555 #[inline(always)]
1556 fn init_frame(
1557 ecx: &mut InterpCx<'tcx, Self>,
1558 frame: Frame<'tcx, Provenance>,
1559 ) -> InterpResult<'tcx, Frame<'tcx, Provenance, FrameExtra<'tcx>>> {
1560 let timing = if let Some(profiler) = ecx.machine.profiler.as_ref() {
1562 let fn_name = frame.instance().to_string();
1563 let entry = ecx.machine.string_cache.entry(fn_name.clone());
1564 let name = entry.or_insert_with(|| profiler.alloc_string(&*fn_name));
1565
1566 Some(profiler.start_recording_interval_event_detached(
1567 *name,
1568 measureme::EventId::from_label(*name),
1569 ecx.active_thread().to_u32(),
1570 ))
1571 } else {
1572 None
1573 };
1574
1575 let borrow_tracker = ecx.machine.borrow_tracker.as_ref();
1576
1577 let extra = FrameExtra {
1578 borrow_tracker: borrow_tracker.map(|bt| bt.borrow_mut().new_frame()),
1579 catch_unwind: None,
1580 timing,
1581 is_user_relevant: ecx.machine.is_user_relevant(&frame),
1582 salt: ecx.machine.rng.borrow_mut().random_range(0..ADDRS_PER_ANON_GLOBAL),
1583 data_race: ecx
1584 .machine
1585 .data_race
1586 .as_vclocks_ref()
1587 .map(|_| data_race::FrameState::default()),
1588 };
1589
1590 interp_ok(frame.with_extra(extra))
1591 }
1592
1593 fn stack<'a>(
1594 ecx: &'a InterpCx<'tcx, Self>,
1595 ) -> &'a [Frame<'tcx, Self::Provenance, Self::FrameExtra>] {
1596 ecx.active_thread_stack()
1597 }
1598
1599 fn stack_mut<'a>(
1600 ecx: &'a mut InterpCx<'tcx, Self>,
1601 ) -> &'a mut Vec<Frame<'tcx, Self::Provenance, Self::FrameExtra>> {
1602 ecx.active_thread_stack_mut()
1603 }
1604
1605 fn before_terminator(ecx: &mut InterpCx<'tcx, Self>) -> InterpResult<'tcx> {
1606 ecx.machine.basic_block_count += 1u64; ecx.machine.since_gc += 1;
1608 if let Some(report_progress) = ecx.machine.report_progress {
1610 if ecx.machine.basic_block_count % u64::from(report_progress) == 0 {
1611 ecx.emit_diagnostic(NonHaltingDiagnostic::ProgressReport {
1612 block_count: ecx.machine.basic_block_count,
1613 });
1614 }
1615 }
1616
1617 if ecx.machine.gc_interval > 0 && ecx.machine.since_gc >= ecx.machine.gc_interval {
1622 ecx.machine.since_gc = 0;
1623 ecx.run_provenance_gc();
1624 }
1625
1626 ecx.maybe_preempt_active_thread();
1629
1630 ecx.machine.monotonic_clock.tick();
1632
1633 interp_ok(())
1634 }
1635
1636 #[inline(always)]
1637 fn after_stack_push(ecx: &mut InterpCx<'tcx, Self>) -> InterpResult<'tcx> {
1638 if ecx.frame().extra.is_user_relevant {
1639 let stack_len = ecx.active_thread_stack().len();
1642 ecx.active_thread_mut().set_top_user_relevant_frame(stack_len - 1);
1643 }
1644 interp_ok(())
1645 }
1646
1647 fn before_stack_pop(ecx: &mut InterpCx<'tcx, Self>) -> InterpResult<'tcx> {
1648 let frame = ecx.frame();
1649 if ecx.machine.borrow_tracker.is_some() {
1652 ecx.on_stack_pop(frame)?;
1653 }
1654 if frame.extra.is_user_relevant {
1655 ecx.active_thread_mut().recompute_top_user_relevant_frame(1);
1661 }
1662 info!("Leaving {}", ecx.frame().instance());
1666 interp_ok(())
1667 }
1668
1669 #[inline(always)]
1670 fn after_stack_pop(
1671 ecx: &mut InterpCx<'tcx, Self>,
1672 frame: Frame<'tcx, Provenance, FrameExtra<'tcx>>,
1673 unwinding: bool,
1674 ) -> InterpResult<'tcx, ReturnAction> {
1675 let res = {
1676 let mut frame = frame;
1678 let timing = frame.extra.timing.take();
1679 let res = ecx.handle_stack_pop_unwind(frame.extra, unwinding);
1680 if let Some(profiler) = ecx.machine.profiler.as_ref() {
1681 profiler.finish_recording_interval_event(timing.unwrap());
1682 }
1683 res
1684 };
1685 if !ecx.active_thread_stack().is_empty() {
1688 info!("Continuing in {}", ecx.frame().instance());
1689 }
1690 res
1691 }
1692
1693 fn after_local_read(
1694 ecx: &InterpCx<'tcx, Self>,
1695 frame: &Frame<'tcx, Provenance, FrameExtra<'tcx>>,
1696 local: mir::Local,
1697 ) -> InterpResult<'tcx> {
1698 if let Some(data_race) = &frame.extra.data_race {
1699 data_race.local_read(local, &ecx.machine);
1700 }
1701 interp_ok(())
1702 }
1703
1704 fn after_local_write(
1705 ecx: &mut InterpCx<'tcx, Self>,
1706 local: mir::Local,
1707 storage_live: bool,
1708 ) -> InterpResult<'tcx> {
1709 if let Some(data_race) = &ecx.frame().extra.data_race {
1710 data_race.local_write(local, storage_live, &ecx.machine);
1711 }
1712 interp_ok(())
1713 }
1714
1715 fn after_local_moved_to_memory(
1716 ecx: &mut InterpCx<'tcx, Self>,
1717 local: mir::Local,
1718 mplace: &MPlaceTy<'tcx>,
1719 ) -> InterpResult<'tcx> {
1720 let Some(Provenance::Concrete { alloc_id, .. }) = mplace.ptr().provenance else {
1721 panic!("after_local_allocated should only be called on fresh allocations");
1722 };
1723 let local_decl = &ecx.frame().body().local_decls[local];
1725 let span = local_decl.source_info.span;
1726 ecx.machine.allocation_spans.borrow_mut().insert(alloc_id, (span, None));
1727 let (alloc_info, machine) = ecx.get_alloc_extra_mut(alloc_id)?;
1729 if let Some(data_race) =
1730 &machine.threads.active_thread_stack().last().unwrap().extra.data_race
1731 {
1732 data_race.local_moved_to_memory(
1733 local,
1734 alloc_info.data_race.as_vclocks_mut().unwrap(),
1735 machine,
1736 );
1737 }
1738 interp_ok(())
1739 }
1740
1741 fn eval_mir_constant<F>(
1742 ecx: &InterpCx<'tcx, Self>,
1743 val: mir::Const<'tcx>,
1744 span: Span,
1745 layout: Option<TyAndLayout<'tcx>>,
1746 eval: F,
1747 ) -> InterpResult<'tcx, OpTy<'tcx>>
1748 where
1749 F: Fn(
1750 &InterpCx<'tcx, Self>,
1751 mir::Const<'tcx>,
1752 Span,
1753 Option<TyAndLayout<'tcx>>,
1754 ) -> InterpResult<'tcx, OpTy<'tcx>>,
1755 {
1756 let frame = ecx.active_thread_stack().last().unwrap();
1757 let mut cache = ecx.machine.const_cache.borrow_mut();
1758 match cache.entry((val, frame.extra.salt)) {
1759 Entry::Vacant(ve) => {
1760 let op = eval(ecx, val, span, layout)?;
1761 ve.insert(op.clone());
1762 interp_ok(op)
1763 }
1764 Entry::Occupied(oe) => interp_ok(oe.get().clone()),
1765 }
1766 }
1767
1768 fn get_global_alloc_salt(
1769 ecx: &InterpCx<'tcx, Self>,
1770 instance: Option<ty::Instance<'tcx>>,
1771 ) -> usize {
1772 let unique = if let Some(instance) = instance {
1773 let is_generic = instance
1786 .args
1787 .into_iter()
1788 .any(|arg| !matches!(arg.kind(), ty::GenericArgKind::Lifetime(_)));
1789 let can_be_inlined = matches!(
1790 ecx.tcx.sess.opts.unstable_opts.cross_crate_inline_threshold,
1791 InliningThreshold::Always
1792 ) || !matches!(
1793 ecx.tcx.codegen_fn_attrs(instance.def_id()).inline,
1794 InlineAttr::Never
1795 );
1796 !is_generic && !can_be_inlined
1797 } else {
1798 false
1800 };
1801 if unique {
1803 CTFE_ALLOC_SALT
1804 } else {
1805 ecx.machine.rng.borrow_mut().random_range(0..ADDRS_PER_ANON_GLOBAL)
1806 }
1807 }
1808
1809 fn cached_union_data_range<'e>(
1810 ecx: &'e mut InterpCx<'tcx, Self>,
1811 ty: Ty<'tcx>,
1812 compute_range: impl FnOnce() -> RangeSet,
1813 ) -> Cow<'e, RangeSet> {
1814 Cow::Borrowed(ecx.machine.union_data_ranges.entry(ty).or_insert_with(compute_range))
1815 }
1816
1817 fn get_default_alloc_params(&self) -> <Self::Bytes as AllocBytes>::AllocParams {
1818 use crate::alloc::MiriAllocParams;
1819
1820 #[cfg(target_os = "linux")]
1821 match &self.allocator {
1822 Some(alloc) => MiriAllocParams::Isolated(alloc.clone()),
1823 None => MiriAllocParams::Global,
1824 }
1825 #[cfg(not(target_os = "linux"))]
1826 MiriAllocParams::Global
1827 }
1828}
1829
1830pub trait MachineCallback<'tcx, T>: VisitProvenance {
1832 fn call(
1834 self: Box<Self>,
1835 ecx: &mut InterpCx<'tcx, MiriMachine<'tcx>>,
1836 arg: T,
1837 ) -> InterpResult<'tcx>;
1838}
1839
1840pub type DynMachineCallback<'tcx, T> = Box<dyn MachineCallback<'tcx, T> + 'tcx>;
1842
1843#[macro_export]
1860macro_rules! callback {
1861 (@capture<$tcx:lifetime $(,)? $($lft:lifetime),*>
1862 { $($name:ident: $type:ty),* $(,)? }
1863 |$this:ident, $arg:ident: $arg_ty:ty| $body:expr $(,)?) => {{
1864 struct Callback<$tcx, $($lft),*> {
1865 $($name: $type,)*
1866 _phantom: std::marker::PhantomData<&$tcx ()>,
1867 }
1868
1869 impl<$tcx, $($lft),*> VisitProvenance for Callback<$tcx, $($lft),*> {
1870 fn visit_provenance(&self, _visit: &mut VisitWith<'_>) {
1871 $(
1872 self.$name.visit_provenance(_visit);
1873 )*
1874 }
1875 }
1876
1877 impl<$tcx, $($lft),*> MachineCallback<$tcx, $arg_ty> for Callback<$tcx, $($lft),*> {
1878 fn call(
1879 self: Box<Self>,
1880 $this: &mut MiriInterpCx<$tcx>,
1881 $arg: $arg_ty
1882 ) -> InterpResult<$tcx> {
1883 #[allow(unused_variables)]
1884 let Callback { $($name,)* _phantom } = *self;
1885 $body
1886 }
1887 }
1888
1889 Box::new(Callback {
1890 $($name,)*
1891 _phantom: std::marker::PhantomData
1892 })
1893 }};
1894}