1use std::mem;
4use std::sync::atomic::Ordering::Relaxed;
5use std::task::Poll;
6use std::time::{Duration, SystemTime};
7
8use either::Either;
9use rand::seq::IteratorRandom;
10use rustc_abi::ExternAbi;
11use rustc_const_eval::CTRL_C_RECEIVED;
12use rustc_data_structures::fx::FxHashMap;
13use rustc_hir::def_id::DefId;
14use rustc_index::{Idx, IndexVec};
15use rustc_middle::mir::Mutability;
16use rustc_middle::ty::layout::TyAndLayout;
17use rustc_span::Span;
18
19use crate::concurrency::GlobalDataRaceHandler;
20use crate::shims::tls;
21use crate::*;
22
23#[derive(Clone, Copy, Debug, PartialEq)]
24enum SchedulingAction {
25 ExecuteStep,
27 ExecuteTimeoutCallback,
29 Sleep(Duration),
31}
32
33#[derive(Clone, Copy, Debug, PartialEq)]
35pub enum TlsAllocAction {
36 Deallocate,
38 Leak,
41}
42
43#[derive(Clone, Copy, Debug, PartialEq)]
45pub enum UnblockKind {
46 Ready,
48 TimedOut,
50}
51
52pub type DynUnblockCallback<'tcx> = DynMachineCallback<'tcx, UnblockKind>;
55
56#[derive(Clone, Copy, Debug, PartialOrd, Ord, PartialEq, Eq, Hash)]
58pub struct ThreadId(u32);
59
60impl ThreadId {
61 pub fn to_u32(self) -> u32 {
62 self.0
63 }
64
65 pub fn new_unchecked(id: u32) -> Self {
67 Self(id)
68 }
69
70 pub const MAIN_THREAD: ThreadId = ThreadId(0);
71}
72
73impl Idx for ThreadId {
74 fn new(idx: usize) -> Self {
75 ThreadId(u32::try_from(idx).unwrap())
76 }
77
78 fn index(self) -> usize {
79 usize::try_from(self.0).unwrap()
80 }
81}
82
83impl From<ThreadId> for u64 {
84 fn from(t: ThreadId) -> Self {
85 t.0.into()
86 }
87}
88
89#[derive(Debug, Copy, Clone, PartialEq, Eq)]
91pub enum BlockReason {
92 Join(ThreadId),
95 Sleep,
97 Mutex,
99 Condvar(CondvarId),
101 RwLock(RwLockId),
103 Futex,
105 InitOnce(InitOnceId),
107 Epoll,
109 Eventfd,
111 UnnamedSocket,
113}
114
115enum ThreadState<'tcx> {
117 Enabled,
119 Blocked { reason: BlockReason, timeout: Option<Timeout>, callback: DynUnblockCallback<'tcx> },
121 Terminated,
124}
125
126impl<'tcx> std::fmt::Debug for ThreadState<'tcx> {
127 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
128 match self {
129 Self::Enabled => write!(f, "Enabled"),
130 Self::Blocked { reason, timeout, .. } =>
131 f.debug_struct("Blocked").field("reason", reason).field("timeout", timeout).finish(),
132 Self::Terminated => write!(f, "Terminated"),
133 }
134 }
135}
136
137impl<'tcx> ThreadState<'tcx> {
138 fn is_enabled(&self) -> bool {
139 matches!(self, ThreadState::Enabled)
140 }
141
142 fn is_terminated(&self) -> bool {
143 matches!(self, ThreadState::Terminated)
144 }
145
146 fn is_blocked_on(&self, reason: BlockReason) -> bool {
147 matches!(*self, ThreadState::Blocked { reason: actual_reason, .. } if actual_reason == reason)
148 }
149}
150
151#[derive(Debug, Copy, Clone, PartialEq, Eq)]
153enum ThreadJoinStatus {
154 Joinable,
156 Detached,
159 Joined,
161}
162
163pub struct Thread<'tcx> {
165 state: ThreadState<'tcx>,
166
167 thread_name: Option<Vec<u8>>,
169
170 stack: Vec<Frame<'tcx, Provenance, FrameExtra<'tcx>>>,
172
173 pub(crate) on_stack_empty: Option<StackEmptyCallback<'tcx>>,
178
179 top_user_relevant_frame: Option<usize>,
185
186 join_status: ThreadJoinStatus,
188
189 pub(crate) panic_payloads: Vec<ImmTy<'tcx>>,
198
199 pub(crate) last_error: Option<MPlaceTy<'tcx>>,
201}
202
203pub type StackEmptyCallback<'tcx> =
204 Box<dyn FnMut(&mut MiriInterpCx<'tcx>) -> InterpResult<'tcx, Poll<()>> + 'tcx>;
205
206impl<'tcx> Thread<'tcx> {
207 fn thread_name(&self) -> Option<&[u8]> {
209 self.thread_name.as_deref()
210 }
211
212 fn thread_display_name(&self, id: ThreadId) -> String {
214 if let Some(ref thread_name) = self.thread_name {
215 String::from_utf8_lossy(thread_name).into_owned()
216 } else {
217 format!("unnamed-{}", id.index())
218 }
219 }
220
221 fn compute_top_user_relevant_frame(&self) -> Option<usize> {
226 self.stack
227 .iter()
228 .enumerate()
229 .rev()
230 .find_map(|(idx, frame)| if frame.extra.is_user_relevant { Some(idx) } else { None })
231 }
232
233 pub fn recompute_top_user_relevant_frame(&mut self) {
235 self.top_user_relevant_frame = self.compute_top_user_relevant_frame();
236 }
237
238 pub fn set_top_user_relevant_frame(&mut self, frame_idx: usize) {
241 debug_assert_eq!(Some(frame_idx), self.compute_top_user_relevant_frame());
242 self.top_user_relevant_frame = Some(frame_idx);
243 }
244
245 pub fn top_user_relevant_frame(&self) -> Option<usize> {
248 debug_assert_eq!(self.top_user_relevant_frame, self.compute_top_user_relevant_frame());
249 self.top_user_relevant_frame.or_else(|| self.stack.len().checked_sub(1))
253 }
254
255 pub fn current_span(&self) -> Span {
256 self.top_user_relevant_frame()
257 .map(|frame_idx| self.stack[frame_idx].current_span())
258 .unwrap_or(rustc_span::DUMMY_SP)
259 }
260}
261
262impl<'tcx> std::fmt::Debug for Thread<'tcx> {
263 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
264 write!(
265 f,
266 "{}({:?}, {:?})",
267 String::from_utf8_lossy(self.thread_name().unwrap_or(b"<unnamed>")),
268 self.state,
269 self.join_status
270 )
271 }
272}
273
274impl<'tcx> Thread<'tcx> {
275 fn new(name: Option<&str>, on_stack_empty: Option<StackEmptyCallback<'tcx>>) -> Self {
276 Self {
277 state: ThreadState::Enabled,
278 thread_name: name.map(|name| Vec::from(name.as_bytes())),
279 stack: Vec::new(),
280 top_user_relevant_frame: None,
281 join_status: ThreadJoinStatus::Joinable,
282 panic_payloads: Vec::new(),
283 last_error: None,
284 on_stack_empty,
285 }
286 }
287}
288
289impl VisitProvenance for Thread<'_> {
290 fn visit_provenance(&self, visit: &mut VisitWith<'_>) {
291 let Thread {
292 panic_payloads: panic_payload,
293 last_error,
294 stack,
295 top_user_relevant_frame: _,
296 state: _,
297 thread_name: _,
298 join_status: _,
299 on_stack_empty: _, } = self;
301
302 for payload in panic_payload {
303 payload.visit_provenance(visit);
304 }
305 last_error.visit_provenance(visit);
306 for frame in stack {
307 frame.visit_provenance(visit)
308 }
309 }
310}
311
312impl VisitProvenance for Frame<'_, Provenance, FrameExtra<'_>> {
313 fn visit_provenance(&self, visit: &mut VisitWith<'_>) {
314 let Frame {
315 return_place,
316 locals,
317 extra,
318 ..
320 } = self;
321
322 return_place.visit_provenance(visit);
324 for local in locals.iter() {
326 match local.as_mplace_or_imm() {
327 None => {}
328 Some(Either::Left((ptr, meta))) => {
329 ptr.visit_provenance(visit);
330 meta.visit_provenance(visit);
331 }
332 Some(Either::Right(imm)) => {
333 imm.visit_provenance(visit);
334 }
335 }
336 }
337
338 extra.visit_provenance(visit);
339 }
340}
341
342#[derive(Debug)]
344enum Timeout {
345 Monotonic(Instant),
346 RealTime(SystemTime),
347}
348
349impl Timeout {
350 fn get_wait_time(&self, clock: &MonotonicClock) -> Duration {
352 match self {
353 Timeout::Monotonic(instant) => instant.duration_since(clock.now()),
354 Timeout::RealTime(time) =>
355 time.duration_since(SystemTime::now()).unwrap_or(Duration::ZERO),
356 }
357 }
358
359 fn add_lossy(&self, duration: Duration) -> Self {
361 match self {
362 Timeout::Monotonic(i) => Timeout::Monotonic(i.add_lossy(duration)),
363 Timeout::RealTime(s) => {
364 Timeout::RealTime(
366 s.checked_add(duration)
367 .unwrap_or_else(|| s.checked_add(Duration::from_secs(3600)).unwrap()),
368 )
369 }
370 }
371 }
372}
373
374#[derive(Debug, Copy, Clone)]
376pub enum TimeoutClock {
377 Monotonic,
378 RealTime,
379}
380
381#[derive(Debug, Copy, Clone)]
383pub enum TimeoutAnchor {
384 Relative,
385 Absolute,
386}
387
388#[derive(Debug, Copy, Clone)]
390pub struct ThreadNotFound;
391
392#[derive(Debug)]
394pub struct ThreadManager<'tcx> {
395 active_thread: ThreadId,
397 threads: IndexVec<ThreadId, Thread<'tcx>>,
401 thread_local_allocs: FxHashMap<(DefId, ThreadId), StrictPointer>,
403 yield_active_thread: bool,
405 fixed_scheduling: bool,
407}
408
409impl VisitProvenance for ThreadManager<'_> {
410 fn visit_provenance(&self, visit: &mut VisitWith<'_>) {
411 let ThreadManager {
412 threads,
413 thread_local_allocs,
414 active_thread: _,
415 yield_active_thread: _,
416 fixed_scheduling: _,
417 } = self;
418
419 for thread in threads {
420 thread.visit_provenance(visit);
421 }
422 for ptr in thread_local_allocs.values() {
423 ptr.visit_provenance(visit);
424 }
425 }
426}
427
428impl<'tcx> ThreadManager<'tcx> {
429 pub(crate) fn new(config: &MiriConfig) -> Self {
430 let mut threads = IndexVec::new();
431 threads.push(Thread::new(Some("main"), None));
433 Self {
434 active_thread: ThreadId::MAIN_THREAD,
435 threads,
436 thread_local_allocs: Default::default(),
437 yield_active_thread: false,
438 fixed_scheduling: config.fixed_scheduling,
439 }
440 }
441
442 pub(crate) fn init(
443 ecx: &mut MiriInterpCx<'tcx>,
444 on_main_stack_empty: StackEmptyCallback<'tcx>,
445 ) {
446 ecx.machine.threads.threads[ThreadId::MAIN_THREAD].on_stack_empty =
447 Some(on_main_stack_empty);
448 if ecx.tcx.sess.target.os.as_ref() != "windows" {
449 ecx.machine.threads.threads[ThreadId::MAIN_THREAD].join_status =
451 ThreadJoinStatus::Detached;
452 }
453 }
454
455 pub fn thread_id_try_from(&self, id: impl TryInto<u32>) -> Result<ThreadId, ThreadNotFound> {
456 if let Ok(id) = id.try_into()
457 && usize::try_from(id).is_ok_and(|id| id < self.threads.len())
458 {
459 Ok(ThreadId(id))
460 } else {
461 Err(ThreadNotFound)
462 }
463 }
464
465 fn get_thread_local_alloc_id(&self, def_id: DefId) -> Option<StrictPointer> {
468 self.thread_local_allocs.get(&(def_id, self.active_thread)).cloned()
469 }
470
471 fn set_thread_local_alloc(&mut self, def_id: DefId, ptr: StrictPointer) {
476 self.thread_local_allocs.try_insert((def_id, self.active_thread), ptr).unwrap();
477 }
478
479 pub fn active_thread_stack(&self) -> &[Frame<'tcx, Provenance, FrameExtra<'tcx>>] {
481 &self.threads[self.active_thread].stack
482 }
483
484 pub fn active_thread_stack_mut(
486 &mut self,
487 ) -> &mut Vec<Frame<'tcx, Provenance, FrameExtra<'tcx>>> {
488 &mut self.threads[self.active_thread].stack
489 }
490
491 pub fn all_stacks(
492 &self,
493 ) -> impl Iterator<Item = (ThreadId, &[Frame<'tcx, Provenance, FrameExtra<'tcx>>])> {
494 self.threads.iter_enumerated().map(|(id, t)| (id, &t.stack[..]))
495 }
496
497 fn create_thread(&mut self, on_stack_empty: StackEmptyCallback<'tcx>) -> ThreadId {
499 let new_thread_id = ThreadId::new(self.threads.len());
500 self.threads.push(Thread::new(None, Some(on_stack_empty)));
501 new_thread_id
502 }
503
504 fn set_active_thread_id(&mut self, id: ThreadId) -> ThreadId {
506 assert!(id.index() < self.threads.len());
507 info!(
508 "---------- Now executing on thread `{}` (previous: `{}`) ----------------------------------------",
509 self.get_thread_display_name(id),
510 self.get_thread_display_name(self.active_thread)
511 );
512 std::mem::replace(&mut self.active_thread, id)
513 }
514
515 pub fn active_thread(&self) -> ThreadId {
517 self.active_thread
518 }
519
520 pub fn get_total_thread_count(&self) -> usize {
522 self.threads.len()
523 }
524
525 pub fn get_live_thread_count(&self) -> usize {
528 self.threads.iter().filter(|t| !t.state.is_terminated()).count()
529 }
530
531 fn has_terminated(&self, thread_id: ThreadId) -> bool {
533 self.threads[thread_id].state.is_terminated()
534 }
535
536 fn have_all_terminated(&self) -> bool {
538 self.threads.iter().all(|thread| thread.state.is_terminated())
539 }
540
541 fn enable_thread(&mut self, thread_id: ThreadId) {
543 assert!(self.has_terminated(thread_id));
544 self.threads[thread_id].state = ThreadState::Enabled;
545 }
546
547 pub fn active_thread_mut(&mut self) -> &mut Thread<'tcx> {
549 &mut self.threads[self.active_thread]
550 }
551
552 pub fn active_thread_ref(&self) -> &Thread<'tcx> {
554 &self.threads[self.active_thread]
555 }
556
557 fn detach_thread(&mut self, id: ThreadId, allow_terminated_joined: bool) -> InterpResult<'tcx> {
566 trace!("detaching {:?}", id);
567
568 let is_ub = if allow_terminated_joined && self.threads[id].state.is_terminated() {
569 self.threads[id].join_status == ThreadJoinStatus::Detached
571 } else {
572 self.threads[id].join_status != ThreadJoinStatus::Joinable
573 };
574 if is_ub {
575 throw_ub_format!("trying to detach thread that was already detached or joined");
576 }
577
578 self.threads[id].join_status = ThreadJoinStatus::Detached;
579 interp_ok(())
580 }
581
582 fn join_thread(
584 &mut self,
585 joined_thread_id: ThreadId,
586 data_race_handler: &mut GlobalDataRaceHandler,
587 ) -> InterpResult<'tcx> {
588 if self.threads[joined_thread_id].join_status == ThreadJoinStatus::Detached {
589 throw_ub_format!("trying to join a detached thread");
591 }
592
593 fn after_join<'tcx>(
594 threads: &mut ThreadManager<'_>,
595 joined_thread_id: ThreadId,
596 data_race_handler: &mut GlobalDataRaceHandler,
597 ) -> InterpResult<'tcx> {
598 match data_race_handler {
599 GlobalDataRaceHandler::None => {}
600 GlobalDataRaceHandler::Vclocks(data_race) =>
601 data_race.thread_joined(threads, joined_thread_id),
602 GlobalDataRaceHandler::Genmc(genmc_ctx) =>
603 genmc_ctx.handle_thread_join(threads.active_thread, joined_thread_id)?,
604 }
605 interp_ok(())
606 }
607
608 self.threads[joined_thread_id].join_status = ThreadJoinStatus::Joined;
611 if !self.threads[joined_thread_id].state.is_terminated() {
612 trace!(
613 "{:?} blocked on {:?} when trying to join",
614 self.active_thread, joined_thread_id
615 );
616 self.block_thread(
619 BlockReason::Join(joined_thread_id),
620 None,
621 callback!(
622 @capture<'tcx> {
623 joined_thread_id: ThreadId,
624 }
625 |this, unblock: UnblockKind| {
626 assert_eq!(unblock, UnblockKind::Ready);
627 after_join(&mut this.machine.threads, joined_thread_id, &mut this.machine.data_race)
628 }
629 ),
630 );
631 } else {
632 after_join(self, joined_thread_id, data_race_handler)?;
634 }
635 interp_ok(())
636 }
637
638 fn join_thread_exclusive(
641 &mut self,
642 joined_thread_id: ThreadId,
643 data_race_handler: &mut GlobalDataRaceHandler,
644 ) -> InterpResult<'tcx> {
645 if self.threads[joined_thread_id].join_status == ThreadJoinStatus::Joined {
646 throw_ub_format!("trying to join an already joined thread");
647 }
648
649 if joined_thread_id == self.active_thread {
650 throw_ub_format!("trying to join itself");
651 }
652
653 assert!(
655 self.threads
656 .iter()
657 .all(|thread| { !thread.state.is_blocked_on(BlockReason::Join(joined_thread_id)) }),
658 "this thread already has threads waiting for its termination"
659 );
660
661 self.join_thread(joined_thread_id, data_race_handler)
662 }
663
664 pub fn set_thread_name(&mut self, thread: ThreadId, new_thread_name: Vec<u8>) {
666 self.threads[thread].thread_name = Some(new_thread_name);
667 }
668
669 pub fn get_thread_name(&self, thread: ThreadId) -> Option<&[u8]> {
671 self.threads[thread].thread_name()
672 }
673
674 pub fn get_thread_display_name(&self, thread: ThreadId) -> String {
675 self.threads[thread].thread_display_name(thread)
676 }
677
678 fn block_thread(
680 &mut self,
681 reason: BlockReason,
682 timeout: Option<Timeout>,
683 callback: DynUnblockCallback<'tcx>,
684 ) {
685 let state = &mut self.threads[self.active_thread].state;
686 assert!(state.is_enabled());
687 *state = ThreadState::Blocked { reason, timeout, callback }
688 }
689
690 fn yield_active_thread(&mut self) {
692 self.yield_active_thread = true;
696 }
697
698 fn next_callback_wait_time(&self, clock: &MonotonicClock) -> Option<Duration> {
700 self.threads
701 .iter()
702 .filter_map(|t| {
703 match &t.state {
704 ThreadState::Blocked { timeout: Some(timeout), .. } =>
705 Some(timeout.get_wait_time(clock)),
706 _ => None,
707 }
708 })
709 .min()
710 }
711}
712
713impl<'tcx> EvalContextPrivExt<'tcx> for MiriInterpCx<'tcx> {}
714trait EvalContextPrivExt<'tcx>: MiriInterpCxExt<'tcx> {
715 #[inline]
717 fn run_timeout_callback(&mut self) -> InterpResult<'tcx> {
718 let this = self.eval_context_mut();
719 let mut found_callback = None;
720 for (id, thread) in this.machine.threads.threads.iter_enumerated_mut() {
722 match &thread.state {
723 ThreadState::Blocked { timeout: Some(timeout), .. }
724 if timeout.get_wait_time(&this.machine.monotonic_clock) == Duration::ZERO =>
725 {
726 let old_state = mem::replace(&mut thread.state, ThreadState::Enabled);
727 let ThreadState::Blocked { callback, .. } = old_state else { unreachable!() };
728 found_callback = Some((id, callback));
729 break;
731 }
732 _ => {}
733 }
734 }
735 if let Some((thread, callback)) = found_callback {
736 let old_thread = this.machine.threads.set_active_thread_id(thread);
743 callback.call(this, UnblockKind::TimedOut)?;
744 this.machine.threads.set_active_thread_id(old_thread);
745 }
746 interp_ok(())
753 }
754
755 #[inline]
756 fn run_on_stack_empty(&mut self) -> InterpResult<'tcx, Poll<()>> {
757 let this = self.eval_context_mut();
758 if let Some(genmc_ctx) = this.machine.data_race.as_genmc_ref() {
760 let thread_id = this.active_thread();
761 genmc_ctx.handle_thread_stack_empty(thread_id);
762 }
763 let mut callback = this
764 .active_thread_mut()
765 .on_stack_empty
766 .take()
767 .expect("`on_stack_empty` not set up, or already running");
768 let res = callback(this)?;
769 this.active_thread_mut().on_stack_empty = Some(callback);
770 interp_ok(res)
771 }
772
773 fn schedule(&mut self) -> InterpResult<'tcx, SchedulingAction> {
782 let this = self.eval_context_mut();
783 if let Some(genmc_ctx) = this.machine.data_race.as_genmc_ref() {
785 let next_thread_id = genmc_ctx.schedule_thread(this)?;
786
787 let thread_manager = &mut this.machine.threads;
788 thread_manager.active_thread = next_thread_id;
789 thread_manager.yield_active_thread = false;
790
791 assert!(thread_manager.threads[thread_manager.active_thread].state.is_enabled());
792 return interp_ok(SchedulingAction::ExecuteStep);
793 }
794
795 let thread_manager = &mut this.machine.threads;
797 let clock = &this.machine.monotonic_clock;
798 let rng = this.machine.rng.get_mut();
799 if thread_manager.threads[thread_manager.active_thread].state.is_enabled()
801 && !thread_manager.yield_active_thread
802 {
803 return interp_ok(SchedulingAction::ExecuteStep);
805 }
806 let potential_sleep_time = thread_manager.next_callback_wait_time(clock);
813 if potential_sleep_time == Some(Duration::ZERO) {
814 return interp_ok(SchedulingAction::ExecuteTimeoutCallback);
815 }
816 let mut threads_iter = thread_manager
823 .threads
824 .iter_enumerated()
825 .skip(thread_manager.active_thread.index() + 1)
826 .chain(
827 thread_manager
828 .threads
829 .iter_enumerated()
830 .take(thread_manager.active_thread.index() + 1),
831 )
832 .filter(|(_id, thread)| thread.state.is_enabled());
833 let new_thread = if thread_manager.fixed_scheduling {
835 threads_iter.next()
836 } else {
837 threads_iter.choose(rng)
838 };
839
840 if let Some((id, _thread)) = new_thread {
841 if thread_manager.active_thread != id {
842 info!(
843 "---------- Now executing on thread `{}` (previous: `{}`) ----------------------------------------",
844 thread_manager.get_thread_display_name(id),
845 thread_manager.get_thread_display_name(thread_manager.active_thread)
846 );
847 thread_manager.active_thread = id;
848 }
849 }
850 thread_manager.yield_active_thread = false;
852
853 if thread_manager.threads[thread_manager.active_thread].state.is_enabled() {
854 return interp_ok(SchedulingAction::ExecuteStep);
855 }
856 if thread_manager.threads.iter().all(|thread| thread.state.is_terminated()) {
858 unreachable!("all threads terminated without the main thread terminating?!");
859 } else if let Some(sleep_time) = potential_sleep_time {
860 interp_ok(SchedulingAction::Sleep(sleep_time))
864 } else {
865 throw_machine_stop!(TerminationInfo::Deadlock);
866 }
867 }
868}
869
870impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {}
872pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
873 #[inline]
874 fn thread_id_try_from(&self, id: impl TryInto<u32>) -> Result<ThreadId, ThreadNotFound> {
875 self.eval_context_ref().machine.threads.thread_id_try_from(id)
876 }
877
878 fn get_or_create_thread_local_alloc(
881 &mut self,
882 def_id: DefId,
883 ) -> InterpResult<'tcx, StrictPointer> {
884 let this = self.eval_context_mut();
885 let tcx = this.tcx;
886 if let Some(old_alloc) = this.machine.threads.get_thread_local_alloc_id(def_id) {
887 interp_ok(old_alloc)
890 } else {
891 if tcx.is_foreign_item(def_id) {
895 throw_unsup_format!("foreign thread-local statics are not supported");
896 }
897 let alloc = this.ctfe_query(|tcx| tcx.eval_static_initializer(def_id))?;
898 let mut alloc = alloc.inner().adjust_from_tcx(
900 &this.tcx,
901 |bytes, align| {
902 interp_ok(MiriAllocBytes::from_bytes(std::borrow::Cow::Borrowed(bytes), align))
903 },
904 |ptr| this.global_root_pointer(ptr),
905 )?;
906 alloc.mutability = Mutability::Mut;
908 let ptr = this.insert_allocation(alloc, MiriMemoryKind::Tls.into())?;
910 this.machine.threads.set_thread_local_alloc(def_id, ptr);
911 interp_ok(ptr)
912 }
913 }
914
915 #[inline]
917 fn start_regular_thread(
918 &mut self,
919 thread: Option<MPlaceTy<'tcx>>,
920 start_routine: Pointer,
921 start_abi: ExternAbi,
922 func_arg: ImmTy<'tcx>,
923 ret_layout: TyAndLayout<'tcx>,
924 ) -> InterpResult<'tcx, ThreadId> {
925 let this = self.eval_context_mut();
926
927 let new_thread_id = this.machine.threads.create_thread({
929 let mut state = tls::TlsDtorsState::default();
930 Box::new(move |m| state.on_stack_empty(m))
931 });
932 let current_span = this.machine.current_span();
933 match &mut this.machine.data_race {
934 GlobalDataRaceHandler::None => {}
935 GlobalDataRaceHandler::Vclocks(data_race) =>
936 data_race.thread_created(&this.machine.threads, new_thread_id, current_span),
937 GlobalDataRaceHandler::Genmc(genmc_ctx) =>
938 genmc_ctx.handle_thread_create(&this.machine.threads, new_thread_id)?,
939 }
940 if let Some(thread_info_place) = thread {
943 this.write_scalar(
944 Scalar::from_uint(new_thread_id.to_u32(), thread_info_place.layout.size),
945 &thread_info_place,
946 )?;
947 }
948
949 let old_thread_id = this.machine.threads.set_active_thread_id(new_thread_id);
952
953 if let Some(cpuset) = this.machine.thread_cpu_affinity.get(&old_thread_id).cloned() {
955 this.machine.thread_cpu_affinity.insert(new_thread_id, cpuset);
956 }
957
958 let instance = this.get_ptr_fn(start_routine)?.as_instance()?;
960
961 let ret_place = this.allocate(ret_layout, MiriMemoryKind::Machine.into())?;
965
966 this.call_function(
967 instance,
968 start_abi,
969 &[func_arg],
970 Some(&ret_place),
971 StackPopCleanup::Root { cleanup: true },
972 )?;
973
974 this.machine.threads.set_active_thread_id(old_thread_id);
976
977 interp_ok(new_thread_id)
978 }
979
980 fn terminate_active_thread(&mut self, tls_alloc_action: TlsAllocAction) -> InterpResult<'tcx> {
985 let this = self.eval_context_mut();
986
987 let thread = this.active_thread_mut();
989 assert!(thread.stack.is_empty(), "only threads with an empty stack can be terminated");
990 thread.state = ThreadState::Terminated;
991 match &mut this.machine.data_race {
992 GlobalDataRaceHandler::None => {}
993 GlobalDataRaceHandler::Vclocks(data_race) =>
994 data_race.thread_terminated(&this.machine.threads),
995 GlobalDataRaceHandler::Genmc(genmc_ctx) =>
996 genmc_ctx.handle_thread_finish(&this.machine.threads)?,
997 }
998 let gone_thread = this.active_thread();
1000 {
1001 let mut free_tls_statics = Vec::new();
1002 this.machine.threads.thread_local_allocs.retain(|&(_def_id, thread), &mut alloc_id| {
1003 if thread != gone_thread {
1004 return true;
1006 }
1007 free_tls_statics.push(alloc_id);
1010 false
1011 });
1012 for ptr in free_tls_statics {
1014 match tls_alloc_action {
1015 TlsAllocAction::Deallocate =>
1016 this.deallocate_ptr(ptr.into(), None, MiriMemoryKind::Tls.into())?,
1017 TlsAllocAction::Leak =>
1018 if let Some(alloc) = ptr.provenance.get_alloc_id() {
1019 trace!(
1020 "Thread-local static leaked and stored as static root: {:?}",
1021 alloc
1022 );
1023 this.machine.static_roots.push(alloc);
1024 },
1025 }
1026 }
1027 }
1028 let unblock_reason = BlockReason::Join(gone_thread);
1030 let threads = &this.machine.threads.threads;
1031 let joining_threads = threads
1032 .iter_enumerated()
1033 .filter(|(_, thread)| thread.state.is_blocked_on(unblock_reason))
1034 .map(|(id, _)| id)
1035 .collect::<Vec<_>>();
1036 for thread in joining_threads {
1037 this.unblock_thread(thread, unblock_reason)?;
1038 }
1039
1040 interp_ok(())
1041 }
1042
1043 #[inline]
1046 fn block_thread(
1047 &mut self,
1048 reason: BlockReason,
1049 timeout: Option<(TimeoutClock, TimeoutAnchor, Duration)>,
1050 callback: DynUnblockCallback<'tcx>,
1051 ) {
1052 let this = self.eval_context_mut();
1053 let timeout = timeout.map(|(clock, anchor, duration)| {
1054 let anchor = match clock {
1055 TimeoutClock::RealTime => {
1056 assert!(
1057 this.machine.communicate(),
1058 "cannot have `RealTime` timeout with isolation enabled!"
1059 );
1060 Timeout::RealTime(match anchor {
1061 TimeoutAnchor::Absolute => SystemTime::UNIX_EPOCH,
1062 TimeoutAnchor::Relative => SystemTime::now(),
1063 })
1064 }
1065 TimeoutClock::Monotonic =>
1066 Timeout::Monotonic(match anchor {
1067 TimeoutAnchor::Absolute => this.machine.monotonic_clock.epoch(),
1068 TimeoutAnchor::Relative => this.machine.monotonic_clock.now(),
1069 }),
1070 };
1071 anchor.add_lossy(duration)
1072 });
1073 this.machine.threads.block_thread(reason, timeout, callback);
1074 }
1075
1076 fn unblock_thread(&mut self, thread: ThreadId, reason: BlockReason) -> InterpResult<'tcx> {
1079 let this = self.eval_context_mut();
1080 let old_state =
1081 mem::replace(&mut this.machine.threads.threads[thread].state, ThreadState::Enabled);
1082 let callback = match old_state {
1083 ThreadState::Blocked { reason: actual_reason, callback, .. } => {
1084 assert_eq!(
1085 reason, actual_reason,
1086 "unblock_thread: thread was blocked for the wrong reason"
1087 );
1088 callback
1089 }
1090 _ => panic!("unblock_thread: thread was not blocked"),
1091 };
1092 let old_thread = this.machine.threads.set_active_thread_id(thread);
1094 callback.call(this, UnblockKind::Ready)?;
1095 this.machine.threads.set_active_thread_id(old_thread);
1096 interp_ok(())
1097 }
1098
1099 #[inline]
1100 fn detach_thread(
1101 &mut self,
1102 thread_id: ThreadId,
1103 allow_terminated_joined: bool,
1104 ) -> InterpResult<'tcx> {
1105 let this = self.eval_context_mut();
1106 this.machine.threads.detach_thread(thread_id, allow_terminated_joined)
1107 }
1108
1109 #[inline]
1110 fn join_thread(&mut self, joined_thread_id: ThreadId) -> InterpResult<'tcx> {
1111 let this = self.eval_context_mut();
1112 this.machine.threads.join_thread(joined_thread_id, &mut this.machine.data_race)?;
1113 interp_ok(())
1114 }
1115
1116 #[inline]
1117 fn join_thread_exclusive(&mut self, joined_thread_id: ThreadId) -> InterpResult<'tcx> {
1118 let this = self.eval_context_mut();
1119 this.machine
1120 .threads
1121 .join_thread_exclusive(joined_thread_id, &mut this.machine.data_race)?;
1122 interp_ok(())
1123 }
1124
1125 #[inline]
1126 fn active_thread(&self) -> ThreadId {
1127 let this = self.eval_context_ref();
1128 this.machine.threads.active_thread()
1129 }
1130
1131 #[inline]
1132 fn active_thread_mut(&mut self) -> &mut Thread<'tcx> {
1133 let this = self.eval_context_mut();
1134 this.machine.threads.active_thread_mut()
1135 }
1136
1137 #[inline]
1138 fn active_thread_ref(&self) -> &Thread<'tcx> {
1139 let this = self.eval_context_ref();
1140 this.machine.threads.active_thread_ref()
1141 }
1142
1143 #[inline]
1144 fn get_total_thread_count(&self) -> usize {
1145 let this = self.eval_context_ref();
1146 this.machine.threads.get_total_thread_count()
1147 }
1148
1149 #[inline]
1150 fn have_all_terminated(&self) -> bool {
1151 let this = self.eval_context_ref();
1152 this.machine.threads.have_all_terminated()
1153 }
1154
1155 #[inline]
1156 fn enable_thread(&mut self, thread_id: ThreadId) {
1157 let this = self.eval_context_mut();
1158 this.machine.threads.enable_thread(thread_id);
1159 }
1160
1161 #[inline]
1162 fn active_thread_stack<'a>(&'a self) -> &'a [Frame<'tcx, Provenance, FrameExtra<'tcx>>] {
1163 let this = self.eval_context_ref();
1164 this.machine.threads.active_thread_stack()
1165 }
1166
1167 #[inline]
1168 fn active_thread_stack_mut<'a>(
1169 &'a mut self,
1170 ) -> &'a mut Vec<Frame<'tcx, Provenance, FrameExtra<'tcx>>> {
1171 let this = self.eval_context_mut();
1172 this.machine.threads.active_thread_stack_mut()
1173 }
1174
1175 #[inline]
1177 fn set_thread_name(&mut self, thread: ThreadId, new_thread_name: Vec<u8>) {
1178 self.eval_context_mut().machine.threads.set_thread_name(thread, new_thread_name);
1179 }
1180
1181 #[inline]
1182 fn get_thread_name<'c>(&'c self, thread: ThreadId) -> Option<&'c [u8]>
1183 where
1184 'tcx: 'c,
1185 {
1186 self.eval_context_ref().machine.threads.get_thread_name(thread)
1187 }
1188
1189 #[inline]
1190 fn yield_active_thread(&mut self) {
1191 self.eval_context_mut().machine.threads.yield_active_thread();
1192 }
1193
1194 #[inline]
1195 fn maybe_preempt_active_thread(&mut self) {
1196 use rand::Rng as _;
1197
1198 let this = self.eval_context_mut();
1199 if !this.machine.threads.fixed_scheduling
1200 && this.machine.rng.get_mut().random_bool(this.machine.preemption_rate)
1201 {
1202 this.yield_active_thread();
1203 }
1204 }
1205
1206 fn run_threads(&mut self) -> InterpResult<'tcx, !> {
1209 let this = self.eval_context_mut();
1210 loop {
1211 if CTRL_C_RECEIVED.load(Relaxed) {
1212 this.machine.handle_abnormal_termination();
1213 throw_machine_stop!(TerminationInfo::Interrupted);
1214 }
1215 match this.schedule()? {
1216 SchedulingAction::ExecuteStep => {
1217 if !this.step()? {
1218 match this.run_on_stack_empty()? {
1220 Poll::Pending => {} Poll::Ready(()) =>
1222 this.terminate_active_thread(TlsAllocAction::Deallocate)?,
1223 }
1224 }
1225 }
1226 SchedulingAction::ExecuteTimeoutCallback => {
1227 this.run_timeout_callback()?;
1228 }
1229 SchedulingAction::Sleep(duration) => {
1230 this.machine.monotonic_clock.sleep(duration);
1231 }
1232 }
1233 }
1234 }
1235}