1use std::collections::BTreeMap;
4use std::collections::btree_map::Entry as BTreeEntry;
5use std::task::Poll;
6
7use rustc_abi::{ExternAbi, HasDataLayout, Size};
8use rustc_middle::ty;
9
10use crate::*;
11
12pub type TlsKey = u128;
13
14#[derive(Clone, Debug)]
15pub struct TlsEntry<'tcx> {
16 data: BTreeMap<ThreadId, Scalar>,
19 dtor: Option<ty::Instance<'tcx>>,
20}
21
22#[derive(Default, Debug)]
23struct RunningDtorState {
24 last_key: Option<TlsKey>,
28}
29
30#[derive(Debug)]
31pub struct TlsData<'tcx> {
32 next_key: TlsKey,
34
35 keys: BTreeMap<TlsKey, TlsEntry<'tcx>>,
37
38 macos_thread_dtors: BTreeMap<ThreadId, Vec<(ty::Instance<'tcx>, Scalar)>>,
41}
42
43impl<'tcx> Default for TlsData<'tcx> {
44 fn default() -> Self {
45 TlsData {
46 next_key: 1, keys: Default::default(),
48 macos_thread_dtors: Default::default(),
49 }
50 }
51}
52
53impl<'tcx> TlsData<'tcx> {
54 #[expect(clippy::arithmetic_side_effects)]
57 pub fn create_tls_key(
58 &mut self,
59 dtor: Option<ty::Instance<'tcx>>,
60 max_size: Size,
61 ) -> InterpResult<'tcx, TlsKey> {
62 let new_key = self.next_key;
63 self.next_key += 1;
64 self.keys.try_insert(new_key, TlsEntry { data: Default::default(), dtor }).unwrap();
65 trace!("New TLS key allocated: {} with dtor {:?}", new_key, dtor);
66
67 if max_size.bits() < 128 && new_key >= (1u128 << max_size.bits()) {
68 throw_unsup_format!("we ran out of TLS key space");
69 }
70 interp_ok(new_key)
71 }
72
73 pub fn delete_tls_key(&mut self, key: TlsKey) -> InterpResult<'tcx> {
74 match self.keys.remove(&key) {
75 Some(_) => {
76 trace!("TLS key {} removed", key);
77 interp_ok(())
78 }
79 None => throw_ub_format!("removing a nonexistent TLS key: {}", key),
80 }
81 }
82
83 pub fn load_tls(
84 &self,
85 key: TlsKey,
86 thread_id: ThreadId,
87 cx: &impl HasDataLayout,
88 ) -> InterpResult<'tcx, Scalar> {
89 match self.keys.get(&key) {
90 Some(TlsEntry { data, .. }) => {
91 let value = data.get(&thread_id).copied();
92 trace!("TLS key {} for thread {:?} loaded: {:?}", key, thread_id, value);
93 interp_ok(value.unwrap_or_else(|| Scalar::null_ptr(cx)))
94 }
95 None => throw_ub_format!("loading from a non-existing TLS key: {}", key),
96 }
97 }
98
99 pub fn store_tls(
100 &mut self,
101 key: TlsKey,
102 thread_id: ThreadId,
103 new_data: Scalar,
104 cx: &impl HasDataLayout,
105 ) -> InterpResult<'tcx> {
106 match self.keys.get_mut(&key) {
107 Some(TlsEntry { data, .. }) => {
108 if new_data.to_target_usize(cx)? != 0 {
109 trace!("TLS key {} for thread {:?} stored: {:?}", key, thread_id, new_data);
110 data.insert(thread_id, new_data);
111 } else {
112 trace!("TLS key {} for thread {:?} removed", key, thread_id);
113 data.remove(&thread_id);
114 }
115 interp_ok(())
116 }
117 None => throw_ub_format!("storing to a non-existing TLS key: {}", key),
118 }
119 }
120
121 pub fn add_macos_thread_dtor(
124 &mut self,
125 thread: ThreadId,
126 dtor: ty::Instance<'tcx>,
127 data: Scalar,
128 ) -> InterpResult<'tcx> {
129 self.macos_thread_dtors.entry(thread).or_default().push((dtor, data));
130 interp_ok(())
131 }
132
133 fn fetch_tls_dtor(
153 &mut self,
154 key: Option<TlsKey>,
155 thread_id: ThreadId,
156 ) -> Option<(ty::Instance<'tcx>, Scalar, TlsKey)> {
157 use std::ops::Bound::*;
158
159 let thread_local = &mut self.keys;
160 let start = match key {
161 Some(key) => Excluded(key),
162 None => Unbounded,
163 };
164 for (&key, TlsEntry { data, dtor }) in thread_local.range_mut((start, Unbounded)) {
172 match data.entry(thread_id) {
173 BTreeEntry::Occupied(entry) => {
174 if let Some(dtor) = dtor {
175 let data_scalar = entry.remove();
177 let ret = Some((*dtor, data_scalar, key));
178 return ret;
179 }
180 }
181 BTreeEntry::Vacant(_) => {}
182 }
183 }
184 None
185 }
186
187 fn delete_all_thread_tls(&mut self, thread_id: ThreadId) {
190 for TlsEntry { data, .. } in self.keys.values_mut() {
191 data.remove(&thread_id);
192 }
193
194 if let Some(dtors) = self.macos_thread_dtors.remove(&thread_id) {
195 assert!(dtors.is_empty(), "the destructors should have already been run");
196 }
197 }
198}
199
200impl VisitProvenance for TlsData<'_> {
201 fn visit_provenance(&self, visit: &mut VisitWith<'_>) {
202 let TlsData { keys, macos_thread_dtors, next_key: _ } = self;
203
204 for scalar in keys.values().flat_map(|v| v.data.values()) {
205 scalar.visit_provenance(visit);
206 }
207 for (_, scalar) in macos_thread_dtors.values().flatten() {
208 scalar.visit_provenance(visit);
209 }
210 }
211}
212
213#[derive(Debug, Default)]
214pub struct TlsDtorsState<'tcx>(TlsDtorsStatePriv<'tcx>);
215
216#[derive(Debug, Default)]
217enum TlsDtorsStatePriv<'tcx> {
218 #[default]
219 Init,
220 MacOsDtors,
221 PthreadDtors(RunningDtorState),
222 WindowsDtors(Vec<ImmTy<'tcx>>),
225 Done,
226}
227
228impl<'tcx> TlsDtorsState<'tcx> {
229 pub fn on_stack_empty(
230 &mut self,
231 this: &mut MiriInterpCx<'tcx>,
232 ) -> InterpResult<'tcx, Poll<()>> {
233 use TlsDtorsStatePriv::*;
234 let new_state = 'new_state: {
235 match &mut self.0 {
236 Init => {
237 match this.tcx.sess.target.os.as_ref() {
238 "macos" => {
239 break 'new_state MacOsDtors;
243 }
244 _ if this.target_os_is_unix() => {
245 break 'new_state PthreadDtors(Default::default());
247 }
248 "windows" => {
249 let dtors = this.lookup_windows_tls_dtors()?;
251 break 'new_state WindowsDtors(dtors);
253 }
254 _ => {
255 break 'new_state Done;
258 }
259 }
260 }
261 MacOsDtors => {
262 match this.schedule_macos_tls_dtor()? {
263 Poll::Pending => return interp_ok(Poll::Pending),
264 Poll::Ready(()) => break 'new_state PthreadDtors(Default::default()),
267 }
268 }
269 PthreadDtors(state) => {
270 match this.schedule_next_pthread_tls_dtor(state)? {
271 Poll::Pending => return interp_ok(Poll::Pending), Poll::Ready(()) => break 'new_state Done,
273 }
274 }
275 WindowsDtors(dtors) => {
276 if let Some(dtor) = dtors.pop() {
277 this.schedule_windows_tls_dtor(dtor)?;
278 return interp_ok(Poll::Pending); } else {
280 break 'new_state Done;
282 }
283 }
284 Done => {
285 this.machine.tls.delete_all_thread_tls(this.active_thread());
286 return interp_ok(Poll::Ready(()));
287 }
288 }
289 };
290
291 self.0 = new_state;
292 interp_ok(Poll::Pending)
293 }
294}
295
296impl<'tcx> EvalContextPrivExt<'tcx> for crate::MiriInterpCx<'tcx> {}
297trait EvalContextPrivExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
298 fn lookup_windows_tls_dtors(&mut self) -> InterpResult<'tcx, Vec<ImmTy<'tcx>>> {
301 let this = self.eval_context_mut();
302
303 interp_ok(this.lookup_link_section(".CRT$XLB")?)
306 }
307
308 fn schedule_windows_tls_dtor(&mut self, dtor: ImmTy<'tcx>) -> InterpResult<'tcx> {
309 let this = self.eval_context_mut();
310
311 let dtor = dtor.to_scalar().to_pointer(this)?;
312 let thread_callback = this.get_ptr_fn(dtor)?.as_instance()?;
313
314 let reason = this.eval_windows("c", "DLL_THREAD_DETACH");
317 let null_ptr =
318 ImmTy::from_scalar(Scalar::null_ptr(this), this.machine.layouts.const_raw_ptr);
319
320 this.call_function(
324 thread_callback,
325 ExternAbi::System { unwind: false },
326 &[null_ptr.clone(), ImmTy::from_scalar(reason, this.machine.layouts.u32), null_ptr],
327 None,
328 StackPopCleanup::Root { cleanup: true },
329 )?;
330 interp_ok(())
331 }
332
333 fn schedule_macos_tls_dtor(&mut self) -> InterpResult<'tcx, Poll<()>> {
335 let this = self.eval_context_mut();
336 let thread_id = this.active_thread();
337 let dtor = this.machine.tls.macos_thread_dtors.get_mut(&thread_id).and_then(Vec::pop);
341 if let Some((instance, data)) = dtor {
342 trace!("Running macos dtor {:?} on {:?} at {:?}", instance, data, thread_id);
343
344 this.call_function(
345 instance,
346 ExternAbi::C { unwind: false },
347 &[ImmTy::from_scalar(data, this.machine.layouts.mut_raw_ptr)],
348 None,
349 StackPopCleanup::Root { cleanup: true },
350 )?;
351
352 return interp_ok(Poll::Pending);
353 }
354
355 interp_ok(Poll::Ready(()))
356 }
357
358 fn schedule_next_pthread_tls_dtor(
361 &mut self,
362 state: &mut RunningDtorState,
363 ) -> InterpResult<'tcx, Poll<()>> {
364 let this = self.eval_context_mut();
365 let active_thread = this.active_thread();
366
367 let dtor = match this.machine.tls.fetch_tls_dtor(state.last_key, active_thread) {
369 dtor @ Some(_) => dtor,
370 None => this.machine.tls.fetch_tls_dtor(None, active_thread),
372 };
373 if let Some((instance, ptr, key)) = dtor {
374 state.last_key = Some(key);
375 trace!("Running TLS dtor {:?} on {:?} at {:?}", instance, ptr, active_thread);
376 assert!(
377 ptr.to_target_usize(this).unwrap() != 0,
378 "data can't be NULL when dtor is called!"
379 );
380
381 this.call_function(
382 instance,
383 ExternAbi::C { unwind: false },
384 &[ImmTy::from_scalar(ptr, this.machine.layouts.mut_raw_ptr)],
385 None,
386 StackPopCleanup::Root { cleanup: true },
387 )?;
388
389 return interp_ok(Poll::Pending);
390 }
391
392 interp_ok(Poll::Ready(()))
393 }
394}