miri/shims/
tls.rs

1//! Implement thread-local storage.
2
3use std::collections::BTreeMap;
4use std::collections::btree_map::Entry as BTreeEntry;
5use std::task::Poll;
6
7use rustc_abi::{ExternAbi, HasDataLayout, Size};
8use rustc_middle::ty;
9
10use crate::*;
11
12pub type TlsKey = u128;
13
14#[derive(Clone, Debug)]
15pub struct TlsEntry<'tcx> {
16    /// The data for this key. None is used to represent NULL.
17    /// (We normalize this early to avoid having to do a NULL-ptr-test each time we access the data.)
18    data: BTreeMap<ThreadId, Scalar>,
19    dtor: Option<ty::Instance<'tcx>>,
20}
21
22#[derive(Default, Debug)]
23struct RunningDtorState {
24    /// The last TlsKey used to retrieve a TLS destructor. `None` means that we
25    /// have not tried to retrieve a TLS destructor yet or that we already tried
26    /// all keys.
27    last_key: Option<TlsKey>,
28}
29
30#[derive(Debug)]
31pub struct TlsData<'tcx> {
32    /// The Key to use for the next thread-local allocation.
33    next_key: TlsKey,
34
35    /// pthreads-style thread-local storage.
36    keys: BTreeMap<TlsKey, TlsEntry<'tcx>>,
37
38    /// On macOS, each thread holds a list of destructor functions with their
39    /// respective data arguments.
40    macos_thread_dtors: BTreeMap<ThreadId, Vec<(ty::Instance<'tcx>, Scalar)>>,
41}
42
43impl<'tcx> Default for TlsData<'tcx> {
44    fn default() -> Self {
45        TlsData {
46            next_key: 1, // start with 1 as we must not use 0 on Windows
47            keys: Default::default(),
48            macos_thread_dtors: Default::default(),
49        }
50    }
51}
52
53impl<'tcx> TlsData<'tcx> {
54    /// Generate a new TLS key with the given destructor.
55    /// `max_size` determines the integer size the key has to fit in.
56    #[expect(clippy::arithmetic_side_effects)]
57    pub fn create_tls_key(
58        &mut self,
59        dtor: Option<ty::Instance<'tcx>>,
60        max_size: Size,
61    ) -> InterpResult<'tcx, TlsKey> {
62        let new_key = self.next_key;
63        self.next_key += 1;
64        self.keys.try_insert(new_key, TlsEntry { data: Default::default(), dtor }).unwrap();
65        trace!("New TLS key allocated: {} with dtor {:?}", new_key, dtor);
66
67        if max_size.bits() < 128 && new_key >= (1u128 << max_size.bits()) {
68            throw_unsup_format!("we ran out of TLS key space");
69        }
70        interp_ok(new_key)
71    }
72
73    pub fn delete_tls_key(&mut self, key: TlsKey) -> InterpResult<'tcx> {
74        match self.keys.remove(&key) {
75            Some(_) => {
76                trace!("TLS key {} removed", key);
77                interp_ok(())
78            }
79            None => throw_ub_format!("removing a nonexistent TLS key: {}", key),
80        }
81    }
82
83    pub fn load_tls(
84        &self,
85        key: TlsKey,
86        thread_id: ThreadId,
87        cx: &impl HasDataLayout,
88    ) -> InterpResult<'tcx, Scalar> {
89        match self.keys.get(&key) {
90            Some(TlsEntry { data, .. }) => {
91                let value = data.get(&thread_id).copied();
92                trace!("TLS key {} for thread {:?} loaded: {:?}", key, thread_id, value);
93                interp_ok(value.unwrap_or_else(|| Scalar::null_ptr(cx)))
94            }
95            None => throw_ub_format!("loading from a non-existing TLS key: {}", key),
96        }
97    }
98
99    pub fn store_tls(
100        &mut self,
101        key: TlsKey,
102        thread_id: ThreadId,
103        new_data: Scalar,
104        cx: &impl HasDataLayout,
105    ) -> InterpResult<'tcx> {
106        match self.keys.get_mut(&key) {
107            Some(TlsEntry { data, .. }) => {
108                if new_data.to_target_usize(cx)? != 0 {
109                    trace!("TLS key {} for thread {:?} stored: {:?}", key, thread_id, new_data);
110                    data.insert(thread_id, new_data);
111                } else {
112                    trace!("TLS key {} for thread {:?} removed", key, thread_id);
113                    data.remove(&thread_id);
114                }
115                interp_ok(())
116            }
117            None => throw_ub_format!("storing to a non-existing TLS key: {}", key),
118        }
119    }
120
121    /// Add a thread local storage destructor for the given thread. This function
122    /// is used to implement the `_tlv_atexit` shim on MacOS.
123    pub fn add_macos_thread_dtor(
124        &mut self,
125        thread: ThreadId,
126        dtor: ty::Instance<'tcx>,
127        data: Scalar,
128    ) -> InterpResult<'tcx> {
129        self.macos_thread_dtors.entry(thread).or_default().push((dtor, data));
130        interp_ok(())
131    }
132
133    /// Returns a dtor, its argument and its index, if one is supposed to run.
134    /// `key` is the last dtors that was run; we return the *next* one after that.
135    ///
136    /// An optional destructor function may be associated with each key value.
137    /// At thread exit, if a key value has a non-NULL destructor pointer,
138    /// and the thread has a non-NULL value associated with that key,
139    /// the value of the key is set to NULL, and then the function pointed
140    /// to is called with the previously associated value as its sole argument.
141    /// **The order of destructor calls is unspecified if more than one destructor
142    /// exists for a thread when it exits.**
143    ///
144    /// If, after all the destructors have been called for all non-NULL values
145    /// with associated destructors, there are still some non-NULL values with
146    /// associated destructors, then the process is repeated.
147    /// If, after at least {PTHREAD_DESTRUCTOR_ITERATIONS} iterations of destructor
148    /// calls for outstanding non-NULL values, there are still some non-NULL values
149    /// with associated destructors, implementations may stop calling destructors,
150    /// or they may continue calling destructors until no non-NULL values with
151    /// associated destructors exist, even though this might result in an infinite loop.
152    fn fetch_tls_dtor(
153        &mut self,
154        key: Option<TlsKey>,
155        thread_id: ThreadId,
156    ) -> Option<(ty::Instance<'tcx>, Scalar, TlsKey)> {
157        use std::ops::Bound::*;
158
159        let thread_local = &mut self.keys;
160        let start = match key {
161            Some(key) => Excluded(key),
162            None => Unbounded,
163        };
164        // We interpret the documentation above (taken from POSIX) as saying that we need to iterate
165        // over all keys and run each destructor at least once before running any destructor a 2nd
166        // time. That's why we have `key` to indicate how far we got in the current iteration. If we
167        // return `None`, `schedule_next_pthread_tls_dtor` will re-try with `ket` set to `None` to
168        // start the next round.
169        // TODO: In the future, we might consider randomizing destructor order, but we still have to
170        // uphold this requirement.
171        for (&key, TlsEntry { data, dtor }) in thread_local.range_mut((start, Unbounded)) {
172            match data.entry(thread_id) {
173                BTreeEntry::Occupied(entry) => {
174                    if let Some(dtor) = dtor {
175                        // Set TLS data to NULL, and call dtor with old value.
176                        let data_scalar = entry.remove();
177                        let ret = Some((*dtor, data_scalar, key));
178                        return ret;
179                    }
180                }
181                BTreeEntry::Vacant(_) => {}
182            }
183        }
184        None
185    }
186
187    /// Delete all TLS entries for the given thread. This function should be
188    /// called after all TLS destructors have already finished.
189    fn delete_all_thread_tls(&mut self, thread_id: ThreadId) {
190        for TlsEntry { data, .. } in self.keys.values_mut() {
191            data.remove(&thread_id);
192        }
193
194        if let Some(dtors) = self.macos_thread_dtors.remove(&thread_id) {
195            assert!(dtors.is_empty(), "the destructors should have already been run");
196        }
197    }
198}
199
200impl VisitProvenance for TlsData<'_> {
201    fn visit_provenance(&self, visit: &mut VisitWith<'_>) {
202        let TlsData { keys, macos_thread_dtors, next_key: _ } = self;
203
204        for scalar in keys.values().flat_map(|v| v.data.values()) {
205            scalar.visit_provenance(visit);
206        }
207        for (_, scalar) in macos_thread_dtors.values().flatten() {
208            scalar.visit_provenance(visit);
209        }
210    }
211}
212
213#[derive(Debug, Default)]
214pub struct TlsDtorsState<'tcx>(TlsDtorsStatePriv<'tcx>);
215
216#[derive(Debug, Default)]
217enum TlsDtorsStatePriv<'tcx> {
218    #[default]
219    Init,
220    MacOsDtors,
221    PthreadDtors(RunningDtorState),
222    /// For Windows Dtors, we store the list of functions that we still have to call.
223    /// These are functions from the magic `.CRT$XLB` linker section.
224    WindowsDtors(Vec<ImmTy<'tcx>>),
225    Done,
226}
227
228impl<'tcx> TlsDtorsState<'tcx> {
229    pub fn on_stack_empty(
230        &mut self,
231        this: &mut MiriInterpCx<'tcx>,
232    ) -> InterpResult<'tcx, Poll<()>> {
233        use TlsDtorsStatePriv::*;
234        let new_state = 'new_state: {
235            match &mut self.0 {
236                Init => {
237                    match this.tcx.sess.target.os.as_ref() {
238                        "macos" => {
239                            // macOS has a _tlv_atexit function that allows
240                            // registering destructors without associated keys.
241                            // These are run first.
242                            break 'new_state MacOsDtors;
243                        }
244                        _ if this.target_os_is_unix() => {
245                            // All other Unixes directly jump to running the pthread dtors.
246                            break 'new_state PthreadDtors(Default::default());
247                        }
248                        "windows" => {
249                            // Determine which destructors to run.
250                            let dtors = this.lookup_windows_tls_dtors()?;
251                            // And move to the next state, that runs them.
252                            break 'new_state WindowsDtors(dtors);
253                        }
254                        _ => {
255                            // No TLS dtor support.
256                            // FIXME: should we do something on wasi?
257                            break 'new_state Done;
258                        }
259                    }
260                }
261                MacOsDtors => {
262                    match this.schedule_macos_tls_dtor()? {
263                        Poll::Pending => return interp_ok(Poll::Pending),
264                        // After all macOS destructors are run, the system switches
265                        // to destroying the pthread destructors.
266                        Poll::Ready(()) => break 'new_state PthreadDtors(Default::default()),
267                    }
268                }
269                PthreadDtors(state) => {
270                    match this.schedule_next_pthread_tls_dtor(state)? {
271                        Poll::Pending => return interp_ok(Poll::Pending), // just keep going
272                        Poll::Ready(()) => break 'new_state Done,
273                    }
274                }
275                WindowsDtors(dtors) => {
276                    if let Some(dtor) = dtors.pop() {
277                        this.schedule_windows_tls_dtor(dtor)?;
278                        return interp_ok(Poll::Pending); // we stay in this state (but `dtors` got shorter)
279                    } else {
280                        // No more destructors to run.
281                        break 'new_state Done;
282                    }
283                }
284                Done => {
285                    this.machine.tls.delete_all_thread_tls(this.active_thread());
286                    return interp_ok(Poll::Ready(()));
287                }
288            }
289        };
290
291        self.0 = new_state;
292        interp_ok(Poll::Pending)
293    }
294}
295
296impl<'tcx> EvalContextPrivExt<'tcx> for crate::MiriInterpCx<'tcx> {}
297trait EvalContextPrivExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
298    /// Schedule TLS destructors for Windows.
299    /// On windows, TLS destructors are managed by std.
300    fn lookup_windows_tls_dtors(&mut self) -> InterpResult<'tcx, Vec<ImmTy<'tcx>>> {
301        let this = self.eval_context_mut();
302
303        // Windows has a special magic linker section that is run on certain events.
304        // We don't support most of that, but just enough to make thread-local dtors in `std` work.
305        interp_ok(this.lookup_link_section(".CRT$XLB")?)
306    }
307
308    fn schedule_windows_tls_dtor(&mut self, dtor: ImmTy<'tcx>) -> InterpResult<'tcx> {
309        let this = self.eval_context_mut();
310
311        let dtor = dtor.to_scalar().to_pointer(this)?;
312        let thread_callback = this.get_ptr_fn(dtor)?.as_instance()?;
313
314        // FIXME: Technically, the reason should be `DLL_PROCESS_DETACH` when the main thread exits
315        // but std treats both the same.
316        let reason = this.eval_windows("c", "DLL_THREAD_DETACH");
317        let null_ptr =
318            ImmTy::from_scalar(Scalar::null_ptr(this), this.machine.layouts.const_raw_ptr);
319
320        // The signature of this function is `unsafe extern "system" fn(h: c::LPVOID, dwReason: c::DWORD, pv: c::LPVOID)`.
321        // FIXME: `h` should be a handle to the current module and what `pv` should be is unknown
322        // but both are ignored by std.
323        this.call_function(
324            thread_callback,
325            ExternAbi::System { unwind: false },
326            &[null_ptr.clone(), ImmTy::from_scalar(reason, this.machine.layouts.u32), null_ptr],
327            None,
328            StackPopCleanup::Root { cleanup: true },
329        )?;
330        interp_ok(())
331    }
332
333    /// Schedule the macOS thread local storage destructors to be executed.
334    fn schedule_macos_tls_dtor(&mut self) -> InterpResult<'tcx, Poll<()>> {
335        let this = self.eval_context_mut();
336        let thread_id = this.active_thread();
337        // macOS keeps track of TLS destructors in a stack. If a destructor
338        // registers another destructor, it will be run next.
339        // See https://github.com/apple-oss-distributions/dyld/blob/d552c40cd1de105f0ec95008e0e0c0972de43456/dyld/DyldRuntimeState.cpp#L2277
340        let dtor = this.machine.tls.macos_thread_dtors.get_mut(&thread_id).and_then(Vec::pop);
341        if let Some((instance, data)) = dtor {
342            trace!("Running macos dtor {:?} on {:?} at {:?}", instance, data, thread_id);
343
344            this.call_function(
345                instance,
346                ExternAbi::C { unwind: false },
347                &[ImmTy::from_scalar(data, this.machine.layouts.mut_raw_ptr)],
348                None,
349                StackPopCleanup::Root { cleanup: true },
350            )?;
351
352            return interp_ok(Poll::Pending);
353        }
354
355        interp_ok(Poll::Ready(()))
356    }
357
358    /// Schedule a pthread TLS destructor. Returns `true` if found
359    /// a destructor to schedule, and `false` otherwise.
360    fn schedule_next_pthread_tls_dtor(
361        &mut self,
362        state: &mut RunningDtorState,
363    ) -> InterpResult<'tcx, Poll<()>> {
364        let this = self.eval_context_mut();
365        let active_thread = this.active_thread();
366
367        // Fetch next dtor after `key`.
368        let dtor = match this.machine.tls.fetch_tls_dtor(state.last_key, active_thread) {
369            dtor @ Some(_) => dtor,
370            // We ran each dtor once, start over from the beginning.
371            None => this.machine.tls.fetch_tls_dtor(None, active_thread),
372        };
373        if let Some((instance, ptr, key)) = dtor {
374            state.last_key = Some(key);
375            trace!("Running TLS dtor {:?} on {:?} at {:?}", instance, ptr, active_thread);
376            assert!(
377                ptr.to_target_usize(this).unwrap() != 0,
378                "data can't be NULL when dtor is called!"
379            );
380
381            this.call_function(
382                instance,
383                ExternAbi::C { unwind: false },
384                &[ImmTy::from_scalar(ptr, this.machine.layouts.mut_raw_ptr)],
385                None,
386                StackPopCleanup::Root { cleanup: true },
387            )?;
388
389            return interp_ok(Poll::Pending);
390        }
391
392        interp_ok(Poll::Ready(()))
393    }
394}