std/sys/sync/futex/
unix.rs

1use crate::sync::atomic::Atomic;
2use crate::time::Duration;
3
4/// An atomic for use as a futex that is at least 32-bits but may be larger
5pub type Futex = Atomic<Primitive>;
6/// Must be the underlying type of Futex
7pub type Primitive = u32;
8
9/// An atomic for use as a futex that is at least 8-bits but may be larger.
10pub type SmallFutex = Atomic<SmallPrimitive>;
11/// Must be the underlying type of SmallFutex
12pub type SmallPrimitive = u32;
13
14#[cfg(any(target_os = "linux", target_os = "android"))]
15cfg_select! {
16    // riscv32 only exposes the `SYS_futex_time64` syscall,
17    // whereas other 32-bit targets also expose `SYS_futex`, which uses 32-bit time.
18    target_arch = "riscv32" => {
19        use libc::SYS_futex_time64 as SYS_futex;
20    }
21    _ => {
22        use libc::SYS_futex;
23    }
24}
25
26/// Waits for a `futex_wake` operation to wake us.
27///
28/// Returns directly if the futex doesn't hold the expected value.
29///
30/// Returns false on timeout, and true in all other cases.
31#[cfg(any(target_os = "linux", target_os = "android", target_os = "freebsd"))]
32pub fn futex_wait(futex: &Atomic<u32>, expected: u32, timeout: Option<Duration>) -> bool {
33    use crate::ptr::null;
34    use crate::sync::atomic::Ordering::Relaxed;
35    use crate::sys::pal::time::Timespec;
36
37    // Calculate the timeout as an absolute timespec.
38    //
39    // Overflows are rounded up to an infinite timeout (None).
40    let timespec = timeout
41        .and_then(|d| Timespec::now(libc::CLOCK_MONOTONIC).checked_add_duration(&d))
42        .and_then(|t| t.to_timespec());
43
44    loop {
45        // No need to wait if the value already changed.
46        if futex.load(Relaxed) != expected {
47            return true;
48        }
49
50        let r = unsafe {
51            cfg_select! {
52                target_os = "freebsd" => {
53                    // FreeBSD doesn't have futex(), but it has
54                    // _umtx_op(UMTX_OP_WAIT_UINT_PRIVATE), which is nearly
55                    // identical. It supports absolute timeouts through a flag
56                    // in the _umtx_time struct.
57                    let umtx_timeout = timespec.map(|t| libc::_umtx_time {
58                        _timeout: t,
59                        _flags: libc::UMTX_ABSTIME,
60                        _clockid: libc::CLOCK_MONOTONIC as u32,
61                    });
62                    let umtx_timeout_ptr = umtx_timeout.as_ref().map_or(null(), |t| t as *const _);
63                    let umtx_timeout_size = umtx_timeout.as_ref().map_or(0, |t| size_of_val(t));
64                    libc::_umtx_op(
65                        futex as *const Atomic<u32> as *mut _,
66                        libc::UMTX_OP_WAIT_UINT_PRIVATE,
67                        expected as libc::c_ulong,
68                        crate::ptr::without_provenance_mut(umtx_timeout_size),
69                        umtx_timeout_ptr as *mut _,
70                    )
71                }
72                any(target_os = "linux", target_os = "android") => {
73                    // Use FUTEX_WAIT_BITSET rather than FUTEX_WAIT to be able to give an
74                    // absolute time rather than a relative time.
75                    libc::syscall(
76                        SYS_futex,
77                        futex as *const Atomic<u32> as *const u32,
78                        libc::FUTEX_WAIT_BITSET | libc::FUTEX_PRIVATE_FLAG,
79                        expected,
80                        timespec.as_ref().map_or(null(), |t| t as *const libc::timespec),
81                        null::<u32>(), // This argument is unused for FUTEX_WAIT_BITSET.
82                        !0u32, // A full bitmask, to make it behave like a regular FUTEX_WAIT.
83                    )
84                }
85                _ => {
86                    compile_error!("unknown target_os");
87                }
88            }
89        };
90
91        match (r < 0).then(crate::sys::io::errno) {
92            Some(libc::ETIMEDOUT) => return false,
93            Some(libc::EINTR) => continue,
94            _ => return true,
95        }
96    }
97}
98
99/// Wakes up one thread that's blocked on `futex_wait` on this futex.
100///
101/// Returns true if this actually woke up such a thread,
102/// or false if no thread was waiting on this futex.
103///
104/// On some platforms, this always returns false.
105#[cfg(any(target_os = "linux", target_os = "android"))]
106pub fn futex_wake(futex: &Atomic<u32>) -> bool {
107    let ptr = futex as *const Atomic<u32> as *const u32;
108    let op = libc::FUTEX_WAKE | libc::FUTEX_PRIVATE_FLAG;
109    unsafe { libc::syscall(SYS_futex, ptr, op, 1) > 0 }
110}
111
112/// Wakes up all threads that are waiting on `futex_wait` on this futex.
113#[cfg(any(target_os = "linux", target_os = "android"))]
114pub fn futex_wake_all(futex: &Atomic<u32>) {
115    let ptr = futex as *const Atomic<u32> as *const u32;
116    let op = libc::FUTEX_WAKE | libc::FUTEX_PRIVATE_FLAG;
117    unsafe {
118        libc::syscall(SYS_futex, ptr, op, i32::MAX);
119    }
120}
121
122// FreeBSD doesn't tell us how many threads are woken up, so this always returns false.
123#[cfg(target_os = "freebsd")]
124pub fn futex_wake(futex: &Atomic<u32>) -> bool {
125    use crate::ptr::null_mut;
126    unsafe {
127        libc::_umtx_op(
128            futex as *const Atomic<u32> as *mut _,
129            libc::UMTX_OP_WAKE_PRIVATE,
130            1,
131            null_mut(),
132            null_mut(),
133        )
134    };
135    false
136}
137
138#[cfg(target_os = "freebsd")]
139pub fn futex_wake_all(futex: &Atomic<u32>) {
140    use crate::ptr::null_mut;
141    unsafe {
142        libc::_umtx_op(
143            futex as *const Atomic<u32> as *mut _,
144            libc::UMTX_OP_WAKE_PRIVATE,
145            i32::MAX as libc::c_ulong,
146            null_mut(),
147            null_mut(),
148        )
149    };
150}
151
152#[cfg(target_os = "openbsd")]
153pub fn futex_wait(futex: &Atomic<u32>, expected: u32, timeout: Option<Duration>) -> bool {
154    use crate::ptr::{null, null_mut};
155    use crate::sys::pal::time::Timespec;
156
157    // Overflows are rounded up to an infinite timeout (None).
158    let timespec = timeout
159        .and_then(|d| Timespec::zero().checked_add_duration(&d))
160        .and_then(|t| t.to_timespec());
161
162    let r = unsafe {
163        libc::futex(
164            futex as *const Atomic<u32> as *mut u32,
165            libc::FUTEX_WAIT,
166            expected as i32,
167            timespec.as_ref().map_or(null(), |t| t as *const libc::timespec),
168            null_mut(),
169        )
170    };
171
172    r == 0 || crate::sys::io::errno() != libc::ETIMEDOUT
173}
174
175#[cfg(target_os = "openbsd")]
176pub fn futex_wake(futex: &Atomic<u32>) -> bool {
177    use crate::ptr::{null, null_mut};
178    unsafe {
179        libc::futex(
180            futex as *const Atomic<u32> as *mut u32,
181            libc::FUTEX_WAKE,
182            1,
183            null(),
184            null_mut(),
185        ) > 0
186    }
187}
188
189#[cfg(target_os = "openbsd")]
190pub fn futex_wake_all(futex: &Atomic<u32>) {
191    use crate::ptr::{null, null_mut};
192    unsafe {
193        libc::futex(
194            futex as *const Atomic<u32> as *mut u32,
195            libc::FUTEX_WAKE,
196            i32::MAX,
197            null(),
198            null_mut(),
199        );
200    }
201}
202
203#[cfg(target_os = "dragonfly")]
204pub fn futex_wait(futex: &Atomic<u32>, expected: u32, timeout: Option<Duration>) -> bool {
205    // A timeout of 0 means infinite.
206    // We round smaller timeouts up to 1 millisecond.
207    // Overflows are rounded up to an infinite timeout.
208    let timeout_ms =
209        timeout.and_then(|d| Some(i32::try_from(d.as_millis()).ok()?.max(1))).unwrap_or(0);
210
211    let r = unsafe {
212        libc::umtx_sleep(futex as *const Atomic<u32> as *const i32, expected as i32, timeout_ms)
213    };
214
215    r == 0 || crate::sys::io::errno() != libc::ETIMEDOUT
216}
217
218// DragonflyBSD doesn't tell us how many threads are woken up, so this always returns false.
219#[cfg(target_os = "dragonfly")]
220pub fn futex_wake(futex: &Atomic<u32>) -> bool {
221    unsafe { libc::umtx_wakeup(futex as *const Atomic<u32> as *const i32, 1) };
222    false
223}
224
225#[cfg(target_os = "dragonfly")]
226pub fn futex_wake_all(futex: &Atomic<u32>) {
227    unsafe { libc::umtx_wakeup(futex as *const Atomic<u32> as *const i32, i32::MAX) };
228}
229
230#[cfg(target_os = "emscripten")]
231unsafe extern "C" {
232    fn emscripten_futex_wake(addr: *const Atomic<u32>, count: libc::c_int) -> libc::c_int;
233    fn emscripten_futex_wait(
234        addr: *const Atomic<u32>,
235        val: libc::c_uint,
236        max_wait_ms: libc::c_double,
237    ) -> libc::c_int;
238}
239
240#[cfg(target_os = "emscripten")]
241pub fn futex_wait(futex: &Atomic<u32>, expected: u32, timeout: Option<Duration>) -> bool {
242    unsafe {
243        emscripten_futex_wait(
244            futex,
245            expected,
246            timeout.map_or(f64::INFINITY, |d| d.as_secs_f64() * 1000.0),
247        ) != -libc::ETIMEDOUT
248    }
249}
250
251#[cfg(target_os = "emscripten")]
252pub fn futex_wake(futex: &Atomic<u32>) -> bool {
253    unsafe { emscripten_futex_wake(futex, 1) > 0 }
254}
255
256#[cfg(target_os = "emscripten")]
257pub fn futex_wake_all(futex: &Atomic<u32>) {
258    unsafe { emscripten_futex_wake(futex, i32::MAX) };
259}
260
261#[cfg(target_os = "fuchsia")]
262pub fn futex_wait(futex: &Atomic<u32>, expected: u32, timeout: Option<Duration>) -> bool {
263    use crate::sys::pal::fuchsia::*;
264
265    // Sleep forever if the timeout is longer than fits in a i64.
266    let deadline = timeout
267        .and_then(|d| i64::try_from(d.as_nanos()).ok()?.checked_add(zx_clock_get_monotonic()))
268        .unwrap_or(ZX_TIME_INFINITE);
269
270    unsafe {
271        zx_futex_wait(futex, zx_futex_t::new(expected), ZX_HANDLE_INVALID, deadline)
272            != ZX_ERR_TIMED_OUT
273    }
274}
275
276// Fuchsia doesn't tell us how many threads are woken up, so this always returns false.
277#[cfg(target_os = "fuchsia")]
278pub fn futex_wake(futex: &Atomic<u32>) -> bool {
279    unsafe { crate::sys::pal::fuchsia::zx_futex_wake(futex, 1) };
280    false
281}
282
283#[cfg(target_os = "fuchsia")]
284pub fn futex_wake_all(futex: &Atomic<u32>) {
285    unsafe { crate::sys::pal::fuchsia::zx_futex_wake(futex, u32::MAX) };
286}
std/sys/sync/futex/unix.rs

std/sys/sync/futex/
unix.rs
