miri/shims/unix/

sync.rs

use rustc_abi::Size;
use rustc_target::spec::Os;

use crate::concurrency::sync::{AccessKind, SyncObj};
use crate::*;

/// Do a bytewise comparison of the two places. This is used to check if
/// a synchronization primitive matches its static initializer value.
///
/// `prefix`, if set, indicates that only the first N bytes should be compared.
fn bytewise_equal<'tcx>(
    ecx: &MiriInterpCx<'tcx>,
    left: &MPlaceTy<'tcx>,
    right: &MPlaceTy<'tcx>,
    prefix: Option<u64>,
) -> InterpResult<'tcx, bool> {
    let size = left.layout.size;
    assert_eq!(size, right.layout.size);
    let cmp_size = prefix.map(Size::from_bytes).unwrap_or(size);

    let left_bytes = ecx.read_bytes_ptr_strip_provenance(left.ptr(), cmp_size)?;
    let right_bytes = ecx.read_bytes_ptr_strip_provenance(right.ptr(), cmp_size)?;

    interp_ok(left_bytes == right_bytes)
}

// The in-memory marker values we use to indicate whether objects have been initialized.
const PTHREAD_UNINIT: u8 = 0;
const PTHREAD_INIT: u8 = 1;

// # pthread_mutexattr_t
// We store some data directly inside the type, ignoring the platform layout:
// - kind: i32

#[inline]
fn mutexattr_kind_offset<'tcx>(ecx: &MiriInterpCx<'tcx>) -> InterpResult<'tcx, u64> {
    interp_ok(match &ecx.tcx.sess.target.os {
        Os::Linux
        | Os::Illumos
        | Os::Solaris
        | Os::MacOs
        | Os::FreeBsd
        | Os::Android
        | Os::NetBsd => 0,
        os => throw_unsup_format!("`pthread_mutexattr` is not supported on {os}"),
    })
}

fn mutexattr_get_kind<'tcx>(
    ecx: &MiriInterpCx<'tcx>,
    attr_ptr: &OpTy<'tcx>,
) -> InterpResult<'tcx, i32> {
    ecx.deref_pointer_and_read(
        attr_ptr,
        mutexattr_kind_offset(ecx)?,
        ecx.libc_ty_layout("pthread_mutexattr_t"),
        ecx.machine.layouts.i32,
    )?
    .to_i32()
}

fn mutexattr_set_kind<'tcx>(
    ecx: &mut MiriInterpCx<'tcx>,
    attr_ptr: &OpTy<'tcx>,
    kind: i32,
) -> InterpResult<'tcx, ()> {
    ecx.deref_pointer_and_write(
        attr_ptr,
        mutexattr_kind_offset(ecx)?,
        Scalar::from_i32(kind),
        ecx.libc_ty_layout("pthread_mutexattr_t"),
        ecx.machine.layouts.i32,
    )
}

/// To differentiate "the mutex kind has not been changed" from
/// "the mutex kind has been set to PTHREAD_MUTEX_DEFAULT and that is
/// equal to some other mutex kind", we make the default value of this
/// field *not* PTHREAD_MUTEX_DEFAULT but this special flag.
const PTHREAD_MUTEX_KIND_UNCHANGED: i32 = 0x8000000;

/// Translates the mutex kind from what is stored in pthread_mutexattr_t to our enum.
fn mutexattr_translate_kind<'tcx>(
    ecx: &MiriInterpCx<'tcx>,
    kind: i32,
) -> InterpResult<'tcx, MutexKind> {
    interp_ok(if kind == (ecx.eval_libc_i32("PTHREAD_MUTEX_NORMAL")) {
        MutexKind::Normal
    } else if kind == ecx.eval_libc_i32("PTHREAD_MUTEX_ERRORCHECK") {
        MutexKind::ErrorCheck
    } else if kind == ecx.eval_libc_i32("PTHREAD_MUTEX_RECURSIVE") {
        MutexKind::Recursive
    } else if kind == ecx.eval_libc_i32("PTHREAD_MUTEX_DEFAULT")
        || kind == PTHREAD_MUTEX_KIND_UNCHANGED
    {
        // We check this *last* since PTHREAD_MUTEX_DEFAULT may be numerically equal to one of the
        // others, and we want an explicit `mutexattr_settype` to work as expected.
        MutexKind::Default
    } else {
        throw_unsup_format!("unsupported type of mutex: {kind}");
    })
}

// # pthread_mutex_t
// We store some data directly inside the type, ignoring the platform layout:
// - init: u8

/// The mutex kind.
#[derive(Debug, Clone, Copy)]
enum MutexKind {
    Normal,
    Default,
    Recursive,
    ErrorCheck,
}

#[derive(Debug, Clone)]
struct PthreadMutex {
    mutex_ref: MutexRef,
    kind: MutexKind,
}

impl SyncObj for PthreadMutex {
    fn on_access<'tcx>(&self, access_kind: AccessKind) -> InterpResult<'tcx> {
        if !self.mutex_ref.queue_is_empty() {
            throw_ub_format!(
                "{access_kind} of `pthread_mutex_t` is forbidden while the queue is non-empty"
            );
        }
        interp_ok(())
    }

    fn delete_on_write(&self) -> bool {
        true
    }
}

/// To ensure an initialized mutex that was moved somewhere else can be distinguished from
/// a statically initialized mutex that is used the first time, we pick some offset within
/// `pthread_mutex_t` and use it as an "initialized" flag.
fn mutex_init_offset<'tcx>(ecx: &MiriInterpCx<'tcx>) -> InterpResult<'tcx, Size> {
    let offset = match &ecx.tcx.sess.target.os {
        Os::Linux | Os::Illumos | Os::Solaris | Os::FreeBsd | Os::Android => 0,
        // macOS and NetBSD store a signature in the first bytes, so we move to offset 4.
        Os::MacOs | Os::NetBsd => 4,
        os => throw_unsup_format!("`pthread_mutex` is not supported on {os}"),
    };
    let offset = Size::from_bytes(offset);

    // Sanity-check this against PTHREAD_MUTEX_INITIALIZER (but only once):
    // the `init` field must start out not equal to INIT_COOKIE.
    if !ecx.machine.pthread_mutex_sanity.replace(true) {
        let check_static_initializer = |name| {
            let static_initializer = ecx.eval_path(&["libc", name]);
            let init_field =
                static_initializer.offset(offset, ecx.machine.layouts.u8, ecx).unwrap();
            let init = ecx.read_scalar(&init_field).unwrap().to_u8().unwrap();
            assert_eq!(
                init, PTHREAD_UNINIT,
                "{name} is incompatible with our initialization logic"
            );
        };

        check_static_initializer("PTHREAD_MUTEX_INITIALIZER");
        // Check non-standard initializers.
        match &ecx.tcx.sess.target.os {
            Os::Linux => {
                check_static_initializer("PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP");
                check_static_initializer("PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP");
                check_static_initializer("PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP");
            }
            Os::Illumos | Os::Solaris | Os::MacOs | Os::FreeBsd | Os::Android | Os::NetBsd => {
                // No non-standard initializers.
            }
            os => throw_unsup_format!("`pthread_mutex` is not supported on {os}"),
        }
    }

    interp_ok(offset)
}

/// Eagerly create and initialize a new mutex.
fn mutex_create<'tcx>(
    ecx: &mut MiriInterpCx<'tcx>,
    mutex_ptr: &OpTy<'tcx>,
    kind: MutexKind,
) -> InterpResult<'tcx> {
    let mutex = ecx.deref_pointer_as(mutex_ptr, ecx.libc_ty_layout("pthread_mutex_t"))?;
    let data = PthreadMutex { mutex_ref: MutexRef::new(), kind };
    ecx.init_immovable_sync(&mutex, mutex_init_offset(ecx)?, PTHREAD_INIT, data)?;
    interp_ok(())
}

/// Returns the mutex data stored at the address that `mutex_ptr` points to.
/// Will raise an error if the mutex has been moved since its first use.
fn mutex_get_data<'tcx, 'a>(
    ecx: &'a mut MiriInterpCx<'tcx>,
    mutex_ptr: &OpTy<'tcx>,
) -> InterpResult<'tcx, &'a PthreadMutex>
where
    'tcx: 'a,
{
    let mutex = ecx.deref_pointer_as(mutex_ptr, ecx.libc_ty_layout("pthread_mutex_t"))?;
    ecx.get_immovable_sync_with_static_init(
        &mutex,
        mutex_init_offset(ecx)?,
        PTHREAD_UNINIT,
        PTHREAD_INIT,
        |ecx| {
            let kind = mutex_kind_from_static_initializer(ecx, &mutex)?;
            interp_ok(PthreadMutex { mutex_ref: MutexRef::new(), kind })
        },
    )
}

/// Returns the kind of a static initializer.
fn mutex_kind_from_static_initializer<'tcx>(
    ecx: &MiriInterpCx<'tcx>,
    mutex: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, MutexKind> {
    let prefix = match &ecx.tcx.sess.target.os {
        // On android, there's a 4-byte `value` header followed by "padding", and some versions
        // of libc leave that uninitialized. Only check the `value` bytes.
        Os::Android => Some(4),
        _ => None,
    };
    let is_initializer = |name| bytewise_equal(ecx, mutex, &ecx.eval_path(&["libc", name]), prefix);

    // All the static initializers recognized here *must* be checked in `mutex_init_offset`!

    // PTHREAD_MUTEX_INITIALIZER is recognized on all targets.
    if is_initializer("PTHREAD_MUTEX_INITIALIZER")? {
        return interp_ok(MutexKind::Default);
    }
    // Support additional platform-specific initializers.
    match &ecx.tcx.sess.target.os {
        Os::Linux =>
            if is_initializer("PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP")? {
                return interp_ok(MutexKind::Recursive);
            } else if is_initializer("PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP")? {
                return interp_ok(MutexKind::ErrorCheck);
            },
        _ => {}
    }
    throw_ub_format!(
        "`pthread_mutex_t` was not properly initialized at this location, or it got overwritten"
    );
}

// # pthread_rwlock_t
// We store some data directly inside the type, ignoring the platform layout:
// - init: u8

#[derive(Debug, Clone)]
struct PthreadRwLock {
    rwlock_ref: RwLockRef,
}

impl SyncObj for PthreadRwLock {
    fn on_access<'tcx>(&self, access_kind: AccessKind) -> InterpResult<'tcx> {
        if !self.rwlock_ref.queue_is_empty() {
            throw_ub_format!(
                "{access_kind} of `pthread_rwlock_t` is forbidden while the queue is non-empty"
            );
        }
        interp_ok(())
    }

    fn delete_on_write(&self) -> bool {
        true
    }
}

fn rwlock_init_offset<'tcx>(ecx: &MiriInterpCx<'tcx>) -> InterpResult<'tcx, Size> {
    let offset = match &ecx.tcx.sess.target.os {
        Os::Linux | Os::Illumos | Os::Solaris | Os::FreeBsd | Os::Android => 0,
        // macOS stores a signature in the first bytes, so we move to offset 4.
        Os::MacOs => 4,
        os => throw_unsup_format!("`pthread_rwlock` is not supported on {os}"),
    };
    let offset = Size::from_bytes(offset);

    // Sanity-check this against PTHREAD_RWLOCK_INITIALIZER (but only once):
    // the `init` field must start out not equal to LAZY_INIT_COOKIE.
    if !ecx.machine.pthread_rwlock_sanity.replace(true) {
        let static_initializer = ecx.eval_path(&["libc", "PTHREAD_RWLOCK_INITIALIZER"]);
        let init_field = static_initializer.offset(offset, ecx.machine.layouts.u8, ecx).unwrap();
        let init = ecx.read_scalar(&init_field).unwrap().to_u8().unwrap();
        assert_eq!(
            init, PTHREAD_UNINIT,
            "PTHREAD_RWLOCK_INITIALIZER is incompatible with our initialization logic"
        );
    }

    interp_ok(offset)
}

fn rwlock_get_data<'tcx, 'a>(
    ecx: &'a mut MiriInterpCx<'tcx>,
    rwlock_ptr: &OpTy<'tcx>,
) -> InterpResult<'tcx, &'a PthreadRwLock>
where
    'tcx: 'a,
{
    let rwlock = ecx.deref_pointer_as(rwlock_ptr, ecx.libc_ty_layout("pthread_rwlock_t"))?;
    ecx.get_immovable_sync_with_static_init(
        &rwlock,
        rwlock_init_offset(ecx)?,
        PTHREAD_UNINIT,
        PTHREAD_INIT,
        |ecx| {
            let prefix = match &ecx.tcx.sess.target.os {
                // On android, there's a 4-byte `value` header followed by "padding", and some
                // versions of libc leave that uninitialized. Only check the `value` bytes.
                Os::Android => Some(4),
                _ => None,
            };
            if !bytewise_equal(
                ecx,
                &rwlock,
                &ecx.eval_path(&["libc", "PTHREAD_RWLOCK_INITIALIZER"]),
                prefix,
            )? {
                throw_ub_format!(
                    "`pthread_rwlock_t` was not properly initialized at this location, or it got overwritten"
                );
            }
            interp_ok(PthreadRwLock { rwlock_ref: RwLockRef::new() })
        },
    )
}

// # pthread_condattr_t
// We store some data directly inside the type, ignoring the platform layout:
// - clock: i32

#[inline]
fn condattr_clock_offset<'tcx>(ecx: &MiriInterpCx<'tcx>) -> InterpResult<'tcx, u64> {
    interp_ok(match &ecx.tcx.sess.target.os {
        Os::Linux | Os::Illumos | Os::Solaris | Os::FreeBsd | Os::Android => 0,
        // macOS does not have a clock attribute.
        os => throw_unsup_format!("`pthread_condattr` clock field is not supported on {os}"),
    })
}

fn condattr_get_clock_id<'tcx>(
    ecx: &MiriInterpCx<'tcx>,
    attr_ptr: &OpTy<'tcx>,
) -> InterpResult<'tcx, Scalar> {
    ecx.deref_pointer_and_read(
        attr_ptr,
        condattr_clock_offset(ecx)?,
        ecx.libc_ty_layout("pthread_condattr_t"),
        ecx.machine.layouts.i32,
    )
}

fn condattr_set_clock_id<'tcx>(
    ecx: &mut MiriInterpCx<'tcx>,
    attr_ptr: &OpTy<'tcx>,
    clock_id: i32,
) -> InterpResult<'tcx, ()> {
    ecx.deref_pointer_and_write(
        attr_ptr,
        condattr_clock_offset(ecx)?,
        Scalar::from_i32(clock_id),
        ecx.libc_ty_layout("pthread_condattr_t"),
        ecx.machine.layouts.i32,
    )
}

// # pthread_cond_t
// We store some data directly inside the type, ignoring the platform layout:
// - init: u8

fn cond_init_offset<'tcx>(ecx: &MiriInterpCx<'tcx>) -> InterpResult<'tcx, Size> {
    let offset = match &ecx.tcx.sess.target.os {
        Os::Linux | Os::Illumos | Os::Solaris | Os::FreeBsd | Os::Android => 0,
        // macOS stores a signature in the first bytes, so we move to offset 4.
        Os::MacOs => 4,
        os => throw_unsup_format!("`pthread_cond` is not supported on {os}"),
    };
    let offset = Size::from_bytes(offset);

    // Sanity-check this against PTHREAD_COND_INITIALIZER (but only once):
    // the `init` field must start out not equal to LAZY_INIT_COOKIE.
    if !ecx.machine.pthread_condvar_sanity.replace(true) {
        let static_initializer = ecx.eval_path(&["libc", "PTHREAD_COND_INITIALIZER"]);
        let init_field = static_initializer.offset(offset, ecx.machine.layouts.u8, ecx).unwrap();
        let init = ecx.read_scalar(&init_field).unwrap().to_u8().unwrap();
        assert_eq!(
            init, PTHREAD_UNINIT,
            "PTHREAD_COND_INITIALIZER is incompatible with our initialization logic"
        );
    }

    interp_ok(offset)
}

#[derive(Debug, Clone)]
struct PthreadCondvar {
    condvar_ref: CondvarRef,
    clock: TimeoutClock,
}

impl SyncObj for PthreadCondvar {
    fn on_access<'tcx>(&self, access_kind: AccessKind) -> InterpResult<'tcx> {
        if !self.condvar_ref.queue_is_empty() {
            throw_ub_format!(
                "{access_kind} of `pthread_cond_t` is forbidden while the queue is non-empty"
            );
        }
        interp_ok(())
    }

    fn delete_on_write(&self) -> bool {
        true
    }
}

fn cond_create<'tcx>(
    ecx: &mut MiriInterpCx<'tcx>,
    cond_ptr: &OpTy<'tcx>,
    clock: TimeoutClock,
) -> InterpResult<'tcx> {
    let cond = ecx.deref_pointer_as(cond_ptr, ecx.libc_ty_layout("pthread_cond_t"))?;
    let data = PthreadCondvar { condvar_ref: CondvarRef::new(), clock };
    ecx.init_immovable_sync(&cond, cond_init_offset(ecx)?, PTHREAD_INIT, data)?;
    interp_ok(())
}

fn cond_get_data<'tcx, 'a>(
    ecx: &'a mut MiriInterpCx<'tcx>,
    cond_ptr: &OpTy<'tcx>,
) -> InterpResult<'tcx, &'a PthreadCondvar>
where
    'tcx: 'a,
{
    let cond = ecx.deref_pointer_as(cond_ptr, ecx.libc_ty_layout("pthread_cond_t"))?;
    ecx.get_immovable_sync_with_static_init(
        &cond,
        cond_init_offset(ecx)?,
        PTHREAD_UNINIT,
        PTHREAD_INIT,
        |ecx| {
            let prefix = match &ecx.tcx.sess.target.os {
                // On android, there's a 4-byte `value` header followed by "padding", and some
                // versions of libc leave that uninitialized. Only check the `value` bytes.
                Os::Android => Some(4),
                _ => None,
            };
            if !bytewise_equal(
                ecx,
                &cond,
                &ecx.eval_path(&["libc", "PTHREAD_COND_INITIALIZER"]),
                prefix,
            )? {
                throw_ub_format!(
                    "`pthread_cond_t` was not properly initialized at this location, or it got overwritten"
                );
            }
            // This used the static initializer. The clock there is always CLOCK_REALTIME.
            interp_ok(PthreadCondvar {
                condvar_ref: CondvarRef::new(),
                clock: TimeoutClock::RealTime,
            })
        },
    )
}

// # sem_t
// We store some data directly inside the type, ignoring the platform layout:
// - init: u8

fn sem_init_offset<'tcx>(ecx: &MiriInterpCx<'tcx>) -> InterpResult<'tcx, Size> {
    let offset = match &ecx.tcx.sess.target.os {
        Os::Linux | Os::Illumos | Os::Solaris | Os::FreeBsd | Os::Android => 0,
        // Explicitly *not* supported on macOS!
        os => throw_unsup_format!("`sem` is not supported on {os}"),
    };
    interp_ok(Size::from_bytes(offset))
}

#[derive(Debug, Clone)]
struct PthreadSem {
    sem_ref: SemaphoreRef,
}

impl SyncObj for PthreadSem {
    fn on_access<'tcx>(&self, access_kind: AccessKind) -> InterpResult<'tcx> {
        if !self.sem_ref.queue_is_empty() {
            throw_ub_format!("{access_kind} of `sem_t` is forbidden while the queue is non-empty");
        }
        interp_ok(())
    }

    fn delete_on_write(&self) -> bool {
        true
    }
}

fn sem_create<'tcx>(
    ecx: &mut MiriInterpCx<'tcx>,
    sem_ptr: &OpTy<'tcx>,
    value: u32,
) -> InterpResult<'tcx> {
    let sem = ecx.deref_pointer_as(sem_ptr, ecx.libc_ty_layout("sem_t"))?;
    let data = PthreadSem { sem_ref: SemaphoreRef::new(value.into()) };
    ecx.init_immovable_sync(&sem, sem_init_offset(ecx)?, PTHREAD_INIT, data)?;
    interp_ok(())
}

fn sem_get_data<'tcx, 'a>(
    ecx: &'a mut MiriInterpCx<'tcx>,
    sem_ptr: &OpTy<'tcx>,
) -> InterpResult<'tcx, &'a PthreadSem>
where
    'tcx: 'a,
{
    let sem = ecx.deref_pointer_as(sem_ptr, ecx.libc_ty_layout("sem_t"))?;
    ecx.get_immovable_sync_with_static_init(
        &sem,
        sem_init_offset(ecx)?,
        PTHREAD_UNINIT,
        PTHREAD_INIT,
        |_ecx| throw_ub_format!("`sem_t` value has not been initialized with `sem_init`"),
    )
}

impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {}
pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
    fn pthread_mutexattr_init(&mut self, attr_op: &OpTy<'tcx>) -> InterpResult<'tcx, ()> {
        let this = self.eval_context_mut();

        mutexattr_set_kind(this, attr_op, PTHREAD_MUTEX_KIND_UNCHANGED)?;

        interp_ok(())
    }

    fn pthread_mutexattr_settype(
        &mut self,
        attr_op: &OpTy<'tcx>,
        kind_op: &OpTy<'tcx>,
    ) -> InterpResult<'tcx, Scalar> {
        let this = self.eval_context_mut();

        let kind = this.read_scalar(kind_op)?.to_i32()?;
        if kind == this.eval_libc_i32("PTHREAD_MUTEX_NORMAL")
            || kind == this.eval_libc_i32("PTHREAD_MUTEX_DEFAULT")
            || kind == this.eval_libc_i32("PTHREAD_MUTEX_ERRORCHECK")
            || kind == this.eval_libc_i32("PTHREAD_MUTEX_RECURSIVE")
        {
            // Make sure we do not mix this up with the "unchanged" kind.
            assert_ne!(kind, PTHREAD_MUTEX_KIND_UNCHANGED);
            mutexattr_set_kind(this, attr_op, kind)?;
        } else {
            let einval = this.eval_libc_i32("EINVAL");
            return interp_ok(Scalar::from_i32(einval));
        }

        interp_ok(Scalar::from_i32(0))
    }

    fn pthread_mutexattr_destroy(&mut self, attr_op: &OpTy<'tcx>) -> InterpResult<'tcx, ()> {
        let this = self.eval_context_mut();

        // Destroying an uninit pthread_mutexattr is UB, so check to make sure it's not uninit.
        mutexattr_get_kind(this, attr_op)?;

        // To catch double-destroys, we de-initialize the mutexattr.
        // This is technically not right and might lead to false positives. For example, the below
        // code is *likely* sound, even assuming uninit numbers are UB, but Miri complains.
        //
        // let mut x: MaybeUninit<libc::pthread_mutexattr_t> = MaybeUninit::zeroed();
        // libc::pthread_mutexattr_init(x.as_mut_ptr());
        // libc::pthread_mutexattr_destroy(x.as_mut_ptr());
        // x.assume_init();
        //
        // However, the way libstd uses the pthread APIs works in our favor here, so we can get away with this.
        // This can always be revisited to have some external state to catch double-destroys
        // but not complain about the above code. See https://github.com/rust-lang/miri/pull/1933
        this.write_uninit(
            &this.deref_pointer_as(attr_op, this.libc_ty_layout("pthread_mutexattr_t"))?,
        )?;

        interp_ok(())
    }

    fn pthread_mutex_init(
        &mut self,
        mutex_op: &OpTy<'tcx>,
        attr_op: &OpTy<'tcx>,
    ) -> InterpResult<'tcx, ()> {
        let this = self.eval_context_mut();

        let attr = this.read_pointer(attr_op)?;
        let kind = if this.ptr_is_null(attr)? {
            MutexKind::Default
        } else {
            mutexattr_translate_kind(this, mutexattr_get_kind(this, attr_op)?)?
        };

        mutex_create(this, mutex_op, kind)?;

        interp_ok(())
    }

    fn pthread_mutex_lock(
        &mut self,
        mutex_op: &OpTy<'tcx>,
        dest: &MPlaceTy<'tcx>,
    ) -> InterpResult<'tcx> {
        let this = self.eval_context_mut();

        let mutex = mutex_get_data(this, mutex_op)?.clone();

        let ret = if let Some(owner_thread) = mutex.mutex_ref.owner() {
            if owner_thread != this.active_thread() {
                this.mutex_enqueue_and_block(
                    mutex.mutex_ref,
                    Some((Scalar::from_i32(0), dest.clone())),
                );
                return interp_ok(());
            } else {
                // Trying to acquire the same mutex again.
                match mutex.kind {
                    MutexKind::Default =>
                        throw_ub_format!(
                            "trying to acquire default mutex already locked by the current thread"
                        ),
                    MutexKind::Normal => throw_machine_stop!(TerminationInfo::LocalDeadlock),
                    MutexKind::ErrorCheck => this.eval_libc_i32("EDEADLK"),
                    MutexKind::Recursive => {
                        this.mutex_lock(&mutex.mutex_ref)?;
                        0
                    }
                }
            }
        } else {
            // The mutex is unlocked. Let's lock it.
            this.mutex_lock(&mutex.mutex_ref)?;
            0
        };
        this.write_scalar(Scalar::from_i32(ret), dest)?;
        interp_ok(())
    }

    fn pthread_mutex_trylock(&mut self, mutex_op: &OpTy<'tcx>) -> InterpResult<'tcx, Scalar> {
        let this = self.eval_context_mut();

        let mutex = mutex_get_data(this, mutex_op)?.clone();

        interp_ok(Scalar::from_i32(if let Some(owner_thread) = mutex.mutex_ref.owner() {
            if owner_thread != this.active_thread() {
                this.eval_libc_i32("EBUSY")
            } else {
                match mutex.kind {
                    MutexKind::Default | MutexKind::Normal | MutexKind::ErrorCheck =>
                        this.eval_libc_i32("EBUSY"),
                    MutexKind::Recursive => {
                        this.mutex_lock(&mutex.mutex_ref)?;
                        0
                    }
                }
            }
        } else {
            // The mutex is unlocked. Let's lock it.
            this.mutex_lock(&mutex.mutex_ref)?;
            0
        }))
    }

    fn pthread_mutex_unlock(&mut self, mutex_op: &OpTy<'tcx>) -> InterpResult<'tcx, Scalar> {
        let this = self.eval_context_mut();

        let mutex = mutex_get_data(this, mutex_op)?.clone();

        if let Some(_old_locked_count) = this.mutex_unlock(&mutex.mutex_ref)? {
            // The mutex was locked by the current thread.
            interp_ok(Scalar::from_i32(0))
        } else {
            // The mutex was locked by another thread or not locked at all. See
            // the “Unlock When Not Owner” column in
            // https://pubs.opengroup.org/onlinepubs/9699919799/functions/pthread_mutex_unlock.html.
            match mutex.kind {
                MutexKind::Default =>
                    throw_ub_format!(
                        "unlocked a default mutex that was not locked by the current thread"
                    ),
                MutexKind::Normal =>
                    throw_ub_format!(
                        "unlocked a PTHREAD_MUTEX_NORMAL mutex that was not locked by the current thread"
                    ),
                MutexKind::ErrorCheck | MutexKind::Recursive =>
                    interp_ok(Scalar::from_i32(this.eval_libc_i32("EPERM"))),
            }
        }
    }

    fn pthread_mutex_destroy(&mut self, mutex_op: &OpTy<'tcx>) -> InterpResult<'tcx, ()> {
        let this = self.eval_context_mut();

        // Reading the field also has the side-effect that we detect double-`destroy`
        // since we make the field uninit below.
        let mutex = mutex_get_data(this, mutex_op)?.clone();

        if mutex.mutex_ref.owner().is_some() {
            throw_ub_format!("destroyed a locked mutex");
        }

        // This write also deletes the interpreter state for this mutex.
        // This might lead to false positives, see comment in pthread_mutexattr_destroy
        let mutex_place =
            this.deref_pointer_as(mutex_op, this.libc_ty_layout("pthread_mutex_t"))?;
        this.write_uninit(&mutex_place)?;

        interp_ok(())
    }

    fn pthread_rwlock_rdlock(
        &mut self,
        rwlock_op: &OpTy<'tcx>,
        dest: &MPlaceTy<'tcx>,
    ) -> InterpResult<'tcx> {
        let this = self.eval_context_mut();

        let rwlock = rwlock_get_data(this, rwlock_op)?.clone();

        if rwlock.rwlock_ref.is_write_locked() {
            this.rwlock_enqueue_and_block_reader(
                rwlock.rwlock_ref,
                Scalar::from_i32(0),
                dest.clone(),
            );
        } else {
            this.rwlock_reader_lock(&rwlock.rwlock_ref)?;
            this.write_null(dest)?;
        }

        interp_ok(())
    }

    fn pthread_rwlock_tryrdlock(&mut self, rwlock_op: &OpTy<'tcx>) -> InterpResult<'tcx, Scalar> {
        let this = self.eval_context_mut();

        let rwlock = rwlock_get_data(this, rwlock_op)?.clone();

        if rwlock.rwlock_ref.is_write_locked() {
            interp_ok(Scalar::from_i32(this.eval_libc_i32("EBUSY")))
        } else {
            this.rwlock_reader_lock(&rwlock.rwlock_ref)?;
            interp_ok(Scalar::from_i32(0))
        }
    }

    fn pthread_rwlock_wrlock(
        &mut self,
        rwlock_op: &OpTy<'tcx>,
        dest: &MPlaceTy<'tcx>,
    ) -> InterpResult<'tcx> {
        let this = self.eval_context_mut();

        let rwlock = rwlock_get_data(this, rwlock_op)?.clone();

        if rwlock.rwlock_ref.is_locked() {
            // Note: this will deadlock if the lock is already locked by this
            // thread in any way.
            //
            // Relevant documentation:
            // https://pubs.opengroup.org/onlinepubs/9699919799/functions/pthread_rwlock_wrlock.html
            // An in-depth discussion on this topic:
            // https://github.com/rust-lang/rust/issues/53127
            //
            // FIXME: Detect and report the deadlock proactively. (We currently
            // report the deadlock only when no thread can continue execution,
            // but we could detect that this lock is already locked and report
            // an error.)
            this.rwlock_enqueue_and_block_writer(
                rwlock.rwlock_ref,
                Scalar::from_i32(0),
                dest.clone(),
            );
        } else {
            this.rwlock_writer_lock(&rwlock.rwlock_ref)?;
            this.write_null(dest)?;
        }

        interp_ok(())
    }

    fn pthread_rwlock_trywrlock(&mut self, rwlock_op: &OpTy<'tcx>) -> InterpResult<'tcx, Scalar> {
        let this = self.eval_context_mut();

        let rwlock = rwlock_get_data(this, rwlock_op)?.clone();

        if rwlock.rwlock_ref.is_locked() {
            interp_ok(Scalar::from_i32(this.eval_libc_i32("EBUSY")))
        } else {
            this.rwlock_writer_lock(&rwlock.rwlock_ref)?;
            interp_ok(Scalar::from_i32(0))
        }
    }

    fn pthread_rwlock_unlock(&mut self, rwlock_op: &OpTy<'tcx>) -> InterpResult<'tcx, ()> {
        let this = self.eval_context_mut();

        let rwlock = rwlock_get_data(this, rwlock_op)?.clone();

        if this.rwlock_reader_unlock(&rwlock.rwlock_ref)?
            || this.rwlock_writer_unlock(&rwlock.rwlock_ref)?
        {
            interp_ok(())
        } else {
            throw_ub_format!("unlocked an rwlock that was not locked by the active thread");
        }
    }

    fn pthread_rwlock_destroy(&mut self, rwlock_op: &OpTy<'tcx>) -> InterpResult<'tcx, ()> {
        let this = self.eval_context_mut();

        // Reading the field also has the side-effect that we detect double-`destroy`
        // since we make the field uninit below.
        let rwlock = rwlock_get_data(this, rwlock_op)?.clone();

        if rwlock.rwlock_ref.is_locked() {
            throw_ub_format!("destroyed a locked rwlock");
        }

        // This write also deletes the interpreter state for this rwlock.
        // This might lead to false positives, see comment in pthread_mutexattr_destroy
        let rwlock_place =
            this.deref_pointer_as(rwlock_op, this.libc_ty_layout("pthread_rwlock_t"))?;
        this.write_uninit(&rwlock_place)?;

        interp_ok(())
    }

    fn pthread_condattr_init(&mut self, attr_op: &OpTy<'tcx>) -> InterpResult<'tcx, ()> {
        let this = self.eval_context_mut();

        // no clock attribute on macOS
        if this.tcx.sess.target.os != Os::MacOs {
            // The default value of the clock attribute shall refer to the system
            // clock.
            // https://pubs.opengroup.org/onlinepubs/9699919799/functions/pthread_condattr_setclock.html
            let default_clock_id = this.eval_libc_i32("CLOCK_REALTIME");
            condattr_set_clock_id(this, attr_op, default_clock_id)?;
        }

        interp_ok(())
    }

    fn pthread_condattr_setclock(
        &mut self,
        attr_op: &OpTy<'tcx>,
        clock_id_op: &OpTy<'tcx>,
    ) -> InterpResult<'tcx, Scalar> {
        let this = self.eval_context_mut();

        let clock_id = this.read_scalar(clock_id_op)?;
        if this.parse_clockid(clock_id).is_some() {
            condattr_set_clock_id(this, attr_op, clock_id.to_i32()?)?;
        } else {
            let einval = this.eval_libc_i32("EINVAL");
            return interp_ok(Scalar::from_i32(einval));
        }

        interp_ok(Scalar::from_i32(0))
    }

    fn pthread_condattr_getclock(
        &mut self,
        attr_op: &OpTy<'tcx>,
        clk_id_op: &OpTy<'tcx>,
    ) -> InterpResult<'tcx, ()> {
        let this = self.eval_context_mut();

        let clock_id = condattr_get_clock_id(this, attr_op)?;
        this.write_scalar(
            clock_id,
            &this.deref_pointer_as(clk_id_op, this.libc_ty_layout("clockid_t"))?,
        )?;

        interp_ok(())
    }

    fn pthread_condattr_destroy(&mut self, attr_op: &OpTy<'tcx>) -> InterpResult<'tcx, ()> {
        let this = self.eval_context_mut();

        // Destroying an uninit pthread_condattr is UB, so check to make sure it's not uninit.
        // There's no clock attribute on macOS.
        if this.tcx.sess.target.os != Os::MacOs {
            condattr_get_clock_id(this, attr_op)?;
        }

        // De-init the entire thing.
        // This might lead to false positives, see comment in pthread_mutexattr_destroy
        this.write_uninit(
            &this.deref_pointer_as(attr_op, this.libc_ty_layout("pthread_condattr_t"))?,
        )?;

        interp_ok(())
    }

    fn pthread_cond_init(
        &mut self,
        cond_op: &OpTy<'tcx>,
        attr_op: &OpTy<'tcx>,
    ) -> InterpResult<'tcx, ()> {
        let this = self.eval_context_mut();

        let attr = this.read_pointer(attr_op)?;
        // Default clock if `attr` is null, and on macOS where there is no clock attribute.
        let clock_id = if this.ptr_is_null(attr)? || this.tcx.sess.target.os == Os::MacOs {
            this.eval_libc("CLOCK_REALTIME")
        } else {
            condattr_get_clock_id(this, attr_op)?
        };
        let Some(clock) = this.parse_clockid(clock_id) else {
            // This is UB since this situation cannot arise when using pthread_condattr_setclock.
            throw_ub_format!("pthread_cond_init: invalid attributes (unsupported clock)")
        };

        cond_create(this, cond_op, clock)?;

        interp_ok(())
    }

    fn pthread_cond_signal(&mut self, cond_op: &OpTy<'tcx>) -> InterpResult<'tcx, ()> {
        let this = self.eval_context_mut();
        let condvar = cond_get_data(this, cond_op)?.condvar_ref.clone();
        this.condvar_signal(&condvar)?;
        interp_ok(())
    }

    fn pthread_cond_broadcast(&mut self, cond_op: &OpTy<'tcx>) -> InterpResult<'tcx, ()> {
        let this = self.eval_context_mut();
        let condvar = cond_get_data(this, cond_op)?.condvar_ref.clone();
        while this.condvar_signal(&condvar)? {}
        interp_ok(())
    }

    fn pthread_cond_wait(
        &mut self,
        cond_op: &OpTy<'tcx>,
        mutex_op: &OpTy<'tcx>,
        dest: &MPlaceTy<'tcx>,
    ) -> InterpResult<'tcx> {
        let this = self.eval_context_mut();

        let data = cond_get_data(this, cond_op)?.clone();
        let mutex_ref = mutex_get_data(this, mutex_op)?.mutex_ref.clone();

        this.condvar_wait(
            data.condvar_ref,
            mutex_ref,
            None, // no timeout
            Scalar::from_i32(0),
            Scalar::from_i32(0), // retval_timeout -- unused
            dest.clone(),
        )?;

        interp_ok(())
    }

    fn pthread_cond_timedwait(
        &mut self,
        cond_op: &OpTy<'tcx>,
        mutex_op: &OpTy<'tcx>,
        timeout_op: &OpTy<'tcx>,
        dest: &MPlaceTy<'tcx>,
        macos_relative_np: bool,
    ) -> InterpResult<'tcx> {
        let this = self.eval_context_mut();

        let data = cond_get_data(this, cond_op)?.clone();
        let mutex_ref = mutex_get_data(this, mutex_op)?.mutex_ref.clone();

        // Extract the timeout.
        let Some(duration) = this
            .read_timespec(&this.deref_pointer_as(timeout_op, this.libc_ty_layout("timespec"))?)?
        else {
            let einval = this.eval_libc("EINVAL");
            this.write_scalar(einval, dest)?;
            return interp_ok(());
        };

        let (clock, style) = if macos_relative_np {
            // `pthread_cond_timedwait_relative_np` always measures time against the
            // monotonic clock, regardless of the condvar clock.
            (TimeoutClock::Monotonic, TimeoutStyle::Relative)
        } else {
            if data.clock == TimeoutClock::RealTime {
                this.check_no_isolation("`pthread_cond_timedwait` with `CLOCK_REALTIME`")?;
            }

            (data.clock, TimeoutStyle::Absolute)
        };

        this.condvar_wait(
            data.condvar_ref,
            mutex_ref,
            Some(this.machine.timeout(clock, style, duration)),
            Scalar::from_i32(0),
            this.eval_libc("ETIMEDOUT"), // retval_timeout
            dest.clone(),
        )?;

        interp_ok(())
    }

    fn pthread_cond_destroy(&mut self, cond_op: &OpTy<'tcx>) -> InterpResult<'tcx, ()> {
        let this = self.eval_context_mut();

        // Reading the field also has the side-effect that we detect double-`destroy`
        // since we make the field uninit below.
        let condvar = &cond_get_data(this, cond_op)?.condvar_ref;
        if !condvar.queue_is_empty() {
            throw_ub_format!("destroying an awaited conditional variable");
        }

        // This write also deletes the interpreter state for this condvar.
        // This might lead to false positives, see comment in pthread_mutexattr_destroy
        let cond_place = this.deref_pointer_as(cond_op, this.libc_ty_layout("pthread_cond_t"))?;
        this.write_uninit(&cond_place)?;

        interp_ok(())
    }

    fn pthread_sem_init(
        &mut self,
        sem_op: &OpTy<'tcx>,
        pshared_op: &OpTy<'tcx>,
        value_op: &OpTy<'tcx>,
        dest: &MPlaceTy<'tcx>,
    ) -> InterpResult<'tcx> {
        let this = self.eval_context_mut();

        let pshared = this.read_scalar(pshared_op)?.to_i32()?;
        let value = this.read_scalar(value_op)?.to_u32()?;

        // We can ignore `pshared` since memory cannot actually be shared with other processes.
        let _ignore = pshared;

        sem_create(this, sem_op, value)?;

        this.write_null(dest)?; // we always succeed
        interp_ok(())
    }

    fn pthread_sem_wait(
        &mut self,
        sem_op: &OpTy<'tcx>,
        dest: &MPlaceTy<'tcx>,
    ) -> InterpResult<'tcx> {
        let this = self.eval_context_mut();

        let data = sem_get_data(this, sem_op)?.clone();
        let dest = dest.clone();
        this.semaphore_decrement(
            data.sem_ref,
            /* deadline */ None,
            callback!(
                @capture<'tcx> {
                    dest: MPlaceTy<'tcx>,
                }
                |this, unblock: UnblockKind| {
                    assert_eq!(unblock, UnblockKind::Ready);
                    this.write_null(&dest)?;
                    interp_ok(())
                }
            ),
        )?;

        interp_ok(())
    }

    fn pthread_sem_timedwait(
        &mut self,
        sem_op: &OpTy<'tcx>,
        abs_timeout_op: &OpTy<'tcx>,
        dest: &MPlaceTy<'tcx>,
    ) -> InterpResult<'tcx> {
        let this = self.eval_context_mut();
        this.check_no_isolation("`pthread_sem_timedwait`")?;

        let abs_timeout = this.deref_pointer_as(abs_timeout_op, this.libc_ty_layout("timespec"))?;

        let Some(duration) = this.read_timespec(&abs_timeout)? else {
            return this.set_errno_and_return_neg1(LibcError("EINVAL"), dest);
        };
        let deadline =
            this.machine.timeout(TimeoutClock::RealTime, TimeoutStyle::Absolute, duration);
        let data = sem_get_data(this, sem_op)?.clone();
        let dest = dest.clone();
        this.semaphore_decrement(
            data.sem_ref,
            Some(deadline),
            callback!(
                @capture<'tcx> {
                    dest: MPlaceTy<'tcx>,
                }
                |this, unblock: UnblockKind| {
                    match unblock {
                        UnblockKind::Ready =>
                            this.write_null(&dest),
                        UnblockKind::TimedOut =>
                            this.set_errno_and_return_neg1(LibcError("ETIMEDOUT"), &dest),
                    }
                }
            ),
        )?;

        interp_ok(())
    }

    fn pthread_sem_post(
        &mut self,
        sem_op: &OpTy<'tcx>,
        dest: &MPlaceTy<'tcx>,
    ) -> InterpResult<'tcx> {
        let this = self.eval_context_mut();

        let data = sem_get_data(this, sem_op)?.clone();
        this.semaphore_increment(&data.sem_ref)?;

        this.write_null(dest)?; // we always succeed
        interp_ok(())
    }

    fn pthread_sem_destroy(
        &mut self,
        sem_op: &OpTy<'tcx>,
        dest: &MPlaceTy<'tcx>,
    ) -> InterpResult<'tcx> {
        let this = self.eval_context_mut();

        let data = sem_get_data(this, sem_op)?.clone();
        if !data.sem_ref.queue_is_empty() {
            throw_ub_format!("destroying an awaited semaphore");
        }

        // This write also deletes the interpreter state for this semaphore.
        // This might lead to false positives, see comment in pthread_mutexattr_destroy
        let sem_place = this.deref_pointer_as(sem_op, this.libc_ty_layout("sem_t"))?;
        this.write_uninit(&sem_place)?;

        this.write_null(dest)?; // we always succeed
        interp_ok(())
    }
}