Adds priority-inheritance futexes for mutexex

This uses FUTEX_LOCK_PI and FUTEX_UNLOCK_PI
on Linux.

Author: ruihe774

library/std/src/sys/pal/unix/mod.rs

@@ -23,6 +23,7 @@ pub mod net;
 #[cfg(target_os = "l4re")]
 pub use self::l4re::net;
 pub mod os;
+pub mod pi_futex;
 pub mod pipe;
 pub mod process;
 pub mod stack_overflow;

library/std/src/sys/pal/unix/pi_futex.rs

@@ -0,0 +1,52 @@
+#![cfg(any(target_os = "linux", target_os = "android"))]
+
+use crate::sync::atomic::AtomicU32;
+use crate::sys::cvt;
+use crate::{io, ptr};
+
+pub const fn unlocked() -> u32 {
+    0
+}
+
+pub fn locked() -> u32 {
+    (unsafe { libc::gettid() }) as _
+}
+
+pub fn is_contended(futex_val: u32) -> bool {
+    (futex_val & libc::FUTEX_WAITERS) != 0
+}
+
+pub fn is_owned_died(futex_val: u32) -> bool {
+    (futex_val & libc::FUTEX_OWNER_DIED) != 0
+}
+
+pub fn futex_lock(futex: &AtomicU32) -> io::Result<()> {
+    loop {
+        match cvt(unsafe {
+            libc::syscall(
+                libc::SYS_futex,
+                ptr::from_ref(futex),
+                libc::FUTEX_LOCK_PI | libc::FUTEX_PRIVATE_FLAG,
+                0,
+                ptr::null::<u32>(),
+                // remaining args are unused
+            )
+        }) {
+            Ok(_) => return Ok(()),
+            Err(e) if e.raw_os_error() == Some(libc::EINTR) => continue,
+            Err(e) => return Err(e),
+        }
+    }
+}
+
+pub fn futex_unlock(futex: &AtomicU32) -> io::Result<()> {
+    cvt(unsafe {
+        libc::syscall(
+            libc::SYS_futex,
+            ptr::from_ref(futex),
+            libc::FUTEX_UNLOCK_PI | libc::FUTEX_PRIVATE_FLAG,
+            // remaining args are unused
+        )
+    })
+    .map(|_| ())
+}

library/std/src/sys/sync/mutex/mod.rs

@@ -1,8 +1,12 @@
 cfg_if::cfg_if! {
     if #[cfg(any(
-        all(target_os = "windows", not(target_vendor = "win7")),
         target_os = "linux",
         target_os = "android",
+    ))] {
+        mod pi_futex;
+        pub use pi_futex::Mutex;
+    } else if #[cfg(any(
+        all(target_os = "windows", not(target_vendor = "win7")),
         target_os = "freebsd",
         target_os = "openbsd",
         target_os = "dragonfly",

library/std/src/sys/sync/mutex/pi_futex.rs

@@ -0,0 +1,81 @@
+use crate::sync::atomic::AtomicU32;
+use crate::sync::atomic::Ordering::{Acquire, Relaxed, Release};
+use crate::sys::pi_futex as pi;
+
+pub struct Mutex {
+    futex: AtomicU32,
+}
+
+impl Mutex {
+    #[inline]
+    pub const fn new() -> Self {
+        Self { futex: AtomicU32::new(pi::unlocked()) }
+    }
+
+    #[inline]
+    pub fn try_lock(&self) -> bool {
+        self.futex.compare_exchange(pi::unlocked(), pi::locked(), Acquire, Relaxed).is_ok()
+    }
+
+    #[inline]
+    pub fn lock(&self) {
+        if self.futex.compare_exchange(pi::unlocked(), pi::locked(), Acquire, Relaxed).is_err() {
+            self.lock_contended();
+        }
+    }
+
+    #[cold]
+    fn lock_contended(&self) {
+        // Spin first to speed things up if the lock is released quickly.
+        let state = self.spin();
+
+        // If it's unlocked now, attempt to take the lock.
+        if state == pi::unlocked() {
+            if self.try_lock() {
+                return;
+            }
+        };
+
+        pi::futex_lock(&self.futex).expect("failed to lock mutex");
+
+        let state = self.futex.load(Relaxed);
+        if pi::is_owned_died(state) {
+            panic!("failed to lock mutex because its owner is died");
+        }
+    }
+
+    fn spin(&self) -> u32 {
+        let mut spin = 100;
+        loop {
+            // We only use `load` (and not `swap` or `compare_exchange`)
+            // while spinning, to be easier on the caches.
+            let state = self.futex.load(Relaxed);
+
+            // We stop spinning when the mutex is unlocked,
+            // but also when it's contended.
+            if state == pi::unlocked() || pi::is_contended(state) || spin == 0 {
+                return state;
+            }
+
+            crate::hint::spin_loop();
+            spin -= 1;
+        }
+    }
+
+    #[inline]
+    pub unsafe fn unlock(&self) {
+        if self.futex.compare_exchange(pi::locked(), pi::unlocked(), Release, Relaxed).is_err() {
+            // We only wake up one thread. When that thread locks the mutex,
+            // the kernel will mark the mutex as contended automatically
+            // (futex != pi::locked() in this case),
+            // which makes sure that any other waiting threads will also be
+            // woken up eventually.
+            self.wake();
+        }
+    }
+
+    #[cold]
+    fn wake(&self) {
+        pi::futex_unlock(&self.futex).unwrap();
+    }
+}