Abstract the state type for futexes

In the same way that we expose SmallAtomic and SmallPrimitive to allow
Windows to use a value other than an AtomicU32 for its futex state, this
patch switches the primary futex state type from AtomicU32 to
futex::Atomic. The futex::Atomic type should be usable as an atomic
value with underlying primitive type equal to futex::Primitive.
This allows supporting the futex API on systems where the underlying
kernel futex implementation requires more state than simply an
AtomicU32.
All in-tree futex implementations simply define {Atomic,Primitive}
directly as {AtomicU32,u32}.

Author: paulmenage

library/std/src/sys/pal/hermit/futex.rs

@@ -3,9 +3,14 @@ use crate::ptr::null;
 usecrate::sync::atomic::AtomicU32;
 usecrate::time::Duration;
 
+/// An atomic for use as a futex that is at least 32-bits but may be larger
+pubtypeFutex = AtomicU32;
+/// Must be the underlying type of Futex
+pubtypePrimitive = u32;
+
 /// An atomic for use as a futex that is at least 8-bits but may be larger.
-pubtypeSmallAtomic = AtomicU32;
-/// Must be the underlying type of SmallAtomic
+pubtypeSmallFutex = AtomicU32;
+/// Must be the underlying type of SmallFutex
 pubtypeSmallPrimitive = u32;
 
 pubfnfutex_wait(futex:&AtomicU32,expected:u32,timeout:Option<Duration>) -> bool{

library/std/src/sys/pal/unix/futex.rs

@@ -11,9 +11,14 @@
 usecrate::sync::atomic::AtomicU32;
 usecrate::time::Duration;
 
+/// An atomic for use as a futex that is at least 32-bits but may be larger
+pubtypeFutex = AtomicU32;
+/// Must be the underlying type of Futex
+pubtypePrimitive = u32;
+
 /// An atomic for use as a futex that is at least 8-bits but may be larger.
-pubtypeSmallAtomic = AtomicU32;
-/// Must be the underlying type of SmallAtomic
+pubtypeSmallFutex = AtomicU32;
+/// Must be the underlying type of SmallFutex
 pubtypeSmallPrimitive = u32;
 
 /// Waits for a `futex_wake` operation to wake us.

library/std/src/sys/pal/wasm/atomics/futex.rs

@@ -6,9 +6,14 @@ use core::arch::wasm64 as wasm;
 usecrate::sync::atomic::AtomicU32;
 usecrate::time::Duration;
 
+/// An atomic for use as a futex that is at least 32-bits but may be larger
+pubtypeFutex = AtomicU32;
+/// Must be the underlying type of Futex
+pubtypePrimitive = u32;
+
 /// An atomic for use as a futex that is at least 8-bits but may be larger.
-pubtypeSmallAtomic = AtomicU32;
-/// Must be the underlying type of SmallAtomic
+pubtypeSmallFutex = AtomicU32;
+/// Must be the underlying type of SmallFutex
 pubtypeSmallPrimitive = u32;
 
 /// Wait for a futex_wake operation to wake us.

library/std/src/sys/pal/windows/futex.rs

@@ -9,22 +9,27 @@ use core::{mem, ptr};
 usesuper::api::{self,WinError};
 usecrate::sys::{c, dur2timeout};
 
+/// An atomic for use as a futex that is at least 32-bits but may be larger
+pubtypeFutex = AtomicU32;
+/// Must be the underlying type of Futex
+pubtypePrimitive = u32;
+
 /// An atomic for use as a futex that is at least 8-bits but may be larger.
-pubtypeSmallAtomic = AtomicU8;
-/// Must be the underlying type of SmallAtomic
+pubtypeSmallFutex = AtomicU8;
+/// Must be the underlying type of SmallFutex
 pubtypeSmallPrimitive = u8;
 
-pubunsafetraitFutex{}
+pubunsafetraitFutexable{}
 pubunsafetraitWaitable{
-typeAtomic;
+typeFutex;
 }
 macro_rules! unsafe_waitable_int {
 ($(($int:ty, $atomic:ty)),*$(,)?) => {
  $(
 unsafeimplWaitablefor $int {
-typeAtomic = $atomic;
+typeFutex = $atomic;
 }
-unsafeimplFutexfor $atomic {}
+unsafeimplFutexablefor $atomic {}
 )*
 };
 }
@@ -42,15 +47,15 @@ unsafe_waitable_int! {
 (usize,AtomicUsize),
 }
 unsafeimpl<T>Waitablefor*constT{
-typeAtomic = AtomicPtr<T>;
+typeFutex = AtomicPtr<T>;
 }
 unsafeimpl<T>Waitablefor*mutT{
-typeAtomic = AtomicPtr<T>;
+typeFutex = AtomicPtr<T>;
 }
-unsafeimpl<T>FutexforAtomicPtr<T>{}
+unsafeimpl<T>FutexableforAtomicPtr<T>{}
 
 pubfnwait_on_address<W:Waitable>(
-address:&W::Atomic,
+address:&W::Futex,
 compare:W,
 timeout:Option<Duration>,
 ) -> bool{
@@ -63,30 +68,30 @@ pub fn wait_on_address<W: Waitable>(
 }
 }
 
-pubfnwake_by_address_single<T:Futex>(address:&T){
+pubfnwake_by_address_single<T:Futexable>(address:&T){
 unsafe{
 let addr = ptr::from_ref(address).cast::<c_void>();
  c::WakeByAddressSingle(addr);
 }
 }
 
-pubfnwake_by_address_all<T:Futex>(address:&T){
+pubfnwake_by_address_all<T:Futexable>(address:&T){
 unsafe{
 let addr = ptr::from_ref(address).cast::<c_void>();
  c::WakeByAddressAll(addr);
 }
 }
 
-pubfnfutex_wait<W:Waitable>(futex:&W::Atomic,expected:W,timeout:Option<Duration>) -> bool{
+pubfnfutex_wait<W:Waitable>(futex:&W::Futex,expected:W,timeout:Option<Duration>) -> bool{
 // return false only on timeout
 wait_on_address(futex, expected, timeout) || api::get_last_error() != WinError::TIMEOUT
 }
 
-pubfnfutex_wake<T:Futex>(futex:&T) -> bool{
+pubfnfutex_wake<T:Futexable>(futex:&T) -> bool{
 wake_by_address_single(futex);
 false
 }
 
-pubfnfutex_wake_all<T:Futex>(futex:&T){
+pubfnfutex_wake_all<T:Futexable>(futex:&T){
 wake_by_address_all(futex)
 }

library/std/src/sys/sync/condvar/futex.rs

@@ -1,20 +1,19 @@
-usecrate::sync::atomic::AtomicU32;
 usecrate::sync::atomic::Ordering::Relaxed;
-usecrate::sys::futex::{futex_wait, futex_wake, futex_wake_all};
+usecrate::sys::futex::{Futex,futex_wait, futex_wake, futex_wake_all};
 usecrate::sys::sync::Mutex;
 usecrate::time::Duration;
 
 pubstructCondvar{
 // The value of this atomic is simply incremented on every notification.
 // This is used by `.wait()` to not miss any notifications after
 // unlocking the mutex and before waiting for notifications.
-futex:AtomicU32,
+futex:Futex,
 }
 
 implCondvar{
 #[inline]
 pubconstfnnew() -> Self{
-Self{futex:AtomicU32::new(0)}
+Self{futex:Futex::new(0)}
 }
 
 // All the memory orderings here are `Relaxed`,

library/std/src/sys/sync/mutex/futex.rs

@@ -1,11 +1,11 @@
 usecrate::sync::atomic::Ordering::{Acquire,Relaxed,Release};
 usecrate::sys::futex::{self, futex_wait, futex_wake};
 
-typeAtomic = futex::SmallAtomic;
+typeFutex = futex::SmallFutex;
 typeState = futex::SmallPrimitive;
 
 pubstructMutex{
-futex:Atomic,
+futex:Futex,
 }
 
 constUNLOCKED:State = 0;
@@ -15,7 +15,7 @@ const CONTENDED: State = 2; // locked, and other threads waiting (contended)
 implMutex{
 #[inline]
 pubconstfnnew() -> Self{
-Self{futex:Atomic::new(UNLOCKED)}
+Self{futex:Futex::new(UNLOCKED)}
 }
 
 #[inline]

library/std/src/sys/sync/once/futex.rs

@@ -1,39 +1,38 @@
 usecrate::cell::Cell;
 usecrate::sync as public;
-usecrate::sync::atomic::AtomicU32;
 usecrate::sync::atomic::Ordering::{Acquire,Relaxed,Release};
 usecrate::sync::once::ExclusiveState;
-usecrate::sys::futex::{futex_wait, futex_wake_all};
+usecrate::sys::futex::{Futex,Primitive,futex_wait, futex_wake_all};
 
 // On some platforms, the OS is very nice and handles the waiter queue for us.
 // This means we only need one atomic value with 4 states:
 
 /// No initialization has run yet, and no thread is currently using the Once.
-constINCOMPLETE:u32 = 0;
+constINCOMPLETE:Primitive = 0;
 /// Some thread has previously attempted to initialize the Once, but it panicked,
 /// so the Once is now poisoned. There are no other threads currently accessing
 /// this Once.
-constPOISONED:u32 = 1;
+constPOISONED:Primitive = 1;
 /// Some thread is currently attempting to run initialization. It may succeed,
 /// so all future threads need to wait for it to finish.
-constRUNNING:u32 = 2;
+constRUNNING:Primitive = 2;
 /// Initialization has completed and all future calls should finish immediately.
-constCOMPLETE:u32 = 3;
+constCOMPLETE:Primitive = 3;
 
 // An additional bit indicates whether there are waiting threads:
 
 /// May only be set if the state is not COMPLETE.
-constQUEUED:u32 = 4;
+constQUEUED:Primitive = 4;
 
 // Threads wait by setting the QUEUED bit and calling `futex_wait` on the state
 // variable. When the running thread finishes, it will wake all waiting threads using
 // `futex_wake_all`.
 
-constSTATE_MASK:u32 = 0b11;
+constSTATE_MASK:Primitive = 0b11;
 
 pubstructOnceState{
 poisoned:bool,
-set_state_to:Cell<u32>,
+set_state_to:Cell<Primitive>,
 }
 
 implOnceState{
@@ -49,8 +48,8 @@ impl OnceState {
 }
 
 structCompletionGuard<'a>{
-state_and_queued:&'aAtomicU32,
-set_state_on_drop_to:u32,
+state_and_queued:&'aFutex,
+set_state_on_drop_to:Primitive,
 }
 
 impl<'a>DropforCompletionGuard<'a>{
@@ -65,13 +64,13 @@ impl<'a> Drop for CompletionGuard<'a> {
 }
 
 pubstructOnce{
-state_and_queued:AtomicU32,
+state_and_queued:Futex,
 }
 
 implOnce{
 #[inline]
 pubconstfnnew() -> Once{
-Once{state_and_queued:AtomicU32::new(INCOMPLETE)}
+Once{state_and_queued:Futex::new(INCOMPLETE)}
 }
 
 #[inline]

library/std/src/sys/sync/once/queue.rs

@@ -23,7 +23,7 @@
 // You'll find a few more details in the implementation, but that's the gist of
 // it!
 //
-// Atomic orderings:
+// Futex orderings:
 // When running `Once` we deal with multiple atomics:
 // `Once.state_and_queue` and an unknown number of `Waiter.signaled`.
 // * `state_and_queue` is used (1) as a state flag, (2) for synchronizing the

library/std/src/sys/sync/rwlock/futex.rs

@@ -1,6 +1,5 @@
-usecrate::sync::atomic::AtomicU32;
 usecrate::sync::atomic::Ordering::{Acquire,Relaxed,Release};
-usecrate::sys::futex::{futex_wait, futex_wake, futex_wake_all};
+usecrate::sys::futex::{Futex,Primitive,futex_wait, futex_wake, futex_wake_all};
 
 pubstructRwLock{
 // The state consists of a 30-bit reader counter, a 'readers waiting' flag, and a 'writers waiting' flag.
@@ -10,41 +9,41 @@ pub struct RwLock {
 // 0x3FFF_FFFF: Write locked
 // Bit 30: Readers are waiting on this futex.
 // Bit 31: Writers are waiting on the writer_notify futex.
-state:AtomicU32,
+state:Futex,
 // The 'condition variable' to notify writers through.
 // Incremented on every signal.
-writer_notify:AtomicU32,
+writer_notify:Futex,
 }
 
-constREAD_LOCKED:u32 = 1;
-constMASK:u32 = (1 << 30) - 1;
-constWRITE_LOCKED:u32 = MASK;
-constMAX_READERS:u32 = MASK - 1;
-constREADERS_WAITING:u32 = 1 << 30;
-constWRITERS_WAITING:u32 = 1 << 31;
+constREAD_LOCKED:Primitive = 1;
+constMASK:Primitive = (1 << 30) - 1;
+constWRITE_LOCKED:Primitive = MASK;
+constMAX_READERS:Primitive = MASK - 1;
+constREADERS_WAITING:Primitive = 1 << 30;
+constWRITERS_WAITING:Primitive = 1 << 31;
 
 #[inline]
-fnis_unlocked(state:u32) -> bool{
+fnis_unlocked(state:Primitive) -> bool{
  state &MASK == 0
 }
 
 #[inline]
-fnis_write_locked(state:u32) -> bool{
+fnis_write_locked(state:Primitive) -> bool{
  state &MASK == WRITE_LOCKED
 }
 
 #[inline]
-fnhas_readers_waiting(state:u32) -> bool{
+fnhas_readers_waiting(state:Primitive) -> bool{
  state &READERS_WAITING != 0
 }
 
 #[inline]
-fnhas_writers_waiting(state:u32) -> bool{
+fnhas_writers_waiting(state:Primitive) -> bool{
  state &WRITERS_WAITING != 0
 }
 
 #[inline]
-fnis_read_lockable(state:u32) -> bool{
+fnis_read_lockable(state:Primitive) -> bool{
 // This also returns false if the counter could overflow if we tried to read lock it.
 //
 // We don't allow read-locking if there's readers waiting, even if the lock is unlocked
@@ -55,14 +54,14 @@ fn is_read_lockable(state: u32) -> bool {
 }
 
 #[inline]
-fnhas_reached_max_readers(state:u32) -> bool{
+fnhas_reached_max_readers(state:Primitive) -> bool{
  state &MASK == MAX_READERS
 }
 
 implRwLock{
 #[inline]
 pubconstfnnew() -> Self{
-Self{state:AtomicU32::new(0),writer_notify:AtomicU32::new(0)}
+Self{state:Futex::new(0),writer_notify:Futex::new(0)}
 }
 
 #[inline]
@@ -225,7 +224,7 @@ impl RwLock {
 /// If both are waiting, this will wake up only one writer, but will fall
 /// back to waking up readers if there was no writer to wake up.
 #[cold]
-fnwake_writer_or_readers(&self,mutstate:u32){
+fnwake_writer_or_readers(&self,mutstate:Primitive){
 assert!(is_unlocked(state));
 
 // The readers waiting bit might be turned on at any point now,
@@ -290,7 +289,7 @@ impl RwLock {
 
 /// Spin for a while, but stop directly at the given condition.
 #[inline]
-fnspin_until(&self,f:implFn(u32) -> bool) -> u32{
+fnspin_until(&self,f:implFn(Primitive) -> bool) -> Primitive{
 letmut spin = 100;// Chosen by fair dice roll.
 loop{
 let state = self.state.load(Relaxed);
@@ -303,13 +302,13 @@ impl RwLock {
 }
 
 #[inline]
-fnspin_write(&self) -> u32{
+fnspin_write(&self) -> Primitive{
 // Stop spinning when it's unlocked or when there's waiting writers, to keep things somewhat fair.
 self.spin_until(|state| is_unlocked(state) || has_writers_waiting(state))
 }
 
 #[inline]
-fnspin_read(&self) -> u32{
+fnspin_read(&self) -> Primitive{
 // Stop spinning when it's unlocked or read locked, or when there's waiting threads.
 self.spin_until(|state| {
  !is_write_locked(state) || has_readers_waiting(state) || has_writers_waiting(state)