10 files changed, 200 insertions, 162 deletions

diff --git a/src/loom/std/atomic_ptr.rs b/src/loom/std/atomic_ptr.rs
deleted file mode 100644
index 236645f..0000000
--- a/src/loom/std/atomic_ptr.rs
+++ /dev/null
@@ -1,34 +0,0 @@
-use std::fmt;
-use std::ops::{Deref, DerefMut};
-
-/// `AtomicPtr` providing an additional `load_unsync` function.
-pub(crate) struct AtomicPtr<T> {
-    inner: std::sync::atomic::AtomicPtr<T>,
-}
-
-impl<T> AtomicPtr<T> {
-    pub(crate) fn new(ptr: *mut T) -> AtomicPtr<T> {
-        let inner = std::sync::atomic::AtomicPtr::new(ptr);
-        AtomicPtr { inner }
-    }
-}
-
-impl<T> Deref for AtomicPtr<T> {
-    type Target = std::sync::atomic::AtomicPtr<T>;
-
-    fn deref(&self) -> &Self::Target {
-        &self.inner
-    }
-}
-
-impl<T> DerefMut for AtomicPtr<T> {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.inner
-    }
-}
-
-impl<T> fmt::Debug for AtomicPtr<T> {
-    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
-        self.deref().fmt(fmt)
-    }
-}
diff --git a/src/loom/std/atomic_u16.rs b/src/loom/std/atomic_u16.rs
index c1c5312..c9e105c 100644
--- a/src/loom/std/atomic_u16.rs
+++ b/src/loom/std/atomic_u16.rs
@@ -2,7 +2,7 @@ use std::cell::UnsafeCell;
 use std::fmt;
 use std::ops::Deref;
 
-/// `AtomicU16` providing an additional `load_unsync` function.
+/// `AtomicU16` providing an additional `unsync_load` function.
 pub(crate) struct AtomicU16 {
     inner: UnsafeCell<std::sync::atomic::AtomicU16>,
 }
@@ -23,7 +23,7 @@ impl AtomicU16 {
     /// All mutations must have happened before the unsynchronized load.
     /// Additionally, there must be no concurrent mutations.
     pub(crate) unsafe fn unsync_load(&self) -> u16 {
-        *(*self.inner.get()).get_mut()
+        core::ptr::read(self.inner.get() as *const u16)
     }
 }
 
diff --git a/src/loom/std/atomic_u32.rs b/src/loom/std/atomic_u32.rs
index 61f95fb..ee0d2d3 100644
--- a/src/loom/std/atomic_u32.rs
+++ b/src/loom/std/atomic_u32.rs
@@ -2,7 +2,7 @@ use std::cell::UnsafeCell;
 use std::fmt;
 use std::ops::Deref;
 
-/// `AtomicU32` providing an additional `load_unsync` function.
+/// `AtomicU32` providing an additional `unsync_load` function.
 pub(crate) struct AtomicU32 {
     inner: UnsafeCell<std::sync::atomic::AtomicU32>,
 }
@@ -15,6 +15,16 @@ impl AtomicU32 {
         let inner = UnsafeCell::new(std::sync::atomic::AtomicU32::new(val));
         AtomicU32 { inner }
     }
+
+    /// Performs an unsynchronized load.
+    ///
+    /// # Safety
+    ///
+    /// All mutations must have happened before the unsynchronized load.
+    /// Additionally, there must be no concurrent mutations.
+    pub(crate) unsafe fn unsync_load(&self) -> u32 {
+        core::ptr::read(self.inner.get() as *const u32)
+    }
 }
 
 impl Deref for AtomicU32 {
diff --git a/src/loom/std/atomic_u64.rs b/src/loom/std/atomic_u64.rs
index 8ea6bd4..5d1d8a8 100644
--- a/src/loom/std/atomic_u64.rs
+++ b/src/loom/std/atomic_u64.rs
@@ -11,61 +11,8 @@ cfg_has_atomic_u64! {
 }
 
 cfg_not_has_atomic_u64! {
-    use crate::loom::sync::Mutex;
-    use std::sync::atomic::Ordering;
+    #[path = "atomic_u64_as_mutex.rs"]
+    mod atomic_u64_as_mutex;
 
-    #[derive(Debug)]
-    pub(crate) struct AtomicU64 {
-        inner: Mutex<u64>,
-    }
-
-    impl AtomicU64 {
-        pub(crate) fn new(val: u64) -> Self {
-            Self {
-                inner: Mutex::new(val),
-            }
-        }
-
-        pub(crate) fn load(&self, _: Ordering) -> u64 {
-            *self.inner.lock()
-        }
-
-        pub(crate) fn store(&self, val: u64, _: Ordering) {
-            *self.inner.lock() = val;
-        }
-
-        pub(crate) fn fetch_or(&self, val: u64, _: Ordering) -> u64 {
-            let mut lock = self.inner.lock();
-            let prev = *lock;
-            *lock = prev | val;
-            prev
-        }
-
-        pub(crate) fn compare_exchange(
-            &self,
-            current: u64,
-            new: u64,
-            _success: Ordering,
-            _failure: Ordering,
-        ) -> Result<u64, u64> {
-            let mut lock = self.inner.lock();
-
-            if *lock == current {
-                *lock = new;
-                Ok(current)
-            } else {
-                Err(*lock)
-            }
-        }
-
-        pub(crate) fn compare_exchange_weak(
-            &self,
-            current: u64,
-            new: u64,
-            success: Ordering,
-            failure: Ordering,
-        ) -> Result<u64, u64> {
-            self.compare_exchange(current, new, success, failure)
-        }
-    }
+    pub(crate) use atomic_u64_as_mutex::AtomicU64;
 }
diff --git a/src/loom/std/atomic_u64_as_mutex.rs b/src/loom/std/atomic_u64_as_mutex.rs
new file mode 100644
index 0000000..84ddff0
--- /dev/null
+++ b/src/loom/std/atomic_u64_as_mutex.rs
@@ -0,0 +1,70 @@
+use crate::loom::sync::Mutex;
+use std::sync::atomic::Ordering;
+
+#[derive(Debug)]
+pub(crate) struct AtomicU64 {
+    inner: Mutex<u64>,
+}
+
+impl AtomicU64 {
+    pub(crate) fn new(val: u64) -> Self {
+        Self {
+            inner: Mutex::new(val),
+        }
+    }
+
+    pub(crate) fn load(&self, _: Ordering) -> u64 {
+        *self.inner.lock()
+    }
+
+    pub(crate) fn store(&self, val: u64, _: Ordering) {
+        *self.inner.lock() = val;
+    }
+
+    pub(crate) fn fetch_add(&self, val: u64, _: Ordering) -> u64 {
+        let mut lock = self.inner.lock();
+        let prev = *lock;
+        *lock = prev + val;
+        prev
+    }
+
+    pub(crate) fn fetch_or(&self, val: u64, _: Ordering) -> u64 {
+        let mut lock = self.inner.lock();
+        let prev = *lock;
+        *lock = prev | val;
+        prev
+    }
+
+    pub(crate) fn compare_exchange(
+        &self,
+        current: u64,
+        new: u64,
+        _success: Ordering,
+        _failure: Ordering,
+    ) -> Result<u64, u64> {
+        let mut lock = self.inner.lock();
+
+        if *lock == current {
+            *lock = new;
+            Ok(current)
+        } else {
+            Err(*lock)
+        }
+    }
+
+    pub(crate) fn compare_exchange_weak(
+        &self,
+        current: u64,
+        new: u64,
+        success: Ordering,
+        failure: Ordering,
+    ) -> Result<u64, u64> {
+        self.compare_exchange(current, new, success, failure)
+    }
+}
+
+impl Default for AtomicU64 {
+    fn default() -> AtomicU64 {
+        AtomicU64::new(u64::default())
+    }
+}
diff --git a/src/loom/std/atomic_u8.rs b/src/loom/std/atomic_u8.rs
deleted file mode 100644
index 408aea3..0000000
--- a/src/loom/std/atomic_u8.rs
+++ /dev/null
@@ -1,34 +0,0 @@
-use std::cell::UnsafeCell;
-use std::fmt;
-use std::ops::Deref;
-
-/// `AtomicU8` providing an additional `load_unsync` function.
-pub(crate) struct AtomicU8 {
-    inner: UnsafeCell<std::sync::atomic::AtomicU8>,
-}
-
-unsafe impl Send for AtomicU8 {}
-unsafe impl Sync for AtomicU8 {}
-
-impl AtomicU8 {
-    pub(crate) const fn new(val: u8) -> AtomicU8 {
-        let inner = UnsafeCell::new(std::sync::atomic::AtomicU8::new(val));
-        AtomicU8 { inner }
-    }
-}
-
-impl Deref for AtomicU8 {
-    type Target = std::sync::atomic::AtomicU8;
-
-    fn deref(&self) -> &Self::Target {
-        // safety: it is always safe to access `&self` fns on the inner value as
-        // we never perform unsafe mutations.
-        unsafe { &*self.inner.get() }
-    }
-}
-
-impl fmt::Debug for AtomicU8 {
-    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
-        self.deref().fmt(fmt)
-    }
-}
diff --git a/src/loom/std/atomic_usize.rs b/src/loom/std/atomic_usize.rs
index 0d5f36e..c5503a2 100644
--- a/src/loom/std/atomic_usize.rs
+++ b/src/loom/std/atomic_usize.rs
@@ -2,7 +2,7 @@ use std::cell::UnsafeCell;
 use std::fmt;
 use std::ops;
 
-/// `AtomicUsize` providing an additional `load_unsync` function.
+/// `AtomicUsize` providing an additional `unsync_load` function.
 pub(crate) struct AtomicUsize {
     inner: UnsafeCell<std::sync::atomic::AtomicUsize>,
 }
@@ -23,7 +23,7 @@ impl AtomicUsize {
     /// All mutations must have happened before the unsynchronized load.
     /// Additionally, there must be no concurrent mutations.
     pub(crate) unsafe fn unsync_load(&self) -> usize {
-        *(*self.inner.get()).get_mut()
+        core::ptr::read(self.inner.get() as *const usize)
     }
 
     pub(crate) fn with_mut<R>(&mut self, f: impl FnOnce(&mut usize) -> R) -> R {
diff --git a/src/loom/std/mod.rs b/src/loom/std/mod.rs
index 8b6e8bc..1fc0032 100644
--- a/src/loom/std/mod.rs
+++ b/src/loom/std/mod.rs
@@ -1,10 +1,8 @@
 #![cfg_attr(any(not(feature = "full"), loom), allow(unused_imports, dead_code))]
 
-mod atomic_ptr;
 mod atomic_u16;
 mod atomic_u32;
 mod atomic_u64;
-mod atomic_u8;
 mod atomic_usize;
 mod mutex;
 #[cfg(feature = "parking_lot")]
@@ -25,6 +23,10 @@ pub(crate) mod future {
     pub(crate) use crate::sync::AtomicWaker;
 }
 
+pub(crate) mod hint {
+    pub(crate) use std::hint::spin_loop;
+}
+
 pub(crate) mod rand {
     use std::collections::hash_map::RandomState;
     use std::hash::{BuildHasher, Hash, Hasher};
@@ -67,17 +69,12 @@ pub(crate) mod sync {
     pub(crate) use crate::loom::std::mutex::Mutex;
 
     pub(crate) mod atomic {
-        pub(crate) use crate::loom::std::atomic_ptr::AtomicPtr;
         pub(crate) use crate::loom::std::atomic_u16::AtomicU16;
         pub(crate) use crate::loom::std::atomic_u32::AtomicU32;
         pub(crate) use crate::loom::std::atomic_u64::AtomicU64;
-        pub(crate) use crate::loom::std::atomic_u8::AtomicU8;
         pub(crate) use crate::loom::std::atomic_usize::AtomicUsize;
 
-        pub(crate) use std::sync::atomic::{fence, AtomicBool, Ordering};
-        // TODO: once we bump MSRV to 1.49+, use `hint::spin_loop` instead.
-        #[allow(deprecated)]
-        pub(crate) use std::sync::atomic::spin_loop_hint;
+        pub(crate) use std::sync::atomic::{fence, AtomicBool, AtomicPtr, AtomicU8, Ordering};
     }
 }
 
@@ -96,14 +93,12 @@ pub(crate) mod sys {
 pub(crate) mod thread {
     #[inline]
     pub(crate) fn yield_now() {
-        // TODO: once we bump MSRV to 1.49+, use `hint::spin_loop` instead.
-        #[allow(deprecated)]
-        std::sync::atomic::spin_loop_hint();
+        std::hint::spin_loop();
     }
 
     #[allow(unused_imports)]
     pub(crate) use std::thread::{
-        current, panicking, park, park_timeout, sleep, spawn, Builder, JoinHandle, LocalKey,
-        Result, Thread, ThreadId,
+        current, panicking, park, park_timeout, sleep, spawn, AccessError, Builder, JoinHandle,
+        LocalKey, Result, Thread, ThreadId,
     };
 }
diff --git a/src/loom/std/mutex.rs b/src/loom/std/mutex.rs
index 3f686e0..076f786 100644
--- a/src/loom/std/mutex.rs
+++ b/src/loom/std/mutex.rs
@@ -13,6 +13,12 @@ impl<T> Mutex<T> {
     }
 
     #[inline]
+    #[cfg(not(tokio_no_const_mutex_new))]
+    pub(crate) const fn const_new(t: T) -> Mutex<T> {
+        Mutex(sync::Mutex::new(t))
+    }
+
+    #[inline]
     pub(crate) fn lock(&self) -> MutexGuard<'_, T> {
         match self.0.lock() {
             Ok(guard) => guard,
diff --git a/src/loom/std/parking_lot.rs b/src/loom/std/parking_lot.rs
index 8448bed..e3af258 100644
--- a/src/loom/std/parking_lot.rs
+++ b/src/loom/std/parking_lot.rs
@@ -3,83 +3,143 @@
 //!
 //! This can be extended to additional types/methods as required.
 
+use std::fmt;
+use std::marker::PhantomData;
+use std::ops::{Deref, DerefMut};
 use std::sync::LockResult;
 use std::time::Duration;
 
+// All types in this file are marked with PhantomData to ensure that
+// parking_lot's send_guard feature does not leak through and affect when Tokio
+// types are Send.
+//
+// See <https://github.com/tokio-rs/tokio/pull/4359> for more info.
+
 // Types that do not need wrapping
-pub(crate) use parking_lot::{MutexGuard, RwLockReadGuard, RwLockWriteGuard, WaitTimeoutResult};
+pub(crate) use parking_lot::WaitTimeoutResult;
+
+#[derive(Debug)]
+pub(crate) struct Mutex<T: ?Sized>(PhantomData<std::sync::Mutex<T>>, parking_lot::Mutex<T>);
+
+#[derive(Debug)]
+pub(crate) struct RwLock<T>(PhantomData<std::sync::RwLock<T>>, parking_lot::RwLock<T>);
+
+#[derive(Debug)]
+pub(crate) struct Condvar(PhantomData<std::sync::Condvar>, parking_lot::Condvar);
 
-/// Adapter for `parking_lot::Mutex` to the `std::sync::Mutex` interface.
 #[derive(Debug)]
-pub(crate) struct Mutex<T: ?Sized>(parking_lot::Mutex<T>);
+pub(crate) struct MutexGuard<'a, T: ?Sized>(
+    PhantomData<std::sync::MutexGuard<'a, T>>,
+    parking_lot::MutexGuard<'a, T>,
+);
 
 #[derive(Debug)]
-pub(crate) struct RwLock<T>(parking_lot::RwLock<T>);
+pub(crate) struct RwLockReadGuard<'a, T: ?Sized>(
+    PhantomData<std::sync::RwLockReadGuard<'a, T>>,
+    parking_lot::RwLockReadGuard<'a, T>,
+);
 
-/// Adapter for `parking_lot::Condvar` to the `std::sync::Condvar` interface.
 #[derive(Debug)]
-pub(crate) struct Condvar(parking_lot::Condvar);
+pub(crate) struct RwLockWriteGuard<'a, T: ?Sized>(
+    PhantomData<std::sync::RwLockWriteGuard<'a, T>>,
+    parking_lot::RwLockWriteGuard<'a, T>,
+);
 
 impl<T> Mutex<T> {
     #[inline]
     pub(crate) fn new(t: T) -> Mutex<T> {
-        Mutex(parking_lot::Mutex::new(t))
+        Mutex(PhantomData, parking_lot::Mutex::new(t))
     }
 
     #[inline]
-    #[cfg(all(feature = "parking_lot", not(all(loom, test)),))]
+    #[cfg(all(feature = "parking_lot", not(all(loom, test))))]
     #[cfg_attr(docsrs, doc(cfg(all(feature = "parking_lot",))))]
     pub(crate) const fn const_new(t: T) -> Mutex<T> {
-        Mutex(parking_lot::const_mutex(t))
+        Mutex(PhantomData, parking_lot::const_mutex(t))
     }
 
     #[inline]
     pub(crate) fn lock(&self) -> MutexGuard<'_, T> {
-        self.0.lock()
+        MutexGuard(PhantomData, self.1.lock())
     }
 
     #[inline]
     pub(crate) fn try_lock(&self) -> Option<MutexGuard<'_, T>> {
-        self.0.try_lock()
+        self.1
+            .try_lock()
+            .map(|guard| MutexGuard(PhantomData, guard))
     }
 
     #[inline]
     pub(crate) fn get_mut(&mut self) -> &mut T {
-        self.0.get_mut()
+        self.1.get_mut()
     }
 
     // Note: Additional methods `is_poisoned` and `into_inner`, can be
     // provided here as needed.
 }
 
+impl<'a, T: ?Sized> Deref for MutexGuard<'a, T> {
+    type Target = T;
+    fn deref(&self) -> &T {
+        self.1.deref()
+    }
+}
+
+impl<'a, T: ?Sized> DerefMut for MutexGuard<'a, T> {
+    fn deref_mut(&mut self) -> &mut T {
+        self.1.deref_mut()
+    }
+}
+
 impl<T> RwLock<T> {
     pub(crate) fn new(t: T) -> RwLock<T> {
-        RwLock(parking_lot::RwLock::new(t))
+        RwLock(PhantomData, parking_lot::RwLock::new(t))
     }
 
     pub(crate) fn read(&self) -> LockResult<RwLockReadGuard<'_, T>> {
-        Ok(self.0.read())
+        Ok(RwLockReadGuard(PhantomData, self.1.read()))
     }
 
     pub(crate) fn write(&self) -> LockResult<RwLockWriteGuard<'_, T>> {
-        Ok(self.0.write())
+        Ok(RwLockWriteGuard(PhantomData, self.1.write()))
+    }
+}
+
+impl<'a, T: ?Sized> Deref for RwLockReadGuard<'a, T> {
+    type Target = T;
+    fn deref(&self) -> &T {
+        self.1.deref()
+    }
+}
+
+impl<'a, T: ?Sized> Deref for RwLockWriteGuard<'a, T> {
+    type Target = T;
+    fn deref(&self) -> &T {
+        self.1.deref()
+    }
+}
+
+impl<'a, T: ?Sized> DerefMut for RwLockWriteGuard<'a, T> {
+    fn deref_mut(&mut self) -> &mut T {
+        self.1.deref_mut()
     }
 }
 
 impl Condvar {
     #[inline]
     pub(crate) fn new() -> Condvar {
-        Condvar(parking_lot::Condvar::new())
+        Condvar(PhantomData, parking_lot::Condvar::new())
     }
 
     #[inline]
     pub(crate) fn notify_one(&self) {
-        self.0.notify_one();
+        self.1.notify_one();
     }
 
     #[inline]
     pub(crate) fn notify_all(&self) {
-        self.0.notify_all();
+        self.1.notify_all();
     }
 
     #[inline]
@@ -87,7 +147,7 @@ impl Condvar {
         &self,
         mut guard: MutexGuard<'a, T>,
     ) -> LockResult<MutexGuard<'a, T>> {
-        self.0.wait(&mut guard);
+        self.1.wait(&mut guard.1);
         Ok(guard)
     }
 
@@ -97,10 +157,28 @@ impl Condvar {
         mut guard: MutexGuard<'a, T>,
         timeout: Duration,
     ) -> LockResult<(MutexGuard<'a, T>, WaitTimeoutResult)> {
-        let wtr = self.0.wait_for(&mut guard, timeout);
+        let wtr = self.1.wait_for(&mut guard.1, timeout);
         Ok((guard, wtr))
     }
 
     // Note: Additional methods `wait_timeout_ms`, `wait_timeout_until`,
     // `wait_until` can be provided here as needed.
 }
+
+impl<'a, T: ?Sized + fmt::Display> fmt::Display for MutexGuard<'a, T> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Display::fmt(&self.1, f)
+    }
+}
+
+impl<'a, T: ?Sized + fmt::Display> fmt::Display for RwLockReadGuard<'a, T> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Display::fmt(&self.1, f)
+    }
+}
+
+impl<'a, T: ?Sized + fmt::Display> fmt::Display for RwLockWriteGuard<'a, T> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Display::fmt(&self.1, f)
+    }
+}