alloc/
sync.rs

1#![stable(feature = "rust1", since = "1.0.0")]
2
3//! Thread-safe reference-counting pointers.
4//!
5//! See the [`Arc<T>`][Arc] documentation for more details.
6//!
7//! **Note**: This module is only available on platforms that support atomic
8//! loads and stores of pointers. This may be detected at compile time using
9//! `#[cfg(target_has_atomic = "ptr")]`.
10
11use core::any::Any;
12use core::cell::CloneFromCell;
13#[cfg(not(no_global_oom_handling))]
14use core::clone::CloneToUninit;
15use core::clone::UseCloned;
16use core::cmp::Ordering;
17use core::hash::{Hash, Hasher};
18use core::intrinsics::abort;
19#[cfg(not(no_global_oom_handling))]
20use core::iter;
21use core::marker::{PhantomData, Unsize};
22use core::mem::{self, ManuallyDrop, align_of_val_raw};
23use core::num::NonZeroUsize;
24use core::ops::{CoerceUnsized, Deref, DerefMut, DerefPure, DispatchFromDyn, LegacyReceiver};
25use core::panic::{RefUnwindSafe, UnwindSafe};
26use core::pin::{Pin, PinCoerceUnsized};
27use core::ptr::{self, NonNull};
28#[cfg(not(no_global_oom_handling))]
29use core::slice::from_raw_parts_mut;
30use core::sync::atomic::Ordering::{Acquire, Relaxed, Release};
31use core::sync::atomic::{self, Atomic};
32use core::{borrow, fmt, hint};
33
34#[cfg(not(no_global_oom_handling))]
35use crate::alloc::handle_alloc_error;
36use crate::alloc::{AllocError, Allocator, Global, Layout};
37use crate::borrow::{Cow, ToOwned};
38use crate::boxed::Box;
39use crate::rc::is_dangling;
40#[cfg(not(no_global_oom_handling))]
41use crate::string::String;
42#[cfg(not(no_global_oom_handling))]
43use crate::vec::Vec;
44
45/// A soft limit on the amount of references that may be made to an `Arc`.
46///
47/// Going above this limit will abort your program (although not
48/// necessarily) at _exactly_ `MAX_REFCOUNT + 1` references.
49/// Trying to go above it might call a `panic` (if not actually going above it).
50///
51/// This is a global invariant, and also applies when using a compare-exchange loop.
52///
53/// See comment in `Arc::clone`.
54const MAX_REFCOUNT: usize = (isize::MAX) as usize;
55
56/// The error in case either counter reaches above `MAX_REFCOUNT`, and we can `panic` safely.
57const INTERNAL_OVERFLOW_ERROR: &str = "Arc counter overflow";
58
59#[cfg(not(sanitize = "thread"))]
60macro_rules! acquire {
61    ($x:expr) => {
62        atomic::fence(Acquire)
63    };
64}
65
66// ThreadSanitizer does not support memory fences. To avoid false positive
67// reports in Arc / Weak implementation use atomic loads for synchronization
68// instead.
69#[cfg(sanitize = "thread")]
70macro_rules! acquire {
71    ($x:expr) => {
72        $x.load(Acquire)
73    };
74}
75
76/// A thread-safe reference-counting pointer. 'Arc' stands for 'Atomically
77/// Reference Counted'.
78///
79/// The type `Arc<T>` provides shared ownership of a value of type `T`,
80/// allocated in the heap. Invoking [`clone`][clone] on `Arc` produces
81/// a new `Arc` instance, which points to the same allocation on the heap as the
82/// source `Arc`, while increasing a reference count. When the last `Arc`
83/// pointer to a given allocation is destroyed, the value stored in that allocation (often
84/// referred to as "inner value") is also dropped.
85///
86/// Shared references in Rust disallow mutation by default, and `Arc` is no
87/// exception: you cannot generally obtain a mutable reference to something
88/// inside an `Arc`. If you do need to mutate through an `Arc`, you have several options:
89///
90/// 1. Use interior mutability with synchronization primitives like [`Mutex`][mutex],
91///    [`RwLock`][rwlock], or one of the [`Atomic`][atomic] types.
92///
93/// 2. Use clone-on-write semantics with [`Arc::make_mut`] which provides efficient mutation
94///    without requiring interior mutability. This approach clones the data only when
95///    needed (when there are multiple references) and can be more efficient when mutations
96///    are infrequent.
97///
98/// 3. Use [`Arc::get_mut`] when you know your `Arc` is not shared (has a reference count of 1),
99///    which provides direct mutable access to the inner value without any cloning.
100///
101/// ```
102/// use std::sync::Arc;
103///
104/// let mut data = Arc::new(vec![1, 2, 3]);
105///
106/// // This will clone the vector only if there are other references to it
107/// Arc::make_mut(&mut data).push(4);
108///
109/// assert_eq!(*data, vec![1, 2, 3, 4]);
110/// ```
111///
112/// **Note**: This type is only available on platforms that support atomic
113/// loads and stores of pointers, which includes all platforms that support
114/// the `std` crate but not all those which only support [`alloc`](crate).
115/// This may be detected at compile time using `#[cfg(target_has_atomic = "ptr")]`.
116///
117/// ## Thread Safety
118///
119/// Unlike [`Rc<T>`], `Arc<T>` uses atomic operations for its reference
120/// counting. This means that it is thread-safe. The disadvantage is that
121/// atomic operations are more expensive than ordinary memory accesses. If you
122/// are not sharing reference-counted allocations between threads, consider using
123/// [`Rc<T>`] for lower overhead. [`Rc<T>`] is a safe default, because the
124/// compiler will catch any attempt to send an [`Rc<T>`] between threads.
125/// However, a library might choose `Arc<T>` in order to give library consumers
126/// more flexibility.
127///
128/// `Arc<T>` will implement [`Send`] and [`Sync`] as long as the `T` implements
129/// [`Send`] and [`Sync`]. Why can't you put a non-thread-safe type `T` in an
130/// `Arc<T>` to make it thread-safe? This may be a bit counter-intuitive at
131/// first: after all, isn't the point of `Arc<T>` thread safety? The key is
132/// this: `Arc<T>` makes it thread safe to have multiple ownership of the same
133/// data, but it  doesn't add thread safety to its data. Consider
134/// <code>Arc<[RefCell\<T>]></code>. [`RefCell<T>`] isn't [`Sync`], and if `Arc<T>` was always
135/// [`Send`], <code>Arc<[RefCell\<T>]></code> would be as well. But then we'd have a problem:
136/// [`RefCell<T>`] is not thread safe; it keeps track of the borrowing count using
137/// non-atomic operations.
138///
139/// In the end, this means that you may need to pair `Arc<T>` with some sort of
140/// [`std::sync`] type, usually [`Mutex<T>`][mutex].
141///
142/// ## Breaking cycles with `Weak`
143///
144/// The [`downgrade`][downgrade] method can be used to create a non-owning
145/// [`Weak`] pointer. A [`Weak`] pointer can be [`upgrade`][upgrade]d
146/// to an `Arc`, but this will return [`None`] if the value stored in the allocation has
147/// already been dropped. In other words, `Weak` pointers do not keep the value
148/// inside the allocation alive; however, they *do* keep the allocation
149/// (the backing store for the value) alive.
150///
151/// A cycle between `Arc` pointers will never be deallocated. For this reason,
152/// [`Weak`] is used to break cycles. For example, a tree could have
153/// strong `Arc` pointers from parent nodes to children, and [`Weak`]
154/// pointers from children back to their parents.
155///
156/// # Cloning references
157///
158/// Creating a new reference from an existing reference-counted pointer is done using the
159/// `Clone` trait implemented for [`Arc<T>`][Arc] and [`Weak<T>`][Weak].
160///
161/// ```
162/// use std::sync::Arc;
163/// let foo = Arc::new(vec![1.0, 2.0, 3.0]);
164/// // The two syntaxes below are equivalent.
165/// let a = foo.clone();
166/// let b = Arc::clone(&foo);
167/// // a, b, and foo are all Arcs that point to the same memory location
168/// ```
169///
170/// ## `Deref` behavior
171///
172/// `Arc<T>` automatically dereferences to `T` (via the [`Deref`] trait),
173/// so you can call `T`'s methods on a value of type `Arc<T>`. To avoid name
174/// clashes with `T`'s methods, the methods of `Arc<T>` itself are associated
175/// functions, called using [fully qualified syntax]:
176///
177/// ```
178/// use std::sync::Arc;
179///
180/// let my_arc = Arc::new(());
181/// let my_weak = Arc::downgrade(&my_arc);
182/// ```
183///
184/// `Arc<T>`'s implementations of traits like `Clone` may also be called using
185/// fully qualified syntax. Some people prefer to use fully qualified syntax,
186/// while others prefer using method-call syntax.
187///
188/// ```
189/// use std::sync::Arc;
190///
191/// let arc = Arc::new(());
192/// // Method-call syntax
193/// let arc2 = arc.clone();
194/// // Fully qualified syntax
195/// let arc3 = Arc::clone(&arc);
196/// ```
197///
198/// [`Weak<T>`][Weak] does not auto-dereference to `T`, because the inner value may have
199/// already been dropped.
200///
201/// [`Rc<T>`]: crate::rc::Rc
202/// [clone]: Clone::clone
203/// [mutex]: ../../std/sync/struct.Mutex.html
204/// [rwlock]: ../../std/sync/struct.RwLock.html
205/// [atomic]: core::sync::atomic
206/// [downgrade]: Arc::downgrade
207/// [upgrade]: Weak::upgrade
208/// [RefCell\<T>]: core::cell::RefCell
209/// [`RefCell<T>`]: core::cell::RefCell
210/// [`std::sync`]: ../../std/sync/index.html
211/// [`Arc::clone(&from)`]: Arc::clone
212/// [fully qualified syntax]: https://doc.rust-lang.org/book/ch19-03-advanced-traits.html#fully-qualified-syntax-for-disambiguation-calling-methods-with-the-same-name
213///
214/// # Examples
215///
216/// Sharing some immutable data between threads:
217///
218/// ```
219/// use std::sync::Arc;
220/// use std::thread;
221///
222/// let five = Arc::new(5);
223///
224/// for _ in 0..10 {
225///     let five = Arc::clone(&five);
226///
227///     thread::spawn(move || {
228///         println!("{five:?}");
229///     });
230/// }
231/// ```
232///
233/// Sharing a mutable [`AtomicUsize`]:
234///
235/// [`AtomicUsize`]: core::sync::atomic::AtomicUsize "sync::atomic::AtomicUsize"
236///
237/// ```
238/// use std::sync::Arc;
239/// use std::sync::atomic::{AtomicUsize, Ordering};
240/// use std::thread;
241///
242/// let val = Arc::new(AtomicUsize::new(5));
243///
244/// for _ in 0..10 {
245///     let val = Arc::clone(&val);
246///
247///     thread::spawn(move || {
248///         let v = val.fetch_add(1, Ordering::Relaxed);
249///         println!("{v:?}");
250///     });
251/// }
252/// ```
253///
254/// See the [`rc` documentation][rc_examples] for more examples of reference
255/// counting in general.
256///
257/// [rc_examples]: crate::rc#examples
258#[doc(search_unbox)]
259#[rustc_diagnostic_item = "Arc"]
260#[stable(feature = "rust1", since = "1.0.0")]
261#[rustc_insignificant_dtor]
262pub struct Arc<
263    T: ?Sized,
264    #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator = Global,
265> {
266    ptr: NonNull<ArcInner<T>>,
267    phantom: PhantomData<ArcInner<T>>,
268    alloc: A,
269}
270
271#[stable(feature = "rust1", since = "1.0.0")]
272unsafe impl<T: ?Sized + Sync + Send, A: Allocator + Send> Send for Arc<T, A> {}
273#[stable(feature = "rust1", since = "1.0.0")]
274unsafe impl<T: ?Sized + Sync + Send, A: Allocator + Sync> Sync for Arc<T, A> {}
275
276#[stable(feature = "catch_unwind", since = "1.9.0")]
277impl<T: RefUnwindSafe + ?Sized, A: Allocator + UnwindSafe> UnwindSafe for Arc<T, A> {}
278
279#[unstable(feature = "coerce_unsized", issue = "18598")]
280impl<T: ?Sized + Unsize<U>, U: ?Sized, A: Allocator> CoerceUnsized<Arc<U, A>> for Arc<T, A> {}
281
282#[unstable(feature = "dispatch_from_dyn", issue = "none")]
283impl<T: ?Sized + Unsize<U>, U: ?Sized> DispatchFromDyn<Arc<U>> for Arc<T> {}
284
285// SAFETY: `Arc::clone` doesn't access any `Cell`s which could contain the `Arc` being cloned.
286#[unstable(feature = "cell_get_cloned", issue = "145329")]
287unsafe impl<T: ?Sized> CloneFromCell for Arc<T> {}
288
289impl<T: ?Sized> Arc<T> {
290    unsafe fn from_inner(ptr: NonNull<ArcInner<T>>) -> Self {
291        unsafe { Self::from_inner_in(ptr, Global) }
292    }
293
294    unsafe fn from_ptr(ptr: *mut ArcInner<T>) -> Self {
295        unsafe { Self::from_ptr_in(ptr, Global) }
296    }
297}
298
299impl<T: ?Sized, A: Allocator> Arc<T, A> {
300    #[inline]
301    fn into_inner_with_allocator(this: Self) -> (NonNull<ArcInner<T>>, A) {
302        let this = mem::ManuallyDrop::new(this);
303        (this.ptr, unsafe { ptr::read(&this.alloc) })
304    }
305
306    #[inline]
307    unsafe fn from_inner_in(ptr: NonNull<ArcInner<T>>, alloc: A) -> Self {
308        Self { ptr, phantom: PhantomData, alloc }
309    }
310
311    #[inline]
312    unsafe fn from_ptr_in(ptr: *mut ArcInner<T>, alloc: A) -> Self {
313        unsafe { Self::from_inner_in(NonNull::new_unchecked(ptr), alloc) }
314    }
315}
316
317/// `Weak` is a version of [`Arc`] that holds a non-owning reference to the
318/// managed allocation.
319///
320/// The allocation is accessed by calling [`upgrade`] on the `Weak`
321/// pointer, which returns an <code>[Option]<[Arc]\<T>></code>.
322///
323/// Since a `Weak` reference does not count towards ownership, it will not
324/// prevent the value stored in the allocation from being dropped, and `Weak` itself makes no
325/// guarantees about the value still being present. Thus it may return [`None`]
326/// when [`upgrade`]d. Note however that a `Weak` reference *does* prevent the allocation
327/// itself (the backing store) from being deallocated.
328///
329/// A `Weak` pointer is useful for keeping a temporary reference to the allocation
330/// managed by [`Arc`] without preventing its inner value from being dropped. It is also used to
331/// prevent circular references between [`Arc`] pointers, since mutual owning references
332/// would never allow either [`Arc`] to be dropped. For example, a tree could
333/// have strong [`Arc`] pointers from parent nodes to children, and `Weak`
334/// pointers from children back to their parents.
335///
336/// The typical way to obtain a `Weak` pointer is to call [`Arc::downgrade`].
337///
338/// [`upgrade`]: Weak::upgrade
339#[stable(feature = "arc_weak", since = "1.4.0")]
340#[rustc_diagnostic_item = "ArcWeak"]
341pub struct Weak<
342    T: ?Sized,
343    #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator = Global,
344> {
345    // This is a `NonNull` to allow optimizing the size of this type in enums,
346    // but it is not necessarily a valid pointer.
347    // `Weak::new` sets this to `usize::MAX` so that it doesn’t need
348    // to allocate space on the heap. That's not a value a real pointer
349    // will ever have because ArcInner has alignment at least 2.
350    ptr: NonNull<ArcInner<T>>,
351    alloc: A,
352}
353
354#[stable(feature = "arc_weak", since = "1.4.0")]
355unsafe impl<T: ?Sized + Sync + Send, A: Allocator + Send> Send for Weak<T, A> {}
356#[stable(feature = "arc_weak", since = "1.4.0")]
357unsafe impl<T: ?Sized + Sync + Send, A: Allocator + Sync> Sync for Weak<T, A> {}
358
359#[unstable(feature = "coerce_unsized", issue = "18598")]
360impl<T: ?Sized + Unsize<U>, U: ?Sized, A: Allocator> CoerceUnsized<Weak<U, A>> for Weak<T, A> {}
361#[unstable(feature = "dispatch_from_dyn", issue = "none")]
362impl<T: ?Sized + Unsize<U>, U: ?Sized> DispatchFromDyn<Weak<U>> for Weak<T> {}
363
364// SAFETY: `Weak::clone` doesn't access any `Cell`s which could contain the `Weak` being cloned.
365#[unstable(feature = "cell_get_cloned", issue = "145329")]
366unsafe impl<T: ?Sized> CloneFromCell for Weak<T> {}
367
368#[stable(feature = "arc_weak", since = "1.4.0")]
369impl<T: ?Sized, A: Allocator> fmt::Debug for Weak<T, A> {
370    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
371        write!(f, "(Weak)")
372    }
373}
374
375// This is repr(C) to future-proof against possible field-reordering, which
376// would interfere with otherwise safe [into|from]_raw() of transmutable
377// inner types.
378// Unlike RcInner, repr(align(2)) is not strictly required because atomic types
379// have the alignment same as its size, but we use it for consistency and clarity.
380#[repr(C, align(2))]
381struct ArcInner<T: ?Sized> {
382    strong: Atomic<usize>,
383
384    // the value usize::MAX acts as a sentinel for temporarily "locking" the
385    // ability to upgrade weak pointers or downgrade strong ones; this is used
386    // to avoid races in `make_mut` and `get_mut`.
387    weak: Atomic<usize>,
388
389    data: T,
390}
391
392/// Calculate layout for `ArcInner<T>` using the inner value's layout
393fn arcinner_layout_for_value_layout(layout: Layout) -> Layout {
394    // Calculate layout using the given value layout.
395    // Previously, layout was calculated on the expression
396    // `&*(ptr as *const ArcInner<T>)`, but this created a misaligned
397    // reference (see #54908).
398    Layout::new::<ArcInner<()>>().extend(layout).unwrap().0.pad_to_align()
399}
400
401unsafe impl<T: ?Sized + Sync + Send> Send for ArcInner<T> {}
402unsafe impl<T: ?Sized + Sync + Send> Sync for ArcInner<T> {}
403
404impl<T> Arc<T> {
405    /// Constructs a new `Arc<T>`.
406    ///
407    /// # Examples
408    ///
409    /// ```
410    /// use std::sync::Arc;
411    ///
412    /// let five = Arc::new(5);
413    /// ```
414    #[cfg(not(no_global_oom_handling))]
415    #[inline]
416    #[stable(feature = "rust1", since = "1.0.0")]
417    pub fn new(data: T) -> Arc<T> {
418        // Start the weak pointer count as 1 which is the weak pointer that's
419        // held by all the strong pointers (kinda), see std/rc.rs for more info
420        let x: Box<_> = Box::new(ArcInner {
421            strong: atomic::AtomicUsize::new(1),
422            weak: atomic::AtomicUsize::new(1),
423            data,
424        });
425        unsafe { Self::from_inner(Box::leak(x).into()) }
426    }
427
428    /// Constructs a new `Arc<T>` while giving you a `Weak<T>` to the allocation,
429    /// to allow you to construct a `T` which holds a weak pointer to itself.
430    ///
431    /// Generally, a structure circularly referencing itself, either directly or
432    /// indirectly, should not hold a strong reference to itself to prevent a memory leak.
433    /// Using this function, you get access to the weak pointer during the
434    /// initialization of `T`, before the `Arc<T>` is created, such that you can
435    /// clone and store it inside the `T`.
436    ///
437    /// `new_cyclic` first allocates the managed allocation for the `Arc<T>`,
438    /// then calls your closure, giving it a `Weak<T>` to this allocation,
439    /// and only afterwards completes the construction of the `Arc<T>` by placing
440    /// the `T` returned from your closure into the allocation.
441    ///
442    /// Since the new `Arc<T>` is not fully-constructed until `Arc<T>::new_cyclic`
443    /// returns, calling [`upgrade`] on the weak reference inside your closure will
444    /// fail and result in a `None` value.
445    ///
446    /// # Panics
447    ///
448    /// If `data_fn` panics, the panic is propagated to the caller, and the
449    /// temporary [`Weak<T>`] is dropped normally.
450    ///
451    /// # Example
452    ///
453    /// ```
454    /// # #![allow(dead_code)]
455    /// use std::sync::{Arc, Weak};
456    ///
457    /// struct Gadget {
458    ///     me: Weak<Gadget>,
459    /// }
460    ///
461    /// impl Gadget {
462    ///     /// Constructs a reference counted Gadget.
463    ///     fn new() -> Arc<Self> {
464    ///         // `me` is a `Weak<Gadget>` pointing at the new allocation of the
465    ///         // `Arc` we're constructing.
466    ///         Arc::new_cyclic(|me| {
467    ///             // Create the actual struct here.
468    ///             Gadget { me: me.clone() }
469    ///         })
470    ///     }
471    ///
472    ///     /// Returns a reference counted pointer to Self.
473    ///     fn me(&self) -> Arc<Self> {
474    ///         self.me.upgrade().unwrap()
475    ///     }
476    /// }
477    /// ```
478    /// [`upgrade`]: Weak::upgrade
479    #[cfg(not(no_global_oom_handling))]
480    #[inline]
481    #[stable(feature = "arc_new_cyclic", since = "1.60.0")]
482    pub fn new_cyclic<F>(data_fn: F) -> Arc<T>
483    where
484        F: FnOnce(&Weak<T>) -> T,
485    {
486        Self::new_cyclic_in(data_fn, Global)
487    }
488
489    /// Constructs a new `Arc` with uninitialized contents.
490    ///
491    /// # Examples
492    ///
493    /// ```
494    /// use std::sync::Arc;
495    ///
496    /// let mut five = Arc::<u32>::new_uninit();
497    ///
498    /// // Deferred initialization:
499    /// Arc::get_mut(&mut five).unwrap().write(5);
500    ///
501    /// let five = unsafe { five.assume_init() };
502    ///
503    /// assert_eq!(*five, 5)
504    /// ```
505    #[cfg(not(no_global_oom_handling))]
506    #[inline]
507    #[stable(feature = "new_uninit", since = "1.82.0")]
508    #[must_use]
509    pub fn new_uninit() -> Arc<mem::MaybeUninit<T>> {
510        unsafe {
511            Arc::from_ptr(Arc::allocate_for_layout(
512                Layout::new::<T>(),
513                |layout| Global.allocate(layout),
514                <*mut u8>::cast,
515            ))
516        }
517    }
518
519    /// Constructs a new `Arc` with uninitialized contents, with the memory
520    /// being filled with `0` bytes.
521    ///
522    /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage
523    /// of this method.
524    ///
525    /// # Examples
526    ///
527    /// ```
528    /// use std::sync::Arc;
529    ///
530    /// let zero = Arc::<u32>::new_zeroed();
531    /// let zero = unsafe { zero.assume_init() };
532    ///
533    /// assert_eq!(*zero, 0)
534    /// ```
535    ///
536    /// [zeroed]: mem::MaybeUninit::zeroed
537    #[cfg(not(no_global_oom_handling))]
538    #[inline]
539    #[stable(feature = "new_zeroed_alloc", since = "CURRENT_RUSTC_VERSION")]
540    #[must_use]
541    pub fn new_zeroed() -> Arc<mem::MaybeUninit<T>> {
542        unsafe {
543            Arc::from_ptr(Arc::allocate_for_layout(
544                Layout::new::<T>(),
545                |layout| Global.allocate_zeroed(layout),
546                <*mut u8>::cast,
547            ))
548        }
549    }
550
551    /// Constructs a new `Pin<Arc<T>>`. If `T` does not implement `Unpin`, then
552    /// `data` will be pinned in memory and unable to be moved.
553    #[cfg(not(no_global_oom_handling))]
554    #[stable(feature = "pin", since = "1.33.0")]
555    #[must_use]
556    pub fn pin(data: T) -> Pin<Arc<T>> {
557        unsafe { Pin::new_unchecked(Arc::new(data)) }
558    }
559
560    /// Constructs a new `Pin<Arc<T>>`, return an error if allocation fails.
561    #[unstable(feature = "allocator_api", issue = "32838")]
562    #[inline]
563    pub fn try_pin(data: T) -> Result<Pin<Arc<T>>, AllocError> {
564        unsafe { Ok(Pin::new_unchecked(Arc::try_new(data)?)) }
565    }
566
567    /// Constructs a new `Arc<T>`, returning an error if allocation fails.
568    ///
569    /// # Examples
570    ///
571    /// ```
572    /// #![feature(allocator_api)]
573    /// use std::sync::Arc;
574    ///
575    /// let five = Arc::try_new(5)?;
576    /// # Ok::<(), std::alloc::AllocError>(())
577    /// ```
578    #[unstable(feature = "allocator_api", issue = "32838")]
579    #[inline]
580    pub fn try_new(data: T) -> Result<Arc<T>, AllocError> {
581        // Start the weak pointer count as 1 which is the weak pointer that's
582        // held by all the strong pointers (kinda), see std/rc.rs for more info
583        let x: Box<_> = Box::try_new(ArcInner {
584            strong: atomic::AtomicUsize::new(1),
585            weak: atomic::AtomicUsize::new(1),
586            data,
587        })?;
588        unsafe { Ok(Self::from_inner(Box::leak(x).into())) }
589    }
590
591    /// Constructs a new `Arc` with uninitialized contents, returning an error
592    /// if allocation fails.
593    ///
594    /// # Examples
595    ///
596    /// ```
597    /// #![feature(allocator_api)]
598    ///
599    /// use std::sync::Arc;
600    ///
601    /// let mut five = Arc::<u32>::try_new_uninit()?;
602    ///
603    /// // Deferred initialization:
604    /// Arc::get_mut(&mut five).unwrap().write(5);
605    ///
606    /// let five = unsafe { five.assume_init() };
607    ///
608    /// assert_eq!(*five, 5);
609    /// # Ok::<(), std::alloc::AllocError>(())
610    /// ```
611    #[unstable(feature = "allocator_api", issue = "32838")]
612    pub fn try_new_uninit() -> Result<Arc<mem::MaybeUninit<T>>, AllocError> {
613        unsafe {
614            Ok(Arc::from_ptr(Arc::try_allocate_for_layout(
615                Layout::new::<T>(),
616                |layout| Global.allocate(layout),
617                <*mut u8>::cast,
618            )?))
619        }
620    }
621
622    /// Constructs a new `Arc` with uninitialized contents, with the memory
623    /// being filled with `0` bytes, returning an error if allocation fails.
624    ///
625    /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage
626    /// of this method.
627    ///
628    /// # Examples
629    ///
630    /// ```
631    /// #![feature( allocator_api)]
632    ///
633    /// use std::sync::Arc;
634    ///
635    /// let zero = Arc::<u32>::try_new_zeroed()?;
636    /// let zero = unsafe { zero.assume_init() };
637    ///
638    /// assert_eq!(*zero, 0);
639    /// # Ok::<(), std::alloc::AllocError>(())
640    /// ```
641    ///
642    /// [zeroed]: mem::MaybeUninit::zeroed
643    #[unstable(feature = "allocator_api", issue = "32838")]
644    pub fn try_new_zeroed() -> Result<Arc<mem::MaybeUninit<T>>, AllocError> {
645        unsafe {
646            Ok(Arc::from_ptr(Arc::try_allocate_for_layout(
647                Layout::new::<T>(),
648                |layout| Global.allocate_zeroed(layout),
649                <*mut u8>::cast,
650            )?))
651        }
652    }
653}
654
655impl<T, A: Allocator> Arc<T, A> {
656    /// Constructs a new `Arc<T>` in the provided allocator.
657    ///
658    /// # Examples
659    ///
660    /// ```
661    /// #![feature(allocator_api)]
662    ///
663    /// use std::sync::Arc;
664    /// use std::alloc::System;
665    ///
666    /// let five = Arc::new_in(5, System);
667    /// ```
668    #[inline]
669    #[cfg(not(no_global_oom_handling))]
670    #[unstable(feature = "allocator_api", issue = "32838")]
671    pub fn new_in(data: T, alloc: A) -> Arc<T, A> {
672        // Start the weak pointer count as 1 which is the weak pointer that's
673        // held by all the strong pointers (kinda), see std/rc.rs for more info
674        let x = Box::new_in(
675            ArcInner {
676                strong: atomic::AtomicUsize::new(1),
677                weak: atomic::AtomicUsize::new(1),
678                data,
679            },
680            alloc,
681        );
682        let (ptr, alloc) = Box::into_unique(x);
683        unsafe { Self::from_inner_in(ptr.into(), alloc) }
684    }
685
686    /// Constructs a new `Arc` with uninitialized contents in the provided allocator.
687    ///
688    /// # Examples
689    ///
690    /// ```
691    /// #![feature(get_mut_unchecked)]
692    /// #![feature(allocator_api)]
693    ///
694    /// use std::sync::Arc;
695    /// use std::alloc::System;
696    ///
697    /// let mut five = Arc::<u32, _>::new_uninit_in(System);
698    ///
699    /// let five = unsafe {
700    ///     // Deferred initialization:
701    ///     Arc::get_mut_unchecked(&mut five).as_mut_ptr().write(5);
702    ///
703    ///     five.assume_init()
704    /// };
705    ///
706    /// assert_eq!(*five, 5)
707    /// ```
708    #[cfg(not(no_global_oom_handling))]
709    #[unstable(feature = "allocator_api", issue = "32838")]
710    #[inline]
711    pub fn new_uninit_in(alloc: A) -> Arc<mem::MaybeUninit<T>, A> {
712        unsafe {
713            Arc::from_ptr_in(
714                Arc::allocate_for_layout(
715                    Layout::new::<T>(),
716                    |layout| alloc.allocate(layout),
717                    <*mut u8>::cast,
718                ),
719                alloc,
720            )
721        }
722    }
723
724    /// Constructs a new `Arc` with uninitialized contents, with the memory
725    /// being filled with `0` bytes, in the provided allocator.
726    ///
727    /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage
728    /// of this method.
729    ///
730    /// # Examples
731    ///
732    /// ```
733    /// #![feature(allocator_api)]
734    ///
735    /// use std::sync::Arc;
736    /// use std::alloc::System;
737    ///
738    /// let zero = Arc::<u32, _>::new_zeroed_in(System);
739    /// let zero = unsafe { zero.assume_init() };
740    ///
741    /// assert_eq!(*zero, 0)
742    /// ```
743    ///
744    /// [zeroed]: mem::MaybeUninit::zeroed
745    #[cfg(not(no_global_oom_handling))]
746    #[unstable(feature = "allocator_api", issue = "32838")]
747    #[inline]
748    pub fn new_zeroed_in(alloc: A) -> Arc<mem::MaybeUninit<T>, A> {
749        unsafe {
750            Arc::from_ptr_in(
751                Arc::allocate_for_layout(
752                    Layout::new::<T>(),
753                    |layout| alloc.allocate_zeroed(layout),
754                    <*mut u8>::cast,
755                ),
756                alloc,
757            )
758        }
759    }
760
761    /// Constructs a new `Arc<T, A>` in the given allocator while giving you a `Weak<T, A>` to the allocation,
762    /// to allow you to construct a `T` which holds a weak pointer to itself.
763    ///
764    /// Generally, a structure circularly referencing itself, either directly or
765    /// indirectly, should not hold a strong reference to itself to prevent a memory leak.
766    /// Using this function, you get access to the weak pointer during the
767    /// initialization of `T`, before the `Arc<T, A>` is created, such that you can
768    /// clone and store it inside the `T`.
769    ///
770    /// `new_cyclic_in` first allocates the managed allocation for the `Arc<T, A>`,
771    /// then calls your closure, giving it a `Weak<T, A>` to this allocation,
772    /// and only afterwards completes the construction of the `Arc<T, A>` by placing
773    /// the `T` returned from your closure into the allocation.
774    ///
775    /// Since the new `Arc<T, A>` is not fully-constructed until `Arc<T, A>::new_cyclic_in`
776    /// returns, calling [`upgrade`] on the weak reference inside your closure will
777    /// fail and result in a `None` value.
778    ///
779    /// # Panics
780    ///
781    /// If `data_fn` panics, the panic is propagated to the caller, and the
782    /// temporary [`Weak<T>`] is dropped normally.
783    ///
784    /// # Example
785    ///
786    /// See [`new_cyclic`]
787    ///
788    /// [`new_cyclic`]: Arc::new_cyclic
789    /// [`upgrade`]: Weak::upgrade
790    #[cfg(not(no_global_oom_handling))]
791    #[inline]
792    #[unstable(feature = "allocator_api", issue = "32838")]
793    pub fn new_cyclic_in<F>(data_fn: F, alloc: A) -> Arc<T, A>
794    where
795        F: FnOnce(&Weak<T, A>) -> T,
796    {
797        // Construct the inner in the "uninitialized" state with a single
798        // weak reference.
799        let (uninit_raw_ptr, alloc) = Box::into_raw_with_allocator(Box::new_in(
800            ArcInner {
801                strong: atomic::AtomicUsize::new(0),
802                weak: atomic::AtomicUsize::new(1),
803                data: mem::MaybeUninit::<T>::uninit(),
804            },
805            alloc,
806        ));
807        let uninit_ptr: NonNull<_> = (unsafe { &mut *uninit_raw_ptr }).into();
808        let init_ptr: NonNull<ArcInner<T>> = uninit_ptr.cast();
809
810        let weak = Weak { ptr: init_ptr, alloc };
811
812        // It's important we don't give up ownership of the weak pointer, or
813        // else the memory might be freed by the time `data_fn` returns. If
814        // we really wanted to pass ownership, we could create an additional
815        // weak pointer for ourselves, but this would result in additional
816        // updates to the weak reference count which might not be necessary
817        // otherwise.
818        let data = data_fn(&weak);
819
820        // Now we can properly initialize the inner value and turn our weak
821        // reference into a strong reference.
822        let strong = unsafe {
823            let inner = init_ptr.as_ptr();
824            ptr::write(&raw mut (*inner).data, data);
825
826            // The above write to the data field must be visible to any threads which
827            // observe a non-zero strong count. Therefore we need at least "Release" ordering
828            // in order to synchronize with the `compare_exchange_weak` in `Weak::upgrade`.
829            //
830            // "Acquire" ordering is not required. When considering the possible behaviors
831            // of `data_fn` we only need to look at what it could do with a reference to a
832            // non-upgradeable `Weak`:
833            // - It can *clone* the `Weak`, increasing the weak reference count.
834            // - It can drop those clones, decreasing the weak reference count (but never to zero).
835            //
836            // These side effects do not impact us in any way, and no other side effects are
837            // possible with safe code alone.
838            let prev_value = (*inner).strong.fetch_add(1, Release);
839            debug_assert_eq!(prev_value, 0, "No prior strong references should exist");
840
841            // Strong references should collectively own a shared weak reference,
842            // so don't run the destructor for our old weak reference.
843            // Calling into_raw_with_allocator has the double effect of giving us back the allocator,
844            // and forgetting the weak reference.
845            let alloc = weak.into_raw_with_allocator().1;
846
847            Arc::from_inner_in(init_ptr, alloc)
848        };
849
850        strong
851    }
852
853    /// Constructs a new `Pin<Arc<T, A>>` in the provided allocator. If `T` does not implement `Unpin`,
854    /// then `data` will be pinned in memory and unable to be moved.
855    #[cfg(not(no_global_oom_handling))]
856    #[unstable(feature = "allocator_api", issue = "32838")]
857    #[inline]
858    pub fn pin_in(data: T, alloc: A) -> Pin<Arc<T, A>>
859    where
860        A: 'static,
861    {
862        unsafe { Pin::new_unchecked(Arc::new_in(data, alloc)) }
863    }
864
865    /// Constructs a new `Pin<Arc<T, A>>` in the provided allocator, return an error if allocation
866    /// fails.
867    #[inline]
868    #[unstable(feature = "allocator_api", issue = "32838")]
869    pub fn try_pin_in(data: T, alloc: A) -> Result<Pin<Arc<T, A>>, AllocError>
870    where
871        A: 'static,
872    {
873        unsafe { Ok(Pin::new_unchecked(Arc::try_new_in(data, alloc)?)) }
874    }
875
876    /// Constructs a new `Arc<T, A>` in the provided allocator, returning an error if allocation fails.
877    ///
878    /// # Examples
879    ///
880    /// ```
881    /// #![feature(allocator_api)]
882    ///
883    /// use std::sync::Arc;
884    /// use std::alloc::System;
885    ///
886    /// let five = Arc::try_new_in(5, System)?;
887    /// # Ok::<(), std::alloc::AllocError>(())
888    /// ```
889    #[unstable(feature = "allocator_api", issue = "32838")]
890    #[inline]
891    pub fn try_new_in(data: T, alloc: A) -> Result<Arc<T, A>, AllocError> {
892        // Start the weak pointer count as 1 which is the weak pointer that's
893        // held by all the strong pointers (kinda), see std/rc.rs for more info
894        let x = Box::try_new_in(
895            ArcInner {
896                strong: atomic::AtomicUsize::new(1),
897                weak: atomic::AtomicUsize::new(1),
898                data,
899            },
900            alloc,
901        )?;
902        let (ptr, alloc) = Box::into_unique(x);
903        Ok(unsafe { Self::from_inner_in(ptr.into(), alloc) })
904    }
905
906    /// Constructs a new `Arc` with uninitialized contents, in the provided allocator, returning an
907    /// error if allocation fails.
908    ///
909    /// # Examples
910    ///
911    /// ```
912    /// #![feature(allocator_api)]
913    /// #![feature(get_mut_unchecked)]
914    ///
915    /// use std::sync::Arc;
916    /// use std::alloc::System;
917    ///
918    /// let mut five = Arc::<u32, _>::try_new_uninit_in(System)?;
919    ///
920    /// let five = unsafe {
921    ///     // Deferred initialization:
922    ///     Arc::get_mut_unchecked(&mut five).as_mut_ptr().write(5);
923    ///
924    ///     five.assume_init()
925    /// };
926    ///
927    /// assert_eq!(*five, 5);
928    /// # Ok::<(), std::alloc::AllocError>(())
929    /// ```
930    #[unstable(feature = "allocator_api", issue = "32838")]
931    #[inline]
932    pub fn try_new_uninit_in(alloc: A) -> Result<Arc<mem::MaybeUninit<T>, A>, AllocError> {
933        unsafe {
934            Ok(Arc::from_ptr_in(
935                Arc::try_allocate_for_layout(
936                    Layout::new::<T>(),
937                    |layout| alloc.allocate(layout),
938                    <*mut u8>::cast,
939                )?,
940                alloc,
941            ))
942        }
943    }
944
945    /// Constructs a new `Arc` with uninitialized contents, with the memory
946    /// being filled with `0` bytes, in the provided allocator, returning an error if allocation
947    /// fails.
948    ///
949    /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage
950    /// of this method.
951    ///
952    /// # Examples
953    ///
954    /// ```
955    /// #![feature(allocator_api)]
956    ///
957    /// use std::sync::Arc;
958    /// use std::alloc::System;
959    ///
960    /// let zero = Arc::<u32, _>::try_new_zeroed_in(System)?;
961    /// let zero = unsafe { zero.assume_init() };
962    ///
963    /// assert_eq!(*zero, 0);
964    /// # Ok::<(), std::alloc::AllocError>(())
965    /// ```
966    ///
967    /// [zeroed]: mem::MaybeUninit::zeroed
968    #[unstable(feature = "allocator_api", issue = "32838")]
969    #[inline]
970    pub fn try_new_zeroed_in(alloc: A) -> Result<Arc<mem::MaybeUninit<T>, A>, AllocError> {
971        unsafe {
972            Ok(Arc::from_ptr_in(
973                Arc::try_allocate_for_layout(
974                    Layout::new::<T>(),
975                    |layout| alloc.allocate_zeroed(layout),
976                    <*mut u8>::cast,
977                )?,
978                alloc,
979            ))
980        }
981    }
982    /// Returns the inner value, if the `Arc` has exactly one strong reference.
983    ///
984    /// Otherwise, an [`Err`] is returned with the same `Arc` that was
985    /// passed in.
986    ///
987    /// This will succeed even if there are outstanding weak references.
988    ///
989    /// It is strongly recommended to use [`Arc::into_inner`] instead if you don't
990    /// keep the `Arc` in the [`Err`] case.
991    /// Immediately dropping the [`Err`]-value, as the expression
992    /// `Arc::try_unwrap(this).ok()` does, can cause the strong count to
993    /// drop to zero and the inner value of the `Arc` to be dropped.
994    /// For instance, if two threads execute such an expression in parallel,
995    /// there is a race condition without the possibility of unsafety:
996    /// The threads could first both check whether they own the last instance
997    /// in `Arc::try_unwrap`, determine that they both do not, and then both
998    /// discard and drop their instance in the call to [`ok`][`Result::ok`].
999    /// In this scenario, the value inside the `Arc` is safely destroyed
1000    /// by exactly one of the threads, but neither thread will ever be able
1001    /// to use the value.
1002    ///
1003    /// # Examples
1004    ///
1005    /// ```
1006    /// use std::sync::Arc;
1007    ///
1008    /// let x = Arc::new(3);
1009    /// assert_eq!(Arc::try_unwrap(x), Ok(3));
1010    ///
1011    /// let x = Arc::new(4);
1012    /// let _y = Arc::clone(&x);
1013    /// assert_eq!(*Arc::try_unwrap(x).unwrap_err(), 4);
1014    /// ```
1015    #[inline]
1016    #[stable(feature = "arc_unique", since = "1.4.0")]
1017    pub fn try_unwrap(this: Self) -> Result<T, Self> {
1018        if this.inner().strong.compare_exchange(1, 0, Relaxed, Relaxed).is_err() {
1019            return Err(this);
1020        }
1021
1022        acquire!(this.inner().strong);
1023
1024        let this = ManuallyDrop::new(this);
1025        let elem: T = unsafe { ptr::read(&this.ptr.as_ref().data) };
1026        let alloc: A = unsafe { ptr::read(&this.alloc) }; // copy the allocator
1027
1028        // Make a weak pointer to clean up the implicit strong-weak reference
1029        let _weak = Weak { ptr: this.ptr, alloc };
1030
1031        Ok(elem)
1032    }
1033
1034    /// Returns the inner value, if the `Arc` has exactly one strong reference.
1035    ///
1036    /// Otherwise, [`None`] is returned and the `Arc` is dropped.
1037    ///
1038    /// This will succeed even if there are outstanding weak references.
1039    ///
1040    /// If `Arc::into_inner` is called on every clone of this `Arc`,
1041    /// it is guaranteed that exactly one of the calls returns the inner value.
1042    /// This means in particular that the inner value is not dropped.
1043    ///
1044    /// [`Arc::try_unwrap`] is conceptually similar to `Arc::into_inner`, but it
1045    /// is meant for different use-cases. If used as a direct replacement
1046    /// for `Arc::into_inner` anyway, such as with the expression
1047    /// <code>[Arc::try_unwrap]\(this).[ok][Result::ok]()</code>, then it does
1048    /// **not** give the same guarantee as described in the previous paragraph.
1049    /// For more information, see the examples below and read the documentation
1050    /// of [`Arc::try_unwrap`].
1051    ///
1052    /// # Examples
1053    ///
1054    /// Minimal example demonstrating the guarantee that `Arc::into_inner` gives.
1055    /// ```
1056    /// use std::sync::Arc;
1057    ///
1058    /// let x = Arc::new(3);
1059    /// let y = Arc::clone(&x);
1060    ///
1061    /// // Two threads calling `Arc::into_inner` on both clones of an `Arc`:
1062    /// let x_thread = std::thread::spawn(|| Arc::into_inner(x));
1063    /// let y_thread = std::thread::spawn(|| Arc::into_inner(y));
1064    ///
1065    /// let x_inner_value = x_thread.join().unwrap();
1066    /// let y_inner_value = y_thread.join().unwrap();
1067    ///
1068    /// // One of the threads is guaranteed to receive the inner value:
1069    /// assert!(matches!(
1070    ///     (x_inner_value, y_inner_value),
1071    ///     (None, Some(3)) | (Some(3), None)
1072    /// ));
1073    /// // The result could also be `(None, None)` if the threads called
1074    /// // `Arc::try_unwrap(x).ok()` and `Arc::try_unwrap(y).ok()` instead.
1075    /// ```
1076    ///
1077    /// A more practical example demonstrating the need for `Arc::into_inner`:
1078    /// ```
1079    /// use std::sync::Arc;
1080    ///
1081    /// // Definition of a simple singly linked list using `Arc`:
1082    /// #[derive(Clone)]
1083    /// struct LinkedList<T>(Option<Arc<Node<T>>>);
1084    /// struct Node<T>(T, Option<Arc<Node<T>>>);
1085    ///
1086    /// // Dropping a long `LinkedList<T>` relying on the destructor of `Arc`
1087    /// // can cause a stack overflow. To prevent this, we can provide a
1088    /// // manual `Drop` implementation that does the destruction in a loop:
1089    /// impl<T> Drop for LinkedList<T> {
1090    ///     fn drop(&mut self) {
1091    ///         let mut link = self.0.take();
1092    ///         while let Some(arc_node) = link.take() {
1093    ///             if let Some(Node(_value, next)) = Arc::into_inner(arc_node) {
1094    ///                 link = next;
1095    ///             }
1096    ///         }
1097    ///     }
1098    /// }
1099    ///
1100    /// // Implementation of `new` and `push` omitted
1101    /// impl<T> LinkedList<T> {
1102    ///     /* ... */
1103    /// #   fn new() -> Self {
1104    /// #       LinkedList(None)
1105    /// #   }
1106    /// #   fn push(&mut self, x: T) {
1107    /// #       self.0 = Some(Arc::new(Node(x, self.0.take())));
1108    /// #   }
1109    /// }
1110    ///
1111    /// // The following code could have still caused a stack overflow
1112    /// // despite the manual `Drop` impl if that `Drop` impl had used
1113    /// // `Arc::try_unwrap(arc).ok()` instead of `Arc::into_inner(arc)`.
1114    ///
1115    /// // Create a long list and clone it
1116    /// let mut x = LinkedList::new();
1117    /// let size = 100000;
1118    /// # let size = if cfg!(miri) { 100 } else { size };
1119    /// for i in 0..size {
1120    ///     x.push(i); // Adds i to the front of x
1121    /// }
1122    /// let y = x.clone();
1123    ///
1124    /// // Drop the clones in parallel
1125    /// let x_thread = std::thread::spawn(|| drop(x));
1126    /// let y_thread = std::thread::spawn(|| drop(y));
1127    /// x_thread.join().unwrap();
1128    /// y_thread.join().unwrap();
1129    /// ```
1130    #[inline]
1131    #[stable(feature = "arc_into_inner", since = "1.70.0")]
1132    pub fn into_inner(this: Self) -> Option<T> {
1133        // Make sure that the ordinary `Drop` implementation isn’t called as well
1134        let mut this = mem::ManuallyDrop::new(this);
1135
1136        // Following the implementation of `drop` and `drop_slow`
1137        if this.inner().strong.fetch_sub(1, Release) != 1 {
1138            return None;
1139        }
1140
1141        acquire!(this.inner().strong);
1142
1143        // SAFETY: This mirrors the line
1144        //
1145        //     unsafe { ptr::drop_in_place(Self::get_mut_unchecked(self)) };
1146        //
1147        // in `drop_slow`. Instead of dropping the value behind the pointer,
1148        // it is read and eventually returned; `ptr::read` has the same
1149        // safety conditions as `ptr::drop_in_place`.
1150
1151        let inner = unsafe { ptr::read(Self::get_mut_unchecked(&mut this)) };
1152        let alloc = unsafe { ptr::read(&this.alloc) };
1153
1154        drop(Weak { ptr: this.ptr, alloc });
1155
1156        Some(inner)
1157    }
1158}
1159
1160impl<T> Arc<[T]> {
1161    /// Constructs a new atomically reference-counted slice with uninitialized contents.
1162    ///
1163    /// # Examples
1164    ///
1165    /// ```
1166    /// use std::sync::Arc;
1167    ///
1168    /// let mut values = Arc::<[u32]>::new_uninit_slice(3);
1169    ///
1170    /// // Deferred initialization:
1171    /// let data = Arc::get_mut(&mut values).unwrap();
1172    /// data[0].write(1);
1173    /// data[1].write(2);
1174    /// data[2].write(3);
1175    ///
1176    /// let values = unsafe { values.assume_init() };
1177    ///
1178    /// assert_eq!(*values, [1, 2, 3])
1179    /// ```
1180    #[cfg(not(no_global_oom_handling))]
1181    #[inline]
1182    #[stable(feature = "new_uninit", since = "1.82.0")]
1183    #[must_use]
1184    pub fn new_uninit_slice(len: usize) -> Arc<[mem::MaybeUninit<T>]> {
1185        unsafe { Arc::from_ptr(Arc::allocate_for_slice(len)) }
1186    }
1187
1188    /// Constructs a new atomically reference-counted slice with uninitialized contents, with the memory being
1189    /// filled with `0` bytes.
1190    ///
1191    /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and
1192    /// incorrect usage of this method.
1193    ///
1194    /// # Examples
1195    ///
1196    /// ```
1197    /// use std::sync::Arc;
1198    ///
1199    /// let values = Arc::<[u32]>::new_zeroed_slice(3);
1200    /// let values = unsafe { values.assume_init() };
1201    ///
1202    /// assert_eq!(*values, [0, 0, 0])
1203    /// ```
1204    ///
1205    /// [zeroed]: mem::MaybeUninit::zeroed
1206    #[cfg(not(no_global_oom_handling))]
1207    #[inline]
1208    #[stable(feature = "new_zeroed_alloc", since = "CURRENT_RUSTC_VERSION")]
1209    #[must_use]
1210    pub fn new_zeroed_slice(len: usize) -> Arc<[mem::MaybeUninit<T>]> {
1211        unsafe {
1212            Arc::from_ptr(Arc::allocate_for_layout(
1213                Layout::array::<T>(len).unwrap(),
1214                |layout| Global.allocate_zeroed(layout),
1215                |mem| {
1216                    ptr::slice_from_raw_parts_mut(mem as *mut T, len)
1217                        as *mut ArcInner<[mem::MaybeUninit<T>]>
1218                },
1219            ))
1220        }
1221    }
1222
1223    /// Converts the reference-counted slice into a reference-counted array.
1224    ///
1225    /// This operation does not reallocate; the underlying array of the slice is simply reinterpreted as an array type.
1226    ///
1227    /// If `N` is not exactly equal to the length of `self`, then this method returns `None`.
1228    #[unstable(feature = "slice_as_array", issue = "133508")]
1229    #[inline]
1230    #[must_use]
1231    pub fn into_array<const N: usize>(self) -> Option<Arc<[T; N]>> {
1232        if self.len() == N {
1233            let ptr = Self::into_raw(self) as *const [T; N];
1234
1235            // SAFETY: The underlying array of a slice has the exact same layout as an actual array `[T; N]` if `N` is equal to the slice's length.
1236            let me = unsafe { Arc::from_raw(ptr) };
1237            Some(me)
1238        } else {
1239            None
1240        }
1241    }
1242}
1243
1244impl<T, A: Allocator> Arc<[T], A> {
1245    /// Constructs a new atomically reference-counted slice with uninitialized contents in the
1246    /// provided allocator.
1247    ///
1248    /// # Examples
1249    ///
1250    /// ```
1251    /// #![feature(get_mut_unchecked)]
1252    /// #![feature(allocator_api)]
1253    ///
1254    /// use std::sync::Arc;
1255    /// use std::alloc::System;
1256    ///
1257    /// let mut values = Arc::<[u32], _>::new_uninit_slice_in(3, System);
1258    ///
1259    /// let values = unsafe {
1260    ///     // Deferred initialization:
1261    ///     Arc::get_mut_unchecked(&mut values)[0].as_mut_ptr().write(1);
1262    ///     Arc::get_mut_unchecked(&mut values)[1].as_mut_ptr().write(2);
1263    ///     Arc::get_mut_unchecked(&mut values)[2].as_mut_ptr().write(3);
1264    ///
1265    ///     values.assume_init()
1266    /// };
1267    ///
1268    /// assert_eq!(*values, [1, 2, 3])
1269    /// ```
1270    #[cfg(not(no_global_oom_handling))]
1271    #[unstable(feature = "allocator_api", issue = "32838")]
1272    #[inline]
1273    pub fn new_uninit_slice_in(len: usize, alloc: A) -> Arc<[mem::MaybeUninit<T>], A> {
1274        unsafe { Arc::from_ptr_in(Arc::allocate_for_slice_in(len, &alloc), alloc) }
1275    }
1276
1277    /// Constructs a new atomically reference-counted slice with uninitialized contents, with the memory being
1278    /// filled with `0` bytes, in the provided allocator.
1279    ///
1280    /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and
1281    /// incorrect usage of this method.
1282    ///
1283    /// # Examples
1284    ///
1285    /// ```
1286    /// #![feature(allocator_api)]
1287    ///
1288    /// use std::sync::Arc;
1289    /// use std::alloc::System;
1290    ///
1291    /// let values = Arc::<[u32], _>::new_zeroed_slice_in(3, System);
1292    /// let values = unsafe { values.assume_init() };
1293    ///
1294    /// assert_eq!(*values, [0, 0, 0])
1295    /// ```
1296    ///
1297    /// [zeroed]: mem::MaybeUninit::zeroed
1298    #[cfg(not(no_global_oom_handling))]
1299    #[unstable(feature = "allocator_api", issue = "32838")]
1300    #[inline]
1301    pub fn new_zeroed_slice_in(len: usize, alloc: A) -> Arc<[mem::MaybeUninit<T>], A> {
1302        unsafe {
1303            Arc::from_ptr_in(
1304                Arc::allocate_for_layout(
1305                    Layout::array::<T>(len).unwrap(),
1306                    |layout| alloc.allocate_zeroed(layout),
1307                    |mem| {
1308                        ptr::slice_from_raw_parts_mut(mem.cast::<T>(), len)
1309                            as *mut ArcInner<[mem::MaybeUninit<T>]>
1310                    },
1311                ),
1312                alloc,
1313            )
1314        }
1315    }
1316}
1317
1318impl<T, A: Allocator> Arc<mem::MaybeUninit<T>, A> {
1319    /// Converts to `Arc<T>`.
1320    ///
1321    /// # Safety
1322    ///
1323    /// As with [`MaybeUninit::assume_init`],
1324    /// it is up to the caller to guarantee that the inner value
1325    /// really is in an initialized state.
1326    /// Calling this when the content is not yet fully initialized
1327    /// causes immediate undefined behavior.
1328    ///
1329    /// [`MaybeUninit::assume_init`]: mem::MaybeUninit::assume_init
1330    ///
1331    /// # Examples
1332    ///
1333    /// ```
1334    /// use std::sync::Arc;
1335    ///
1336    /// let mut five = Arc::<u32>::new_uninit();
1337    ///
1338    /// // Deferred initialization:
1339    /// Arc::get_mut(&mut five).unwrap().write(5);
1340    ///
1341    /// let five = unsafe { five.assume_init() };
1342    ///
1343    /// assert_eq!(*five, 5)
1344    /// ```
1345    #[stable(feature = "new_uninit", since = "1.82.0")]
1346    #[must_use = "`self` will be dropped if the result is not used"]
1347    #[inline]
1348    pub unsafe fn assume_init(self) -> Arc<T, A> {
1349        let (ptr, alloc) = Arc::into_inner_with_allocator(self);
1350        unsafe { Arc::from_inner_in(ptr.cast(), alloc) }
1351    }
1352}
1353
1354impl<T, A: Allocator> Arc<[mem::MaybeUninit<T>], A> {
1355    /// Converts to `Arc<[T]>`.
1356    ///
1357    /// # Safety
1358    ///
1359    /// As with [`MaybeUninit::assume_init`],
1360    /// it is up to the caller to guarantee that the inner value
1361    /// really is in an initialized state.
1362    /// Calling this when the content is not yet fully initialized
1363    /// causes immediate undefined behavior.
1364    ///
1365    /// [`MaybeUninit::assume_init`]: mem::MaybeUninit::assume_init
1366    ///
1367    /// # Examples
1368    ///
1369    /// ```
1370    /// use std::sync::Arc;
1371    ///
1372    /// let mut values = Arc::<[u32]>::new_uninit_slice(3);
1373    ///
1374    /// // Deferred initialization:
1375    /// let data = Arc::get_mut(&mut values).unwrap();
1376    /// data[0].write(1);
1377    /// data[1].write(2);
1378    /// data[2].write(3);
1379    ///
1380    /// let values = unsafe { values.assume_init() };
1381    ///
1382    /// assert_eq!(*values, [1, 2, 3])
1383    /// ```
1384    #[stable(feature = "new_uninit", since = "1.82.0")]
1385    #[must_use = "`self` will be dropped if the result is not used"]
1386    #[inline]
1387    pub unsafe fn assume_init(self) -> Arc<[T], A> {
1388        let (ptr, alloc) = Arc::into_inner_with_allocator(self);
1389        unsafe { Arc::from_ptr_in(ptr.as_ptr() as _, alloc) }
1390    }
1391}
1392
1393impl<T: ?Sized> Arc<T> {
1394    /// Constructs an `Arc<T>` from a raw pointer.
1395    ///
1396    /// The raw pointer must have been previously returned by a call to
1397    /// [`Arc<U>::into_raw`][into_raw] with the following requirements:
1398    ///
1399    /// * If `U` is sized, it must have the same size and alignment as `T`. This
1400    ///   is trivially true if `U` is `T`.
1401    /// * If `U` is unsized, its data pointer must have the same size and
1402    ///   alignment as `T`. This is trivially true if `Arc<U>` was constructed
1403    ///   through `Arc<T>` and then converted to `Arc<U>` through an [unsized
1404    ///   coercion].
1405    ///
1406    /// Note that if `U` or `U`'s data pointer is not `T` but has the same size
1407    /// and alignment, this is basically like transmuting references of
1408    /// different types. See [`mem::transmute`][transmute] for more information
1409    /// on what restrictions apply in this case.
1410    ///
1411    /// The raw pointer must point to a block of memory allocated by the global allocator.
1412    ///
1413    /// The user of `from_raw` has to make sure a specific value of `T` is only
1414    /// dropped once.
1415    ///
1416    /// This function is unsafe because improper use may lead to memory unsafety,
1417    /// even if the returned `Arc<T>` is never accessed.
1418    ///
1419    /// [into_raw]: Arc::into_raw
1420    /// [transmute]: core::mem::transmute
1421    /// [unsized coercion]: https://doc.rust-lang.org/reference/type-coercions.html#unsized-coercions
1422    ///
1423    /// # Examples
1424    ///
1425    /// ```
1426    /// use std::sync::Arc;
1427    ///
1428    /// let x = Arc::new("hello".to_owned());
1429    /// let x_ptr = Arc::into_raw(x);
1430    ///
1431    /// unsafe {
1432    ///     // Convert back to an `Arc` to prevent leak.
1433    ///     let x = Arc::from_raw(x_ptr);
1434    ///     assert_eq!(&*x, "hello");
1435    ///
1436    ///     // Further calls to `Arc::from_raw(x_ptr)` would be memory-unsafe.
1437    /// }
1438    ///
1439    /// // The memory was freed when `x` went out of scope above, so `x_ptr` is now dangling!
1440    /// ```
1441    ///
1442    /// Convert a slice back into its original array:
1443    ///
1444    /// ```
1445    /// use std::sync::Arc;
1446    ///
1447    /// let x: Arc<[u32]> = Arc::new([1, 2, 3]);
1448    /// let x_ptr: *const [u32] = Arc::into_raw(x);
1449    ///
1450    /// unsafe {
1451    ///     let x: Arc<[u32; 3]> = Arc::from_raw(x_ptr.cast::<[u32; 3]>());
1452    ///     assert_eq!(&*x, &[1, 2, 3]);
1453    /// }
1454    /// ```
1455    #[inline]
1456    #[stable(feature = "rc_raw", since = "1.17.0")]
1457    pub unsafe fn from_raw(ptr: *const T) -> Self {
1458        unsafe { Arc::from_raw_in(ptr, Global) }
1459    }
1460
1461    /// Consumes the `Arc`, returning the wrapped pointer.
1462    ///
1463    /// To avoid a memory leak the pointer must be converted back to an `Arc` using
1464    /// [`Arc::from_raw`].
1465    ///
1466    /// # Examples
1467    ///
1468    /// ```
1469    /// use std::sync::Arc;
1470    ///
1471    /// let x = Arc::new("hello".to_owned());
1472    /// let x_ptr = Arc::into_raw(x);
1473    /// assert_eq!(unsafe { &*x_ptr }, "hello");
1474    /// # // Prevent leaks for Miri.
1475    /// # drop(unsafe { Arc::from_raw(x_ptr) });
1476    /// ```
1477    #[must_use = "losing the pointer will leak memory"]
1478    #[stable(feature = "rc_raw", since = "1.17.0")]
1479    #[rustc_never_returns_null_ptr]
1480    pub fn into_raw(this: Self) -> *const T {
1481        let this = ManuallyDrop::new(this);
1482        Self::as_ptr(&*this)
1483    }
1484
1485    /// Increments the strong reference count on the `Arc<T>` associated with the
1486    /// provided pointer by one.
1487    ///
1488    /// # Safety
1489    ///
1490    /// The pointer must have been obtained through `Arc::into_raw` and must satisfy the
1491    /// same layout requirements specified in [`Arc::from_raw_in`][from_raw_in].
1492    /// The associated `Arc` instance must be valid (i.e. the strong count must be at
1493    /// least 1) for the duration of this method, and `ptr` must point to a block of memory
1494    /// allocated by the global allocator.
1495    ///
1496    /// [from_raw_in]: Arc::from_raw_in
1497    ///
1498    /// # Examples
1499    ///
1500    /// ```
1501    /// use std::sync::Arc;
1502    ///
1503    /// let five = Arc::new(5);
1504    ///
1505    /// unsafe {
1506    ///     let ptr = Arc::into_raw(five);
1507    ///     Arc::increment_strong_count(ptr);
1508    ///
1509    ///     // This assertion is deterministic because we haven't shared
1510    ///     // the `Arc` between threads.
1511    ///     let five = Arc::from_raw(ptr);
1512    ///     assert_eq!(2, Arc::strong_count(&five));
1513    /// #   // Prevent leaks for Miri.
1514    /// #   Arc::decrement_strong_count(ptr);
1515    /// }
1516    /// ```
1517    #[inline]
1518    #[stable(feature = "arc_mutate_strong_count", since = "1.51.0")]
1519    pub unsafe fn increment_strong_count(ptr: *const T) {
1520        unsafe { Arc::increment_strong_count_in(ptr, Global) }
1521    }
1522
1523    /// Decrements the strong reference count on the `Arc<T>` associated with the
1524    /// provided pointer by one.
1525    ///
1526    /// # Safety
1527    ///
1528    /// The pointer must have been obtained through `Arc::into_raw` and must satisfy the
1529    /// same layout requirements specified in [`Arc::from_raw_in`][from_raw_in].
1530    /// The associated `Arc` instance must be valid (i.e. the strong count must be at
1531    /// least 1) when invoking this method, and `ptr` must point to a block of memory
1532    /// allocated by the global allocator. This method can be used to release the final
1533    /// `Arc` and backing storage, but **should not** be called after the final `Arc` has been
1534    /// released.
1535    ///
1536    /// [from_raw_in]: Arc::from_raw_in
1537    ///
1538    /// # Examples
1539    ///
1540    /// ```
1541    /// use std::sync::Arc;
1542    ///
1543    /// let five = Arc::new(5);
1544    ///
1545    /// unsafe {
1546    ///     let ptr = Arc::into_raw(five);
1547    ///     Arc::increment_strong_count(ptr);
1548    ///
1549    ///     // Those assertions are deterministic because we haven't shared
1550    ///     // the `Arc` between threads.
1551    ///     let five = Arc::from_raw(ptr);
1552    ///     assert_eq!(2, Arc::strong_count(&five));
1553    ///     Arc::decrement_strong_count(ptr);
1554    ///     assert_eq!(1, Arc::strong_count(&five));
1555    /// }
1556    /// ```
1557    #[inline]
1558    #[stable(feature = "arc_mutate_strong_count", since = "1.51.0")]
1559    pub unsafe fn decrement_strong_count(ptr: *const T) {
1560        unsafe { Arc::decrement_strong_count_in(ptr, Global) }
1561    }
1562}
1563
1564impl<T: ?Sized, A: Allocator> Arc<T, A> {
1565    /// Returns a reference to the underlying allocator.
1566    ///
1567    /// Note: this is an associated function, which means that you have
1568    /// to call it as `Arc::allocator(&a)` instead of `a.allocator()`. This
1569    /// is so that there is no conflict with a method on the inner type.
1570    #[inline]
1571    #[unstable(feature = "allocator_api", issue = "32838")]
1572    pub fn allocator(this: &Self) -> &A {
1573        &this.alloc
1574    }
1575
1576    /// Consumes the `Arc`, returning the wrapped pointer and allocator.
1577    ///
1578    /// To avoid a memory leak the pointer must be converted back to an `Arc` using
1579    /// [`Arc::from_raw_in`].
1580    ///
1581    /// # Examples
1582    ///
1583    /// ```
1584    /// #![feature(allocator_api)]
1585    /// use std::sync::Arc;
1586    /// use std::alloc::System;
1587    ///
1588    /// let x = Arc::new_in("hello".to_owned(), System);
1589    /// let (ptr, alloc) = Arc::into_raw_with_allocator(x);
1590    /// assert_eq!(unsafe { &*ptr }, "hello");
1591    /// let x = unsafe { Arc::from_raw_in(ptr, alloc) };
1592    /// assert_eq!(&*x, "hello");
1593    /// ```
1594    #[must_use = "losing the pointer will leak memory"]
1595    #[unstable(feature = "allocator_api", issue = "32838")]
1596    pub fn into_raw_with_allocator(this: Self) -> (*const T, A) {
1597        let this = mem::ManuallyDrop::new(this);
1598        let ptr = Self::as_ptr(&this);
1599        // Safety: `this` is ManuallyDrop so the allocator will not be double-dropped
1600        let alloc = unsafe { ptr::read(&this.alloc) };
1601        (ptr, alloc)
1602    }
1603
1604    /// Provides a raw pointer to the data.
1605    ///
1606    /// The counts are not affected in any way and the `Arc` is not consumed. The pointer is valid for
1607    /// as long as there are strong counts in the `Arc`.
1608    ///
1609    /// # Examples
1610    ///
1611    /// ```
1612    /// use std::sync::Arc;
1613    ///
1614    /// let x = Arc::new("hello".to_owned());
1615    /// let y = Arc::clone(&x);
1616    /// let x_ptr = Arc::as_ptr(&x);
1617    /// assert_eq!(x_ptr, Arc::as_ptr(&y));
1618    /// assert_eq!(unsafe { &*x_ptr }, "hello");
1619    /// ```
1620    #[must_use]
1621    #[stable(feature = "rc_as_ptr", since = "1.45.0")]
1622    #[rustc_never_returns_null_ptr]
1623    pub fn as_ptr(this: &Self) -> *const T {
1624        let ptr: *mut ArcInner<T> = NonNull::as_ptr(this.ptr);
1625
1626        // SAFETY: This cannot go through Deref::deref or ArcInnerPtr::inner because
1627        // this is required to retain raw/mut provenance such that e.g. `get_mut` can
1628        // write through the pointer after the Arc is recovered through `from_raw`.
1629        unsafe { &raw mut (*ptr).data }
1630    }
1631
1632    /// Constructs an `Arc<T, A>` from a raw pointer.
1633    ///
1634    /// The raw pointer must have been previously returned by a call to [`Arc<U,
1635    /// A>::into_raw`][into_raw] with the following requirements:
1636    ///
1637    /// * If `U` is sized, it must have the same size and alignment as `T`. This
1638    ///   is trivially true if `U` is `T`.
1639    /// * If `U` is unsized, its data pointer must have the same size and
1640    ///   alignment as `T`. This is trivially true if `Arc<U>` was constructed
1641    ///   through `Arc<T>` and then converted to `Arc<U>` through an [unsized
1642    ///   coercion].
1643    ///
1644    /// Note that if `U` or `U`'s data pointer is not `T` but has the same size
1645    /// and alignment, this is basically like transmuting references of
1646    /// different types. See [`mem::transmute`][transmute] for more information
1647    /// on what restrictions apply in this case.
1648    ///
1649    /// The raw pointer must point to a block of memory allocated by `alloc`
1650    ///
1651    /// The user of `from_raw` has to make sure a specific value of `T` is only
1652    /// dropped once.
1653    ///
1654    /// This function is unsafe because improper use may lead to memory unsafety,
1655    /// even if the returned `Arc<T>` is never accessed.
1656    ///
1657    /// [into_raw]: Arc::into_raw
1658    /// [transmute]: core::mem::transmute
1659    /// [unsized coercion]: https://doc.rust-lang.org/reference/type-coercions.html#unsized-coercions
1660    ///
1661    /// # Examples
1662    ///
1663    /// ```
1664    /// #![feature(allocator_api)]
1665    ///
1666    /// use std::sync::Arc;
1667    /// use std::alloc::System;
1668    ///
1669    /// let x = Arc::new_in("hello".to_owned(), System);
1670    /// let (x_ptr, alloc) = Arc::into_raw_with_allocator(x);
1671    ///
1672    /// unsafe {
1673    ///     // Convert back to an `Arc` to prevent leak.
1674    ///     let x = Arc::from_raw_in(x_ptr, System);
1675    ///     assert_eq!(&*x, "hello");
1676    ///
1677    ///     // Further calls to `Arc::from_raw(x_ptr)` would be memory-unsafe.
1678    /// }
1679    ///
1680    /// // The memory was freed when `x` went out of scope above, so `x_ptr` is now dangling!
1681    /// ```
1682    ///
1683    /// Convert a slice back into its original array:
1684    ///
1685    /// ```
1686    /// #![feature(allocator_api)]
1687    ///
1688    /// use std::sync::Arc;
1689    /// use std::alloc::System;
1690    ///
1691    /// let x: Arc<[u32], _> = Arc::new_in([1, 2, 3], System);
1692    /// let x_ptr: *const [u32] = Arc::into_raw_with_allocator(x).0;
1693    ///
1694    /// unsafe {
1695    ///     let x: Arc<[u32; 3], _> = Arc::from_raw_in(x_ptr.cast::<[u32; 3]>(), System);
1696    ///     assert_eq!(&*x, &[1, 2, 3]);
1697    /// }
1698    /// ```
1699    #[inline]
1700    #[unstable(feature = "allocator_api", issue = "32838")]
1701    pub unsafe fn from_raw_in(ptr: *const T, alloc: A) -> Self {
1702        unsafe {
1703            let offset = data_offset(ptr);
1704
1705            // Reverse the offset to find the original ArcInner.
1706            let arc_ptr = ptr.byte_sub(offset) as *mut ArcInner<T>;
1707
1708            Self::from_ptr_in(arc_ptr, alloc)
1709        }
1710    }
1711
1712    /// Creates a new [`Weak`] pointer to this allocation.
1713    ///
1714    /// # Examples
1715    ///
1716    /// ```
1717    /// use std::sync::Arc;
1718    ///
1719    /// let five = Arc::new(5);
1720    ///
1721    /// let weak_five = Arc::downgrade(&five);
1722    /// ```
1723    #[must_use = "this returns a new `Weak` pointer, \
1724                  without modifying the original `Arc`"]
1725    #[stable(feature = "arc_weak", since = "1.4.0")]
1726    pub fn downgrade(this: &Self) -> Weak<T, A>
1727    where
1728        A: Clone,
1729    {
1730        // This Relaxed is OK because we're checking the value in the CAS
1731        // below.
1732        let mut cur = this.inner().weak.load(Relaxed);
1733
1734        loop {
1735            // check if the weak counter is currently "locked"; if so, spin.
1736            if cur == usize::MAX {
1737                hint::spin_loop();
1738                cur = this.inner().weak.load(Relaxed);
1739                continue;
1740            }
1741
1742            // We can't allow the refcount to increase much past `MAX_REFCOUNT`.
1743            assert!(cur <= MAX_REFCOUNT, "{}", INTERNAL_OVERFLOW_ERROR);
1744
1745            // NOTE: this code currently ignores the possibility of overflow
1746            // into usize::MAX; in general both Rc and Arc need to be adjusted
1747            // to deal with overflow.
1748
1749            // Unlike with Clone(), we need this to be an Acquire read to
1750            // synchronize with the write coming from `is_unique`, so that the
1751            // events prior to that write happen before this read.
1752            match this.inner().weak.compare_exchange_weak(cur, cur + 1, Acquire, Relaxed) {
1753                Ok(_) => {
1754                    // Make sure we do not create a dangling Weak
1755                    debug_assert!(!is_dangling(this.ptr.as_ptr()));
1756                    return Weak { ptr: this.ptr, alloc: this.alloc.clone() };
1757                }
1758                Err(old) => cur = old,
1759            }
1760        }
1761    }
1762
1763    /// Gets the number of [`Weak`] pointers to this allocation.
1764    ///
1765    /// # Safety
1766    ///
1767    /// This method by itself is safe, but using it correctly requires extra care.
1768    /// Another thread can change the weak count at any time,
1769    /// including potentially between calling this method and acting on the result.
1770    ///
1771    /// # Examples
1772    ///
1773    /// ```
1774    /// use std::sync::Arc;
1775    ///
1776    /// let five = Arc::new(5);
1777    /// let _weak_five = Arc::downgrade(&five);
1778    ///
1779    /// // This assertion is deterministic because we haven't shared
1780    /// // the `Arc` or `Weak` between threads.
1781    /// assert_eq!(1, Arc::weak_count(&five));
1782    /// ```
1783    #[inline]
1784    #[must_use]
1785    #[stable(feature = "arc_counts", since = "1.15.0")]
1786    pub fn weak_count(this: &Self) -> usize {
1787        let cnt = this.inner().weak.load(Relaxed);
1788        // If the weak count is currently locked, the value of the
1789        // count was 0 just before taking the lock.
1790        if cnt == usize::MAX { 0 } else { cnt - 1 }
1791    }
1792
1793    /// Gets the number of strong (`Arc`) pointers to this allocation.
1794    ///
1795    /// # Safety
1796    ///
1797    /// This method by itself is safe, but using it correctly requires extra care.
1798    /// Another thread can change the strong count at any time,
1799    /// including potentially between calling this method and acting on the result.
1800    ///
1801    /// # Examples
1802    ///
1803    /// ```
1804    /// use std::sync::Arc;
1805    ///
1806    /// let five = Arc::new(5);
1807    /// let _also_five = Arc::clone(&five);
1808    ///
1809    /// // This assertion is deterministic because we haven't shared
1810    /// // the `Arc` between threads.
1811    /// assert_eq!(2, Arc::strong_count(&five));
1812    /// ```
1813    #[inline]
1814    #[must_use]
1815    #[stable(feature = "arc_counts", since = "1.15.0")]
1816    pub fn strong_count(this: &Self) -> usize {
1817        this.inner().strong.load(Relaxed)
1818    }
1819
1820    /// Increments the strong reference count on the `Arc<T>` associated with the
1821    /// provided pointer by one.
1822    ///
1823    /// # Safety
1824    ///
1825    /// The pointer must have been obtained through `Arc::into_raw` and must satisfy the
1826    /// same layout requirements specified in [`Arc::from_raw_in`][from_raw_in].
1827    /// The associated `Arc` instance must be valid (i.e. the strong count must be at
1828    /// least 1) for the duration of this method, and `ptr` must point to a block of memory
1829    /// allocated by `alloc`.
1830    ///
1831    /// [from_raw_in]: Arc::from_raw_in
1832    ///
1833    /// # Examples
1834    ///
1835    /// ```
1836    /// #![feature(allocator_api)]
1837    ///
1838    /// use std::sync::Arc;
1839    /// use std::alloc::System;
1840    ///
1841    /// let five = Arc::new_in(5, System);
1842    ///
1843    /// unsafe {
1844    ///     let (ptr, _alloc) = Arc::into_raw_with_allocator(five);
1845    ///     Arc::increment_strong_count_in(ptr, System);
1846    ///
1847    ///     // This assertion is deterministic because we haven't shared
1848    ///     // the `Arc` between threads.
1849    ///     let five = Arc::from_raw_in(ptr, System);
1850    ///     assert_eq!(2, Arc::strong_count(&five));
1851    /// #   // Prevent leaks for Miri.
1852    /// #   Arc::decrement_strong_count_in(ptr, System);
1853    /// }
1854    /// ```
1855    #[inline]
1856    #[unstable(feature = "allocator_api", issue = "32838")]
1857    pub unsafe fn increment_strong_count_in(ptr: *const T, alloc: A)
1858    where
1859        A: Clone,
1860    {
1861        // Retain Arc, but don't touch refcount by wrapping in ManuallyDrop
1862        let arc = unsafe { mem::ManuallyDrop::new(Arc::from_raw_in(ptr, alloc)) };
1863        // Now increase refcount, but don't drop new refcount either
1864        let _arc_clone: mem::ManuallyDrop<_> = arc.clone();
1865    }
1866
1867    /// Decrements the strong reference count on the `Arc<T>` associated with the
1868    /// provided pointer by one.
1869    ///
1870    /// # Safety
1871    ///
1872    /// The pointer must have been obtained through `Arc::into_raw` and must satisfy the
1873    /// same layout requirements specified in [`Arc::from_raw_in`][from_raw_in].
1874    /// The associated `Arc` instance must be valid (i.e. the strong count must be at
1875    /// least 1) when invoking this method, and `ptr` must point to a block of memory
1876    /// allocated by `alloc`. This method can be used to release the final
1877    /// `Arc` and backing storage, but **should not** be called after the final `Arc` has been
1878    /// released.
1879    ///
1880    /// [from_raw_in]: Arc::from_raw_in
1881    ///
1882    /// # Examples
1883    ///
1884    /// ```
1885    /// #![feature(allocator_api)]
1886    ///
1887    /// use std::sync::Arc;
1888    /// use std::alloc::System;
1889    ///
1890    /// let five = Arc::new_in(5, System);
1891    ///
1892    /// unsafe {
1893    ///     let (ptr, _alloc) = Arc::into_raw_with_allocator(five);
1894    ///     Arc::increment_strong_count_in(ptr, System);
1895    ///
1896    ///     // Those assertions are deterministic because we haven't shared
1897    ///     // the `Arc` between threads.
1898    ///     let five = Arc::from_raw_in(ptr, System);
1899    ///     assert_eq!(2, Arc::strong_count(&five));
1900    ///     Arc::decrement_strong_count_in(ptr, System);
1901    ///     assert_eq!(1, Arc::strong_count(&five));
1902    /// }
1903    /// ```
1904    #[inline]
1905    #[unstable(feature = "allocator_api", issue = "32838")]
1906    pub unsafe fn decrement_strong_count_in(ptr: *const T, alloc: A) {
1907        unsafe { drop(Arc::from_raw_in(ptr, alloc)) };
1908    }
1909
1910    #[inline]
1911    fn inner(&self) -> &ArcInner<T> {
1912        // This unsafety is ok because while this arc is alive we're guaranteed
1913        // that the inner pointer is valid. Furthermore, we know that the
1914        // `ArcInner` structure itself is `Sync` because the inner data is
1915        // `Sync` as well, so we're ok loaning out an immutable pointer to these
1916        // contents.
1917        unsafe { self.ptr.as_ref() }
1918    }
1919
1920    // Non-inlined part of `drop`.
1921    #[inline(never)]
1922    unsafe fn drop_slow(&mut self) {
1923        // Drop the weak ref collectively held by all strong references when this
1924        // variable goes out of scope. This ensures that the memory is deallocated
1925        // even if the destructor of `T` panics.
1926        // Take a reference to `self.alloc` instead of cloning because 1. it'll last long
1927        // enough, and 2. you should be able to drop `Arc`s with unclonable allocators
1928        let _weak = Weak { ptr: self.ptr, alloc: &self.alloc };
1929
1930        // Destroy the data at this time, even though we must not free the box
1931        // allocation itself (there might still be weak pointers lying around).
1932        // We cannot use `get_mut_unchecked` here, because `self.alloc` is borrowed.
1933        unsafe { ptr::drop_in_place(&mut (*self.ptr.as_ptr()).data) };
1934    }
1935
1936    /// Returns `true` if the two `Arc`s point to the same allocation in a vein similar to
1937    /// [`ptr::eq`]. This function ignores the metadata of  `dyn Trait` pointers.
1938    ///
1939    /// # Examples
1940    ///
1941    /// ```
1942    /// use std::sync::Arc;
1943    ///
1944    /// let five = Arc::new(5);
1945    /// let same_five = Arc::clone(&five);
1946    /// let other_five = Arc::new(5);
1947    ///
1948    /// assert!(Arc::ptr_eq(&five, &same_five));
1949    /// assert!(!Arc::ptr_eq(&five, &other_five));
1950    /// ```
1951    ///
1952    /// [`ptr::eq`]: core::ptr::eq "ptr::eq"
1953    #[inline]
1954    #[must_use]
1955    #[stable(feature = "ptr_eq", since = "1.17.0")]
1956    pub fn ptr_eq(this: &Self, other: &Self) -> bool {
1957        ptr::addr_eq(this.ptr.as_ptr(), other.ptr.as_ptr())
1958    }
1959}
1960
1961impl<T: ?Sized> Arc<T> {
1962    /// Allocates an `ArcInner<T>` with sufficient space for
1963    /// a possibly-unsized inner value where the value has the layout provided.
1964    ///
1965    /// The function `mem_to_arcinner` is called with the data pointer
1966    /// and must return back a (potentially fat)-pointer for the `ArcInner<T>`.
1967    #[cfg(not(no_global_oom_handling))]
1968    unsafe fn allocate_for_layout(
1969        value_layout: Layout,
1970        allocate: impl FnOnce(Layout) -> Result<NonNull<[u8]>, AllocError>,
1971        mem_to_arcinner: impl FnOnce(*mut u8) -> *mut ArcInner<T>,
1972    ) -> *mut ArcInner<T> {
1973        let layout = arcinner_layout_for_value_layout(value_layout);
1974
1975        let ptr = allocate(layout).unwrap_or_else(|_| handle_alloc_error(layout));
1976
1977        unsafe { Self::initialize_arcinner(ptr, layout, mem_to_arcinner) }
1978    }
1979
1980    /// Allocates an `ArcInner<T>` with sufficient space for
1981    /// a possibly-unsized inner value where the value has the layout provided,
1982    /// returning an error if allocation fails.
1983    ///
1984    /// The function `mem_to_arcinner` is called with the data pointer
1985    /// and must return back a (potentially fat)-pointer for the `ArcInner<T>`.
1986    unsafe fn try_allocate_for_layout(
1987        value_layout: Layout,
1988        allocate: impl FnOnce(Layout) -> Result<NonNull<[u8]>, AllocError>,
1989        mem_to_arcinner: impl FnOnce(*mut u8) -> *mut ArcInner<T>,
1990    ) -> Result<*mut ArcInner<T>, AllocError> {
1991        let layout = arcinner_layout_for_value_layout(value_layout);
1992
1993        let ptr = allocate(layout)?;
1994
1995        let inner = unsafe { Self::initialize_arcinner(ptr, layout, mem_to_arcinner) };
1996
1997        Ok(inner)
1998    }
1999
2000    unsafe fn initialize_arcinner(
2001        ptr: NonNull<[u8]>,
2002        layout: Layout,
2003        mem_to_arcinner: impl FnOnce(*mut u8) -> *mut ArcInner<T>,
2004    ) -> *mut ArcInner<T> {
2005        let inner = mem_to_arcinner(ptr.as_non_null_ptr().as_ptr());
2006        debug_assert_eq!(unsafe { Layout::for_value_raw(inner) }, layout);
2007
2008        unsafe {
2009            (&raw mut (*inner).strong).write(atomic::AtomicUsize::new(1));
2010            (&raw mut (*inner).weak).write(atomic::AtomicUsize::new(1));
2011        }
2012
2013        inner
2014    }
2015}
2016
2017impl<T: ?Sized, A: Allocator> Arc<T, A> {
2018    /// Allocates an `ArcInner<T>` with sufficient space for an unsized inner value.
2019    #[inline]
2020    #[cfg(not(no_global_oom_handling))]
2021    unsafe fn allocate_for_ptr_in(ptr: *const T, alloc: &A) -> *mut ArcInner<T> {
2022        // Allocate for the `ArcInner<T>` using the given value.
2023        unsafe {
2024            Arc::allocate_for_layout(
2025                Layout::for_value_raw(ptr),
2026                |layout| alloc.allocate(layout),
2027                |mem| mem.with_metadata_of(ptr as *const ArcInner<T>),
2028            )
2029        }
2030    }
2031
2032    #[cfg(not(no_global_oom_handling))]
2033    fn from_box_in(src: Box<T, A>) -> Arc<T, A> {
2034        unsafe {
2035            let value_size = size_of_val(&*src);
2036            let ptr = Self::allocate_for_ptr_in(&*src, Box::allocator(&src));
2037
2038            // Copy value as bytes
2039            ptr::copy_nonoverlapping(
2040                (&raw const *src) as *const u8,
2041                (&raw mut (*ptr).data) as *mut u8,
2042                value_size,
2043            );
2044
2045            // Free the allocation without dropping its contents
2046            let (bptr, alloc) = Box::into_raw_with_allocator(src);
2047            let src = Box::from_raw_in(bptr as *mut mem::ManuallyDrop<T>, alloc.by_ref());
2048            drop(src);
2049
2050            Self::from_ptr_in(ptr, alloc)
2051        }
2052    }
2053}
2054
2055impl<T> Arc<[T]> {
2056    /// Allocates an `ArcInner<[T]>` with the given length.
2057    #[cfg(not(no_global_oom_handling))]
2058    unsafe fn allocate_for_slice(len: usize) -> *mut ArcInner<[T]> {
2059        unsafe {
2060            Self::allocate_for_layout(
2061                Layout::array::<T>(len).unwrap(),
2062                |layout| Global.allocate(layout),
2063                |mem| ptr::slice_from_raw_parts_mut(mem.cast::<T>(), len) as *mut ArcInner<[T]>,
2064            )
2065        }
2066    }
2067
2068    /// Copy elements from slice into newly allocated `Arc<[T]>`
2069    ///
2070    /// Unsafe because the caller must either take ownership or bind `T: Copy`.
2071    #[cfg(not(no_global_oom_handling))]
2072    unsafe fn copy_from_slice(v: &[T]) -> Arc<[T]> {
2073        unsafe {
2074            let ptr = Self::allocate_for_slice(v.len());
2075
2076            ptr::copy_nonoverlapping(v.as_ptr(), (&raw mut (*ptr).data) as *mut T, v.len());
2077
2078            Self::from_ptr(ptr)
2079        }
2080    }
2081
2082    /// Constructs an `Arc<[T]>` from an iterator known to be of a certain size.
2083    ///
2084    /// Behavior is undefined should the size be wrong.
2085    #[cfg(not(no_global_oom_handling))]
2086    unsafe fn from_iter_exact(iter: impl Iterator<Item = T>, len: usize) -> Arc<[T]> {
2087        // Panic guard while cloning T elements.
2088        // In the event of a panic, elements that have been written
2089        // into the new ArcInner will be dropped, then the memory freed.
2090        struct Guard<T> {
2091            mem: NonNull<u8>,
2092            elems: *mut T,
2093            layout: Layout,
2094            n_elems: usize,
2095        }
2096
2097        impl<T> Drop for Guard<T> {
2098            fn drop(&mut self) {
2099                unsafe {
2100                    let slice = from_raw_parts_mut(self.elems, self.n_elems);
2101                    ptr::drop_in_place(slice);
2102
2103                    Global.deallocate(self.mem, self.layout);
2104                }
2105            }
2106        }
2107
2108        unsafe {
2109            let ptr = Self::allocate_for_slice(len);
2110
2111            let mem = ptr as *mut _ as *mut u8;
2112            let layout = Layout::for_value_raw(ptr);
2113
2114            // Pointer to first element
2115            let elems = (&raw mut (*ptr).data) as *mut T;
2116
2117            let mut guard = Guard { mem: NonNull::new_unchecked(mem), elems, layout, n_elems: 0 };
2118
2119            for (i, item) in iter.enumerate() {
2120                ptr::write(elems.add(i), item);
2121                guard.n_elems += 1;
2122            }
2123
2124            // All clear. Forget the guard so it doesn't free the new ArcInner.
2125            mem::forget(guard);
2126
2127            Self::from_ptr(ptr)
2128        }
2129    }
2130}
2131
2132impl<T, A: Allocator> Arc<[T], A> {
2133    /// Allocates an `ArcInner<[T]>` with the given length.
2134    #[inline]
2135    #[cfg(not(no_global_oom_handling))]
2136    unsafe fn allocate_for_slice_in(len: usize, alloc: &A) -> *mut ArcInner<[T]> {
2137        unsafe {
2138            Arc::allocate_for_layout(
2139                Layout::array::<T>(len).unwrap(),
2140                |layout| alloc.allocate(layout),
2141                |mem| ptr::slice_from_raw_parts_mut(mem.cast::<T>(), len) as *mut ArcInner<[T]>,
2142            )
2143        }
2144    }
2145}
2146
2147/// Specialization trait used for `From<&[T]>`.
2148#[cfg(not(no_global_oom_handling))]
2149trait ArcFromSlice<T> {
2150    fn from_slice(slice: &[T]) -> Self;
2151}
2152
2153#[cfg(not(no_global_oom_handling))]
2154impl<T: Clone> ArcFromSlice<T> for Arc<[T]> {
2155    #[inline]
2156    default fn from_slice(v: &[T]) -> Self {
2157        unsafe { Self::from_iter_exact(v.iter().cloned(), v.len()) }
2158    }
2159}
2160
2161#[cfg(not(no_global_oom_handling))]
2162impl<T: Copy> ArcFromSlice<T> for Arc<[T]> {
2163    #[inline]
2164    fn from_slice(v: &[T]) -> Self {
2165        unsafe { Arc::copy_from_slice(v) }
2166    }
2167}
2168
2169#[stable(feature = "rust1", since = "1.0.0")]
2170impl<T: ?Sized, A: Allocator + Clone> Clone for Arc<T, A> {
2171    /// Makes a clone of the `Arc` pointer.
2172    ///
2173    /// This creates another pointer to the same allocation, increasing the
2174    /// strong reference count.
2175    ///
2176    /// # Examples
2177    ///
2178    /// ```
2179    /// use std::sync::Arc;
2180    ///
2181    /// let five = Arc::new(5);
2182    ///
2183    /// let _ = Arc::clone(&five);
2184    /// ```
2185    #[inline]
2186    fn clone(&self) -> Arc<T, A> {
2187        // Using a relaxed ordering is alright here, as knowledge of the
2188        // original reference prevents other threads from erroneously deleting
2189        // the object.
2190        //
2191        // As explained in the [Boost documentation][1], Increasing the
2192        // reference counter can always be done with memory_order_relaxed: New
2193        // references to an object can only be formed from an existing
2194        // reference, and passing an existing reference from one thread to
2195        // another must already provide any required synchronization.
2196        //
2197        // [1]: (www.boost.org/doc/libs/1_55_0/doc/html/atomic/usage_examples.html)
2198        let old_size = self.inner().strong.fetch_add(1, Relaxed);
2199
2200        // However we need to guard against massive refcounts in case someone is `mem::forget`ing
2201        // Arcs. If we don't do this the count can overflow and users will use-after free. This
2202        // branch will never be taken in any realistic program. We abort because such a program is
2203        // incredibly degenerate, and we don't care to support it.
2204        //
2205        // This check is not 100% water-proof: we error when the refcount grows beyond `isize::MAX`.
2206        // But we do that check *after* having done the increment, so there is a chance here that
2207        // the worst already happened and we actually do overflow the `usize` counter. However, that
2208        // requires the counter to grow from `isize::MAX` to `usize::MAX` between the increment
2209        // above and the `abort` below, which seems exceedingly unlikely.
2210        //
2211        // This is a global invariant, and also applies when using a compare-exchange loop to increment
2212        // counters in other methods.
2213        // Otherwise, the counter could be brought to an almost-overflow using a compare-exchange loop,
2214        // and then overflow using a few `fetch_add`s.
2215        if old_size > MAX_REFCOUNT {
2216            abort();
2217        }
2218
2219        unsafe { Self::from_inner_in(self.ptr, self.alloc.clone()) }
2220    }
2221}
2222
2223#[unstable(feature = "ergonomic_clones", issue = "132290")]
2224impl<T: ?Sized, A: Allocator + Clone> UseCloned for Arc<T, A> {}
2225
2226#[stable(feature = "rust1", since = "1.0.0")]
2227impl<T: ?Sized, A: Allocator> Deref for Arc<T, A> {
2228    type Target = T;
2229
2230    #[inline]
2231    fn deref(&self) -> &T {
2232        &self.inner().data
2233    }
2234}
2235
2236#[unstable(feature = "pin_coerce_unsized_trait", issue = "123430")]
2237unsafe impl<T: ?Sized, A: Allocator> PinCoerceUnsized for Arc<T, A> {}
2238
2239#[unstable(feature = "pin_coerce_unsized_trait", issue = "123430")]
2240unsafe impl<T: ?Sized, A: Allocator> PinCoerceUnsized for Weak<T, A> {}
2241
2242#[unstable(feature = "deref_pure_trait", issue = "87121")]
2243unsafe impl<T: ?Sized, A: Allocator> DerefPure for Arc<T, A> {}
2244
2245#[unstable(feature = "legacy_receiver_trait", issue = "none")]
2246impl<T: ?Sized> LegacyReceiver for Arc<T> {}
2247
2248#[cfg(not(no_global_oom_handling))]
2249impl<T: ?Sized + CloneToUninit, A: Allocator + Clone> Arc<T, A> {
2250    /// Makes a mutable reference into the given `Arc`.
2251    ///
2252    /// If there are other `Arc` pointers to the same allocation, then `make_mut` will
2253    /// [`clone`] the inner value to a new allocation to ensure unique ownership.  This is also
2254    /// referred to as clone-on-write.
2255    ///
2256    /// However, if there are no other `Arc` pointers to this allocation, but some [`Weak`]
2257    /// pointers, then the [`Weak`] pointers will be dissociated and the inner value will not
2258    /// be cloned.
2259    ///
2260    /// See also [`get_mut`], which will fail rather than cloning the inner value
2261    /// or dissociating [`Weak`] pointers.
2262    ///
2263    /// [`clone`]: Clone::clone
2264    /// [`get_mut`]: Arc::get_mut
2265    ///
2266    /// # Examples
2267    ///
2268    /// ```
2269    /// use std::sync::Arc;
2270    ///
2271    /// let mut data = Arc::new(5);
2272    ///
2273    /// *Arc::make_mut(&mut data) += 1;         // Won't clone anything
2274    /// let mut other_data = Arc::clone(&data); // Won't clone inner data
2275    /// *Arc::make_mut(&mut data) += 1;         // Clones inner data
2276    /// *Arc::make_mut(&mut data) += 1;         // Won't clone anything
2277    /// *Arc::make_mut(&mut other_data) *= 2;   // Won't clone anything
2278    ///
2279    /// // Now `data` and `other_data` point to different allocations.
2280    /// assert_eq!(*data, 8);
2281    /// assert_eq!(*other_data, 12);
2282    /// ```
2283    ///
2284    /// [`Weak`] pointers will be dissociated:
2285    ///
2286    /// ```
2287    /// use std::sync::Arc;
2288    ///
2289    /// let mut data = Arc::new(75);
2290    /// let weak = Arc::downgrade(&data);
2291    ///
2292    /// assert!(75 == *data);
2293    /// assert!(75 == *weak.upgrade().unwrap());
2294    ///
2295    /// *Arc::make_mut(&mut data) += 1;
2296    ///
2297    /// assert!(76 == *data);
2298    /// assert!(weak.upgrade().is_none());
2299    /// ```
2300    #[inline]
2301    #[stable(feature = "arc_unique", since = "1.4.0")]
2302    pub fn make_mut(this: &mut Self) -> &mut T {
2303        let size_of_val = size_of_val::<T>(&**this);
2304
2305        // Note that we hold both a strong reference and a weak reference.
2306        // Thus, releasing our strong reference only will not, by itself, cause
2307        // the memory to be deallocated.
2308        //
2309        // Use Acquire to ensure that we see any writes to `weak` that happen
2310        // before release writes (i.e., decrements) to `strong`. Since we hold a
2311        // weak count, there's no chance the ArcInner itself could be
2312        // deallocated.
2313        if this.inner().strong.compare_exchange(1, 0, Acquire, Relaxed).is_err() {
2314            // Another strong pointer exists, so we must clone.
2315
2316            let this_data_ref: &T = &**this;
2317            // `in_progress` drops the allocation if we panic before finishing initializing it.
2318            let mut in_progress: UniqueArcUninit<T, A> =
2319                UniqueArcUninit::new(this_data_ref, this.alloc.clone());
2320
2321            let initialized_clone = unsafe {
2322                // Clone. If the clone panics, `in_progress` will be dropped and clean up.
2323                this_data_ref.clone_to_uninit(in_progress.data_ptr().cast());
2324                // Cast type of pointer, now that it is initialized.
2325                in_progress.into_arc()
2326            };
2327            *this = initialized_clone;
2328        } else if this.inner().weak.load(Relaxed) != 1 {
2329            // Relaxed suffices in the above because this is fundamentally an
2330            // optimization: we are always racing with weak pointers being
2331            // dropped. Worst case, we end up allocated a new Arc unnecessarily.
2332
2333            // We removed the last strong ref, but there are additional weak
2334            // refs remaining. We'll move the contents to a new Arc, and
2335            // invalidate the other weak refs.
2336
2337            // Note that it is not possible for the read of `weak` to yield
2338            // usize::MAX (i.e., locked), since the weak count can only be
2339            // locked by a thread with a strong reference.
2340
2341            // Materialize our own implicit weak pointer, so that it can clean
2342            // up the ArcInner as needed.
2343            let _weak = Weak { ptr: this.ptr, alloc: this.alloc.clone() };
2344
2345            // Can just steal the data, all that's left is Weaks
2346            //
2347            // We don't need panic-protection like the above branch does, but we might as well
2348            // use the same mechanism.
2349            let mut in_progress: UniqueArcUninit<T, A> =
2350                UniqueArcUninit::new(&**this, this.alloc.clone());
2351            unsafe {
2352                // Initialize `in_progress` with move of **this.
2353                // We have to express this in terms of bytes because `T: ?Sized`; there is no
2354                // operation that just copies a value based on its `size_of_val()`.
2355                ptr::copy_nonoverlapping(
2356                    ptr::from_ref(&**this).cast::<u8>(),
2357                    in_progress.data_ptr().cast::<u8>(),
2358                    size_of_val,
2359                );
2360
2361                ptr::write(this, in_progress.into_arc());
2362            }
2363        } else {
2364            // We were the sole reference of either kind; bump back up the
2365            // strong ref count.
2366            this.inner().strong.store(1, Release);
2367        }
2368
2369        // As with `get_mut()`, the unsafety is ok because our reference was
2370        // either unique to begin with, or became one upon cloning the contents.
2371        unsafe { Self::get_mut_unchecked(this) }
2372    }
2373}
2374
2375impl<T: Clone, A: Allocator> Arc<T, A> {
2376    /// If we have the only reference to `T` then unwrap it. Otherwise, clone `T` and return the
2377    /// clone.
2378    ///
2379    /// Assuming `arc_t` is of type `Arc<T>`, this function is functionally equivalent to
2380    /// `(*arc_t).clone()`, but will avoid cloning the inner value where possible.
2381    ///
2382    /// # Examples
2383    ///
2384    /// ```
2385    /// # use std::{ptr, sync::Arc};
2386    /// let inner = String::from("test");
2387    /// let ptr = inner.as_ptr();
2388    ///
2389    /// let arc = Arc::new(inner);
2390    /// let inner = Arc::unwrap_or_clone(arc);
2391    /// // The inner value was not cloned
2392    /// assert!(ptr::eq(ptr, inner.as_ptr()));
2393    ///
2394    /// let arc = Arc::new(inner);
2395    /// let arc2 = arc.clone();
2396    /// let inner = Arc::unwrap_or_clone(arc);
2397    /// // Because there were 2 references, we had to clone the inner value.
2398    /// assert!(!ptr::eq(ptr, inner.as_ptr()));
2399    /// // `arc2` is the last reference, so when we unwrap it we get back
2400    /// // the original `String`.
2401    /// let inner = Arc::unwrap_or_clone(arc2);
2402    /// assert!(ptr::eq(ptr, inner.as_ptr()));
2403    /// ```
2404    #[inline]
2405    #[stable(feature = "arc_unwrap_or_clone", since = "1.76.0")]
2406    pub fn unwrap_or_clone(this: Self) -> T {
2407        Arc::try_unwrap(this).unwrap_or_else(|arc| (*arc).clone())
2408    }
2409}
2410
2411impl<T: ?Sized, A: Allocator> Arc<T, A> {
2412    /// Returns a mutable reference into the given `Arc`, if there are
2413    /// no other `Arc` or [`Weak`] pointers to the same allocation.
2414    ///
2415    /// Returns [`None`] otherwise, because it is not safe to
2416    /// mutate a shared value.
2417    ///
2418    /// See also [`make_mut`][make_mut], which will [`clone`][clone]
2419    /// the inner value when there are other `Arc` pointers.
2420    ///
2421    /// [make_mut]: Arc::make_mut
2422    /// [clone]: Clone::clone
2423    ///
2424    /// # Examples
2425    ///
2426    /// ```
2427    /// use std::sync::Arc;
2428    ///
2429    /// let mut x = Arc::new(3);
2430    /// *Arc::get_mut(&mut x).unwrap() = 4;
2431    /// assert_eq!(*x, 4);
2432    ///
2433    /// let _y = Arc::clone(&x);
2434    /// assert!(Arc::get_mut(&mut x).is_none());
2435    /// ```
2436    #[inline]
2437    #[stable(feature = "arc_unique", since = "1.4.0")]
2438    pub fn get_mut(this: &mut Self) -> Option<&mut T> {
2439        if Self::is_unique(this) {
2440            // This unsafety is ok because we're guaranteed that the pointer
2441            // returned is the *only* pointer that will ever be returned to T. Our
2442            // reference count is guaranteed to be 1 at this point, and we required
2443            // the Arc itself to be `mut`, so we're returning the only possible
2444            // reference to the inner data.
2445            unsafe { Some(Arc::get_mut_unchecked(this)) }
2446        } else {
2447            None
2448        }
2449    }
2450
2451    /// Returns a mutable reference into the given `Arc`,
2452    /// without any check.
2453    ///
2454    /// See also [`get_mut`], which is safe and does appropriate checks.
2455    ///
2456    /// [`get_mut`]: Arc::get_mut
2457    ///
2458    /// # Safety
2459    ///
2460    /// If any other `Arc` or [`Weak`] pointers to the same allocation exist, then
2461    /// they must not be dereferenced or have active borrows for the duration
2462    /// of the returned borrow, and their inner type must be exactly the same as the
2463    /// inner type of this Arc (including lifetimes). This is trivially the case if no
2464    /// such pointers exist, for example immediately after `Arc::new`.
2465    ///
2466    /// # Examples
2467    ///
2468    /// ```
2469    /// #![feature(get_mut_unchecked)]
2470    ///
2471    /// use std::sync::Arc;
2472    ///
2473    /// let mut x = Arc::new(String::new());
2474    /// unsafe {
2475    ///     Arc::get_mut_unchecked(&mut x).push_str("foo")
2476    /// }
2477    /// assert_eq!(*x, "foo");
2478    /// ```
2479    /// Other `Arc` pointers to the same allocation must be to the same type.
2480    /// ```no_run
2481    /// #![feature(get_mut_unchecked)]
2482    ///
2483    /// use std::sync::Arc;
2484    ///
2485    /// let x: Arc<str> = Arc::from("Hello, world!");
2486    /// let mut y: Arc<[u8]> = x.clone().into();
2487    /// unsafe {
2488    ///     // this is Undefined Behavior, because x's inner type is str, not [u8]
2489    ///     Arc::get_mut_unchecked(&mut y).fill(0xff); // 0xff is invalid in UTF-8
2490    /// }
2491    /// println!("{}", &*x); // Invalid UTF-8 in a str
2492    /// ```
2493    /// Other `Arc` pointers to the same allocation must be to the exact same type, including lifetimes.
2494    /// ```no_run
2495    /// #![feature(get_mut_unchecked)]
2496    ///
2497    /// use std::sync::Arc;
2498    ///
2499    /// let x: Arc<&str> = Arc::new("Hello, world!");
2500    /// {
2501    ///     let s = String::from("Oh, no!");
2502    ///     let mut y: Arc<&str> = x.clone();
2503    ///     unsafe {
2504    ///         // this is Undefined Behavior, because x's inner type
2505    ///         // is &'long str, not &'short str
2506    ///         *Arc::get_mut_unchecked(&mut y) = &s;
2507    ///     }
2508    /// }
2509    /// println!("{}", &*x); // Use-after-free
2510    /// ```
2511    #[inline]
2512    #[unstable(feature = "get_mut_unchecked", issue = "63292")]
2513    pub unsafe fn get_mut_unchecked(this: &mut Self) -> &mut T {
2514        // We are careful to *not* create a reference covering the "count" fields, as
2515        // this would alias with concurrent access to the reference counts (e.g. by `Weak`).
2516        unsafe { &mut (*this.ptr.as_ptr()).data }
2517    }
2518
2519    /// Determine whether this is the unique reference to the underlying data.
2520    ///
2521    /// Returns `true` if there are no other `Arc` or [`Weak`] pointers to the same allocation;
2522    /// returns `false` otherwise.
2523    ///
2524    /// If this function returns `true`, then is guaranteed to be safe to call [`get_mut_unchecked`]
2525    /// on this `Arc`, so long as no clones occur in between.
2526    ///
2527    /// # Examples
2528    ///
2529    /// ```
2530    /// #![feature(arc_is_unique)]
2531    ///
2532    /// use std::sync::Arc;
2533    ///
2534    /// let x = Arc::new(3);
2535    /// assert!(Arc::is_unique(&x));
2536    ///
2537    /// let y = Arc::clone(&x);
2538    /// assert!(!Arc::is_unique(&x));
2539    /// drop(y);
2540    ///
2541    /// // Weak references also count, because they could be upgraded at any time.
2542    /// let z = Arc::downgrade(&x);
2543    /// assert!(!Arc::is_unique(&x));
2544    /// ```
2545    ///
2546    /// # Pointer invalidation
2547    ///
2548    /// This function will always return the same value as `Arc::get_mut(arc).is_some()`. However,
2549    /// unlike that operation it does not produce any mutable references to the underlying data,
2550    /// meaning no pointers to the data inside the `Arc` are invalidated by the call. Thus, the
2551    /// following code is valid, even though it would be UB if it used `Arc::get_mut`:
2552    ///
2553    /// ```
2554    /// #![feature(arc_is_unique)]
2555    ///
2556    /// use std::sync::Arc;
2557    ///
2558    /// let arc = Arc::new(5);
2559    /// let pointer: *const i32 = &*arc;
2560    /// assert!(Arc::is_unique(&arc));
2561    /// assert_eq!(unsafe { *pointer }, 5);
2562    /// ```
2563    ///
2564    /// # Atomic orderings
2565    ///
2566    /// Concurrent drops to other `Arc` pointers to the same allocation will synchronize with this
2567    /// call - that is, this call performs an `Acquire` operation on the underlying strong and weak
2568    /// ref counts. This ensures that calling `get_mut_unchecked` is safe.
2569    ///
2570    /// Note that this operation requires locking the weak ref count, so concurrent calls to
2571    /// `downgrade` may spin-loop for a short period of time.
2572    ///
2573    /// [`get_mut_unchecked`]: Self::get_mut_unchecked
2574    #[inline]
2575    #[unstable(feature = "arc_is_unique", issue = "138938")]
2576    pub fn is_unique(this: &Self) -> bool {
2577        // lock the weak pointer count if we appear to be the sole weak pointer
2578        // holder.
2579        //
2580        // The acquire label here ensures a happens-before relationship with any
2581        // writes to `strong` (in particular in `Weak::upgrade`) prior to decrements
2582        // of the `weak` count (via `Weak::drop`, which uses release). If the upgraded
2583        // weak ref was never dropped, the CAS here will fail so we do not care to synchronize.
2584        if this.inner().weak.compare_exchange(1, usize::MAX, Acquire, Relaxed).is_ok() {
2585            // This needs to be an `Acquire` to synchronize with the decrement of the `strong`
2586            // counter in `drop` -- the only access that happens when any but the last reference
2587            // is being dropped.
2588            let unique = this.inner().strong.load(Acquire) == 1;
2589
2590            // The release write here synchronizes with a read in `downgrade`,
2591            // effectively preventing the above read of `strong` from happening
2592            // after the write.
2593            this.inner().weak.store(1, Release); // release the lock
2594            unique
2595        } else {
2596            false
2597        }
2598    }
2599}
2600
2601#[stable(feature = "rust1", since = "1.0.0")]
2602unsafe impl<#[may_dangle] T: ?Sized, A: Allocator> Drop for Arc<T, A> {
2603    /// Drops the `Arc`.
2604    ///
2605    /// This will decrement the strong reference count. If the strong reference
2606    /// count reaches zero then the only other references (if any) are
2607    /// [`Weak`], so we `drop` the inner value.
2608    ///
2609    /// # Examples
2610    ///
2611    /// ```
2612    /// use std::sync::Arc;
2613    ///
2614    /// struct Foo;
2615    ///
2616    /// impl Drop for Foo {
2617    ///     fn drop(&mut self) {
2618    ///         println!("dropped!");
2619    ///     }
2620    /// }
2621    ///
2622    /// let foo  = Arc::new(Foo);
2623    /// let foo2 = Arc::clone(&foo);
2624    ///
2625    /// drop(foo);    // Doesn't print anything
2626    /// drop(foo2);   // Prints "dropped!"
2627    /// ```
2628    #[inline]
2629    fn drop(&mut self) {
2630        // Because `fetch_sub` is already atomic, we do not need to synchronize
2631        // with other threads unless we are going to delete the object. This
2632        // same logic applies to the below `fetch_sub` to the `weak` count.
2633        if self.inner().strong.fetch_sub(1, Release) != 1 {
2634            return;
2635        }
2636
2637        // This fence is needed to prevent reordering of use of the data and
2638        // deletion of the data. Because it is marked `Release`, the decreasing
2639        // of the reference count synchronizes with this `Acquire` fence. This
2640        // means that use of the data happens before decreasing the reference
2641        // count, which happens before this fence, which happens before the
2642        // deletion of the data.
2643        //
2644        // As explained in the [Boost documentation][1],
2645        //
2646        // > It is important to enforce any possible access to the object in one
2647        // > thread (through an existing reference) to *happen before* deleting
2648        // > the object in a different thread. This is achieved by a "release"
2649        // > operation after dropping a reference (any access to the object
2650        // > through this reference must obviously happened before), and an
2651        // > "acquire" operation before deleting the object.
2652        //
2653        // In particular, while the contents of an Arc are usually immutable, it's
2654        // possible to have interior writes to something like a Mutex<T>. Since a
2655        // Mutex is not acquired when it is deleted, we can't rely on its
2656        // synchronization logic to make writes in thread A visible to a destructor
2657        // running in thread B.
2658        //
2659        // Also note that the Acquire fence here could probably be replaced with an
2660        // Acquire load, which could improve performance in highly-contended
2661        // situations. See [2].
2662        //
2663        // [1]: (www.boost.org/doc/libs/1_55_0/doc/html/atomic/usage_examples.html)
2664        // [2]: (https://github.com/rust-lang/rust/pull/41714)
2665        acquire!(self.inner().strong);
2666
2667        // Make sure we aren't trying to "drop" the shared static for empty slices
2668        // used by Default::default.
2669        debug_assert!(
2670            !ptr::addr_eq(self.ptr.as_ptr(), &STATIC_INNER_SLICE.inner),
2671            "Arcs backed by a static should never reach a strong count of 0. \
2672            Likely decrement_strong_count or from_raw were called too many times.",
2673        );
2674
2675        unsafe {
2676            self.drop_slow();
2677        }
2678    }
2679}
2680
2681impl<A: Allocator> Arc<dyn Any + Send + Sync, A> {
2682    /// Attempts to downcast the `Arc<dyn Any + Send + Sync>` to a concrete type.
2683    ///
2684    /// # Examples
2685    ///
2686    /// ```
2687    /// use std::any::Any;
2688    /// use std::sync::Arc;
2689    ///
2690    /// fn print_if_string(value: Arc<dyn Any + Send + Sync>) {
2691    ///     if let Ok(string) = value.downcast::<String>() {
2692    ///         println!("String ({}): {}", string.len(), string);
2693    ///     }
2694    /// }
2695    ///
2696    /// let my_string = "Hello World".to_string();
2697    /// print_if_string(Arc::new(my_string));
2698    /// print_if_string(Arc::new(0i8));
2699    /// ```
2700    #[inline]
2701    #[stable(feature = "rc_downcast", since = "1.29.0")]
2702    pub fn downcast<T>(self) -> Result<Arc<T, A>, Self>
2703    where
2704        T: Any + Send + Sync,
2705    {
2706        if (*self).is::<T>() {
2707            unsafe {
2708                let (ptr, alloc) = Arc::into_inner_with_allocator(self);
2709                Ok(Arc::from_inner_in(ptr.cast(), alloc))
2710            }
2711        } else {
2712            Err(self)
2713        }
2714    }
2715
2716    /// Downcasts the `Arc<dyn Any + Send + Sync>` to a concrete type.
2717    ///
2718    /// For a safe alternative see [`downcast`].
2719    ///
2720    /// # Examples
2721    ///
2722    /// ```
2723    /// #![feature(downcast_unchecked)]
2724    ///
2725    /// use std::any::Any;
2726    /// use std::sync::Arc;
2727    ///
2728    /// let x: Arc<dyn Any + Send + Sync> = Arc::new(1_usize);
2729    ///
2730    /// unsafe {
2731    ///     assert_eq!(*x.downcast_unchecked::<usize>(), 1);
2732    /// }
2733    /// ```
2734    ///
2735    /// # Safety
2736    ///
2737    /// The contained value must be of type `T`. Calling this method
2738    /// with the incorrect type is *undefined behavior*.
2739    ///
2740    ///
2741    /// [`downcast`]: Self::downcast
2742    #[inline]
2743    #[unstable(feature = "downcast_unchecked", issue = "90850")]
2744    pub unsafe fn downcast_unchecked<T>(self) -> Arc<T, A>
2745    where
2746        T: Any + Send + Sync,
2747    {
2748        unsafe {
2749            let (ptr, alloc) = Arc::into_inner_with_allocator(self);
2750            Arc::from_inner_in(ptr.cast(), alloc)
2751        }
2752    }
2753}
2754
2755impl<T> Weak<T> {
2756    /// Constructs a new `Weak<T>`, without allocating any memory.
2757    /// Calling [`upgrade`] on the return value always gives [`None`].
2758    ///
2759    /// [`upgrade`]: Weak::upgrade
2760    ///
2761    /// # Examples
2762    ///
2763    /// ```
2764    /// use std::sync::Weak;
2765    ///
2766    /// let empty: Weak<i64> = Weak::new();
2767    /// assert!(empty.upgrade().is_none());
2768    /// ```
2769    #[inline]
2770    #[stable(feature = "downgraded_weak", since = "1.10.0")]
2771    #[rustc_const_stable(feature = "const_weak_new", since = "1.73.0")]
2772    #[must_use]
2773    pub const fn new() -> Weak<T> {
2774        Weak { ptr: NonNull::without_provenance(NonZeroUsize::MAX), alloc: Global }
2775    }
2776}
2777
2778impl<T, A: Allocator> Weak<T, A> {
2779    /// Constructs a new `Weak<T, A>`, without allocating any memory, technically in the provided
2780    /// allocator.
2781    /// Calling [`upgrade`] on the return value always gives [`None`].
2782    ///
2783    /// [`upgrade`]: Weak::upgrade
2784    ///
2785    /// # Examples
2786    ///
2787    /// ```
2788    /// #![feature(allocator_api)]
2789    ///
2790    /// use std::sync::Weak;
2791    /// use std::alloc::System;
2792    ///
2793    /// let empty: Weak<i64, _> = Weak::new_in(System);
2794    /// assert!(empty.upgrade().is_none());
2795    /// ```
2796    #[inline]
2797    #[unstable(feature = "allocator_api", issue = "32838")]
2798    pub fn new_in(alloc: A) -> Weak<T, A> {
2799        Weak { ptr: NonNull::without_provenance(NonZeroUsize::MAX), alloc }
2800    }
2801}
2802
2803/// Helper type to allow accessing the reference counts without
2804/// making any assertions about the data field.
2805struct WeakInner<'a> {
2806    weak: &'a Atomic<usize>,
2807    strong: &'a Atomic<usize>,
2808}
2809
2810impl<T: ?Sized> Weak<T> {
2811    /// Converts a raw pointer previously created by [`into_raw`] back into `Weak<T>`.
2812    ///
2813    /// This can be used to safely get a strong reference (by calling [`upgrade`]
2814    /// later) or to deallocate the weak count by dropping the `Weak<T>`.
2815    ///
2816    /// It takes ownership of one weak reference (with the exception of pointers created by [`new`],
2817    /// as these don't own anything; the method still works on them).
2818    ///
2819    /// # Safety
2820    ///
2821    /// The pointer must have originated from the [`into_raw`] and must still own its potential
2822    /// weak reference, and must point to a block of memory allocated by global allocator.
2823    ///
2824    /// It is allowed for the strong count to be 0 at the time of calling this. Nevertheless, this
2825    /// takes ownership of one weak reference currently represented as a raw pointer (the weak
2826    /// count is not modified by this operation) and therefore it must be paired with a previous
2827    /// call to [`into_raw`].
2828    /// # Examples
2829    ///
2830    /// ```
2831    /// use std::sync::{Arc, Weak};
2832    ///
2833    /// let strong = Arc::new("hello".to_owned());
2834    ///
2835    /// let raw_1 = Arc::downgrade(&strong).into_raw();
2836    /// let raw_2 = Arc::downgrade(&strong).into_raw();
2837    ///
2838    /// assert_eq!(2, Arc::weak_count(&strong));
2839    ///
2840    /// assert_eq!("hello", &*unsafe { Weak::from_raw(raw_1) }.upgrade().unwrap());
2841    /// assert_eq!(1, Arc::weak_count(&strong));
2842    ///
2843    /// drop(strong);
2844    ///
2845    /// // Decrement the last weak count.
2846    /// assert!(unsafe { Weak::from_raw(raw_2) }.upgrade().is_none());
2847    /// ```
2848    ///
2849    /// [`new`]: Weak::new
2850    /// [`into_raw`]: Weak::into_raw
2851    /// [`upgrade`]: Weak::upgrade
2852    #[inline]
2853    #[stable(feature = "weak_into_raw", since = "1.45.0")]
2854    pub unsafe fn from_raw(ptr: *const T) -> Self {
2855        unsafe { Weak::from_raw_in(ptr, Global) }
2856    }
2857
2858    /// Consumes the `Weak<T>` and turns it into a raw pointer.
2859    ///
2860    /// This converts the weak pointer into a raw pointer, while still preserving the ownership of
2861    /// one weak reference (the weak count is not modified by this operation). It can be turned
2862    /// back into the `Weak<T>` with [`from_raw`].
2863    ///
2864    /// The same restrictions of accessing the target of the pointer as with
2865    /// [`as_ptr`] apply.
2866    ///
2867    /// # Examples
2868    ///
2869    /// ```
2870    /// use std::sync::{Arc, Weak};
2871    ///
2872    /// let strong = Arc::new("hello".to_owned());
2873    /// let weak = Arc::downgrade(&strong);
2874    /// let raw = weak.into_raw();
2875    ///
2876    /// assert_eq!(1, Arc::weak_count(&strong));
2877    /// assert_eq!("hello", unsafe { &*raw });
2878    ///
2879    /// drop(unsafe { Weak::from_raw(raw) });
2880    /// assert_eq!(0, Arc::weak_count(&strong));
2881    /// ```
2882    ///
2883    /// [`from_raw`]: Weak::from_raw
2884    /// [`as_ptr`]: Weak::as_ptr
2885    #[must_use = "losing the pointer will leak memory"]
2886    #[stable(feature = "weak_into_raw", since = "1.45.0")]
2887    pub fn into_raw(self) -> *const T {
2888        ManuallyDrop::new(self).as_ptr()
2889    }
2890}
2891
2892impl<T: ?Sized, A: Allocator> Weak<T, A> {
2893    /// Returns a reference to the underlying allocator.
2894    #[inline]
2895    #[unstable(feature = "allocator_api", issue = "32838")]
2896    pub fn allocator(&self) -> &A {
2897        &self.alloc
2898    }
2899
2900    /// Returns a raw pointer to the object `T` pointed to by this `Weak<T>`.
2901    ///
2902    /// The pointer is valid only if there are some strong references. The pointer may be dangling,
2903    /// unaligned or even [`null`] otherwise.
2904    ///
2905    /// # Examples
2906    ///
2907    /// ```
2908    /// use std::sync::Arc;
2909    /// use std::ptr;
2910    ///
2911    /// let strong = Arc::new("hello".to_owned());
2912    /// let weak = Arc::downgrade(&strong);
2913    /// // Both point to the same object
2914    /// assert!(ptr::eq(&*strong, weak.as_ptr()));
2915    /// // The strong here keeps it alive, so we can still access the object.
2916    /// assert_eq!("hello", unsafe { &*weak.as_ptr() });
2917    ///
2918    /// drop(strong);
2919    /// // But not any more. We can do weak.as_ptr(), but accessing the pointer would lead to
2920    /// // undefined behavior.
2921    /// // assert_eq!("hello", unsafe { &*weak.as_ptr() });
2922    /// ```
2923    ///
2924    /// [`null`]: core::ptr::null "ptr::null"
2925    #[must_use]
2926    #[stable(feature = "weak_into_raw", since = "1.45.0")]
2927    pub fn as_ptr(&self) -> *const T {
2928        let ptr: *mut ArcInner<T> = NonNull::as_ptr(self.ptr);
2929
2930        if is_dangling(ptr) {
2931            // If the pointer is dangling, we return the sentinel directly. This cannot be
2932            // a valid payload address, as the payload is at least as aligned as ArcInner (usize).
2933            ptr as *const T
2934        } else {
2935            // SAFETY: if is_dangling returns false, then the pointer is dereferenceable.
2936            // The payload may be dropped at this point, and we have to maintain provenance,
2937            // so use raw pointer manipulation.
2938            unsafe { &raw mut (*ptr).data }
2939        }
2940    }
2941
2942    /// Consumes the `Weak<T>`, returning the wrapped pointer and allocator.
2943    ///
2944    /// This converts the weak pointer into a raw pointer, while still preserving the ownership of
2945    /// one weak reference (the weak count is not modified by this operation). It can be turned
2946    /// back into the `Weak<T>` with [`from_raw_in`].
2947    ///
2948    /// The same restrictions of accessing the target of the pointer as with
2949    /// [`as_ptr`] apply.
2950    ///
2951    /// # Examples
2952    ///
2953    /// ```
2954    /// #![feature(allocator_api)]
2955    /// use std::sync::{Arc, Weak};
2956    /// use std::alloc::System;
2957    ///
2958    /// let strong = Arc::new_in("hello".to_owned(), System);
2959    /// let weak = Arc::downgrade(&strong);
2960    /// let (raw, alloc) = weak.into_raw_with_allocator();
2961    ///
2962    /// assert_eq!(1, Arc::weak_count(&strong));
2963    /// assert_eq!("hello", unsafe { &*raw });
2964    ///
2965    /// drop(unsafe { Weak::from_raw_in(raw, alloc) });
2966    /// assert_eq!(0, Arc::weak_count(&strong));
2967    /// ```
2968    ///
2969    /// [`from_raw_in`]: Weak::from_raw_in
2970    /// [`as_ptr`]: Weak::as_ptr
2971    #[must_use = "losing the pointer will leak memory"]
2972    #[unstable(feature = "allocator_api", issue = "32838")]
2973    pub fn into_raw_with_allocator(self) -> (*const T, A) {
2974        let this = mem::ManuallyDrop::new(self);
2975        let result = this.as_ptr();
2976        // Safety: `this` is ManuallyDrop so the allocator will not be double-dropped
2977        let alloc = unsafe { ptr::read(&this.alloc) };
2978        (result, alloc)
2979    }
2980
2981    /// Converts a raw pointer previously created by [`into_raw`] back into `Weak<T>` in the provided
2982    /// allocator.
2983    ///
2984    /// This can be used to safely get a strong reference (by calling [`upgrade`]
2985    /// later) or to deallocate the weak count by dropping the `Weak<T>`.
2986    ///
2987    /// It takes ownership of one weak reference (with the exception of pointers created by [`new`],
2988    /// as these don't own anything; the method still works on them).
2989    ///
2990    /// # Safety
2991    ///
2992    /// The pointer must have originated from the [`into_raw`] and must still own its potential
2993    /// weak reference, and must point to a block of memory allocated by `alloc`.
2994    ///
2995    /// It is allowed for the strong count to be 0 at the time of calling this. Nevertheless, this
2996    /// takes ownership of one weak reference currently represented as a raw pointer (the weak
2997    /// count is not modified by this operation) and therefore it must be paired with a previous
2998    /// call to [`into_raw`].
2999    /// # Examples
3000    ///
3001    /// ```
3002    /// use std::sync::{Arc, Weak};
3003    ///
3004    /// let strong = Arc::new("hello".to_owned());
3005    ///
3006    /// let raw_1 = Arc::downgrade(&strong).into_raw();
3007    /// let raw_2 = Arc::downgrade(&strong).into_raw();
3008    ///
3009    /// assert_eq!(2, Arc::weak_count(&strong));
3010    ///
3011    /// assert_eq!("hello", &*unsafe { Weak::from_raw(raw_1) }.upgrade().unwrap());
3012    /// assert_eq!(1, Arc::weak_count(&strong));
3013    ///
3014    /// drop(strong);
3015    ///
3016    /// // Decrement the last weak count.
3017    /// assert!(unsafe { Weak::from_raw(raw_2) }.upgrade().is_none());
3018    /// ```
3019    ///
3020    /// [`new`]: Weak::new
3021    /// [`into_raw`]: Weak::into_raw
3022    /// [`upgrade`]: Weak::upgrade
3023    #[inline]
3024    #[unstable(feature = "allocator_api", issue = "32838")]
3025    pub unsafe fn from_raw_in(ptr: *const T, alloc: A) -> Self {
3026        // See Weak::as_ptr for context on how the input pointer is derived.
3027
3028        let ptr = if is_dangling(ptr) {
3029            // This is a dangling Weak.
3030            ptr as *mut ArcInner<T>
3031        } else {
3032            // Otherwise, we're guaranteed the pointer came from a nondangling Weak.
3033            // SAFETY: data_offset is safe to call, as ptr references a real (potentially dropped) T.
3034            let offset = unsafe { data_offset(ptr) };
3035            // Thus, we reverse the offset to get the whole ArcInner.
3036            // SAFETY: the pointer originated from a Weak, so this offset is safe.
3037            unsafe { ptr.byte_sub(offset) as *mut ArcInner<T> }
3038        };
3039
3040        // SAFETY: we now have recovered the original Weak pointer, so can create the Weak.
3041        Weak { ptr: unsafe { NonNull::new_unchecked(ptr) }, alloc }
3042    }
3043}
3044
3045impl<T: ?Sized, A: Allocator> Weak<T, A> {
3046    /// Attempts to upgrade the `Weak` pointer to an [`Arc`], delaying
3047    /// dropping of the inner value if successful.
3048    ///
3049    /// Returns [`None`] if the inner value has since been dropped.
3050    ///
3051    /// # Examples
3052    ///
3053    /// ```
3054    /// use std::sync::Arc;
3055    ///
3056    /// let five = Arc::new(5);
3057    ///
3058    /// let weak_five = Arc::downgrade(&five);
3059    ///
3060    /// let strong_five: Option<Arc<_>> = weak_five.upgrade();
3061    /// assert!(strong_five.is_some());
3062    ///
3063    /// // Destroy all strong pointers.
3064    /// drop(strong_five);
3065    /// drop(five);
3066    ///
3067    /// assert!(weak_five.upgrade().is_none());
3068    /// ```
3069    #[must_use = "this returns a new `Arc`, \
3070                  without modifying the original weak pointer"]
3071    #[stable(feature = "arc_weak", since = "1.4.0")]
3072    pub fn upgrade(&self) -> Option<Arc<T, A>>
3073    where
3074        A: Clone,
3075    {
3076        #[inline]
3077        fn checked_increment(n: usize) -> Option<usize> {
3078            // Any write of 0 we can observe leaves the field in permanently zero state.
3079            if n == 0 {
3080                return None;
3081            }
3082            // See comments in `Arc::clone` for why we do this (for `mem::forget`).
3083            assert!(n <= MAX_REFCOUNT, "{}", INTERNAL_OVERFLOW_ERROR);
3084            Some(n + 1)
3085        }
3086
3087        // We use a CAS loop to increment the strong count instead of a
3088        // fetch_add as this function should never take the reference count
3089        // from zero to one.
3090        //
3091        // Relaxed is fine for the failure case because we don't have any expectations about the new state.
3092        // Acquire is necessary for the success case to synchronise with `Arc::new_cyclic`, when the inner
3093        // value can be initialized after `Weak` references have already been created. In that case, we
3094        // expect to observe the fully initialized value.
3095        if self.inner()?.strong.fetch_update(Acquire, Relaxed, checked_increment).is_ok() {
3096            // SAFETY: pointer is not null, verified in checked_increment
3097            unsafe { Some(Arc::from_inner_in(self.ptr, self.alloc.clone())) }
3098        } else {
3099            None
3100        }
3101    }
3102
3103    /// Gets the number of strong (`Arc`) pointers pointing to this allocation.
3104    ///
3105    /// If `self` was created using [`Weak::new`], this will return 0.
3106    #[must_use]
3107    #[stable(feature = "weak_counts", since = "1.41.0")]
3108    pub fn strong_count(&self) -> usize {
3109        if let Some(inner) = self.inner() { inner.strong.load(Relaxed) } else { 0 }
3110    }
3111
3112    /// Gets an approximation of the number of `Weak` pointers pointing to this
3113    /// allocation.
3114    ///
3115    /// If `self` was created using [`Weak::new`], or if there are no remaining
3116    /// strong pointers, this will return 0.
3117    ///
3118    /// # Accuracy
3119    ///
3120    /// Due to implementation details, the returned value can be off by 1 in
3121    /// either direction when other threads are manipulating any `Arc`s or
3122    /// `Weak`s pointing to the same allocation.
3123    #[must_use]
3124    #[stable(feature = "weak_counts", since = "1.41.0")]
3125    pub fn weak_count(&self) -> usize {
3126        if let Some(inner) = self.inner() {
3127            let weak = inner.weak.load(Acquire);
3128            let strong = inner.strong.load(Relaxed);
3129            if strong == 0 {
3130                0
3131            } else {
3132                // Since we observed that there was at least one strong pointer
3133                // after reading the weak count, we know that the implicit weak
3134                // reference (present whenever any strong references are alive)
3135                // was still around when we observed the weak count, and can
3136                // therefore safely subtract it.
3137                weak - 1
3138            }
3139        } else {
3140            0
3141        }
3142    }
3143
3144    /// Returns `None` when the pointer is dangling and there is no allocated `ArcInner`,
3145    /// (i.e., when this `Weak` was created by `Weak::new`).
3146    #[inline]
3147    fn inner(&self) -> Option<WeakInner<'_>> {
3148        let ptr = self.ptr.as_ptr();
3149        if is_dangling(ptr) {
3150            None
3151        } else {
3152            // We are careful to *not* create a reference covering the "data" field, as
3153            // the field may be mutated concurrently (for example, if the last `Arc`
3154            // is dropped, the data field will be dropped in-place).
3155            Some(unsafe { WeakInner { strong: &(*ptr).strong, weak: &(*ptr).weak } })
3156        }
3157    }
3158
3159    /// Returns `true` if the two `Weak`s point to the same allocation similar to [`ptr::eq`], or if
3160    /// both don't point to any allocation (because they were created with `Weak::new()`). However,
3161    /// this function ignores the metadata of  `dyn Trait` pointers.
3162    ///
3163    /// # Notes
3164    ///
3165    /// Since this compares pointers it means that `Weak::new()` will equal each
3166    /// other, even though they don't point to any allocation.
3167    ///
3168    /// # Examples
3169    ///
3170    /// ```
3171    /// use std::sync::Arc;
3172    ///
3173    /// let first_rc = Arc::new(5);
3174    /// let first = Arc::downgrade(&first_rc);
3175    /// let second = Arc::downgrade(&first_rc);
3176    ///
3177    /// assert!(first.ptr_eq(&second));
3178    ///
3179    /// let third_rc = Arc::new(5);
3180    /// let third = Arc::downgrade(&third_rc);
3181    ///
3182    /// assert!(!first.ptr_eq(&third));
3183    /// ```
3184    ///
3185    /// Comparing `Weak::new`.
3186    ///
3187    /// ```
3188    /// use std::sync::{Arc, Weak};
3189    ///
3190    /// let first = Weak::new();
3191    /// let second = Weak::new();
3192    /// assert!(first.ptr_eq(&second));
3193    ///
3194    /// let third_rc = Arc::new(());
3195    /// let third = Arc::downgrade(&third_rc);
3196    /// assert!(!first.ptr_eq(&third));
3197    /// ```
3198    ///
3199    /// [`ptr::eq`]: core::ptr::eq "ptr::eq"
3200    #[inline]
3201    #[must_use]
3202    #[stable(feature = "weak_ptr_eq", since = "1.39.0")]
3203    pub fn ptr_eq(&self, other: &Self) -> bool {
3204        ptr::addr_eq(self.ptr.as_ptr(), other.ptr.as_ptr())
3205    }
3206}
3207
3208#[stable(feature = "arc_weak", since = "1.4.0")]
3209impl<T: ?Sized, A: Allocator + Clone> Clone for Weak<T, A> {
3210    /// Makes a clone of the `Weak` pointer that points to the same allocation.
3211    ///
3212    /// # Examples
3213    ///
3214    /// ```
3215    /// use std::sync::{Arc, Weak};
3216    ///
3217    /// let weak_five = Arc::downgrade(&Arc::new(5));
3218    ///
3219    /// let _ = Weak::clone(&weak_five);
3220    /// ```
3221    #[inline]
3222    fn clone(&self) -> Weak<T, A> {
3223        if let Some(inner) = self.inner() {
3224            // See comments in Arc::clone() for why this is relaxed. This can use a
3225            // fetch_add (ignoring the lock) because the weak count is only locked
3226            // where are *no other* weak pointers in existence. (So we can't be
3227            // running this code in that case).
3228            let old_size = inner.weak.fetch_add(1, Relaxed);
3229
3230            // See comments in Arc::clone() for why we do this (for mem::forget).
3231            if old_size > MAX_REFCOUNT {
3232                abort();
3233            }
3234        }
3235
3236        Weak { ptr: self.ptr, alloc: self.alloc.clone() }
3237    }
3238}
3239
3240#[unstable(feature = "ergonomic_clones", issue = "132290")]
3241impl<T: ?Sized, A: Allocator + Clone> UseCloned for Weak<T, A> {}
3242
3243#[stable(feature = "downgraded_weak", since = "1.10.0")]
3244impl<T> Default for Weak<T> {
3245    /// Constructs a new `Weak<T>`, without allocating memory.
3246    /// Calling [`upgrade`] on the return value always
3247    /// gives [`None`].
3248    ///
3249    /// [`upgrade`]: Weak::upgrade
3250    ///
3251    /// # Examples
3252    ///
3253    /// ```
3254    /// use std::sync::Weak;
3255    ///
3256    /// let empty: Weak<i64> = Default::default();
3257    /// assert!(empty.upgrade().is_none());
3258    /// ```
3259    fn default() -> Weak<T> {
3260        Weak::new()
3261    }
3262}
3263
3264#[stable(feature = "arc_weak", since = "1.4.0")]
3265unsafe impl<#[may_dangle] T: ?Sized, A: Allocator> Drop for Weak<T, A> {
3266    /// Drops the `Weak` pointer.
3267    ///
3268    /// # Examples
3269    ///
3270    /// ```
3271    /// use std::sync::{Arc, Weak};
3272    ///
3273    /// struct Foo;
3274    ///
3275    /// impl Drop for Foo {
3276    ///     fn drop(&mut self) {
3277    ///         println!("dropped!");
3278    ///     }
3279    /// }
3280    ///
3281    /// let foo = Arc::new(Foo);
3282    /// let weak_foo = Arc::downgrade(&foo);
3283    /// let other_weak_foo = Weak::clone(&weak_foo);
3284    ///
3285    /// drop(weak_foo);   // Doesn't print anything
3286    /// drop(foo);        // Prints "dropped!"
3287    ///
3288    /// assert!(other_weak_foo.upgrade().is_none());
3289    /// ```
3290    fn drop(&mut self) {
3291        // If we find out that we were the last weak pointer, then its time to
3292        // deallocate the data entirely. See the discussion in Arc::drop() about
3293        // the memory orderings
3294        //
3295        // It's not necessary to check for the locked state here, because the
3296        // weak count can only be locked if there was precisely one weak ref,
3297        // meaning that drop could only subsequently run ON that remaining weak
3298        // ref, which can only happen after the lock is released.
3299        let inner = if let Some(inner) = self.inner() { inner } else { return };
3300
3301        if inner.weak.fetch_sub(1, Release) == 1 {
3302            acquire!(inner.weak);
3303
3304            // Make sure we aren't trying to "deallocate" the shared static for empty slices
3305            // used by Default::default.
3306            debug_assert!(
3307                !ptr::addr_eq(self.ptr.as_ptr(), &STATIC_INNER_SLICE.inner),
3308                "Arc/Weaks backed by a static should never be deallocated. \
3309                Likely decrement_strong_count or from_raw were called too many times.",
3310            );
3311
3312            unsafe {
3313                self.alloc.deallocate(self.ptr.cast(), Layout::for_value_raw(self.ptr.as_ptr()))
3314            }
3315        }
3316    }
3317}
3318
3319#[stable(feature = "rust1", since = "1.0.0")]
3320trait ArcEqIdent<T: ?Sized + PartialEq, A: Allocator> {
3321    fn eq(&self, other: &Arc<T, A>) -> bool;
3322    fn ne(&self, other: &Arc<T, A>) -> bool;
3323}
3324
3325#[stable(feature = "rust1", since = "1.0.0")]
3326impl<T: ?Sized + PartialEq, A: Allocator> ArcEqIdent<T, A> for Arc<T, A> {
3327    #[inline]
3328    default fn eq(&self, other: &Arc<T, A>) -> bool {
3329        **self == **other
3330    }
3331    #[inline]
3332    default fn ne(&self, other: &Arc<T, A>) -> bool {
3333        **self != **other
3334    }
3335}
3336
3337/// We're doing this specialization here, and not as a more general optimization on `&T`, because it
3338/// would otherwise add a cost to all equality checks on refs. We assume that `Arc`s are used to
3339/// store large values, that are slow to clone, but also heavy to check for equality, causing this
3340/// cost to pay off more easily. It's also more likely to have two `Arc` clones, that point to
3341/// the same value, than two `&T`s.
3342///
3343/// We can only do this when `T: Eq` as a `PartialEq` might be deliberately irreflexive.
3344#[stable(feature = "rust1", since = "1.0.0")]
3345impl<T: ?Sized + crate::rc::MarkerEq, A: Allocator> ArcEqIdent<T, A> for Arc<T, A> {
3346    #[inline]
3347    fn eq(&self, other: &Arc<T, A>) -> bool {
3348        Arc::ptr_eq(self, other) || **self == **other
3349    }
3350
3351    #[inline]
3352    fn ne(&self, other: &Arc<T, A>) -> bool {
3353        !Arc::ptr_eq(self, other) && **self != **other
3354    }
3355}
3356
3357#[stable(feature = "rust1", since = "1.0.0")]
3358impl<T: ?Sized + PartialEq, A: Allocator> PartialEq for Arc<T, A> {
3359    /// Equality for two `Arc`s.
3360    ///
3361    /// Two `Arc`s are equal if their inner values are equal, even if they are
3362    /// stored in different allocation.
3363    ///
3364    /// If `T` also implements `Eq` (implying reflexivity of equality),
3365    /// two `Arc`s that point to the same allocation are always equal.
3366    ///
3367    /// # Examples
3368    ///
3369    /// ```
3370    /// use std::sync::Arc;
3371    ///
3372    /// let five = Arc::new(5);
3373    ///
3374    /// assert!(five == Arc::new(5));
3375    /// ```
3376    #[inline]
3377    fn eq(&self, other: &Arc<T, A>) -> bool {
3378        ArcEqIdent::eq(self, other)
3379    }
3380
3381    /// Inequality for two `Arc`s.
3382    ///
3383    /// Two `Arc`s are not equal if their inner values are not equal.
3384    ///
3385    /// If `T` also implements `Eq` (implying reflexivity of equality),
3386    /// two `Arc`s that point to the same value are always equal.
3387    ///
3388    /// # Examples
3389    ///
3390    /// ```
3391    /// use std::sync::Arc;
3392    ///
3393    /// let five = Arc::new(5);
3394    ///
3395    /// assert!(five != Arc::new(6));
3396    /// ```
3397    #[inline]
3398    fn ne(&self, other: &Arc<T, A>) -> bool {
3399        ArcEqIdent::ne(self, other)
3400    }
3401}
3402
3403#[stable(feature = "rust1", since = "1.0.0")]
3404impl<T: ?Sized + PartialOrd, A: Allocator> PartialOrd for Arc<T, A> {
3405    /// Partial comparison for two `Arc`s.
3406    ///
3407    /// The two are compared by calling `partial_cmp()` on their inner values.
3408    ///
3409    /// # Examples
3410    ///
3411    /// ```
3412    /// use std::sync::Arc;
3413    /// use std::cmp::Ordering;
3414    ///
3415    /// let five = Arc::new(5);
3416    ///
3417    /// assert_eq!(Some(Ordering::Less), five.partial_cmp(&Arc::new(6)));
3418    /// ```
3419    fn partial_cmp(&self, other: &Arc<T, A>) -> Option<Ordering> {
3420        (**self).partial_cmp(&**other)
3421    }
3422
3423    /// Less-than comparison for two `Arc`s.
3424    ///
3425    /// The two are compared by calling `<` on their inner values.
3426    ///
3427    /// # Examples
3428    ///
3429    /// ```
3430    /// use std::sync::Arc;
3431    ///
3432    /// let five = Arc::new(5);
3433    ///
3434    /// assert!(five < Arc::new(6));
3435    /// ```
3436    fn lt(&self, other: &Arc<T, A>) -> bool {
3437        *(*self) < *(*other)
3438    }
3439
3440    /// 'Less than or equal to' comparison for two `Arc`s.
3441    ///
3442    /// The two are compared by calling `<=` on their inner values.
3443    ///
3444    /// # Examples
3445    ///
3446    /// ```
3447    /// use std::sync::Arc;
3448    ///
3449    /// let five = Arc::new(5);
3450    ///
3451    /// assert!(five <= Arc::new(5));
3452    /// ```
3453    fn le(&self, other: &Arc<T, A>) -> bool {
3454        *(*self) <= *(*other)
3455    }
3456
3457    /// Greater-than comparison for two `Arc`s.
3458    ///
3459    /// The two are compared by calling `>` on their inner values.
3460    ///
3461    /// # Examples
3462    ///
3463    /// ```
3464    /// use std::sync::Arc;
3465    ///
3466    /// let five = Arc::new(5);
3467    ///
3468    /// assert!(five > Arc::new(4));
3469    /// ```
3470    fn gt(&self, other: &Arc<T, A>) -> bool {
3471        *(*self) > *(*other)
3472    }
3473
3474    /// 'Greater than or equal to' comparison for two `Arc`s.
3475    ///
3476    /// The two are compared by calling `>=` on their inner values.
3477    ///
3478    /// # Examples
3479    ///
3480    /// ```
3481    /// use std::sync::Arc;
3482    ///
3483    /// let five = Arc::new(5);
3484    ///
3485    /// assert!(five >= Arc::new(5));
3486    /// ```
3487    fn ge(&self, other: &Arc<T, A>) -> bool {
3488        *(*self) >= *(*other)
3489    }
3490}
3491#[stable(feature = "rust1", since = "1.0.0")]
3492impl<T: ?Sized + Ord, A: Allocator> Ord for Arc<T, A> {
3493    /// Comparison for two `Arc`s.
3494    ///
3495    /// The two are compared by calling `cmp()` on their inner values.
3496    ///
3497    /// # Examples
3498    ///
3499    /// ```
3500    /// use std::sync::Arc;
3501    /// use std::cmp::Ordering;
3502    ///
3503    /// let five = Arc::new(5);
3504    ///
3505    /// assert_eq!(Ordering::Less, five.cmp(&Arc::new(6)));
3506    /// ```
3507    fn cmp(&self, other: &Arc<T, A>) -> Ordering {
3508        (**self).cmp(&**other)
3509    }
3510}
3511#[stable(feature = "rust1", since = "1.0.0")]
3512impl<T: ?Sized + Eq, A: Allocator> Eq for Arc<T, A> {}
3513
3514#[stable(feature = "rust1", since = "1.0.0")]
3515impl<T: ?Sized + fmt::Display, A: Allocator> fmt::Display for Arc<T, A> {
3516    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3517        fmt::Display::fmt(&**self, f)
3518    }
3519}
3520
3521#[stable(feature = "rust1", since = "1.0.0")]
3522impl<T: ?Sized + fmt::Debug, A: Allocator> fmt::Debug for Arc<T, A> {
3523    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3524        fmt::Debug::fmt(&**self, f)
3525    }
3526}
3527
3528#[stable(feature = "rust1", since = "1.0.0")]
3529impl<T: ?Sized, A: Allocator> fmt::Pointer for Arc<T, A> {
3530    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3531        fmt::Pointer::fmt(&(&raw const **self), f)
3532    }
3533}
3534
3535#[cfg(not(no_global_oom_handling))]
3536#[stable(feature = "rust1", since = "1.0.0")]
3537impl<T: Default> Default for Arc<T> {
3538    /// Creates a new `Arc<T>`, with the `Default` value for `T`.
3539    ///
3540    /// # Examples
3541    ///
3542    /// ```
3543    /// use std::sync::Arc;
3544    ///
3545    /// let x: Arc<i32> = Default::default();
3546    /// assert_eq!(*x, 0);
3547    /// ```
3548    fn default() -> Arc<T> {
3549        unsafe {
3550            Self::from_inner(
3551                Box::leak(Box::write(
3552                    Box::new_uninit(),
3553                    ArcInner {
3554                        strong: atomic::AtomicUsize::new(1),
3555                        weak: atomic::AtomicUsize::new(1),
3556                        data: T::default(),
3557                    },
3558                ))
3559                .into(),
3560            )
3561        }
3562    }
3563}
3564
3565/// Struct to hold the static `ArcInner` used for empty `Arc<str/CStr/[T]>` as
3566/// returned by `Default::default`.
3567///
3568/// Layout notes:
3569/// * `repr(align(16))` so we can use it for `[T]` with `align_of::<T>() <= 16`.
3570/// * `repr(C)` so `inner` is at offset 0 (and thus guaranteed to actually be aligned to 16).
3571/// * `[u8; 1]` (to be initialized with 0) so it can be used for `Arc<CStr>`.
3572#[repr(C, align(16))]
3573struct SliceArcInnerForStatic {
3574    inner: ArcInner<[u8; 1]>,
3575}
3576#[cfg(not(no_global_oom_handling))]
3577const MAX_STATIC_INNER_SLICE_ALIGNMENT: usize = 16;
3578
3579static STATIC_INNER_SLICE: SliceArcInnerForStatic = SliceArcInnerForStatic {
3580    inner: ArcInner {
3581        strong: atomic::AtomicUsize::new(1),
3582        weak: atomic::AtomicUsize::new(1),
3583        data: [0],
3584    },
3585};
3586
3587#[cfg(not(no_global_oom_handling))]
3588#[stable(feature = "more_rc_default_impls", since = "1.80.0")]
3589impl Default for Arc<str> {
3590    /// Creates an empty str inside an Arc
3591    ///
3592    /// This may or may not share an allocation with other Arcs.
3593    #[inline]
3594    fn default() -> Self {
3595        let arc: Arc<[u8]> = Default::default();
3596        debug_assert!(core::str::from_utf8(&*arc).is_ok());
3597        let (ptr, alloc) = Arc::into_inner_with_allocator(arc);
3598        unsafe { Arc::from_ptr_in(ptr.as_ptr() as *mut ArcInner<str>, alloc) }
3599    }
3600}
3601
3602#[cfg(not(no_global_oom_handling))]
3603#[stable(feature = "more_rc_default_impls", since = "1.80.0")]
3604impl Default for Arc<core::ffi::CStr> {
3605    /// Creates an empty CStr inside an Arc
3606    ///
3607    /// This may or may not share an allocation with other Arcs.
3608    #[inline]
3609    fn default() -> Self {
3610        use core::ffi::CStr;
3611        let inner: NonNull<ArcInner<[u8]>> = NonNull::from(&STATIC_INNER_SLICE.inner);
3612        let inner: NonNull<ArcInner<CStr>> =
3613            NonNull::new(inner.as_ptr() as *mut ArcInner<CStr>).unwrap();
3614        // `this` semantically is the Arc "owned" by the static, so make sure not to drop it.
3615        let this: mem::ManuallyDrop<Arc<CStr>> =
3616            unsafe { mem::ManuallyDrop::new(Arc::from_inner(inner)) };
3617        (*this).clone()
3618    }
3619}
3620
3621#[cfg(not(no_global_oom_handling))]
3622#[stable(feature = "more_rc_default_impls", since = "1.80.0")]
3623impl<T> Default for Arc<[T]> {
3624    /// Creates an empty `[T]` inside an Arc
3625    ///
3626    /// This may or may not share an allocation with other Arcs.
3627    #[inline]
3628    fn default() -> Self {
3629        if align_of::<T>() <= MAX_STATIC_INNER_SLICE_ALIGNMENT {
3630            // We take a reference to the whole struct instead of the ArcInner<[u8; 1]> inside it so
3631            // we don't shrink the range of bytes the ptr is allowed to access under Stacked Borrows.
3632            // (Miri complains on 32-bit targets with Arc<[Align16]> otherwise.)
3633            // (Note that NonNull::from(&STATIC_INNER_SLICE.inner) is fine under Tree Borrows.)
3634            let inner: NonNull<SliceArcInnerForStatic> = NonNull::from(&STATIC_INNER_SLICE);
3635            let inner: NonNull<ArcInner<[T; 0]>> = inner.cast();
3636            // `this` semantically is the Arc "owned" by the static, so make sure not to drop it.
3637            let this: mem::ManuallyDrop<Arc<[T; 0]>> =
3638                unsafe { mem::ManuallyDrop::new(Arc::from_inner(inner)) };
3639            return (*this).clone();
3640        }
3641
3642        // If T's alignment is too large for the static, make a new unique allocation.
3643        let arr: [T; 0] = [];
3644        Arc::from(arr)
3645    }
3646}
3647
3648#[cfg(not(no_global_oom_handling))]
3649#[stable(feature = "pin_default_impls", since = "1.91.0")]
3650impl<T> Default for Pin<Arc<T>>
3651where
3652    T: ?Sized,
3653    Arc<T>: Default,
3654{
3655    #[inline]
3656    fn default() -> Self {
3657        unsafe { Pin::new_unchecked(Arc::<T>::default()) }
3658    }
3659}
3660
3661#[stable(feature = "rust1", since = "1.0.0")]
3662impl<T: ?Sized + Hash, A: Allocator> Hash for Arc<T, A> {
3663    fn hash<H: Hasher>(&self, state: &mut H) {
3664        (**self).hash(state)
3665    }
3666}
3667
3668#[cfg(not(no_global_oom_handling))]
3669#[stable(feature = "from_for_ptrs", since = "1.6.0")]
3670impl<T> From<T> for Arc<T> {
3671    /// Converts a `T` into an `Arc<T>`
3672    ///
3673    /// The conversion moves the value into a
3674    /// newly allocated `Arc`. It is equivalent to
3675    /// calling `Arc::new(t)`.
3676    ///
3677    /// # Example
3678    /// ```rust
3679    /// # use std::sync::Arc;
3680    /// let x = 5;
3681    /// let arc = Arc::new(5);
3682    ///
3683    /// assert_eq!(Arc::from(x), arc);
3684    /// ```
3685    fn from(t: T) -> Self {
3686        Arc::new(t)
3687    }
3688}
3689
3690#[cfg(not(no_global_oom_handling))]
3691#[stable(feature = "shared_from_array", since = "1.74.0")]
3692impl<T, const N: usize> From<[T; N]> for Arc<[T]> {
3693    /// Converts a [`[T; N]`](prim@array) into an `Arc<[T]>`.
3694    ///
3695    /// The conversion moves the array into a newly allocated `Arc`.
3696    ///
3697    /// # Example
3698    ///
3699    /// ```
3700    /// # use std::sync::Arc;
3701    /// let original: [i32; 3] = [1, 2, 3];
3702    /// let shared: Arc<[i32]> = Arc::from(original);
3703    /// assert_eq!(&[1, 2, 3], &shared[..]);
3704    /// ```
3705    #[inline]
3706    fn from(v: [T; N]) -> Arc<[T]> {
3707        Arc::<[T; N]>::from(v)
3708    }
3709}
3710
3711#[cfg(not(no_global_oom_handling))]
3712#[stable(feature = "shared_from_slice", since = "1.21.0")]
3713impl<T: Clone> From<&[T]> for Arc<[T]> {
3714    /// Allocates a reference-counted slice and fills it by cloning `v`'s items.
3715    ///
3716    /// # Example
3717    ///
3718    /// ```
3719    /// # use std::sync::Arc;
3720    /// let original: &[i32] = &[1, 2, 3];
3721    /// let shared: Arc<[i32]> = Arc::from(original);
3722    /// assert_eq!(&[1, 2, 3], &shared[..]);
3723    /// ```
3724    #[inline]
3725    fn from(v: &[T]) -> Arc<[T]> {
3726        <Self as ArcFromSlice<T>>::from_slice(v)
3727    }
3728}
3729
3730#[cfg(not(no_global_oom_handling))]
3731#[stable(feature = "shared_from_mut_slice", since = "1.84.0")]
3732impl<T: Clone> From<&mut [T]> for Arc<[T]> {
3733    /// Allocates a reference-counted slice and fills it by cloning `v`'s items.
3734    ///
3735    /// # Example
3736    ///
3737    /// ```
3738    /// # use std::sync::Arc;
3739    /// let mut original = [1, 2, 3];
3740    /// let original: &mut [i32] = &mut original;
3741    /// let shared: Arc<[i32]> = Arc::from(original);
3742    /// assert_eq!(&[1, 2, 3], &shared[..]);
3743    /// ```
3744    #[inline]
3745    fn from(v: &mut [T]) -> Arc<[T]> {
3746        Arc::from(&*v)
3747    }
3748}
3749
3750#[cfg(not(no_global_oom_handling))]
3751#[stable(feature = "shared_from_slice", since = "1.21.0")]
3752impl From<&str> for Arc<str> {
3753    /// Allocates a reference-counted `str` and copies `v` into it.
3754    ///
3755    /// # Example
3756    ///
3757    /// ```
3758    /// # use std::sync::Arc;
3759    /// let shared: Arc<str> = Arc::from("eggplant");
3760    /// assert_eq!("eggplant", &shared[..]);
3761    /// ```
3762    #[inline]
3763    fn from(v: &str) -> Arc<str> {
3764        let arc = Arc::<[u8]>::from(v.as_bytes());
3765        unsafe { Arc::from_raw(Arc::into_raw(arc) as *const str) }
3766    }
3767}
3768
3769#[cfg(not(no_global_oom_handling))]
3770#[stable(feature = "shared_from_mut_slice", since = "1.84.0")]
3771impl From<&mut str> for Arc<str> {
3772    /// Allocates a reference-counted `str` and copies `v` into it.
3773    ///
3774    /// # Example
3775    ///
3776    /// ```
3777    /// # use std::sync::Arc;
3778    /// let mut original = String::from("eggplant");
3779    /// let original: &mut str = &mut original;
3780    /// let shared: Arc<str> = Arc::from(original);
3781    /// assert_eq!("eggplant", &shared[..]);
3782    /// ```
3783    #[inline]
3784    fn from(v: &mut str) -> Arc<str> {
3785        Arc::from(&*v)
3786    }
3787}
3788
3789#[cfg(not(no_global_oom_handling))]
3790#[stable(feature = "shared_from_slice", since = "1.21.0")]
3791impl From<String> for Arc<str> {
3792    /// Allocates a reference-counted `str` and copies `v` into it.
3793    ///
3794    /// # Example
3795    ///
3796    /// ```
3797    /// # use std::sync::Arc;
3798    /// let unique: String = "eggplant".to_owned();
3799    /// let shared: Arc<str> = Arc::from(unique);
3800    /// assert_eq!("eggplant", &shared[..]);
3801    /// ```
3802    #[inline]
3803    fn from(v: String) -> Arc<str> {
3804        Arc::from(&v[..])
3805    }
3806}
3807
3808#[cfg(not(no_global_oom_handling))]
3809#[stable(feature = "shared_from_slice", since = "1.21.0")]
3810impl<T: ?Sized, A: Allocator> From<Box<T, A>> for Arc<T, A> {
3811    /// Move a boxed object to a new, reference-counted allocation.
3812    ///
3813    /// # Example
3814    ///
3815    /// ```
3816    /// # use std::sync::Arc;
3817    /// let unique: Box<str> = Box::from("eggplant");
3818    /// let shared: Arc<str> = Arc::from(unique);
3819    /// assert_eq!("eggplant", &shared[..]);
3820    /// ```
3821    #[inline]
3822    fn from(v: Box<T, A>) -> Arc<T, A> {
3823        Arc::from_box_in(v)
3824    }
3825}
3826
3827#[cfg(not(no_global_oom_handling))]
3828#[stable(feature = "shared_from_slice", since = "1.21.0")]
3829impl<T, A: Allocator + Clone> From<Vec<T, A>> for Arc<[T], A> {
3830    /// Allocates a reference-counted slice and moves `v`'s items into it.
3831    ///
3832    /// # Example
3833    ///
3834    /// ```
3835    /// # use std::sync::Arc;
3836    /// let unique: Vec<i32> = vec![1, 2, 3];
3837    /// let shared: Arc<[i32]> = Arc::from(unique);
3838    /// assert_eq!(&[1, 2, 3], &shared[..]);
3839    /// ```
3840    #[inline]
3841    fn from(v: Vec<T, A>) -> Arc<[T], A> {
3842        unsafe {
3843            let (vec_ptr, len, cap, alloc) = v.into_raw_parts_with_alloc();
3844
3845            let rc_ptr = Self::allocate_for_slice_in(len, &alloc);
3846            ptr::copy_nonoverlapping(vec_ptr, (&raw mut (*rc_ptr).data) as *mut T, len);
3847
3848            // Create a `Vec<T, &A>` with length 0, to deallocate the buffer
3849            // without dropping its contents or the allocator
3850            let _ = Vec::from_raw_parts_in(vec_ptr, 0, cap, &alloc);
3851
3852            Self::from_ptr_in(rc_ptr, alloc)
3853        }
3854    }
3855}
3856
3857#[stable(feature = "shared_from_cow", since = "1.45.0")]
3858impl<'a, B> From<Cow<'a, B>> for Arc<B>
3859where
3860    B: ToOwned + ?Sized,
3861    Arc<B>: From<&'a B> + From<B::Owned>,
3862{
3863    /// Creates an atomically reference-counted pointer from a clone-on-write
3864    /// pointer by copying its content.
3865    ///
3866    /// # Example
3867    ///
3868    /// ```rust
3869    /// # use std::sync::Arc;
3870    /// # use std::borrow::Cow;
3871    /// let cow: Cow<'_, str> = Cow::Borrowed("eggplant");
3872    /// let shared: Arc<str> = Arc::from(cow);
3873    /// assert_eq!("eggplant", &shared[..]);
3874    /// ```
3875    #[inline]
3876    fn from(cow: Cow<'a, B>) -> Arc<B> {
3877        match cow {
3878            Cow::Borrowed(s) => Arc::from(s),
3879            Cow::Owned(s) => Arc::from(s),
3880        }
3881    }
3882}
3883
3884#[stable(feature = "shared_from_str", since = "1.62.0")]
3885impl From<Arc<str>> for Arc<[u8]> {
3886    /// Converts an atomically reference-counted string slice into a byte slice.
3887    ///
3888    /// # Example
3889    ///
3890    /// ```
3891    /// # use std::sync::Arc;
3892    /// let string: Arc<str> = Arc::from("eggplant");
3893    /// let bytes: Arc<[u8]> = Arc::from(string);
3894    /// assert_eq!("eggplant".as_bytes(), bytes.as_ref());
3895    /// ```
3896    #[inline]
3897    fn from(rc: Arc<str>) -> Self {
3898        // SAFETY: `str` has the same layout as `[u8]`.
3899        unsafe { Arc::from_raw(Arc::into_raw(rc) as *const [u8]) }
3900    }
3901}
3902
3903#[stable(feature = "boxed_slice_try_from", since = "1.43.0")]
3904impl<T, A: Allocator, const N: usize> TryFrom<Arc<[T], A>> for Arc<[T; N], A> {
3905    type Error = Arc<[T], A>;
3906
3907    fn try_from(boxed_slice: Arc<[T], A>) -> Result<Self, Self::Error> {
3908        if boxed_slice.len() == N {
3909            let (ptr, alloc) = Arc::into_inner_with_allocator(boxed_slice);
3910            Ok(unsafe { Arc::from_inner_in(ptr.cast(), alloc) })
3911        } else {
3912            Err(boxed_slice)
3913        }
3914    }
3915}
3916
3917#[cfg(not(no_global_oom_handling))]
3918#[stable(feature = "shared_from_iter", since = "1.37.0")]
3919impl<T> FromIterator<T> for Arc<[T]> {
3920    /// Takes each element in the `Iterator` and collects it into an `Arc<[T]>`.
3921    ///
3922    /// # Performance characteristics
3923    ///
3924    /// ## The general case
3925    ///
3926    /// In the general case, collecting into `Arc<[T]>` is done by first
3927    /// collecting into a `Vec<T>`. That is, when writing the following:
3928    ///
3929    /// ```rust
3930    /// # use std::sync::Arc;
3931    /// let evens: Arc<[u8]> = (0..10).filter(|&x| x % 2 == 0).collect();
3932    /// # assert_eq!(&*evens, &[0, 2, 4, 6, 8]);
3933    /// ```
3934    ///
3935    /// this behaves as if we wrote:
3936    ///
3937    /// ```rust
3938    /// # use std::sync::Arc;
3939    /// let evens: Arc<[u8]> = (0..10).filter(|&x| x % 2 == 0)
3940    ///     .collect::<Vec<_>>() // The first set of allocations happens here.
3941    ///     .into(); // A second allocation for `Arc<[T]>` happens here.
3942    /// # assert_eq!(&*evens, &[0, 2, 4, 6, 8]);
3943    /// ```
3944    ///
3945    /// This will allocate as many times as needed for constructing the `Vec<T>`
3946    /// and then it will allocate once for turning the `Vec<T>` into the `Arc<[T]>`.
3947    ///
3948    /// ## Iterators of known length
3949    ///
3950    /// When your `Iterator` implements `TrustedLen` and is of an exact size,
3951    /// a single allocation will be made for the `Arc<[T]>`. For example:
3952    ///
3953    /// ```rust
3954    /// # use std::sync::Arc;
3955    /// let evens: Arc<[u8]> = (0..10).collect(); // Just a single allocation happens here.
3956    /// # assert_eq!(&*evens, &*(0..10).collect::<Vec<_>>());
3957    /// ```
3958    fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self {
3959        ToArcSlice::to_arc_slice(iter.into_iter())
3960    }
3961}
3962
3963#[cfg(not(no_global_oom_handling))]
3964/// Specialization trait used for collecting into `Arc<[T]>`.
3965trait ToArcSlice<T>: Iterator<Item = T> + Sized {
3966    fn to_arc_slice(self) -> Arc<[T]>;
3967}
3968
3969#[cfg(not(no_global_oom_handling))]
3970impl<T, I: Iterator<Item = T>> ToArcSlice<T> for I {
3971    default fn to_arc_slice(self) -> Arc<[T]> {
3972        self.collect::<Vec<T>>().into()
3973    }
3974}
3975
3976#[cfg(not(no_global_oom_handling))]
3977impl<T, I: iter::TrustedLen<Item = T>> ToArcSlice<T> for I {
3978    fn to_arc_slice(self) -> Arc<[T]> {
3979        // This is the case for a `TrustedLen` iterator.
3980        let (low, high) = self.size_hint();
3981        if let Some(high) = high {
3982            debug_assert_eq!(
3983                low,
3984                high,
3985                "TrustedLen iterator's size hint is not exact: {:?}",
3986                (low, high)
3987            );
3988
3989            unsafe {
3990                // SAFETY: We need to ensure that the iterator has an exact length and we have.
3991                Arc::from_iter_exact(self, low)
3992            }
3993        } else {
3994            // TrustedLen contract guarantees that `upper_bound == None` implies an iterator
3995            // length exceeding `usize::MAX`.
3996            // The default implementation would collect into a vec which would panic.
3997            // Thus we panic here immediately without invoking `Vec` code.
3998            panic!("capacity overflow");
3999        }
4000    }
4001}
4002
4003#[stable(feature = "rust1", since = "1.0.0")]
4004impl<T: ?Sized, A: Allocator> borrow::Borrow<T> for Arc<T, A> {
4005    fn borrow(&self) -> &T {
4006        &**self
4007    }
4008}
4009
4010#[stable(since = "1.5.0", feature = "smart_ptr_as_ref")]
4011impl<T: ?Sized, A: Allocator> AsRef<T> for Arc<T, A> {
4012    fn as_ref(&self) -> &T {
4013        &**self
4014    }
4015}
4016
4017#[stable(feature = "pin", since = "1.33.0")]
4018impl<T: ?Sized, A: Allocator> Unpin for Arc<T, A> {}
4019
4020/// Gets the offset within an `ArcInner` for the payload behind a pointer.
4021///
4022/// # Safety
4023///
4024/// The pointer must point to (and have valid metadata for) a previously
4025/// valid instance of T, but the T is allowed to be dropped.
4026unsafe fn data_offset<T: ?Sized>(ptr: *const T) -> usize {
4027    // Align the unsized value to the end of the ArcInner.
4028    // Because ArcInner is repr(C), it will always be the last field in memory.
4029    // SAFETY: since the only unsized types possible are slices, trait objects,
4030    // and extern types, the input safety requirement is currently enough to
4031    // satisfy the requirements of align_of_val_raw; this is an implementation
4032    // detail of the language that must not be relied upon outside of std.
4033    unsafe { data_offset_align(align_of_val_raw(ptr)) }
4034}
4035
4036#[inline]
4037fn data_offset_align(align: usize) -> usize {
4038    let layout = Layout::new::<ArcInner<()>>();
4039    layout.size() + layout.padding_needed_for(align)
4040}
4041
4042/// A unique owning pointer to an [`ArcInner`] **that does not imply the contents are initialized,**
4043/// but will deallocate it (without dropping the value) when dropped.
4044///
4045/// This is a helper for [`Arc::make_mut()`] to ensure correct cleanup on panic.
4046#[cfg(not(no_global_oom_handling))]
4047struct UniqueArcUninit<T: ?Sized, A: Allocator> {
4048    ptr: NonNull<ArcInner<T>>,
4049    layout_for_value: Layout,
4050    alloc: Option<A>,
4051}
4052
4053#[cfg(not(no_global_oom_handling))]
4054impl<T: ?Sized, A: Allocator> UniqueArcUninit<T, A> {
4055    /// Allocates an ArcInner with layout suitable to contain `for_value` or a clone of it.
4056    fn new(for_value: &T, alloc: A) -> UniqueArcUninit<T, A> {
4057        let layout = Layout::for_value(for_value);
4058        let ptr = unsafe {
4059            Arc::allocate_for_layout(
4060                layout,
4061                |layout_for_arcinner| alloc.allocate(layout_for_arcinner),
4062                |mem| mem.with_metadata_of(ptr::from_ref(for_value) as *const ArcInner<T>),
4063            )
4064        };
4065        Self { ptr: NonNull::new(ptr).unwrap(), layout_for_value: layout, alloc: Some(alloc) }
4066    }
4067
4068    /// Returns the pointer to be written into to initialize the [`Arc`].
4069    fn data_ptr(&mut self) -> *mut T {
4070        let offset = data_offset_align(self.layout_for_value.align());
4071        unsafe { self.ptr.as_ptr().byte_add(offset) as *mut T }
4072    }
4073
4074    /// Upgrade this into a normal [`Arc`].
4075    ///
4076    /// # Safety
4077    ///
4078    /// The data must have been initialized (by writing to [`Self::data_ptr()`]).
4079    unsafe fn into_arc(self) -> Arc<T, A> {
4080        let mut this = ManuallyDrop::new(self);
4081        let ptr = this.ptr.as_ptr();
4082        let alloc = this.alloc.take().unwrap();
4083
4084        // SAFETY: The pointer is valid as per `UniqueArcUninit::new`, and the caller is responsible
4085        // for having initialized the data.
4086        unsafe { Arc::from_ptr_in(ptr, alloc) }
4087    }
4088}
4089
4090#[cfg(not(no_global_oom_handling))]
4091impl<T: ?Sized, A: Allocator> Drop for UniqueArcUninit<T, A> {
4092    fn drop(&mut self) {
4093        // SAFETY:
4094        // * new() produced a pointer safe to deallocate.
4095        // * We own the pointer unless into_arc() was called, which forgets us.
4096        unsafe {
4097            self.alloc.take().unwrap().deallocate(
4098                self.ptr.cast(),
4099                arcinner_layout_for_value_layout(self.layout_for_value),
4100            );
4101        }
4102    }
4103}
4104
4105#[stable(feature = "arc_error", since = "1.52.0")]
4106impl<T: core::error::Error + ?Sized> core::error::Error for Arc<T> {
4107    #[allow(deprecated)]
4108    fn cause(&self) -> Option<&dyn core::error::Error> {
4109        core::error::Error::cause(&**self)
4110    }
4111
4112    fn source(&self) -> Option<&(dyn core::error::Error + 'static)> {
4113        core::error::Error::source(&**self)
4114    }
4115
4116    fn provide<'a>(&'a self, req: &mut core::error::Request<'a>) {
4117        core::error::Error::provide(&**self, req);
4118    }
4119}
4120
4121/// A uniquely owned [`Arc`].
4122///
4123/// This represents an `Arc` that is known to be uniquely owned -- that is, have exactly one strong
4124/// reference. Multiple weak pointers can be created, but attempts to upgrade those to strong
4125/// references will fail unless the `UniqueArc` they point to has been converted into a regular `Arc`.
4126///
4127/// Because it is uniquely owned, the contents of a `UniqueArc` can be freely mutated. A common
4128/// use case is to have an object be mutable during its initialization phase but then have it become
4129/// immutable and converted to a normal `Arc`.
4130///
4131/// This can be used as a flexible way to create cyclic data structures, as in the example below.
4132///
4133/// ```
4134/// #![feature(unique_rc_arc)]
4135/// use std::sync::{Arc, Weak, UniqueArc};
4136///
4137/// struct Gadget {
4138///     me: Weak<Gadget>,
4139/// }
4140///
4141/// fn create_gadget() -> Option<Arc<Gadget>> {
4142///     let mut rc = UniqueArc::new(Gadget {
4143///         me: Weak::new(),
4144///     });
4145///     rc.me = UniqueArc::downgrade(&rc);
4146///     Some(UniqueArc::into_arc(rc))
4147/// }
4148///
4149/// create_gadget().unwrap();
4150/// ```
4151///
4152/// An advantage of using `UniqueArc` over [`Arc::new_cyclic`] to build cyclic data structures is that
4153/// [`Arc::new_cyclic`]'s `data_fn` parameter cannot be async or return a [`Result`]. As shown in the
4154/// previous example, `UniqueArc` allows for more flexibility in the construction of cyclic data,
4155/// including fallible or async constructors.
4156#[unstable(feature = "unique_rc_arc", issue = "112566")]
4157pub struct UniqueArc<
4158    T: ?Sized,
4159    #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator = Global,
4160> {
4161    ptr: NonNull<ArcInner<T>>,
4162    // Define the ownership of `ArcInner<T>` for drop-check
4163    _marker: PhantomData<ArcInner<T>>,
4164    // Invariance is necessary for soundness: once other `Weak`
4165    // references exist, we already have a form of shared mutability!
4166    _marker2: PhantomData<*mut T>,
4167    alloc: A,
4168}
4169
4170#[unstable(feature = "unique_rc_arc", issue = "112566")]
4171unsafe impl<T: ?Sized + Sync + Send, A: Allocator + Send> Send for UniqueArc<T, A> {}
4172
4173#[unstable(feature = "unique_rc_arc", issue = "112566")]
4174unsafe impl<T: ?Sized + Sync + Send, A: Allocator + Sync> Sync for UniqueArc<T, A> {}
4175
4176#[unstable(feature = "unique_rc_arc", issue = "112566")]
4177// #[unstable(feature = "coerce_unsized", issue = "18598")]
4178impl<T: ?Sized + Unsize<U>, U: ?Sized, A: Allocator> CoerceUnsized<UniqueArc<U, A>>
4179    for UniqueArc<T, A>
4180{
4181}
4182
4183//#[unstable(feature = "unique_rc_arc", issue = "112566")]
4184#[unstable(feature = "dispatch_from_dyn", issue = "none")]
4185impl<T: ?Sized + Unsize<U>, U: ?Sized> DispatchFromDyn<UniqueArc<U>> for UniqueArc<T> {}
4186
4187#[unstable(feature = "unique_rc_arc", issue = "112566")]
4188impl<T: ?Sized + fmt::Display, A: Allocator> fmt::Display for UniqueArc<T, A> {
4189    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4190        fmt::Display::fmt(&**self, f)
4191    }
4192}
4193
4194#[unstable(feature = "unique_rc_arc", issue = "112566")]
4195impl<T: ?Sized + fmt::Debug, A: Allocator> fmt::Debug for UniqueArc<T, A> {
4196    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4197        fmt::Debug::fmt(&**self, f)
4198    }
4199}
4200
4201#[unstable(feature = "unique_rc_arc", issue = "112566")]
4202impl<T: ?Sized, A: Allocator> fmt::Pointer for UniqueArc<T, A> {
4203    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4204        fmt::Pointer::fmt(&(&raw const **self), f)
4205    }
4206}
4207
4208#[unstable(feature = "unique_rc_arc", issue = "112566")]
4209impl<T: ?Sized, A: Allocator> borrow::Borrow<T> for UniqueArc<T, A> {
4210    fn borrow(&self) -> &T {
4211        &**self
4212    }
4213}
4214
4215#[unstable(feature = "unique_rc_arc", issue = "112566")]
4216impl<T: ?Sized, A: Allocator> borrow::BorrowMut<T> for UniqueArc<T, A> {
4217    fn borrow_mut(&mut self) -> &mut T {
4218        &mut **self
4219    }
4220}
4221
4222#[unstable(feature = "unique_rc_arc", issue = "112566")]
4223impl<T: ?Sized, A: Allocator> AsRef<T> for UniqueArc<T, A> {
4224    fn as_ref(&self) -> &T {
4225        &**self
4226    }
4227}
4228
4229#[unstable(feature = "unique_rc_arc", issue = "112566")]
4230impl<T: ?Sized, A: Allocator> AsMut<T> for UniqueArc<T, A> {
4231    fn as_mut(&mut self) -> &mut T {
4232        &mut **self
4233    }
4234}
4235
4236#[unstable(feature = "unique_rc_arc", issue = "112566")]
4237impl<T: ?Sized, A: Allocator> Unpin for UniqueArc<T, A> {}
4238
4239#[unstable(feature = "unique_rc_arc", issue = "112566")]
4240impl<T: ?Sized + PartialEq, A: Allocator> PartialEq for UniqueArc<T, A> {
4241    /// Equality for two `UniqueArc`s.
4242    ///
4243    /// Two `UniqueArc`s are equal if their inner values are equal.
4244    ///
4245    /// # Examples
4246    ///
4247    /// ```
4248    /// #![feature(unique_rc_arc)]
4249    /// use std::sync::UniqueArc;
4250    ///
4251    /// let five = UniqueArc::new(5);
4252    ///
4253    /// assert!(five == UniqueArc::new(5));
4254    /// ```
4255    #[inline]
4256    fn eq(&self, other: &Self) -> bool {
4257        PartialEq::eq(&**self, &**other)
4258    }
4259}
4260
4261#[unstable(feature = "unique_rc_arc", issue = "112566")]
4262impl<T: ?Sized + PartialOrd, A: Allocator> PartialOrd for UniqueArc<T, A> {
4263    /// Partial comparison for two `UniqueArc`s.
4264    ///
4265    /// The two are compared by calling `partial_cmp()` on their inner values.
4266    ///
4267    /// # Examples
4268    ///
4269    /// ```
4270    /// #![feature(unique_rc_arc)]
4271    /// use std::sync::UniqueArc;
4272    /// use std::cmp::Ordering;
4273    ///
4274    /// let five = UniqueArc::new(5);
4275    ///
4276    /// assert_eq!(Some(Ordering::Less), five.partial_cmp(&UniqueArc::new(6)));
4277    /// ```
4278    #[inline(always)]
4279    fn partial_cmp(&self, other: &UniqueArc<T, A>) -> Option<Ordering> {
4280        (**self).partial_cmp(&**other)
4281    }
4282
4283    /// Less-than comparison for two `UniqueArc`s.
4284    ///
4285    /// The two are compared by calling `<` on their inner values.
4286    ///
4287    /// # Examples
4288    ///
4289    /// ```
4290    /// #![feature(unique_rc_arc)]
4291    /// use std::sync::UniqueArc;
4292    ///
4293    /// let five = UniqueArc::new(5);
4294    ///
4295    /// assert!(five < UniqueArc::new(6));
4296    /// ```
4297    #[inline(always)]
4298    fn lt(&self, other: &UniqueArc<T, A>) -> bool {
4299        **self < **other
4300    }
4301
4302    /// 'Less than or equal to' comparison for two `UniqueArc`s.
4303    ///
4304    /// The two are compared by calling `<=` on their inner values.
4305    ///
4306    /// # Examples
4307    ///
4308    /// ```
4309    /// #![feature(unique_rc_arc)]
4310    /// use std::sync::UniqueArc;
4311    ///
4312    /// let five = UniqueArc::new(5);
4313    ///
4314    /// assert!(five <= UniqueArc::new(5));
4315    /// ```
4316    #[inline(always)]
4317    fn le(&self, other: &UniqueArc<T, A>) -> bool {
4318        **self <= **other
4319    }
4320
4321    /// Greater-than comparison for two `UniqueArc`s.
4322    ///
4323    /// The two are compared by calling `>` on their inner values.
4324    ///
4325    /// # Examples
4326    ///
4327    /// ```
4328    /// #![feature(unique_rc_arc)]
4329    /// use std::sync::UniqueArc;
4330    ///
4331    /// let five = UniqueArc::new(5);
4332    ///
4333    /// assert!(five > UniqueArc::new(4));
4334    /// ```
4335    #[inline(always)]
4336    fn gt(&self, other: &UniqueArc<T, A>) -> bool {
4337        **self > **other
4338    }
4339
4340    /// 'Greater than or equal to' comparison for two `UniqueArc`s.
4341    ///
4342    /// The two are compared by calling `>=` on their inner values.
4343    ///
4344    /// # Examples
4345    ///
4346    /// ```
4347    /// #![feature(unique_rc_arc)]
4348    /// use std::sync::UniqueArc;
4349    ///
4350    /// let five = UniqueArc::new(5);
4351    ///
4352    /// assert!(five >= UniqueArc::new(5));
4353    /// ```
4354    #[inline(always)]
4355    fn ge(&self, other: &UniqueArc<T, A>) -> bool {
4356        **self >= **other
4357    }
4358}
4359
4360#[unstable(feature = "unique_rc_arc", issue = "112566")]
4361impl<T: ?Sized + Ord, A: Allocator> Ord for UniqueArc<T, A> {
4362    /// Comparison for two `UniqueArc`s.
4363    ///
4364    /// The two are compared by calling `cmp()` on their inner values.
4365    ///
4366    /// # Examples
4367    ///
4368    /// ```
4369    /// #![feature(unique_rc_arc)]
4370    /// use std::sync::UniqueArc;
4371    /// use std::cmp::Ordering;
4372    ///
4373    /// let five = UniqueArc::new(5);
4374    ///
4375    /// assert_eq!(Ordering::Less, five.cmp(&UniqueArc::new(6)));
4376    /// ```
4377    #[inline]
4378    fn cmp(&self, other: &UniqueArc<T, A>) -> Ordering {
4379        (**self).cmp(&**other)
4380    }
4381}
4382
4383#[unstable(feature = "unique_rc_arc", issue = "112566")]
4384impl<T: ?Sized + Eq, A: Allocator> Eq for UniqueArc<T, A> {}
4385
4386#[unstable(feature = "unique_rc_arc", issue = "112566")]
4387impl<T: ?Sized + Hash, A: Allocator> Hash for UniqueArc<T, A> {
4388    fn hash<H: Hasher>(&self, state: &mut H) {
4389        (**self).hash(state);
4390    }
4391}
4392
4393impl<T> UniqueArc<T, Global> {
4394    /// Creates a new `UniqueArc`.
4395    ///
4396    /// Weak references to this `UniqueArc` can be created with [`UniqueArc::downgrade`]. Upgrading
4397    /// these weak references will fail before the `UniqueArc` has been converted into an [`Arc`].
4398    /// After converting the `UniqueArc` into an [`Arc`], any weak references created beforehand will
4399    /// point to the new [`Arc`].
4400    #[cfg(not(no_global_oom_handling))]
4401    #[unstable(feature = "unique_rc_arc", issue = "112566")]
4402    #[must_use]
4403    pub fn new(value: T) -> Self {
4404        Self::new_in(value, Global)
4405    }
4406}
4407
4408impl<T, A: Allocator> UniqueArc<T, A> {
4409    /// Creates a new `UniqueArc` in the provided allocator.
4410    ///
4411    /// Weak references to this `UniqueArc` can be created with [`UniqueArc::downgrade`]. Upgrading
4412    /// these weak references will fail before the `UniqueArc` has been converted into an [`Arc`].
4413    /// After converting the `UniqueArc` into an [`Arc`], any weak references created beforehand will
4414    /// point to the new [`Arc`].
4415    #[cfg(not(no_global_oom_handling))]
4416    #[unstable(feature = "unique_rc_arc", issue = "112566")]
4417    #[must_use]
4418    // #[unstable(feature = "allocator_api", issue = "32838")]
4419    pub fn new_in(data: T, alloc: A) -> Self {
4420        let (ptr, alloc) = Box::into_unique(Box::new_in(
4421            ArcInner {
4422                strong: atomic::AtomicUsize::new(0),
4423                // keep one weak reference so if all the weak pointers that are created are dropped
4424                // the UniqueArc still stays valid.
4425                weak: atomic::AtomicUsize::new(1),
4426                data,
4427            },
4428            alloc,
4429        ));
4430        Self { ptr: ptr.into(), _marker: PhantomData, _marker2: PhantomData, alloc }
4431    }
4432}
4433
4434impl<T: ?Sized, A: Allocator> UniqueArc<T, A> {
4435    /// Converts the `UniqueArc` into a regular [`Arc`].
4436    ///
4437    /// This consumes the `UniqueArc` and returns a regular [`Arc`] that contains the `value` that
4438    /// is passed to `into_arc`.
4439    ///
4440    /// Any weak references created before this method is called can now be upgraded to strong
4441    /// references.
4442    #[unstable(feature = "unique_rc_arc", issue = "112566")]
4443    #[must_use]
4444    pub fn into_arc(this: Self) -> Arc<T, A> {
4445        let this = ManuallyDrop::new(this);
4446
4447        // Move the allocator out.
4448        // SAFETY: `this.alloc` will not be accessed again, nor dropped because it is in
4449        // a `ManuallyDrop`.
4450        let alloc: A = unsafe { ptr::read(&this.alloc) };
4451
4452        // SAFETY: This pointer was allocated at creation time so we know it is valid.
4453        unsafe {
4454            // Convert our weak reference into a strong reference
4455            (*this.ptr.as_ptr()).strong.store(1, Release);
4456            Arc::from_inner_in(this.ptr, alloc)
4457        }
4458    }
4459}
4460
4461impl<T: ?Sized, A: Allocator + Clone> UniqueArc<T, A> {
4462    /// Creates a new weak reference to the `UniqueArc`.
4463    ///
4464    /// Attempting to upgrade this weak reference will fail before the `UniqueArc` has been converted
4465    /// to a [`Arc`] using [`UniqueArc::into_arc`].
4466    #[unstable(feature = "unique_rc_arc", issue = "112566")]
4467    #[must_use]
4468    pub fn downgrade(this: &Self) -> Weak<T, A> {
4469        // Using a relaxed ordering is alright here, as knowledge of the
4470        // original reference prevents other threads from erroneously deleting
4471        // the object or converting the object to a normal `Arc<T, A>`.
4472        //
4473        // Note that we don't need to test if the weak counter is locked because there
4474        // are no such operations like `Arc::get_mut` or `Arc::make_mut` that will lock
4475        // the weak counter.
4476        //
4477        // SAFETY: This pointer was allocated at creation time so we know it is valid.
4478        let old_size = unsafe { (*this.ptr.as_ptr()).weak.fetch_add(1, Relaxed) };
4479
4480        // See comments in Arc::clone() for why we do this (for mem::forget).
4481        if old_size > MAX_REFCOUNT {
4482            abort();
4483        }
4484
4485        Weak { ptr: this.ptr, alloc: this.alloc.clone() }
4486    }
4487}
4488
4489#[unstable(feature = "unique_rc_arc", issue = "112566")]
4490impl<T: ?Sized, A: Allocator> Deref for UniqueArc<T, A> {
4491    type Target = T;
4492
4493    fn deref(&self) -> &T {
4494        // SAFETY: This pointer was allocated at creation time so we know it is valid.
4495        unsafe { &self.ptr.as_ref().data }
4496    }
4497}
4498
4499// #[unstable(feature = "unique_rc_arc", issue = "112566")]
4500#[unstable(feature = "pin_coerce_unsized_trait", issue = "123430")]
4501unsafe impl<T: ?Sized> PinCoerceUnsized for UniqueArc<T> {}
4502
4503#[unstable(feature = "unique_rc_arc", issue = "112566")]
4504impl<T: ?Sized, A: Allocator> DerefMut for UniqueArc<T, A> {
4505    fn deref_mut(&mut self) -> &mut T {
4506        // SAFETY: This pointer was allocated at creation time so we know it is valid. We know we
4507        // have unique ownership and therefore it's safe to make a mutable reference because
4508        // `UniqueArc` owns the only strong reference to itself.
4509        // We also need to be careful to only create a mutable reference to the `data` field,
4510        // as a mutable reference to the entire `ArcInner` would assert uniqueness over the
4511        // ref count fields too, invalidating any attempt by `Weak`s to access the ref count.
4512        unsafe { &mut (*self.ptr.as_ptr()).data }
4513    }
4514}
4515
4516#[unstable(feature = "unique_rc_arc", issue = "112566")]
4517// #[unstable(feature = "deref_pure_trait", issue = "87121")]
4518unsafe impl<T: ?Sized, A: Allocator> DerefPure for UniqueArc<T, A> {}
4519
4520#[unstable(feature = "unique_rc_arc", issue = "112566")]
4521unsafe impl<#[may_dangle] T: ?Sized, A: Allocator> Drop for UniqueArc<T, A> {
4522    fn drop(&mut self) {
4523        // See `Arc::drop_slow` which drops an `Arc` with a strong count of 0.
4524        // SAFETY: This pointer was allocated at creation time so we know it is valid.
4525        let _weak = Weak { ptr: self.ptr, alloc: &self.alloc };
4526
4527        unsafe { ptr::drop_in_place(&mut (*self.ptr.as_ptr()).data) };
4528    }
4529}