Skip to main content

rustc_abi/
layout.rs

1use std::cmp;
2use std::fmt::{self, Write};
3use std::num::NonZero;
4use std::ops::Deref;
5use std::range::{RangeFrom, RangeToInclusive};
6
7pub use coroutine::PackCoroutineLayout;
8use rustc_hashes::Hash64;
9use rustc_index::Idx;
10use rustc_index::bit_set::BitMatrix;
11use tracing::debug;
12
13use crate::{
14    Align, BackendLaneCount, BackendRepr, FieldsShape, HasDataLayout, IndexSlice, IndexVec,
15    Integer, LayoutData, NicheOptimizations, NumScalableVectors, ReprOptions, Scalar, Size,
16    StructKind, TargetDataLayout, Variants, WrappingRange,
17};
18
19mod coroutine;
20mod enum_layout;
21mod simple;
22
23#[cfg(feature = "nightly")]
24mod ty;
25
26#[cfg(feature = "nightly")]
27pub use ty::{Layout, TyAbiInterface, TyAndLayout};
28
29#[automatically_derived]impl ::core::marker::Copy for FieldIdx { }
impl FieldIdx {
    #[doc = r" Maximum value the index can take, as a `u32`."]
    pub const MAX_AS_U32: u32 = 0xFFFF_FF00;
    #[doc = r" Maximum value the index can take."]
    pub const MAX: Self = Self::from_u32(0xFFFF_FF00);
    #[doc = r" Zero value of the index."]
    pub const ZERO: Self = Self::from_u32(0);
    #[doc = r" Creates a new index from a given `usize`."]
    #[doc = r""]
    #[doc = r" # Panics"]
    #[doc = r""]
    #[doc = r" Will panic if `value` exceeds `MAX`."]
    #[inline]
    pub const fn from_usize(value: usize) -> Self {
        if !(value <= (0xFFFF_FF00 as usize)) {
            ::core::panicking::panic("assertion failed: value <= (0xFFFF_FF00 as usize)")
        };
        unsafe { Self::from_u32_unchecked(value as u32) }
    }
    #[doc = r" Creates a new index from a given `u32`."]
    #[doc = r""]
    #[doc = r" # Panics"]
    #[doc = r""]
    #[doc = r" Will panic if `value` exceeds `MAX`."]
    #[inline]
    pub const fn from_u32(value: u32) -> Self {
        if !(value <= 0xFFFF_FF00) {
            ::core::panicking::panic("assertion failed: value <= 0xFFFF_FF00")
        };
        unsafe { Self::from_u32_unchecked(value) }
    }
    #[doc = r" Creates a new index from a given `u16`."]
    #[doc = r""]
    #[doc = r" # Panics"]
    #[doc = r""]
    #[doc = r" Will panic if `value` exceeds `MAX`."]
    #[inline]
    pub const fn from_u16(value: u16) -> Self {
        let value = value as u32;
        if !(value <= 0xFFFF_FF00) {
            ::core::panicking::panic("assertion failed: value <= 0xFFFF_FF00")
        };
        unsafe { Self::from_u32_unchecked(value) }
    }
    #[doc = r" Creates a new index from a given `u32`."]
    #[doc = r""]
    #[doc = r" # Safety"]
    #[doc = r""]
    #[doc =
    r" The provided value must be less than or equal to the maximum value for the newtype."]
    #[doc =
    r" Providing a value outside this range is undefined due to layout restrictions."]
    #[doc = r""]
    #[doc = r" Prefer using `from_u32`."]
    #[inline]
    pub const unsafe fn from_u32_unchecked(value: u32) -> Self {
        Self {
            private_use_as_methods_instead: unsafe {
                std::mem::transmute(value)
            },
        }
    }
    #[doc = r" Extracts the value of this index as a `usize`."]
    #[inline]
    pub const fn index(self) -> usize { self.as_usize() }
    #[doc = r" Extracts the value of this index as a `u32`."]
    #[inline]
    pub const fn as_u32(self) -> u32 {
        unsafe { std::mem::transmute(self.private_use_as_methods_instead) }
    }
    #[doc = r" Extracts the value of this index as a `usize`."]
    #[inline]
    pub const fn as_usize(self) -> usize { self.as_u32() as usize }
}
impl std::ops::Add<usize> for FieldIdx {
    type Output = Self;
    #[inline]
    fn add(self, other: usize) -> Self {
        Self::from_usize(self.index() + other)
    }
}
impl std::ops::AddAssign<usize> for FieldIdx {
    #[inline]
    fn add_assign(&mut self, other: usize) { *self = *self + other; }
}
impl rustc_index::Idx for FieldIdx {
    #[inline]
    fn new(value: usize) -> Self { Self::from_usize(value) }
    #[inline]
    fn index(self) -> usize { self.as_usize() }
}
impl ::std::iter::Step for FieldIdx {
    #[inline]
    fn steps_between(start: &Self, end: &Self) -> (usize, Option<usize>) {
        <usize as
                ::std::iter::Step>::steps_between(&Self::index(*start),
            &Self::index(*end))
    }
    #[inline]
    fn forward_checked(start: Self, u: usize) -> Option<Self> {
        Self::index(start).checked_add(u).map(Self::from_usize)
    }
    #[inline]
    fn backward_checked(start: Self, u: usize) -> Option<Self> {
        Self::index(start).checked_sub(u).map(Self::from_usize)
    }
    #[inline]
    fn forward_overflowing(start: Self, u: usize) -> (Self, bool) {
        let (s, o) = Self::index(start).overflowing_add(u);
        (Self::from_usize(s), o)
    }
    #[inline]
    fn backward_overflowing(start: Self, u: usize) -> (Self, bool) {
        let (s, o) = Self::index(start).overflowing_sub(u);
        (Self::from_usize(s), o)
    }
}
impl ::std::cmp::Ord for FieldIdx {
    #[inline]
    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
        self.as_u32().cmp(&other.as_u32())
    }
}
impl ::std::cmp::PartialOrd for FieldIdx {
    #[inline]
    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
        Some(self.cmp(other))
    }
}
impl ::rustc_data_structures::stable_hash::StableHash for FieldIdx {
    fn stable_hash<__Hcx: ::rustc_data_structures::stable_hash::StableHashCtxt>(&self,
        hcx: &mut __Hcx,
        hasher: &mut ::rustc_data_structures::stable_hash::StableHasher) {
        self.as_u32().stable_hash(hcx, hasher)
    }
}
impl From<FieldIdx> for u32 {
    #[inline]
    fn from(v: FieldIdx) -> u32 { v.as_u32() }
}
impl From<FieldIdx> for usize {
    #[inline]
    fn from(v: FieldIdx) -> usize { v.as_usize() }
}
impl From<usize> for FieldIdx {
    #[inline]
    fn from(value: usize) -> Self { Self::from_usize(value) }
}
impl From<u32> for FieldIdx {
    #[inline]
    fn from(value: u32) -> Self { Self::from_u32(value) }
}
impl ::std::cmp::Eq for FieldIdx {}
impl ::std::cmp::PartialEq for FieldIdx {
    fn eq(&self, other: &Self) -> bool { self.as_u32().eq(&other.as_u32()) }
}
impl ::std::marker::StructuralPartialEq for FieldIdx { }
impl ::std::hash::Hash for FieldIdx {
    fn hash<H: ::std::hash::Hasher>(&self, state: &mut H) {
        self.as_u32().hash(state)
    }
}
impl<D: ::rustc_serialize::Decoder> ::rustc_serialize::Decodable<D> for
    FieldIdx {
    fn decode(d: &mut D) -> Self { Self::from_u32(d.read_u32()) }
}
impl<E: ::rustc_serialize::Encoder> ::rustc_serialize::Encodable<E> for
    FieldIdx {
    fn encode(&self, e: &mut E) { e.emit_u32(self.as_u32()); }
}
impl ::std::fmt::Debug for FieldIdx {
    fn fmt(&self, fmt: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result {
        fmt.write_fmt(format_args!("{0}", self.as_u32()))
    }
}rustc_index::newtype_index! {
30    /// The *source-order* index of a field in a variant.
31    ///
32    /// This is how most code after type checking refers to fields, rather than
33    /// using names (as names have hygiene complications and more complex lookup).
34    ///
35    /// Particularly for `repr(Rust)` types, this may not be the same as *layout* order.
36    /// (It is for `repr(C)` `struct`s, however.)
37    ///
38    /// For example, in the following types,
39    /// ```rust
40    /// # enum Never {}
41    /// # #[repr(u16)]
42    /// enum Demo1 {
43    ///    Variant0 { a: Never, b: i32 } = 100,
44    ///    Variant1 { c: u8, d: u64 } = 10,
45    /// }
46    /// struct Demo2 { e: u8, f: u16, g: u8 }
47    /// ```
48    /// `b` is `FieldIdx(1)` in `VariantIdx(0)`,
49    /// `d` is `FieldIdx(1)` in `VariantIdx(1)`, and
50    /// `f` is `FieldIdx(1)` in `VariantIdx(0)`.
51    #[stable_hash]
52    #[encodable]
53    #[orderable]
54    #[gate_rustc_only]
55    pub struct FieldIdx {}
56}
57
58impl FieldIdx {
59    /// The second field, at index 1.
60    ///
61    /// For use alongside [`FieldIdx::ZERO`], particularly with scalar pairs.
62    pub const ONE: FieldIdx = FieldIdx::from_u32(1);
63}
64
65#[automatically_derived]impl ::core::marker::Copy for VariantIdx { }
#[doc = " Equivalent to `VariantIdx(0)`."]
pub const FIRST_VARIANT: VariantIdx = VariantIdx::from_u32(0);
impl VariantIdx {
    #[doc = r" Maximum value the index can take, as a `u32`."]
    pub const MAX_AS_U32: u32 = 0xFFFF_FF00;
    #[doc = r" Maximum value the index can take."]
    pub const MAX: Self = Self::from_u32(0xFFFF_FF00);
    #[doc = r" Zero value of the index."]
    pub const ZERO: Self = Self::from_u32(0);
    #[doc = r" Creates a new index from a given `usize`."]
    #[doc = r""]
    #[doc = r" # Panics"]
    #[doc = r""]
    #[doc = r" Will panic if `value` exceeds `MAX`."]
    #[inline]
    pub const fn from_usize(value: usize) -> Self {
        if !(value <= (0xFFFF_FF00 as usize)) {
            ::core::panicking::panic("assertion failed: value <= (0xFFFF_FF00 as usize)")
        };
        unsafe { Self::from_u32_unchecked(value as u32) }
    }
    #[doc = r" Creates a new index from a given `u32`."]
    #[doc = r""]
    #[doc = r" # Panics"]
    #[doc = r""]
    #[doc = r" Will panic if `value` exceeds `MAX`."]
    #[inline]
    pub const fn from_u32(value: u32) -> Self {
        if !(value <= 0xFFFF_FF00) {
            ::core::panicking::panic("assertion failed: value <= 0xFFFF_FF00")
        };
        unsafe { Self::from_u32_unchecked(value) }
    }
    #[doc = r" Creates a new index from a given `u16`."]
    #[doc = r""]
    #[doc = r" # Panics"]
    #[doc = r""]
    #[doc = r" Will panic if `value` exceeds `MAX`."]
    #[inline]
    pub const fn from_u16(value: u16) -> Self {
        let value = value as u32;
        if !(value <= 0xFFFF_FF00) {
            ::core::panicking::panic("assertion failed: value <= 0xFFFF_FF00")
        };
        unsafe { Self::from_u32_unchecked(value) }
    }
    #[doc = r" Creates a new index from a given `u32`."]
    #[doc = r""]
    #[doc = r" # Safety"]
    #[doc = r""]
    #[doc =
    r" The provided value must be less than or equal to the maximum value for the newtype."]
    #[doc =
    r" Providing a value outside this range is undefined due to layout restrictions."]
    #[doc = r""]
    #[doc = r" Prefer using `from_u32`."]
    #[inline]
    pub const unsafe fn from_u32_unchecked(value: u32) -> Self {
        Self {
            private_use_as_methods_instead: unsafe {
                std::mem::transmute(value)
            },
        }
    }
    #[doc = r" Extracts the value of this index as a `usize`."]
    #[inline]
    pub const fn index(self) -> usize { self.as_usize() }
    #[doc = r" Extracts the value of this index as a `u32`."]
    #[inline]
    pub const fn as_u32(self) -> u32 {
        unsafe { std::mem::transmute(self.private_use_as_methods_instead) }
    }
    #[doc = r" Extracts the value of this index as a `usize`."]
    #[inline]
    pub const fn as_usize(self) -> usize { self.as_u32() as usize }
}
impl std::ops::Add<usize> for VariantIdx {
    type Output = Self;
    #[inline]
    fn add(self, other: usize) -> Self {
        Self::from_usize(self.index() + other)
    }
}
impl std::ops::AddAssign<usize> for VariantIdx {
    #[inline]
    fn add_assign(&mut self, other: usize) { *self = *self + other; }
}
impl rustc_index::Idx for VariantIdx {
    #[inline]
    fn new(value: usize) -> Self { Self::from_usize(value) }
    #[inline]
    fn index(self) -> usize { self.as_usize() }
}
impl ::std::iter::Step for VariantIdx {
    #[inline]
    fn steps_between(start: &Self, end: &Self) -> (usize, Option<usize>) {
        <usize as
                ::std::iter::Step>::steps_between(&Self::index(*start),
            &Self::index(*end))
    }
    #[inline]
    fn forward_checked(start: Self, u: usize) -> Option<Self> {
        Self::index(start).checked_add(u).map(Self::from_usize)
    }
    #[inline]
    fn backward_checked(start: Self, u: usize) -> Option<Self> {
        Self::index(start).checked_sub(u).map(Self::from_usize)
    }
    #[inline]
    fn forward_overflowing(start: Self, u: usize) -> (Self, bool) {
        let (s, o) = Self::index(start).overflowing_add(u);
        (Self::from_usize(s), o)
    }
    #[inline]
    fn backward_overflowing(start: Self, u: usize) -> (Self, bool) {
        let (s, o) = Self::index(start).overflowing_sub(u);
        (Self::from_usize(s), o)
    }
}
impl ::std::cmp::Ord for VariantIdx {
    #[inline]
    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
        self.as_u32().cmp(&other.as_u32())
    }
}
impl ::std::cmp::PartialOrd for VariantIdx {
    #[inline]
    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
        Some(self.cmp(other))
    }
}
impl ::rustc_data_structures::stable_hash::StableHash for VariantIdx {
    fn stable_hash<__Hcx: ::rustc_data_structures::stable_hash::StableHashCtxt>(&self,
        hcx: &mut __Hcx,
        hasher: &mut ::rustc_data_structures::stable_hash::StableHasher) {
        self.as_u32().stable_hash(hcx, hasher)
    }
}
impl From<VariantIdx> for u32 {
    #[inline]
    fn from(v: VariantIdx) -> u32 { v.as_u32() }
}
impl From<VariantIdx> for usize {
    #[inline]
    fn from(v: VariantIdx) -> usize { v.as_usize() }
}
impl From<usize> for VariantIdx {
    #[inline]
    fn from(value: usize) -> Self { Self::from_usize(value) }
}
impl From<u32> for VariantIdx {
    #[inline]
    fn from(value: u32) -> Self { Self::from_u32(value) }
}
impl ::std::cmp::Eq for VariantIdx {}
impl ::std::cmp::PartialEq for VariantIdx {
    fn eq(&self, other: &Self) -> bool { self.as_u32().eq(&other.as_u32()) }
}
impl ::std::marker::StructuralPartialEq for VariantIdx { }
impl ::std::hash::Hash for VariantIdx {
    fn hash<H: ::std::hash::Hasher>(&self, state: &mut H) {
        self.as_u32().hash(state)
    }
}
impl<D: ::rustc_serialize::Decoder> ::rustc_serialize::Decodable<D> for
    VariantIdx {
    fn decode(d: &mut D) -> Self { Self::from_u32(d.read_u32()) }
}
impl<E: ::rustc_serialize::Encoder> ::rustc_serialize::Encodable<E> for
    VariantIdx {
    fn encode(&self, e: &mut E) { e.emit_u32(self.as_u32()); }
}
impl ::std::fmt::Debug for VariantIdx {
    fn fmt(&self, fmt: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result {
        fmt.write_fmt(format_args!("{0}", self.as_u32()))
    }
}rustc_index::newtype_index! {
66    /// The *source-order* index of a variant in a type.
67    ///
68    /// For enums, these are always `0..variant_count`, regardless of any
69    /// custom discriminants that may have been defined, and including any
70    /// variants that may end up uninhabited due to field types.  (Some of the
71    /// variants may not be present in a monomorphized ABI [`Variants`], but
72    /// those skipped variants are always counted when determining the *index*.)
73    ///
74    /// `struct`s, `tuples`, and `unions`s are considered to have a single variant
75    /// with variant index zero, aka [`FIRST_VARIANT`].
76    #[stable_hash]
77    #[encodable]
78    #[orderable]
79    #[gate_rustc_only]
80    pub struct VariantIdx {
81        /// Equivalent to `VariantIdx(0)`.
82        const FIRST_VARIANT = 0;
83    }
84}
85
86// A variant is absent if it's uninhabited and only has ZST fields.
87// Present uninhabited variants only require space for their fields,
88// but *not* an encoding of the discriminant (e.g., a tag value).
89// See issue #49298 for more details on the need to leave space
90// for non-ZST uninhabited data (mostly partial initialization).
91fn absent<'a, FieldIdx, VariantIdx, F>(fields: &IndexSlice<FieldIdx, F>) -> bool
92where
93    FieldIdx: Idx,
94    VariantIdx: Idx,
95    F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,
96{
97    let uninhabited = fields.iter().any(|f| f.is_uninhabited());
98    // We cannot ignore alignment; that might lead us to entirely discard a variant and
99    // produce an enum that is less aligned than it should be!
100    let is_1zst = fields.iter().all(|f| f.is_1zst());
101    uninhabited && is_1zst
102}
103
104/// Determines towards which end of a struct layout optimizations will try to place the best niches.
105enum NicheBias {
106    Start,
107    End,
108}
109
110#[derive(#[automatically_derived]impl<F: ::core::marker::Copy> ::core::marker::Copy for
    LayoutCalculatorError<F> {
}Copy, #[automatically_derived]impl<F: ::core::clone::Clone> ::core::clone::Clone for
    LayoutCalculatorError<F> {
    #[inline]
    #[rustc_trivial_field_reads]
    fn clone(&self) -> Self {
        match self {
            Self::UnexpectedUnsized(__self_0) =>
                LayoutCalculatorError::UnexpectedUnsized(::core::clone::Clone::clone(__self_0)),
            Self::SizeOverflow => LayoutCalculatorError::SizeOverflow,
            Self::EmptyUnion => LayoutCalculatorError::EmptyUnion,
            Self::ReprConflict => LayoutCalculatorError::ReprConflict,
            Self::ZeroLengthSimdType =>
                LayoutCalculatorError::ZeroLengthSimdType,
            Self::OversizedSimdType { max_lanes: __self_0 } =>
                LayoutCalculatorError::OversizedSimdType {
                    max_lanes: ::core::clone::Clone::clone(__self_0),
                },
            Self::NonPrimitiveSimdType(__self_0) =>
                LayoutCalculatorError::NonPrimitiveSimdType(::core::clone::Clone::clone(__self_0)),
        }
    }
}Clone, #[automatically_derived]impl<F: ::core::fmt::Debug> ::core::fmt::Debug for
    LayoutCalculatorError<F> {
    #[inline]
    #[rustc_trivial_field_reads]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        match self {
            Self::UnexpectedUnsized(__self_0) =>
                ::core::fmt::Formatter::debug_struct_field1_finish(f,
                    "UnexpectedUnsized", &__self_0),
            Self::SizeOverflow =>
                ::core::fmt::Formatter::write_str(f, "SizeOverflow"),
            Self::EmptyUnion =>
                ::core::fmt::Formatter::write_str(f, "EmptyUnion"),
            Self::ReprConflict =>
                ::core::fmt::Formatter::write_str(f, "ReprConflict"),
            Self::ZeroLengthSimdType =>
                ::core::fmt::Formatter::write_str(f, "ZeroLengthSimdType"),
            Self::OversizedSimdType { max_lanes: __self_0 } =>
                ::core::fmt::Formatter::debug_struct_field1_finish(f,
                    "OversizedSimdType max_lanes", &__self_0),
            Self::NonPrimitiveSimdType(__self_0) =>
                ::core::fmt::Formatter::debug_struct_field1_finish(f,
                    "NonPrimitiveSimdType", &__self_0),
        }
    }
}Debug, #[automatically_derived]impl<F: ::core::cmp::PartialEq>
    ::core::marker::StructuralPartialEq for LayoutCalculatorError<F> {
}
#[automatically_derived]impl<F: ::core::cmp::PartialEq> ::core::cmp::PartialEq
    for LayoutCalculatorError<F> {
    #[inline]
    fn eq(&self, other: &Self) -> bool {
        ::core::intrinsics::discriminant_value(self) ==
                ::core::intrinsics::discriminant_value(other) &&
            match (self, other) {
                (Self::UnexpectedUnsized(__self_0),
                    Self::UnexpectedUnsized(__arg1_0)) => __self_0 == __arg1_0,
                (Self::OversizedSimdType { max_lanes: __self_0 },
                    Self::OversizedSimdType { max_lanes: __arg1_0 }) =>
                    __self_0 == __arg1_0,
                (Self::NonPrimitiveSimdType(__self_0),
                    Self::NonPrimitiveSimdType(__arg1_0)) =>
                    __self_0 == __arg1_0,
                _ => true,
            }
    }
}PartialEq, #[automatically_derived]impl<F: ::core::cmp::Eq> ::core::cmp::Eq for
    LayoutCalculatorError<F> {
    #[inline]
    #[doc(hidden)]
    #[coverage(off)]
    fn assert_fields_are_eq(&self) {
        let _: ::core::cmp::AssertParamIsEq<F>;
        let _: ::core::cmp::AssertParamIsEq<usize>;
    }
}Eq)]
111pub enum LayoutCalculatorError<F> {
112    /// An unsized type was found in a location where a sized type was expected.
113    ///
114    /// This is not always a compile error, for example if there is a `[T]: Sized`
115    /// bound in a where clause.
116    ///
117    /// Contains the field that was unexpectedly unsized.
118    UnexpectedUnsized(F),
119
120    /// A type was too large for the target platform.
121    SizeOverflow,
122
123    /// A union had no fields.
124    EmptyUnion,
125
126    /// The fields or variants have irreconcilable reprs
127    ReprConflict,
128
129    /// The length of an SIMD type is zero
130    ZeroLengthSimdType,
131
132    /// The length of an SIMD type exceeds the maximum number of lanes
133    OversizedSimdType { max_lanes: usize },
134
135    /// An element type of an SIMD type isn't a primitive
136    NonPrimitiveSimdType(F),
137}
138
139impl<F> LayoutCalculatorError<F> {
140    pub fn without_payload(&self) -> LayoutCalculatorError<()> {
141        use LayoutCalculatorError::*;
142        match *self {
143            UnexpectedUnsized(_) => UnexpectedUnsized(()),
144            SizeOverflow => SizeOverflow,
145            EmptyUnion => EmptyUnion,
146            ReprConflict => ReprConflict,
147            ZeroLengthSimdType => ZeroLengthSimdType,
148            OversizedSimdType { max_lanes } => OversizedSimdType { max_lanes },
149            NonPrimitiveSimdType(_) => NonPrimitiveSimdType(()),
150        }
151    }
152
153    /// Format an untranslated diagnostic for this type
154    ///
155    /// Intended for use by rust-analyzer, as neither it nor `rustc_abi` depend on fluent infra.
156    pub fn fallback_fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
157        use LayoutCalculatorError::*;
158        f.write_str(match self {
159            UnexpectedUnsized(_) => "an unsized type was found where a sized type was expected",
160            SizeOverflow => "size overflow",
161            EmptyUnion => "type is a union with no fields",
162            ReprConflict => "type has an invalid repr",
163            ZeroLengthSimdType | OversizedSimdType { .. } | NonPrimitiveSimdType(_) => {
164                "invalid simd type definition"
165            }
166        })
167    }
168}
169
170type LayoutCalculatorResult<FieldIdx, VariantIdx, F> =
171    Result<LayoutData<FieldIdx, VariantIdx>, LayoutCalculatorError<F>>;
172
173#[derive(#[automatically_derived]impl<Cx: ::core::clone::Clone> ::core::clone::Clone
    for LayoutCalculator<Cx> {
    #[inline]
    #[rustc_trivial_field_reads]
    fn clone(&self) -> Self {
        LayoutCalculator { cx: ::core::clone::Clone::clone(&self.cx) }
    }
}Clone, #[automatically_derived]impl<Cx: ::core::marker::Copy> ::core::marker::Copy
    for LayoutCalculator<Cx> {
}Copy, #[automatically_derived]impl<Cx: ::core::fmt::Debug> ::core::fmt::Debug for
    LayoutCalculator<Cx> {
    #[inline]
    #[rustc_trivial_field_reads]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        ::core::fmt::Formatter::debug_struct_field1_finish(f,
            "LayoutCalculator cx", &&self.cx)
    }
}Debug)]
174pub struct LayoutCalculator<Cx> {
175    pub cx: Cx,
176}
177
178impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
179    pub fn new(cx: Cx) -> Self {
180        Self { cx }
181    }
182
183    pub fn layout_of_array_like<FieldIdx: Idx, VariantIdx: Idx, F>(
184        &self,
185        element: &LayoutData<FieldIdx, VariantIdx>,
186        count_if_sized: Option<u64>, // None for slices
187    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F> {
188        let count = count_if_sized.unwrap_or(0);
189        let size =
190            element.size.checked_mul(count, &self.cx).ok_or(LayoutCalculatorError::SizeOverflow)?;
191
192        Ok(LayoutData {
193            variants: Variants::Single { index: VariantIdx::new(0) },
194            fields: FieldsShape::Array { stride: element.size, count },
195            backend_repr: BackendRepr::Memory { sized: count_if_sized.is_some() },
196            largest_niche: element.largest_niche.filter(|_| count != 0),
197            uninhabited: element.uninhabited && count != 0,
198            align: element.align,
199            size,
200            max_repr_align: None,
201            unadjusted_abi_align: element.align,
202            repr_c: element.repr_c,
203            randomization_seed: element.randomization_seed.wrapping_add(Hash64::new(count)),
204        })
205    }
206
207    pub fn layout_of_scalable_vector_type<FieldIdx, VariantIdx, F>(
208        &self,
209        element: F,
210        count: u64,
211        number_of_vectors: NumScalableVectors,
212    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
213    where
214        FieldIdx: Idx,
215        VariantIdx: Idx,
216        F: AsRef<LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,
217    {
218        vector_type_layout(
219            SimdVectorKind::Scalable(number_of_vectors),
220            self.cx.data_layout(),
221            element,
222            count,
223        )
224    }
225
226    pub fn layout_of_simd_type<FieldIdx, VariantIdx, F>(
227        &self,
228        element: F,
229        count: u64,
230        repr_packed: bool,
231    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
232    where
233        FieldIdx: Idx,
234        VariantIdx: Idx,
235        F: AsRef<LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,
236    {
237        let kind = if repr_packed { SimdVectorKind::PackedFixed } else { SimdVectorKind::Fixed };
238        vector_type_layout(kind, self.cx.data_layout(), element, count)
239    }
240
241    /// Compute the layout for a coroutine.
242    ///
243    /// This uses dedicated code instead of [`Self::layout_of_struct_or_enum`], as coroutine
244    /// fields may be shared between multiple variants (see the [`coroutine`] module for details).
245    pub fn layout_of_coroutine<'a, F, VariantIdx, FieldIdx, LocalIdx>(
246        &self,
247        local_layouts: &IndexSlice<LocalIdx, F>,
248        prefix_layouts: IndexVec<FieldIdx, F>,
249        variant_fields: &IndexSlice<VariantIdx, IndexVec<FieldIdx, LocalIdx>>,
250        storage_conflicts: &BitMatrix<LocalIdx, LocalIdx>,
251        tag_to_layout: impl Fn(Scalar) -> F,
252    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
253    where
254        F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,
255        VariantIdx: Idx,
256        FieldIdx: Idx,
257        LocalIdx: Idx,
258    {
259        coroutine::layout(
260            self,
261            local_layouts,
262            prefix_layouts,
263            variant_fields,
264            storage_conflicts,
265            tag_to_layout,
266        )
267    }
268
269    /// Compute the layout for a univariant (see [`Variants::Single`]).
270    ///
271    /// As a consumer of `rustc_abi`, you should only use this method for non-ADTs.
272    /// For structs and univariant enums, use [`Self::layout_of_struct`] instead
273    /// (it uses this function internally).
274    pub fn layout_of_univariant<'a, FieldIdx, VariantIdx, F>(
275        &self,
276        fields: &IndexSlice<FieldIdx, F>,
277        repr: &ReprOptions,
278        kind: StructKind,
279    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
280    where
281        FieldIdx: Idx,
282        VariantIdx: Idx,
283        F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,
284    {
285        let dl = self.cx.data_layout();
286        let layout = self.layout_of_univariant_biased(fields, repr, kind, NicheBias::Start);
287        // Enums prefer niches close to the beginning or the end of the variants so that other
288        // (smaller) data-carrying variants can be packed into the space after/before the niche.
289        // If the default field ordering does not give us a niche at the front then we do a second
290        // run and bias niches to the right and then check which one is closer to one of the
291        // struct's edges.
292        if let Ok(layout) = &layout
293            // Don't try to calculate an end-biased layout for unsizable structs,
294            // otherwise we could end up with different layouts for
295            // Foo<Type> and Foo<dyn Trait> which would break unsizing.
296            && !#[allow(non_exhaustive_omitted_patterns)] match kind {
    StructKind::MaybeUnsized => true,
    _ => false,
}matches!(kind, StructKind::MaybeUnsized)
297            && let Some(niche) = layout.largest_niche
298            && let head_space = niche.offset.bytes()
299            && let niche_len = niche.value.size(dl).bytes()
300            && let tail_space = layout.size.bytes() - head_space - niche_len
301            // This may end up doing redundant work if the niche is already in the last
302            // field (e.g. a trailing bool) and there is tail padding. But it's non-trivial
303            // to get the unpadded size so we try anyway.
304            && (fields.len() > 1 && head_space != 0 && tail_space > 0)
305        {
306            let alt_layout = self
307                .layout_of_univariant_biased(fields, repr, kind, NicheBias::End)
308                .expect("alt layout should always work");
309            let alt_niche = alt_layout
310                .largest_niche
311                .expect("alt layout should have a niche like the regular one");
312            let alt_head_space = alt_niche.offset.bytes();
313            let alt_niche_len = alt_niche.value.size(dl).bytes();
314            let alt_tail_space = alt_layout.size.bytes() - alt_head_space - alt_niche_len;
315
316            if true {
    {
        match (&layout.size.bytes(), &alt_layout.size.bytes()) {
            (left_val, right_val) => {
                if !(*left_val == *right_val) {
                    let kind = ::core::panicking::AssertKind::Eq;
                    ::core::panicking::assert_failed(kind, &*left_val,
                        &*right_val, ::core::option::Option::None);
                }
            }
        }
    };
};debug_assert_eq!(layout.size.bytes(), alt_layout.size.bytes());
317
318            let prefer_alt_layout = alt_head_space > head_space && alt_head_space > tail_space;
319
320            {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event /rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout.rs:320",
                        "rustc_abi::layout", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("/rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout.rs"),
                        ::tracing_core::__macro_support::Option::Some(320u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_abi::layout"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                __CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("sz: {0}, default_niche_at: {1}+{2}, default_tail_space: {3}, alt_niche_at/head_space: {4}+{5}, alt_tail: {6}, num_fields: {7}, better: {8}\nlayout: {9}\nalt_layout: {10}\n",
                                                    layout.size.bytes(), head_space, niche_len, tail_space,
                                                    alt_head_space, alt_niche_len, alt_tail_space,
                                                    layout.fields.count(), prefer_alt_layout,
                                                    self.format_field_niches(layout, fields),
                                                    self.format_field_niches(&alt_layout, fields)) as
                                            &dyn ::tracing::field::Value))])
            });
    } else { ; }
};debug!(
321                "sz: {}, default_niche_at: {}+{}, default_tail_space: {}, alt_niche_at/head_space: {}+{}, alt_tail: {}, num_fields: {}, better: {}\n\
322                layout: {}\n\
323                alt_layout: {}\n",
324                layout.size.bytes(),
325                head_space,
326                niche_len,
327                tail_space,
328                alt_head_space,
329                alt_niche_len,
330                alt_tail_space,
331                layout.fields.count(),
332                prefer_alt_layout,
333                self.format_field_niches(layout, fields),
334                self.format_field_niches(&alt_layout, fields),
335            );
336
337            if prefer_alt_layout {
338                return Ok(alt_layout);
339            }
340        }
341        layout
342    }
343
344    pub fn layout_of_struct_or_enum<'a, FieldIdx, VariantIdx, F>(
345        &self,
346        repr: &ReprOptions,
347        variants: &IndexSlice<VariantIdx, IndexVec<FieldIdx, F>>,
348        is_enum: bool,
349        niche_optimizations: NicheOptimizations,
350        discr_range_of_repr: impl Fn(RangeFrom<i128>, RangeToInclusive<u128>) -> (Integer, bool),
351        discriminants: impl Iterator<Item = (VariantIdx, u128)>,
352        always_sized: bool,
353    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
354    where
355        FieldIdx: Idx,
356        VariantIdx: Idx,
357        F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,
358    {
359        let (present_first, present_second) = {
360            let mut present_variants = variants.iter_enumerated().filter_map(|(i, v)| {
361                if !repr.inhibit_enum_layout_opt() && absent(v) { None } else { Some(i) }
362            });
363            (present_variants.next(), present_variants.next())
364        };
365        let present_first = match present_first {
366            Some(present_first) => present_first,
367            // Uninhabited because it has no variants, or only absent ones.
368            None if is_enum => {
369                return Ok(LayoutData::never_type());
370            }
371            // If it's a struct, still compute a layout so that we can still compute the
372            // field offsets.
373            None => VariantIdx::new(0),
374        };
375
376        // take the struct path if it is an actual struct
377        if !is_enum ||
378            // or for optimizing univariant enums
379            (present_second.is_none() && !repr.inhibit_enum_layout_opt())
380        {
381            self.layout_of_struct(
382                repr,
383                variants,
384                present_first,
385                is_enum,
386                niche_optimizations,
387                always_sized,
388            )
389        } else {
390            // At this point, we have handled all unions and
391            // structs. (We have also handled univariant enums
392            // that allow representation optimization.)
393            if !is_enum { ::core::panicking::panic("assertion failed: is_enum") };assert!(is_enum);
394
395            self::enum_layout::layout_of_enum(
396                self,
397                repr,
398                variants,
399                discr_range_of_repr,
400                discriminants,
401            )
402        }
403    }
404
405    pub fn layout_of_union<'a, FieldIdx, VariantIdx, F>(
406        &self,
407        repr: &ReprOptions,
408        variants: &IndexSlice<VariantIdx, IndexVec<FieldIdx, F>>,
409    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
410    where
411        FieldIdx: Idx,
412        VariantIdx: Idx,
413        F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,
414    {
415        let dl = self.cx.data_layout();
416        let mut align = if repr.pack.is_some() { Align::ONE } else { dl.aggregate_align };
417        let mut max_repr_align = repr.align;
418
419        // If all the non-ZST fields have the same repr and union repr optimizations aren't
420        // disabled, we can use that common repr for the union as a whole.
421        struct AbiMismatch;
422        let mut common_non_zst_repr_and_align = if repr.inhibits_union_abi_opt() {
423            // Can't optimize
424            Err(AbiMismatch)
425        } else {
426            Ok(None)
427        };
428
429        let mut size = Size::ZERO;
430        let only_variant_idx = VariantIdx::new(0);
431        let only_variant = &variants[only_variant_idx];
432        for field in only_variant {
433            if field.is_unsized() {
434                return Err(LayoutCalculatorError::UnexpectedUnsized(*field));
435            }
436
437            align = align.max(field.align);
438            max_repr_align = max_repr_align.max(field.max_repr_align);
439            size = cmp::max(size, field.size);
440
441            if field.is_zst() {
442                // Nothing more to do for ZST fields
443                continue;
444            }
445
446            if let Ok(common) = common_non_zst_repr_and_align {
447                // Discard valid range information and allow undef
448                let field_abi = field.backend_repr.to_union();
449
450                if let Some((common_abi, common_align)) = common {
451                    if common_abi != field_abi {
452                        // Different fields have different ABI: disable opt
453                        common_non_zst_repr_and_align = Err(AbiMismatch);
454                    } else {
455                        // Fields with the same non-Aggregate ABI should also
456                        // have the same alignment
457                        if !#[allow(non_exhaustive_omitted_patterns)] match common_abi {
    BackendRepr::Memory { .. } => true,
    _ => false,
}matches!(common_abi, BackendRepr::Memory { .. }) {
458                            {
    match (&common_align, &field.align) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val,
                    ::core::option::Option::Some(format_args!("non-Aggregate field with matching ABI but differing alignment")));
            }
        }
    }
};assert_eq!(
459                                common_align, field.align,
460                                "non-Aggregate field with matching ABI but differing alignment"
461                            );
462                        }
463                    }
464                } else {
465                    // First non-ZST field: record its ABI and alignment
466                    common_non_zst_repr_and_align = Ok(Some((field_abi, field.align)));
467                }
468            }
469        }
470
471        if let Some(pack) = repr.pack {
472            align = align.min(pack);
473        }
474        // The unadjusted ABI alignment does not include repr(align), but does include repr(pack).
475        // See documentation on `LayoutData::unadjusted_abi_align`.
476        let unadjusted_abi_align = align;
477        if let Some(repr_align) = repr.align {
478            align = align.max(repr_align);
479        }
480        // `align` must not be modified after this, or `unadjusted_abi_align` could be inaccurate.
481        let align = align;
482
483        // If all non-ZST fields have the same ABI, we may forward that ABI
484        // for the union as a whole, unless otherwise inhibited.
485        let backend_repr = match common_non_zst_repr_and_align {
486            Err(AbiMismatch) | Ok(None) => BackendRepr::Memory { sized: true },
487            Ok(Some((repr, _))) => match repr {
488                // Mismatched alignment (e.g. union is #[repr(packed)]): disable opt
489                BackendRepr::Scalar(_) | BackendRepr::ScalarPair { .. }
490                    if repr.scalar_platform_align(dl).unwrap() != align =>
491                {
492                    BackendRepr::Memory { sized: true }
493                }
494                // Vectors require at least element alignment, else disable the opt
495                BackendRepr::SimdVector { element, count: _ }
496                    if element.default_align(dl) > align =>
497                {
498                    BackendRepr::Memory { sized: true }
499                }
500                // the alignment tests passed and we can use this
501                BackendRepr::Scalar(..)
502                | BackendRepr::ScalarPair { .. }
503                | BackendRepr::SimdVector { .. }
504                | BackendRepr::SimdScalableVector { .. }
505                | BackendRepr::Memory { .. } => repr,
506            },
507        };
508
509        let Some(union_field_count) = NonZero::new(only_variant.len()) else {
510            return Err(LayoutCalculatorError::EmptyUnion);
511        };
512
513        // We also need to check for a `repr(transparent)` wrapper around `repr(C)`
514        let repr_c =
515            repr.c() || (repr.transparent() && only_variant.iter().any(|field| field.repr_c));
516
517        let combined_seed = only_variant
518            .iter()
519            .map(|v| v.randomization_seed)
520            .fold(repr.field_shuffle_seed, |acc, seed| acc.wrapping_add(seed));
521
522        Ok(LayoutData {
523            variants: Variants::Single { index: only_variant_idx },
524            fields: FieldsShape::Union(union_field_count),
525            backend_repr,
526            largest_niche: None,
527            uninhabited: false,
528            align,
529            size: size.align_to(align),
530            max_repr_align,
531            unadjusted_abi_align,
532            repr_c,
533            randomization_seed: combined_seed,
534        })
535    }
536
537    /// Calculate the layout for a struct, or a single-variant enum.
538    ///
539    /// They are the same thing, if you think about it
540    /// (Typechecking will reject discriminant-sizing attrs.)
541    fn layout_of_struct<'a, FieldIdx, VariantIdx, F>(
542        &self,
543        repr: &ReprOptions,
544        variants: &IndexSlice<VariantIdx, IndexVec<FieldIdx, F>>,
545        variant_idx: VariantIdx,
546        is_enum: bool,
547        niche_optimizations: NicheOptimizations,
548        always_sized: bool,
549    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
550    where
551        FieldIdx: Idx,
552        VariantIdx: Idx,
553        F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,
554    {
555        let dl = self.cx.data_layout();
556        let v = variant_idx;
557        let kind = if is_enum || variants[v].is_empty() || always_sized {
558            StructKind::AlwaysSized
559        } else {
560            StructKind::MaybeUnsized
561        };
562
563        let mut st = self.layout_of_univariant(&variants[v], repr, kind)?;
564        st.variants = Variants::Single { index: v };
565
566        if niche_optimizations == NicheOptimizations::Disabled {
567            let hide_niches = |scalar: &mut _| match scalar {
568                Scalar::Initialized { value, valid_range } => {
569                    *valid_range = WrappingRange::full(value.size(dl))
570                }
571                // Already doesn't have any niches
572                Scalar::Union { .. } => {}
573            };
574            match &mut st.backend_repr {
575                BackendRepr::Scalar(scalar) => hide_niches(scalar),
576                BackendRepr::ScalarPair { a, b, b_offset: _ } => {
577                    hide_niches(a);
578                    hide_niches(b);
579                }
580                BackendRepr::SimdVector { element, .. }
581                | BackendRepr::SimdScalableVector { element, .. } => hide_niches(element),
582                BackendRepr::Memory { sized: _ } => {}
583            }
584            st.largest_niche = None;
585            return Ok(st);
586        }
587
588        Ok(st)
589    }
590
591    fn layout_of_univariant_biased<'a, FieldIdx, VariantIdx, F>(
592        &self,
593        fields: &IndexSlice<FieldIdx, F>,
594        repr: &ReprOptions,
595        kind: StructKind,
596        niche_bias: NicheBias,
597    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
598    where
599        FieldIdx: Idx,
600        VariantIdx: Idx,
601        F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,
602    {
603        let dl = self.cx.data_layout();
604        let pack = repr.pack;
605        let mut align = if pack.is_some() { Align::ONE } else { dl.aggregate_align };
606        let mut max_repr_align = repr.align;
607        let mut in_memory_order: IndexVec<u32, FieldIdx> = fields.indices().collect();
608        let optimize_field_order = !repr.inhibit_struct_field_reordering();
609        let end = if let StructKind::MaybeUnsized = kind { fields.len() - 1 } else { fields.len() };
610        let optimizing = &mut in_memory_order.raw[..end];
611        let fields_excluding_tail = &fields.raw[..end];
612        // unsizable tail fields are excluded so that we use the same seed for the sized and unsized layouts.
613        let field_seed = fields_excluding_tail
614            .iter()
615            .fold(Hash64::ZERO, |acc, f| acc.wrapping_add(f.randomization_seed));
616
617        if optimize_field_order && fields.len() > 1 {
618            // If `-Z randomize-layout` was enabled for the type definition we can shuffle
619            // the field ordering to try and catch some code making assumptions about layouts
620            // we don't guarantee.
621            // In the future, we might do more than shuffle field order (e.g. introduce extra padding),
622            // but never for `repr(Rust)` structs with only zero-sized fields, single-variant
623            // `repr(Rust)` enums with only zero-sized fields, or zero-variant `repr(Rust)` enums,
624            // which must remain zero-sized as per T-lang decisions in
625            // https://github.com/rust-lang/reference/pull/2262 and https://github.com/rust-lang/reference/pull/2293
626            if repr.can_randomize_type_layout() && truecfg!(feature = "randomize") {
627                #[cfg(feature = "randomize")]
628                {
629                    use rand::SeedableRng;
630                    use rand::seq::SliceRandom;
631                    // `ReprOptions.field_shuffle_seed` is a deterministic seed we can use to randomize field
632                    // ordering.
633                    let mut rng = rand_xoshiro::Xoshiro128StarStar::seed_from_u64(
634                        field_seed.wrapping_add(repr.field_shuffle_seed).as_u64(),
635                    );
636
637                    // Shuffle the ordering of the fields.
638                    optimizing.shuffle(&mut rng);
639                }
640                // Otherwise we just leave things alone and actually optimize the type's fields
641            } else {
642                // To allow unsizing `&Foo<Type>` -> `&Foo<dyn Trait>`, the layout of the struct must
643                // not depend on the layout of the tail.
644                let max_field_align =
645                    fields_excluding_tail.iter().map(|f| f.align.bytes()).max().unwrap_or(1);
646                let largest_niche_size = fields_excluding_tail
647                    .iter()
648                    .filter_map(|f| f.largest_niche)
649                    .map(|n| n.available(dl))
650                    .max()
651                    .unwrap_or(0);
652
653                // Calculates a sort key to group fields by their alignment or possibly some
654                // size-derived pseudo-alignment.
655                let alignment_group_key = |layout: &F| {
656                    // The two branches here return values that cannot be meaningfully compared with
657                    // each other. However, we know that consistently for all executions of
658                    // `alignment_group_key`, one or the other branch will be taken, so this is okay.
659                    if let Some(pack) = pack {
660                        // Return the packed alignment in bytes.
661                        layout.align.min(pack).bytes()
662                    } else {
663                        // Returns `log2(effective-align)`. The calculation assumes that size is an
664                        // integer multiple of align, except for ZSTs.
665                        let align = layout.align.bytes();
666                        let size = layout.size.bytes();
667                        let niche_size = layout.largest_niche.map(|n| n.available(dl)).unwrap_or(0);
668                        // Group [u8; 4] with align-4 or [u8; 6] with align-2 fields.
669                        let size_as_align = align.max(size).trailing_zeros();
670                        let size_as_align = if largest_niche_size > 0 {
671                            match niche_bias {
672                                // Given `A(u8, [u8; 16])` and `B(bool, [u8; 16])` we want to bump the
673                                // array to the front in the first case (for aligned loads) but keep
674                                // the bool in front in the second case for its niches.
675                                NicheBias::Start => {
676                                    max_field_align.trailing_zeros().min(size_as_align)
677                                }
678                                // When moving niches towards the end of the struct then for
679                                // A((u8, u8, u8, bool), (u8, bool, u8)) we want to keep the first tuple
680                                // in the align-1 group because its bool can be moved closer to the end.
681                                NicheBias::End if niche_size == largest_niche_size => {
682                                    align.trailing_zeros()
683                                }
684                                NicheBias::End => size_as_align,
685                            }
686                        } else {
687                            size_as_align
688                        };
689                        size_as_align as u64
690                    }
691                };
692
693                match kind {
694                    StructKind::AlwaysSized | StructKind::MaybeUnsized => {
695                        // Currently `LayoutData` only exposes a single niche so sorting is usually
696                        // sufficient to get one niche into the preferred position. If it ever
697                        // supported multiple niches then a more advanced pick-and-pack approach could
698                        // provide better results. But even for the single-niche cache it's not
699                        // optimal. E.g. for A(u32, (bool, u8), u16) it would be possible to move the
700                        // bool to the front but it would require packing the tuple together with the
701                        // u16 to build a 4-byte group so that the u32 can be placed after it without
702                        // padding. This kind of packing can't be achieved by sorting.
703                        optimizing.sort_by_key(|&x| {
704                            let f = &fields[x];
705                            let field_size = f.size.bytes();
706                            let niche_size = f.largest_niche.map_or(0, |n| n.available(dl));
707                            let niche_size_key = match niche_bias {
708                                // large niche first
709                                NicheBias::Start => !niche_size,
710                                // large niche last
711                                NicheBias::End => niche_size,
712                            };
713                            let inner_niche_offset_key = match niche_bias {
714                                NicheBias::Start => f.largest_niche.map_or(0, |n| n.offset.bytes()),
715                                NicheBias::End => f.largest_niche.map_or(0, |n| {
716                                    !(field_size - n.value.size(dl).bytes() - n.offset.bytes())
717                                }),
718                            };
719
720                            (
721                                // Then place largest alignments first.
722                                cmp::Reverse(alignment_group_key(f)),
723                                // Then prioritize niche placement within alignment group according to
724                                // `niche_bias_start`.
725                                niche_size_key,
726                                // Then among fields with equally-sized niches prefer the ones
727                                // closer to the start/end of the field.
728                                inner_niche_offset_key,
729                            )
730                        });
731                    }
732
733                    StructKind::Prefixed(..) => {
734                        // Sort in ascending alignment so that the layout stays optimal
735                        // regardless of the prefix.
736                        // And put the largest niche in an alignment group at the end
737                        // so it can be used as discriminant in jagged enums
738                        optimizing.sort_by_key(|&x| {
739                            let f = &fields[x];
740                            let niche_size = f.largest_niche.map_or(0, |n| n.available(dl));
741                            (alignment_group_key(f), niche_size)
742                        });
743                    }
744                }
745
746                // FIXME(Kixiron): We can always shuffle fields within a given alignment class
747                //                 regardless of the status of `-Z randomize-layout`
748            }
749        }
750        // in_memory_order holds field indices by increasing memory offset.
751        // That is, if field 5 has offset 0, the first element of in_memory_order is 5.
752        // We now write field offsets to the corresponding offset slot;
753        // field 5 with offset 0 puts 0 in offsets[5].
754        let mut unsized_field = None::<&F>;
755        let mut offsets = IndexVec::from_elem(Size::ZERO, fields);
756        let mut offset = Size::ZERO;
757        let mut largest_niche = None;
758        let mut largest_niche_available = 0;
759        if let StructKind::Prefixed(prefix_size, prefix_align) = kind {
760            let prefix_align =
761                if let Some(pack) = pack { prefix_align.min(pack) } else { prefix_align };
762            align = align.max(prefix_align);
763            offset = prefix_size.align_to(prefix_align);
764        }
765        for &i in &in_memory_order {
766            let field = &fields[i];
767            if let Some(unsized_field) = unsized_field {
768                return Err(LayoutCalculatorError::UnexpectedUnsized(*unsized_field));
769            }
770
771            if field.is_unsized() {
772                if let StructKind::MaybeUnsized = kind {
773                    unsized_field = Some(field);
774                } else {
775                    return Err(LayoutCalculatorError::UnexpectedUnsized(*field));
776                }
777            }
778
779            // Invariant: offset < dl.obj_size_bound() <= 1<<61
780            let field_align =
781                if let Some(pack) = pack { field.align.min(pack) } else { field.align };
782            offset = offset.align_to(field_align);
783            align = align.max(field_align);
784            max_repr_align = max_repr_align.max(field.max_repr_align);
785
786            {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event /rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout.rs:786",
                        "rustc_abi::layout", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("/rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout.rs"),
                        ::tracing_core::__macro_support::Option::Some(786u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_abi::layout"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                __CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("univariant offset: {0:?} field: {1:#?}",
                                                    offset, field) as &dyn ::tracing::field::Value))])
            });
    } else { ; }
};debug!("univariant offset: {:?} field: {:#?}", offset, field);
787            offsets[i] = offset;
788
789            if let Some(mut niche) = field.largest_niche {
790                let available = niche.available(dl);
791                // Pick up larger niches.
792                let prefer_new_niche = match niche_bias {
793                    NicheBias::Start => available > largest_niche_available,
794                    // if there are several niches of the same size then pick the last one
795                    NicheBias::End => available >= largest_niche_available,
796                };
797                if prefer_new_niche {
798                    largest_niche_available = available;
799                    niche.offset += offset;
800                    largest_niche = Some(niche);
801                }
802            }
803
804            offset =
805                offset.checked_add(field.size, dl).ok_or(LayoutCalculatorError::SizeOverflow)?;
806        }
807
808        // The unadjusted ABI alignment does not include repr(align), but does include repr(pack).
809        // See documentation on `LayoutData::unadjusted_abi_align`.
810        let unadjusted_abi_align = align;
811        if let Some(repr_align) = repr.align {
812            align = align.max(repr_align);
813        }
814        // `align` must not be modified after this point, or `unadjusted_abi_align` could be inaccurate.
815        let align = align;
816
817        {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event /rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout.rs:817",
                        "rustc_abi::layout", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("/rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout.rs"),
                        ::tracing_core::__macro_support::Option::Some(817u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_abi::layout"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                __CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("univariant min_size: {0:?}",
                                                    offset) as &dyn ::tracing::field::Value))])
            });
    } else { ; }
};debug!("univariant min_size: {:?}", offset);
818        let min_size = offset;
819        let size = min_size.align_to(align);
820        // FIXME(oli-obk): deduplicate and harden these checks
821        if size.bytes() >= dl.obj_size_bound() {
822            return Err(LayoutCalculatorError::SizeOverflow);
823        }
824        let mut layout_of_single_non_zst_field = None;
825        let sized = unsized_field.is_none();
826        let mut abi = BackendRepr::Memory { sized };
827
828        let optimize_abi = !repr.inhibit_newtype_abi_optimization();
829
830        // Try to make this a Scalar/ScalarPair.
831        if sized && size.bytes() > 0 {
832            // We skip *all* ZST here and later check if we are good in terms of alignment.
833            // This lets us handle some cases involving aligned ZST.
834            let mut non_zst_fields = fields.iter_enumerated().filter(|&(_, f)| !f.is_zst());
835
836            match (non_zst_fields.next(), non_zst_fields.next(), non_zst_fields.next()) {
837                // We have exactly one non-ZST field.
838                (Some((i, field)), None, None) => {
839                    layout_of_single_non_zst_field = Some(field);
840
841                    // Field fills the struct and it has a scalar or scalar pair ABI.
842                    if offsets[i].bytes() == 0 && align == field.align && size == field.size {
843                        match field.backend_repr {
844                            // For plain scalars, or vectors of them, we can't unpack
845                            // newtypes for `#[repr(C)]`, as that affects C ABIs.
846                            BackendRepr::Scalar(_) | BackendRepr::SimdVector { .. }
847                                if optimize_abi =>
848                            {
849                                abi = field.backend_repr;
850                            }
851                            // But scalar pairs are Rust-specific and get
852                            // treated as aggregates by C ABIs anyway.
853                            BackendRepr::ScalarPair { .. } => {
854                                abi = field.backend_repr;
855                            }
856                            _ => {}
857                        }
858                    }
859                }
860
861                // Two non-ZST fields, and they're both scalars.
862                (Some((i, a)), Some((j, b)), None) => {
863                    match (a.backend_repr, b.backend_repr) {
864                        (BackendRepr::Scalar(a), BackendRepr::Scalar(b)) => {
865                            // Order by the memory placement, not source order.
866                            let ((i, a), (j, b)) = if offsets[i] < offsets[j] {
867                                ((i, a), (j, b))
868                            } else {
869                                ((j, b), (i, a))
870                            };
871                            let pair =
872                                LayoutData::<FieldIdx, VariantIdx>::scalar_pair(&self.cx, a, b);
873                            let pair_offsets = match pair.fields {
874                                FieldsShape::Arbitrary { ref offsets, ref in_memory_order } => {
875                                    {
    match (&in_memory_order.raw, &[FieldIdx::new(0), FieldIdx::new(1)]) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val, ::core::option::Option::None);
            }
        }
    }
};assert_eq!(
876                                        in_memory_order.raw,
877                                        [FieldIdx::new(0), FieldIdx::new(1)]
878                                    );
879                                    offsets
880                                }
881                                FieldsShape::Primitive
882                                | FieldsShape::Array { .. }
883                                | FieldsShape::Union(..) => {
884                                    {
    ::core::panicking::panic_fmt(format_args!("encountered a non-arbitrary layout during enum layout"));
}panic!("encountered a non-arbitrary layout during enum layout")
885                                }
886                            };
887                            if offsets[i] == pair_offsets[FieldIdx::new(0)]
888                                && offsets[j] == pair_offsets[FieldIdx::new(1)]
889                                && align == pair.align
890                                && size == pair.size
891                            {
892                                // We can use `ScalarPair` only when it matches our
893                                // already computed layout (including `#[repr(C)]`).
894                                abi = pair.backend_repr;
895                            }
896                        }
897                        _ => {}
898                    }
899                }
900
901                _ => {}
902            }
903        }
904        let uninhabited = fields.iter().any(|f| f.is_uninhabited());
905
906        let unadjusted_abi_align = if repr.transparent() {
907            match layout_of_single_non_zst_field {
908                Some(l) => l.unadjusted_abi_align,
909                None => {
910                    // `repr(transparent)` with all ZST fields.
911                    align
912                }
913            }
914        } else {
915            unadjusted_abi_align
916        };
917
918        // We also need to check for a `repr(transparent)` wrapper around `repr(C)`
919        let repr_c = repr.c() || (repr.transparent() && fields.iter().any(|field| field.repr_c));
920
921        let seed = field_seed.wrapping_add(repr.field_shuffle_seed);
922
923        Ok(LayoutData {
924            variants: Variants::Single { index: VariantIdx::new(0) },
925            fields: FieldsShape::Arbitrary { offsets, in_memory_order },
926            backend_repr: abi,
927            largest_niche,
928            uninhabited,
929            align,
930            size,
931            max_repr_align,
932            unadjusted_abi_align,
933            repr_c,
934            randomization_seed: seed,
935        })
936    }
937
938    fn format_field_niches<'a, FieldIdx, VariantIdx, F>(
939        &self,
940        layout: &LayoutData<FieldIdx, VariantIdx>,
941        fields: &IndexSlice<FieldIdx, F>,
942    ) -> String
943    where
944        FieldIdx: Idx,
945        VariantIdx: Idx,
946        F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,
947    {
948        let dl = self.cx.data_layout();
949        let mut s = String::new();
950        for i in layout.fields.index_by_increasing_offset() {
951            let offset = layout.fields.offset(i);
952            let f = &fields[FieldIdx::new(i)];
953            s.write_fmt(format_args!("[o{0}a{1}s{2}", offset.bytes(), f.align.bytes(),
        f.size.bytes()))write!(s, "[o{}a{}s{}", offset.bytes(), f.align.bytes(), f.size.bytes()).unwrap();
954            if let Some(n) = f.largest_niche {
955                s.write_fmt(format_args!(" n{0}b{1}s{2}", n.offset.bytes(),
        n.available(dl).ilog2(), n.value.size(dl).bytes()))write!(
956                    s,
957                    " n{}b{}s{}",
958                    n.offset.bytes(),
959                    n.available(dl).ilog2(),
960                    n.value.size(dl).bytes()
961                )
962                .unwrap();
963            }
964            s.write_fmt(format_args!("] "))write!(s, "] ").unwrap();
965        }
966        s
967    }
968}
969
970enum SimdVectorKind {
971    /// `#[rustc_scalable_vector]`
972    Scalable(NumScalableVectors),
973    /// `#[repr(simd, packed)]`
974    PackedFixed,
975    /// `#[repr(simd)]`
976    Fixed,
977}
978
979fn vector_type_layout<FieldIdx, VariantIdx, F>(
980    kind: SimdVectorKind,
981    dl: &TargetDataLayout,
982    element: F,
983    count: u64,
984) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
985where
986    FieldIdx: Idx,
987    VariantIdx: Idx,
988    F: AsRef<LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,
989{
990    let elt = element.as_ref();
991    let count = BackendLaneCount::new(count)?;
992
993    let BackendRepr::Scalar(element) = elt.backend_repr else {
994        return Err(LayoutCalculatorError::NonPrimitiveSimdType(element));
995    };
996
997    // Compute the size and alignment of the vector
998    let size =
999        elt.size.checked_mul(count.as_u64(), dl).ok_or(LayoutCalculatorError::SizeOverflow)?;
1000    let (repr, size, align) = match kind {
1001        SimdVectorKind::Scalable(number_of_vectors) => (
1002            BackendRepr::SimdScalableVector { element, count, number_of_vectors },
1003            size.checked_mul(number_of_vectors.0 as u64, dl)
1004                .ok_or(LayoutCalculatorError::SizeOverflow)?,
1005            dl.rust_vector_align(size),
1006        ),
1007        // Non-power-of-two vectors have padding up to the next power-of-two.
1008        // If we're a packed repr, remove the padding while keeping the alignment as close
1009        // to a vector as possible.
1010        SimdVectorKind::PackedFixed if !count.is_power_of_two() => {
1011            (BackendRepr::Memory { sized: true }, size, Align::max_aligned_factor(size))
1012        }
1013        SimdVectorKind::PackedFixed | SimdVectorKind::Fixed => {
1014            (BackendRepr::SimdVector { element, count }, size, dl.rust_vector_align(size))
1015        }
1016    };
1017    let size = size.align_to(align);
1018
1019    Ok(LayoutData {
1020        variants: Variants::Single { index: VariantIdx::new(0) },
1021        fields: FieldsShape::Arbitrary {
1022            offsets: [Size::ZERO].into(),
1023            in_memory_order: [FieldIdx::new(0)].into(),
1024        },
1025        backend_repr: repr,
1026        largest_niche: elt.largest_niche,
1027        uninhabited: false,
1028        size,
1029        align,
1030        max_repr_align: None,
1031        unadjusted_abi_align: elt.align,
1032        repr_c: false,
1033        randomization_seed: elt.randomization_seed.wrapping_add(Hash64::new(count.as_u64())),
1034    })
1035}