1use std::cmp;
2use std::fmt::{self, Write};
3use std::num::NonZero;
4use std::ops::Deref;
5use std::range::{RangeFrom, RangeToInclusive};
6
7pub use coroutine::PackCoroutineLayout;
8use rustc_hashes::Hash64;
9use rustc_index::Idx;
10use rustc_index::bit_set::BitMatrix;
11use tracing::debug;
12
13use crate::{
14 Align, BackendLaneCount, BackendRepr, FieldsShape, HasDataLayout, IndexSlice, IndexVec,
15 Integer, LayoutData, NicheOptimizations, NumScalableVectors, ReprOptions, Scalar, Size,
16 StructKind, TargetDataLayout, Variants, WrappingRange,
17};
18
19mod coroutine;
20mod enum_layout;
21mod simple;
22
23#[cfg(feature = "nightly")]
24mod ty;
25
26#[cfg(feature = "nightly")]
27pub use ty::{Layout, TyAbiInterface, TyAndLayout};
28
29#[automatically_derived]impl ::core::marker::Copy for FieldIdx { }
impl FieldIdx {
#[doc = r" Maximum value the index can take, as a `u32`."]
pub const MAX_AS_U32: u32 = 0xFFFF_FF00;
#[doc = r" Maximum value the index can take."]
pub const MAX: Self = Self::from_u32(0xFFFF_FF00);
#[doc = r" Zero value of the index."]
pub const ZERO: Self = Self::from_u32(0);
#[doc = r" Creates a new index from a given `usize`."]
#[doc = r""]
#[doc = r" # Panics"]
#[doc = r""]
#[doc = r" Will panic if `value` exceeds `MAX`."]
#[inline]
pub const fn from_usize(value: usize) -> Self {
if !(value <= (0xFFFF_FF00 as usize)) {
::core::panicking::panic("assertion failed: value <= (0xFFFF_FF00 as usize)")
};
unsafe { Self::from_u32_unchecked(value as u32) }
}
#[doc = r" Creates a new index from a given `u32`."]
#[doc = r""]
#[doc = r" # Panics"]
#[doc = r""]
#[doc = r" Will panic if `value` exceeds `MAX`."]
#[inline]
pub const fn from_u32(value: u32) -> Self {
if !(value <= 0xFFFF_FF00) {
::core::panicking::panic("assertion failed: value <= 0xFFFF_FF00")
};
unsafe { Self::from_u32_unchecked(value) }
}
#[doc = r" Creates a new index from a given `u16`."]
#[doc = r""]
#[doc = r" # Panics"]
#[doc = r""]
#[doc = r" Will panic if `value` exceeds `MAX`."]
#[inline]
pub const fn from_u16(value: u16) -> Self {
let value = value as u32;
if !(value <= 0xFFFF_FF00) {
::core::panicking::panic("assertion failed: value <= 0xFFFF_FF00")
};
unsafe { Self::from_u32_unchecked(value) }
}
#[doc = r" Creates a new index from a given `u32`."]
#[doc = r""]
#[doc = r" # Safety"]
#[doc = r""]
#[doc =
r" The provided value must be less than or equal to the maximum value for the newtype."]
#[doc =
r" Providing a value outside this range is undefined due to layout restrictions."]
#[doc = r""]
#[doc = r" Prefer using `from_u32`."]
#[inline]
pub const unsafe fn from_u32_unchecked(value: u32) -> Self {
Self {
private_use_as_methods_instead: unsafe {
std::mem::transmute(value)
},
}
}
#[doc = r" Extracts the value of this index as a `usize`."]
#[inline]
pub const fn index(self) -> usize { self.as_usize() }
#[doc = r" Extracts the value of this index as a `u32`."]
#[inline]
pub const fn as_u32(self) -> u32 {
unsafe { std::mem::transmute(self.private_use_as_methods_instead) }
}
#[doc = r" Extracts the value of this index as a `usize`."]
#[inline]
pub const fn as_usize(self) -> usize { self.as_u32() as usize }
}
impl std::ops::Add<usize> for FieldIdx {
type Output = Self;
#[inline]
fn add(self, other: usize) -> Self {
Self::from_usize(self.index() + other)
}
}
impl std::ops::AddAssign<usize> for FieldIdx {
#[inline]
fn add_assign(&mut self, other: usize) { *self = *self + other; }
}
impl rustc_index::Idx for FieldIdx {
#[inline]
fn new(value: usize) -> Self { Self::from_usize(value) }
#[inline]
fn index(self) -> usize { self.as_usize() }
}
impl ::std::iter::Step for FieldIdx {
#[inline]
fn steps_between(start: &Self, end: &Self) -> (usize, Option<usize>) {
<usize as
::std::iter::Step>::steps_between(&Self::index(*start),
&Self::index(*end))
}
#[inline]
fn forward_checked(start: Self, u: usize) -> Option<Self> {
Self::index(start).checked_add(u).map(Self::from_usize)
}
#[inline]
fn backward_checked(start: Self, u: usize) -> Option<Self> {
Self::index(start).checked_sub(u).map(Self::from_usize)
}
#[inline]
fn forward_overflowing(start: Self, u: usize) -> (Self, bool) {
let (s, o) = Self::index(start).overflowing_add(u);
(Self::from_usize(s), o)
}
#[inline]
fn backward_overflowing(start: Self, u: usize) -> (Self, bool) {
let (s, o) = Self::index(start).overflowing_sub(u);
(Self::from_usize(s), o)
}
}
impl ::std::cmp::Ord for FieldIdx {
#[inline]
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.as_u32().cmp(&other.as_u32())
}
}
impl ::std::cmp::PartialOrd for FieldIdx {
#[inline]
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
impl ::rustc_data_structures::stable_hash::StableHash for FieldIdx {
fn stable_hash<__Hcx: ::rustc_data_structures::stable_hash::StableHashCtxt>(&self,
hcx: &mut __Hcx,
hasher: &mut ::rustc_data_structures::stable_hash::StableHasher) {
self.as_u32().stable_hash(hcx, hasher)
}
}
impl From<FieldIdx> for u32 {
#[inline]
fn from(v: FieldIdx) -> u32 { v.as_u32() }
}
impl From<FieldIdx> for usize {
#[inline]
fn from(v: FieldIdx) -> usize { v.as_usize() }
}
impl From<usize> for FieldIdx {
#[inline]
fn from(value: usize) -> Self { Self::from_usize(value) }
}
impl From<u32> for FieldIdx {
#[inline]
fn from(value: u32) -> Self { Self::from_u32(value) }
}
impl ::std::cmp::Eq for FieldIdx {}
impl ::std::cmp::PartialEq for FieldIdx {
fn eq(&self, other: &Self) -> bool { self.as_u32().eq(&other.as_u32()) }
}
impl ::std::marker::StructuralPartialEq for FieldIdx { }
impl ::std::hash::Hash for FieldIdx {
fn hash<H: ::std::hash::Hasher>(&self, state: &mut H) {
self.as_u32().hash(state)
}
}
impl<D: ::rustc_serialize::Decoder> ::rustc_serialize::Decodable<D> for
FieldIdx {
fn decode(d: &mut D) -> Self { Self::from_u32(d.read_u32()) }
}
impl<E: ::rustc_serialize::Encoder> ::rustc_serialize::Encodable<E> for
FieldIdx {
fn encode(&self, e: &mut E) { e.emit_u32(self.as_u32()); }
}
impl ::std::fmt::Debug for FieldIdx {
fn fmt(&self, fmt: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result {
fmt.write_fmt(format_args!("{0}", self.as_u32()))
}
}rustc_index::newtype_index! {
30 #[stable_hash]
52 #[encodable]
53 #[orderable]
54 #[gate_rustc_only]
55 pub struct FieldIdx {}
56}
57
58impl FieldIdx {
59 pub const ONE: FieldIdx = FieldIdx::from_u32(1);
63}
64
65#[automatically_derived]impl ::core::marker::Copy for VariantIdx { }
#[doc = " Equivalent to `VariantIdx(0)`."]
pub const FIRST_VARIANT: VariantIdx = VariantIdx::from_u32(0);
impl VariantIdx {
#[doc = r" Maximum value the index can take, as a `u32`."]
pub const MAX_AS_U32: u32 = 0xFFFF_FF00;
#[doc = r" Maximum value the index can take."]
pub const MAX: Self = Self::from_u32(0xFFFF_FF00);
#[doc = r" Zero value of the index."]
pub const ZERO: Self = Self::from_u32(0);
#[doc = r" Creates a new index from a given `usize`."]
#[doc = r""]
#[doc = r" # Panics"]
#[doc = r""]
#[doc = r" Will panic if `value` exceeds `MAX`."]
#[inline]
pub const fn from_usize(value: usize) -> Self {
if !(value <= (0xFFFF_FF00 as usize)) {
::core::panicking::panic("assertion failed: value <= (0xFFFF_FF00 as usize)")
};
unsafe { Self::from_u32_unchecked(value as u32) }
}
#[doc = r" Creates a new index from a given `u32`."]
#[doc = r""]
#[doc = r" # Panics"]
#[doc = r""]
#[doc = r" Will panic if `value` exceeds `MAX`."]
#[inline]
pub const fn from_u32(value: u32) -> Self {
if !(value <= 0xFFFF_FF00) {
::core::panicking::panic("assertion failed: value <= 0xFFFF_FF00")
};
unsafe { Self::from_u32_unchecked(value) }
}
#[doc = r" Creates a new index from a given `u16`."]
#[doc = r""]
#[doc = r" # Panics"]
#[doc = r""]
#[doc = r" Will panic if `value` exceeds `MAX`."]
#[inline]
pub const fn from_u16(value: u16) -> Self {
let value = value as u32;
if !(value <= 0xFFFF_FF00) {
::core::panicking::panic("assertion failed: value <= 0xFFFF_FF00")
};
unsafe { Self::from_u32_unchecked(value) }
}
#[doc = r" Creates a new index from a given `u32`."]
#[doc = r""]
#[doc = r" # Safety"]
#[doc = r""]
#[doc =
r" The provided value must be less than or equal to the maximum value for the newtype."]
#[doc =
r" Providing a value outside this range is undefined due to layout restrictions."]
#[doc = r""]
#[doc = r" Prefer using `from_u32`."]
#[inline]
pub const unsafe fn from_u32_unchecked(value: u32) -> Self {
Self {
private_use_as_methods_instead: unsafe {
std::mem::transmute(value)
},
}
}
#[doc = r" Extracts the value of this index as a `usize`."]
#[inline]
pub const fn index(self) -> usize { self.as_usize() }
#[doc = r" Extracts the value of this index as a `u32`."]
#[inline]
pub const fn as_u32(self) -> u32 {
unsafe { std::mem::transmute(self.private_use_as_methods_instead) }
}
#[doc = r" Extracts the value of this index as a `usize`."]
#[inline]
pub const fn as_usize(self) -> usize { self.as_u32() as usize }
}
impl std::ops::Add<usize> for VariantIdx {
type Output = Self;
#[inline]
fn add(self, other: usize) -> Self {
Self::from_usize(self.index() + other)
}
}
impl std::ops::AddAssign<usize> for VariantIdx {
#[inline]
fn add_assign(&mut self, other: usize) { *self = *self + other; }
}
impl rustc_index::Idx for VariantIdx {
#[inline]
fn new(value: usize) -> Self { Self::from_usize(value) }
#[inline]
fn index(self) -> usize { self.as_usize() }
}
impl ::std::iter::Step for VariantIdx {
#[inline]
fn steps_between(start: &Self, end: &Self) -> (usize, Option<usize>) {
<usize as
::std::iter::Step>::steps_between(&Self::index(*start),
&Self::index(*end))
}
#[inline]
fn forward_checked(start: Self, u: usize) -> Option<Self> {
Self::index(start).checked_add(u).map(Self::from_usize)
}
#[inline]
fn backward_checked(start: Self, u: usize) -> Option<Self> {
Self::index(start).checked_sub(u).map(Self::from_usize)
}
#[inline]
fn forward_overflowing(start: Self, u: usize) -> (Self, bool) {
let (s, o) = Self::index(start).overflowing_add(u);
(Self::from_usize(s), o)
}
#[inline]
fn backward_overflowing(start: Self, u: usize) -> (Self, bool) {
let (s, o) = Self::index(start).overflowing_sub(u);
(Self::from_usize(s), o)
}
}
impl ::std::cmp::Ord for VariantIdx {
#[inline]
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.as_u32().cmp(&other.as_u32())
}
}
impl ::std::cmp::PartialOrd for VariantIdx {
#[inline]
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
impl ::rustc_data_structures::stable_hash::StableHash for VariantIdx {
fn stable_hash<__Hcx: ::rustc_data_structures::stable_hash::StableHashCtxt>(&self,
hcx: &mut __Hcx,
hasher: &mut ::rustc_data_structures::stable_hash::StableHasher) {
self.as_u32().stable_hash(hcx, hasher)
}
}
impl From<VariantIdx> for u32 {
#[inline]
fn from(v: VariantIdx) -> u32 { v.as_u32() }
}
impl From<VariantIdx> for usize {
#[inline]
fn from(v: VariantIdx) -> usize { v.as_usize() }
}
impl From<usize> for VariantIdx {
#[inline]
fn from(value: usize) -> Self { Self::from_usize(value) }
}
impl From<u32> for VariantIdx {
#[inline]
fn from(value: u32) -> Self { Self::from_u32(value) }
}
impl ::std::cmp::Eq for VariantIdx {}
impl ::std::cmp::PartialEq for VariantIdx {
fn eq(&self, other: &Self) -> bool { self.as_u32().eq(&other.as_u32()) }
}
impl ::std::marker::StructuralPartialEq for VariantIdx { }
impl ::std::hash::Hash for VariantIdx {
fn hash<H: ::std::hash::Hasher>(&self, state: &mut H) {
self.as_u32().hash(state)
}
}
impl<D: ::rustc_serialize::Decoder> ::rustc_serialize::Decodable<D> for
VariantIdx {
fn decode(d: &mut D) -> Self { Self::from_u32(d.read_u32()) }
}
impl<E: ::rustc_serialize::Encoder> ::rustc_serialize::Encodable<E> for
VariantIdx {
fn encode(&self, e: &mut E) { e.emit_u32(self.as_u32()); }
}
impl ::std::fmt::Debug for VariantIdx {
fn fmt(&self, fmt: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result {
fmt.write_fmt(format_args!("{0}", self.as_u32()))
}
}rustc_index::newtype_index! {
66 #[stable_hash]
77 #[encodable]
78 #[orderable]
79 #[gate_rustc_only]
80 pub struct VariantIdx {
81 const FIRST_VARIANT = 0;
83 }
84}
85
86fn absent<'a, FieldIdx, VariantIdx, F>(fields: &IndexSlice<FieldIdx, F>) -> bool
92where
93 FieldIdx: Idx,
94 VariantIdx: Idx,
95 F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,
96{
97 let uninhabited = fields.iter().any(|f| f.is_uninhabited());
98 let is_1zst = fields.iter().all(|f| f.is_1zst());
101 uninhabited && is_1zst
102}
103
104enum NicheBias {
106 Start,
107 End,
108}
109
110#[derive(#[automatically_derived]impl<F: ::core::marker::Copy> ::core::marker::Copy for
LayoutCalculatorError<F> {
}Copy, #[automatically_derived]impl<F: ::core::clone::Clone> ::core::clone::Clone for
LayoutCalculatorError<F> {
#[inline]
#[rustc_trivial_field_reads]
fn clone(&self) -> Self {
match self {
Self::UnexpectedUnsized(__self_0) =>
LayoutCalculatorError::UnexpectedUnsized(::core::clone::Clone::clone(__self_0)),
Self::SizeOverflow => LayoutCalculatorError::SizeOverflow,
Self::EmptyUnion => LayoutCalculatorError::EmptyUnion,
Self::ReprConflict => LayoutCalculatorError::ReprConflict,
Self::ZeroLengthSimdType =>
LayoutCalculatorError::ZeroLengthSimdType,
Self::OversizedSimdType { max_lanes: __self_0 } =>
LayoutCalculatorError::OversizedSimdType {
max_lanes: ::core::clone::Clone::clone(__self_0),
},
Self::NonPrimitiveSimdType(__self_0) =>
LayoutCalculatorError::NonPrimitiveSimdType(::core::clone::Clone::clone(__self_0)),
}
}
}Clone, #[automatically_derived]impl<F: ::core::fmt::Debug> ::core::fmt::Debug for
LayoutCalculatorError<F> {
#[inline]
#[rustc_trivial_field_reads]
fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
match self {
Self::UnexpectedUnsized(__self_0) =>
::core::fmt::Formatter::debug_struct_field1_finish(f,
"UnexpectedUnsized", &__self_0),
Self::SizeOverflow =>
::core::fmt::Formatter::write_str(f, "SizeOverflow"),
Self::EmptyUnion =>
::core::fmt::Formatter::write_str(f, "EmptyUnion"),
Self::ReprConflict =>
::core::fmt::Formatter::write_str(f, "ReprConflict"),
Self::ZeroLengthSimdType =>
::core::fmt::Formatter::write_str(f, "ZeroLengthSimdType"),
Self::OversizedSimdType { max_lanes: __self_0 } =>
::core::fmt::Formatter::debug_struct_field1_finish(f,
"OversizedSimdType max_lanes", &__self_0),
Self::NonPrimitiveSimdType(__self_0) =>
::core::fmt::Formatter::debug_struct_field1_finish(f,
"NonPrimitiveSimdType", &__self_0),
}
}
}Debug, #[automatically_derived]impl<F: ::core::cmp::PartialEq>
::core::marker::StructuralPartialEq for LayoutCalculatorError<F> {
}
#[automatically_derived]impl<F: ::core::cmp::PartialEq> ::core::cmp::PartialEq
for LayoutCalculatorError<F> {
#[inline]
fn eq(&self, other: &Self) -> bool {
::core::intrinsics::discriminant_value(self) ==
::core::intrinsics::discriminant_value(other) &&
match (self, other) {
(Self::UnexpectedUnsized(__self_0),
Self::UnexpectedUnsized(__arg1_0)) => __self_0 == __arg1_0,
(Self::OversizedSimdType { max_lanes: __self_0 },
Self::OversizedSimdType { max_lanes: __arg1_0 }) =>
__self_0 == __arg1_0,
(Self::NonPrimitiveSimdType(__self_0),
Self::NonPrimitiveSimdType(__arg1_0)) =>
__self_0 == __arg1_0,
_ => true,
}
}
}PartialEq, #[automatically_derived]impl<F: ::core::cmp::Eq> ::core::cmp::Eq for
LayoutCalculatorError<F> {
#[inline]
#[doc(hidden)]
#[coverage(off)]
fn assert_fields_are_eq(&self) {
let _: ::core::cmp::AssertParamIsEq<F>;
let _: ::core::cmp::AssertParamIsEq<usize>;
}
}Eq)]
111pub enum LayoutCalculatorError<F> {
112 UnexpectedUnsized(F),
119
120 SizeOverflow,
122
123 EmptyUnion,
125
126 ReprConflict,
128
129 ZeroLengthSimdType,
131
132 OversizedSimdType { max_lanes: usize },
134
135 NonPrimitiveSimdType(F),
137}
138
139impl<F> LayoutCalculatorError<F> {
140 pub fn without_payload(&self) -> LayoutCalculatorError<()> {
141 use LayoutCalculatorError::*;
142 match *self {
143 UnexpectedUnsized(_) => UnexpectedUnsized(()),
144 SizeOverflow => SizeOverflow,
145 EmptyUnion => EmptyUnion,
146 ReprConflict => ReprConflict,
147 ZeroLengthSimdType => ZeroLengthSimdType,
148 OversizedSimdType { max_lanes } => OversizedSimdType { max_lanes },
149 NonPrimitiveSimdType(_) => NonPrimitiveSimdType(()),
150 }
151 }
152
153 pub fn fallback_fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
157 use LayoutCalculatorError::*;
158 f.write_str(match self {
159 UnexpectedUnsized(_) => "an unsized type was found where a sized type was expected",
160 SizeOverflow => "size overflow",
161 EmptyUnion => "type is a union with no fields",
162 ReprConflict => "type has an invalid repr",
163 ZeroLengthSimdType | OversizedSimdType { .. } | NonPrimitiveSimdType(_) => {
164 "invalid simd type definition"
165 }
166 })
167 }
168}
169
170type LayoutCalculatorResult<FieldIdx, VariantIdx, F> =
171 Result<LayoutData<FieldIdx, VariantIdx>, LayoutCalculatorError<F>>;
172
173#[derive(#[automatically_derived]impl<Cx: ::core::clone::Clone> ::core::clone::Clone
for LayoutCalculator<Cx> {
#[inline]
#[rustc_trivial_field_reads]
fn clone(&self) -> Self {
LayoutCalculator { cx: ::core::clone::Clone::clone(&self.cx) }
}
}Clone, #[automatically_derived]impl<Cx: ::core::marker::Copy> ::core::marker::Copy
for LayoutCalculator<Cx> {
}Copy, #[automatically_derived]impl<Cx: ::core::fmt::Debug> ::core::fmt::Debug for
LayoutCalculator<Cx> {
#[inline]
#[rustc_trivial_field_reads]
fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
::core::fmt::Formatter::debug_struct_field1_finish(f,
"LayoutCalculator cx", &&self.cx)
}
}Debug)]
174pub struct LayoutCalculator<Cx> {
175 pub cx: Cx,
176}
177
178impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
179 pub fn new(cx: Cx) -> Self {
180 Self { cx }
181 }
182
183 pub fn layout_of_array_like<FieldIdx: Idx, VariantIdx: Idx, F>(
184 &self,
185 element: &LayoutData<FieldIdx, VariantIdx>,
186 count_if_sized: Option<u64>, ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F> {
188 let count = count_if_sized.unwrap_or(0);
189 let size =
190 element.size.checked_mul(count, &self.cx).ok_or(LayoutCalculatorError::SizeOverflow)?;
191
192 Ok(LayoutData {
193 variants: Variants::Single { index: VariantIdx::new(0) },
194 fields: FieldsShape::Array { stride: element.size, count },
195 backend_repr: BackendRepr::Memory { sized: count_if_sized.is_some() },
196 largest_niche: element.largest_niche.filter(|_| count != 0),
197 uninhabited: element.uninhabited && count != 0,
198 align: element.align,
199 size,
200 max_repr_align: None,
201 unadjusted_abi_align: element.align,
202 repr_c: element.repr_c,
203 randomization_seed: element.randomization_seed.wrapping_add(Hash64::new(count)),
204 })
205 }
206
207 pub fn layout_of_scalable_vector_type<FieldIdx, VariantIdx, F>(
208 &self,
209 element: F,
210 count: u64,
211 number_of_vectors: NumScalableVectors,
212 ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
213 where
214 FieldIdx: Idx,
215 VariantIdx: Idx,
216 F: AsRef<LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,
217 {
218 vector_type_layout(
219 SimdVectorKind::Scalable(number_of_vectors),
220 self.cx.data_layout(),
221 element,
222 count,
223 )
224 }
225
226 pub fn layout_of_simd_type<FieldIdx, VariantIdx, F>(
227 &self,
228 element: F,
229 count: u64,
230 repr_packed: bool,
231 ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
232 where
233 FieldIdx: Idx,
234 VariantIdx: Idx,
235 F: AsRef<LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,
236 {
237 let kind = if repr_packed { SimdVectorKind::PackedFixed } else { SimdVectorKind::Fixed };
238 vector_type_layout(kind, self.cx.data_layout(), element, count)
239 }
240
241 pub fn layout_of_coroutine<'a, F, VariantIdx, FieldIdx, LocalIdx>(
246 &self,
247 local_layouts: &IndexSlice<LocalIdx, F>,
248 prefix_layouts: IndexVec<FieldIdx, F>,
249 variant_fields: &IndexSlice<VariantIdx, IndexVec<FieldIdx, LocalIdx>>,
250 storage_conflicts: &BitMatrix<LocalIdx, LocalIdx>,
251 tag_to_layout: impl Fn(Scalar) -> F,
252 ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
253 where
254 F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,
255 VariantIdx: Idx,
256 FieldIdx: Idx,
257 LocalIdx: Idx,
258 {
259 coroutine::layout(
260 self,
261 local_layouts,
262 prefix_layouts,
263 variant_fields,
264 storage_conflicts,
265 tag_to_layout,
266 )
267 }
268
269 pub fn layout_of_univariant<'a, FieldIdx, VariantIdx, F>(
275 &self,
276 fields: &IndexSlice<FieldIdx, F>,
277 repr: &ReprOptions,
278 kind: StructKind,
279 ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
280 where
281 FieldIdx: Idx,
282 VariantIdx: Idx,
283 F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,
284 {
285 let dl = self.cx.data_layout();
286 let layout = self.layout_of_univariant_biased(fields, repr, kind, NicheBias::Start);
287 if let Ok(layout) = &layout
293 && !#[allow(non_exhaustive_omitted_patterns)] match kind {
StructKind::MaybeUnsized => true,
_ => false,
}matches!(kind, StructKind::MaybeUnsized)
297 && let Some(niche) = layout.largest_niche
298 && let head_space = niche.offset.bytes()
299 && let niche_len = niche.value.size(dl).bytes()
300 && let tail_space = layout.size.bytes() - head_space - niche_len
301 && (fields.len() > 1 && head_space != 0 && tail_space > 0)
305 {
306 let alt_layout = self
307 .layout_of_univariant_biased(fields, repr, kind, NicheBias::End)
308 .expect("alt layout should always work");
309 let alt_niche = alt_layout
310 .largest_niche
311 .expect("alt layout should have a niche like the regular one");
312 let alt_head_space = alt_niche.offset.bytes();
313 let alt_niche_len = alt_niche.value.size(dl).bytes();
314 let alt_tail_space = alt_layout.size.bytes() - alt_head_space - alt_niche_len;
315
316 if true {
{
match (&layout.size.bytes(), &alt_layout.size.bytes()) {
(left_val, right_val) => {
if !(*left_val == *right_val) {
let kind = ::core::panicking::AssertKind::Eq;
::core::panicking::assert_failed(kind, &*left_val,
&*right_val, ::core::option::Option::None);
}
}
}
};
};debug_assert_eq!(layout.size.bytes(), alt_layout.size.bytes());
317
318 let prefer_alt_layout = alt_head_space > head_space && alt_head_space > tail_space;
319
320 {
use ::tracing::__macro_support::Callsite as _;
static __CALLSITE: ::tracing::callsite::DefaultCallsite =
{
static META: ::tracing::Metadata<'static> =
{
::tracing_core::metadata::Metadata::new("event /rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout.rs:320",
"rustc_abi::layout", ::tracing::Level::DEBUG,
::tracing_core::__macro_support::Option::Some("/rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout.rs"),
::tracing_core::__macro_support::Option::Some(320u32),
::tracing_core::__macro_support::Option::Some("rustc_abi::layout"),
::tracing_core::field::FieldSet::new(&["message"],
::tracing_core::callsite::Identifier(&__CALLSITE)),
::tracing::metadata::Kind::EVENT)
};
::tracing::callsite::DefaultCallsite::new(&META)
};
let enabled =
::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
&&
::tracing::Level::DEBUG <=
::tracing::level_filters::LevelFilter::current() &&
{
let interest = __CALLSITE.interest();
!interest.is_never() &&
::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
interest)
};
if enabled {
(|value_set: ::tracing::field::ValueSet|
{
let meta = __CALLSITE.metadata();
::tracing::Event::dispatch(meta, &value_set);
;
})({
#[allow(unused_imports)]
use ::tracing::field::{debug, display, Value};
__CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("sz: {0}, default_niche_at: {1}+{2}, default_tail_space: {3}, alt_niche_at/head_space: {4}+{5}, alt_tail: {6}, num_fields: {7}, better: {8}\nlayout: {9}\nalt_layout: {10}\n",
layout.size.bytes(), head_space, niche_len, tail_space,
alt_head_space, alt_niche_len, alt_tail_space,
layout.fields.count(), prefer_alt_layout,
self.format_field_niches(layout, fields),
self.format_field_niches(&alt_layout, fields)) as
&dyn ::tracing::field::Value))])
});
} else { ; }
};debug!(
321 "sz: {}, default_niche_at: {}+{}, default_tail_space: {}, alt_niche_at/head_space: {}+{}, alt_tail: {}, num_fields: {}, better: {}\n\
322 layout: {}\n\
323 alt_layout: {}\n",
324 layout.size.bytes(),
325 head_space,
326 niche_len,
327 tail_space,
328 alt_head_space,
329 alt_niche_len,
330 alt_tail_space,
331 layout.fields.count(),
332 prefer_alt_layout,
333 self.format_field_niches(layout, fields),
334 self.format_field_niches(&alt_layout, fields),
335 );
336
337 if prefer_alt_layout {
338 return Ok(alt_layout);
339 }
340 }
341 layout
342 }
343
344 pub fn layout_of_struct_or_enum<'a, FieldIdx, VariantIdx, F>(
345 &self,
346 repr: &ReprOptions,
347 variants: &IndexSlice<VariantIdx, IndexVec<FieldIdx, F>>,
348 is_enum: bool,
349 niche_optimizations: NicheOptimizations,
350 discr_range_of_repr: impl Fn(RangeFrom<i128>, RangeToInclusive<u128>) -> (Integer, bool),
351 discriminants: impl Iterator<Item = (VariantIdx, u128)>,
352 always_sized: bool,
353 ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
354 where
355 FieldIdx: Idx,
356 VariantIdx: Idx,
357 F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,
358 {
359 let (present_first, present_second) = {
360 let mut present_variants = variants.iter_enumerated().filter_map(|(i, v)| {
361 if !repr.inhibit_enum_layout_opt() && absent(v) { None } else { Some(i) }
362 });
363 (present_variants.next(), present_variants.next())
364 };
365 let present_first = match present_first {
366 Some(present_first) => present_first,
367 None if is_enum => {
369 return Ok(LayoutData::never_type());
370 }
371 None => VariantIdx::new(0),
374 };
375
376 if !is_enum ||
378 (present_second.is_none() && !repr.inhibit_enum_layout_opt())
380 {
381 self.layout_of_struct(
382 repr,
383 variants,
384 present_first,
385 is_enum,
386 niche_optimizations,
387 always_sized,
388 )
389 } else {
390 if !is_enum { ::core::panicking::panic("assertion failed: is_enum") };assert!(is_enum);
394
395 self::enum_layout::layout_of_enum(
396 self,
397 repr,
398 variants,
399 discr_range_of_repr,
400 discriminants,
401 )
402 }
403 }
404
405 pub fn layout_of_union<'a, FieldIdx, VariantIdx, F>(
406 &self,
407 repr: &ReprOptions,
408 variants: &IndexSlice<VariantIdx, IndexVec<FieldIdx, F>>,
409 ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
410 where
411 FieldIdx: Idx,
412 VariantIdx: Idx,
413 F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,
414 {
415 let dl = self.cx.data_layout();
416 let mut align = if repr.pack.is_some() { Align::ONE } else { dl.aggregate_align };
417 let mut max_repr_align = repr.align;
418
419 struct AbiMismatch;
422 let mut common_non_zst_repr_and_align = if repr.inhibits_union_abi_opt() {
423 Err(AbiMismatch)
425 } else {
426 Ok(None)
427 };
428
429 let mut size = Size::ZERO;
430 let only_variant_idx = VariantIdx::new(0);
431 let only_variant = &variants[only_variant_idx];
432 for field in only_variant {
433 if field.is_unsized() {
434 return Err(LayoutCalculatorError::UnexpectedUnsized(*field));
435 }
436
437 align = align.max(field.align);
438 max_repr_align = max_repr_align.max(field.max_repr_align);
439 size = cmp::max(size, field.size);
440
441 if field.is_zst() {
442 continue;
444 }
445
446 if let Ok(common) = common_non_zst_repr_and_align {
447 let field_abi = field.backend_repr.to_union();
449
450 if let Some((common_abi, common_align)) = common {
451 if common_abi != field_abi {
452 common_non_zst_repr_and_align = Err(AbiMismatch);
454 } else {
455 if !#[allow(non_exhaustive_omitted_patterns)] match common_abi {
BackendRepr::Memory { .. } => true,
_ => false,
}matches!(common_abi, BackendRepr::Memory { .. }) {
458 {
match (&common_align, &field.align) {
(left_val, right_val) => {
if !(*left_val == *right_val) {
let kind = ::core::panicking::AssertKind::Eq;
::core::panicking::assert_failed(kind, &*left_val,
&*right_val,
::core::option::Option::Some(format_args!("non-Aggregate field with matching ABI but differing alignment")));
}
}
}
};assert_eq!(
459 common_align, field.align,
460 "non-Aggregate field with matching ABI but differing alignment"
461 );
462 }
463 }
464 } else {
465 common_non_zst_repr_and_align = Ok(Some((field_abi, field.align)));
467 }
468 }
469 }
470
471 if let Some(pack) = repr.pack {
472 align = align.min(pack);
473 }
474 let unadjusted_abi_align = align;
477 if let Some(repr_align) = repr.align {
478 align = align.max(repr_align);
479 }
480 let align = align;
482
483 let backend_repr = match common_non_zst_repr_and_align {
486 Err(AbiMismatch) | Ok(None) => BackendRepr::Memory { sized: true },
487 Ok(Some((repr, _))) => match repr {
488 BackendRepr::Scalar(_) | BackendRepr::ScalarPair { .. }
490 if repr.scalar_platform_align(dl).unwrap() != align =>
491 {
492 BackendRepr::Memory { sized: true }
493 }
494 BackendRepr::SimdVector { element, count: _ }
496 if element.default_align(dl) > align =>
497 {
498 BackendRepr::Memory { sized: true }
499 }
500 BackendRepr::Scalar(..)
502 | BackendRepr::ScalarPair { .. }
503 | BackendRepr::SimdVector { .. }
504 | BackendRepr::SimdScalableVector { .. }
505 | BackendRepr::Memory { .. } => repr,
506 },
507 };
508
509 let Some(union_field_count) = NonZero::new(only_variant.len()) else {
510 return Err(LayoutCalculatorError::EmptyUnion);
511 };
512
513 let repr_c =
515 repr.c() || (repr.transparent() && only_variant.iter().any(|field| field.repr_c));
516
517 let combined_seed = only_variant
518 .iter()
519 .map(|v| v.randomization_seed)
520 .fold(repr.field_shuffle_seed, |acc, seed| acc.wrapping_add(seed));
521
522 Ok(LayoutData {
523 variants: Variants::Single { index: only_variant_idx },
524 fields: FieldsShape::Union(union_field_count),
525 backend_repr,
526 largest_niche: None,
527 uninhabited: false,
528 align,
529 size: size.align_to(align),
530 max_repr_align,
531 unadjusted_abi_align,
532 repr_c,
533 randomization_seed: combined_seed,
534 })
535 }
536
537 fn layout_of_struct<'a, FieldIdx, VariantIdx, F>(
542 &self,
543 repr: &ReprOptions,
544 variants: &IndexSlice<VariantIdx, IndexVec<FieldIdx, F>>,
545 variant_idx: VariantIdx,
546 is_enum: bool,
547 niche_optimizations: NicheOptimizations,
548 always_sized: bool,
549 ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
550 where
551 FieldIdx: Idx,
552 VariantIdx: Idx,
553 F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,
554 {
555 let dl = self.cx.data_layout();
556 let v = variant_idx;
557 let kind = if is_enum || variants[v].is_empty() || always_sized {
558 StructKind::AlwaysSized
559 } else {
560 StructKind::MaybeUnsized
561 };
562
563 let mut st = self.layout_of_univariant(&variants[v], repr, kind)?;
564 st.variants = Variants::Single { index: v };
565
566 if niche_optimizations == NicheOptimizations::Disabled {
567 let hide_niches = |scalar: &mut _| match scalar {
568 Scalar::Initialized { value, valid_range } => {
569 *valid_range = WrappingRange::full(value.size(dl))
570 }
571 Scalar::Union { .. } => {}
573 };
574 match &mut st.backend_repr {
575 BackendRepr::Scalar(scalar) => hide_niches(scalar),
576 BackendRepr::ScalarPair { a, b, b_offset: _ } => {
577 hide_niches(a);
578 hide_niches(b);
579 }
580 BackendRepr::SimdVector { element, .. }
581 | BackendRepr::SimdScalableVector { element, .. } => hide_niches(element),
582 BackendRepr::Memory { sized: _ } => {}
583 }
584 st.largest_niche = None;
585 return Ok(st);
586 }
587
588 Ok(st)
589 }
590
591 fn layout_of_univariant_biased<'a, FieldIdx, VariantIdx, F>(
592 &self,
593 fields: &IndexSlice<FieldIdx, F>,
594 repr: &ReprOptions,
595 kind: StructKind,
596 niche_bias: NicheBias,
597 ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
598 where
599 FieldIdx: Idx,
600 VariantIdx: Idx,
601 F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,
602 {
603 let dl = self.cx.data_layout();
604 let pack = repr.pack;
605 let mut align = if pack.is_some() { Align::ONE } else { dl.aggregate_align };
606 let mut max_repr_align = repr.align;
607 let mut in_memory_order: IndexVec<u32, FieldIdx> = fields.indices().collect();
608 let optimize_field_order = !repr.inhibit_struct_field_reordering();
609 let end = if let StructKind::MaybeUnsized = kind { fields.len() - 1 } else { fields.len() };
610 let optimizing = &mut in_memory_order.raw[..end];
611 let fields_excluding_tail = &fields.raw[..end];
612 let field_seed = fields_excluding_tail
614 .iter()
615 .fold(Hash64::ZERO, |acc, f| acc.wrapping_add(f.randomization_seed));
616
617 if optimize_field_order && fields.len() > 1 {
618 if repr.can_randomize_type_layout() && truecfg!(feature = "randomize") {
627 #[cfg(feature = "randomize")]
628 {
629 use rand::SeedableRng;
630 use rand::seq::SliceRandom;
631 let mut rng = rand_xoshiro::Xoshiro128StarStar::seed_from_u64(
634 field_seed.wrapping_add(repr.field_shuffle_seed).as_u64(),
635 );
636
637 optimizing.shuffle(&mut rng);
639 }
640 } else {
642 let max_field_align =
645 fields_excluding_tail.iter().map(|f| f.align.bytes()).max().unwrap_or(1);
646 let largest_niche_size = fields_excluding_tail
647 .iter()
648 .filter_map(|f| f.largest_niche)
649 .map(|n| n.available(dl))
650 .max()
651 .unwrap_or(0);
652
653 let alignment_group_key = |layout: &F| {
656 if let Some(pack) = pack {
660 layout.align.min(pack).bytes()
662 } else {
663 let align = layout.align.bytes();
666 let size = layout.size.bytes();
667 let niche_size = layout.largest_niche.map(|n| n.available(dl)).unwrap_or(0);
668 let size_as_align = align.max(size).trailing_zeros();
670 let size_as_align = if largest_niche_size > 0 {
671 match niche_bias {
672 NicheBias::Start => {
676 max_field_align.trailing_zeros().min(size_as_align)
677 }
678 NicheBias::End if niche_size == largest_niche_size => {
682 align.trailing_zeros()
683 }
684 NicheBias::End => size_as_align,
685 }
686 } else {
687 size_as_align
688 };
689 size_as_align as u64
690 }
691 };
692
693 match kind {
694 StructKind::AlwaysSized | StructKind::MaybeUnsized => {
695 optimizing.sort_by_key(|&x| {
704 let f = &fields[x];
705 let field_size = f.size.bytes();
706 let niche_size = f.largest_niche.map_or(0, |n| n.available(dl));
707 let niche_size_key = match niche_bias {
708 NicheBias::Start => !niche_size,
710 NicheBias::End => niche_size,
712 };
713 let inner_niche_offset_key = match niche_bias {
714 NicheBias::Start => f.largest_niche.map_or(0, |n| n.offset.bytes()),
715 NicheBias::End => f.largest_niche.map_or(0, |n| {
716 !(field_size - n.value.size(dl).bytes() - n.offset.bytes())
717 }),
718 };
719
720 (
721 cmp::Reverse(alignment_group_key(f)),
723 niche_size_key,
726 inner_niche_offset_key,
729 )
730 });
731 }
732
733 StructKind::Prefixed(..) => {
734 optimizing.sort_by_key(|&x| {
739 let f = &fields[x];
740 let niche_size = f.largest_niche.map_or(0, |n| n.available(dl));
741 (alignment_group_key(f), niche_size)
742 });
743 }
744 }
745
746 }
749 }
750 let mut unsized_field = None::<&F>;
755 let mut offsets = IndexVec::from_elem(Size::ZERO, fields);
756 let mut offset = Size::ZERO;
757 let mut largest_niche = None;
758 let mut largest_niche_available = 0;
759 if let StructKind::Prefixed(prefix_size, prefix_align) = kind {
760 let prefix_align =
761 if let Some(pack) = pack { prefix_align.min(pack) } else { prefix_align };
762 align = align.max(prefix_align);
763 offset = prefix_size.align_to(prefix_align);
764 }
765 for &i in &in_memory_order {
766 let field = &fields[i];
767 if let Some(unsized_field) = unsized_field {
768 return Err(LayoutCalculatorError::UnexpectedUnsized(*unsized_field));
769 }
770
771 if field.is_unsized() {
772 if let StructKind::MaybeUnsized = kind {
773 unsized_field = Some(field);
774 } else {
775 return Err(LayoutCalculatorError::UnexpectedUnsized(*field));
776 }
777 }
778
779 let field_align =
781 if let Some(pack) = pack { field.align.min(pack) } else { field.align };
782 offset = offset.align_to(field_align);
783 align = align.max(field_align);
784 max_repr_align = max_repr_align.max(field.max_repr_align);
785
786 {
use ::tracing::__macro_support::Callsite as _;
static __CALLSITE: ::tracing::callsite::DefaultCallsite =
{
static META: ::tracing::Metadata<'static> =
{
::tracing_core::metadata::Metadata::new("event /rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout.rs:786",
"rustc_abi::layout", ::tracing::Level::DEBUG,
::tracing_core::__macro_support::Option::Some("/rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout.rs"),
::tracing_core::__macro_support::Option::Some(786u32),
::tracing_core::__macro_support::Option::Some("rustc_abi::layout"),
::tracing_core::field::FieldSet::new(&["message"],
::tracing_core::callsite::Identifier(&__CALLSITE)),
::tracing::metadata::Kind::EVENT)
};
::tracing::callsite::DefaultCallsite::new(&META)
};
let enabled =
::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
&&
::tracing::Level::DEBUG <=
::tracing::level_filters::LevelFilter::current() &&
{
let interest = __CALLSITE.interest();
!interest.is_never() &&
::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
interest)
};
if enabled {
(|value_set: ::tracing::field::ValueSet|
{
let meta = __CALLSITE.metadata();
::tracing::Event::dispatch(meta, &value_set);
;
})({
#[allow(unused_imports)]
use ::tracing::field::{debug, display, Value};
__CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("univariant offset: {0:?} field: {1:#?}",
offset, field) as &dyn ::tracing::field::Value))])
});
} else { ; }
};debug!("univariant offset: {:?} field: {:#?}", offset, field);
787 offsets[i] = offset;
788
789 if let Some(mut niche) = field.largest_niche {
790 let available = niche.available(dl);
791 let prefer_new_niche = match niche_bias {
793 NicheBias::Start => available > largest_niche_available,
794 NicheBias::End => available >= largest_niche_available,
796 };
797 if prefer_new_niche {
798 largest_niche_available = available;
799 niche.offset += offset;
800 largest_niche = Some(niche);
801 }
802 }
803
804 offset =
805 offset.checked_add(field.size, dl).ok_or(LayoutCalculatorError::SizeOverflow)?;
806 }
807
808 let unadjusted_abi_align = align;
811 if let Some(repr_align) = repr.align {
812 align = align.max(repr_align);
813 }
814 let align = align;
816
817 {
use ::tracing::__macro_support::Callsite as _;
static __CALLSITE: ::tracing::callsite::DefaultCallsite =
{
static META: ::tracing::Metadata<'static> =
{
::tracing_core::metadata::Metadata::new("event /rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout.rs:817",
"rustc_abi::layout", ::tracing::Level::DEBUG,
::tracing_core::__macro_support::Option::Some("/rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout.rs"),
::tracing_core::__macro_support::Option::Some(817u32),
::tracing_core::__macro_support::Option::Some("rustc_abi::layout"),
::tracing_core::field::FieldSet::new(&["message"],
::tracing_core::callsite::Identifier(&__CALLSITE)),
::tracing::metadata::Kind::EVENT)
};
::tracing::callsite::DefaultCallsite::new(&META)
};
let enabled =
::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
&&
::tracing::Level::DEBUG <=
::tracing::level_filters::LevelFilter::current() &&
{
let interest = __CALLSITE.interest();
!interest.is_never() &&
::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
interest)
};
if enabled {
(|value_set: ::tracing::field::ValueSet|
{
let meta = __CALLSITE.metadata();
::tracing::Event::dispatch(meta, &value_set);
;
})({
#[allow(unused_imports)]
use ::tracing::field::{debug, display, Value};
__CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("univariant min_size: {0:?}",
offset) as &dyn ::tracing::field::Value))])
});
} else { ; }
};debug!("univariant min_size: {:?}", offset);
818 let min_size = offset;
819 let size = min_size.align_to(align);
820 if size.bytes() >= dl.obj_size_bound() {
822 return Err(LayoutCalculatorError::SizeOverflow);
823 }
824 let mut layout_of_single_non_zst_field = None;
825 let sized = unsized_field.is_none();
826 let mut abi = BackendRepr::Memory { sized };
827
828 let optimize_abi = !repr.inhibit_newtype_abi_optimization();
829
830 if sized && size.bytes() > 0 {
832 let mut non_zst_fields = fields.iter_enumerated().filter(|&(_, f)| !f.is_zst());
835
836 match (non_zst_fields.next(), non_zst_fields.next(), non_zst_fields.next()) {
837 (Some((i, field)), None, None) => {
839 layout_of_single_non_zst_field = Some(field);
840
841 if offsets[i].bytes() == 0 && align == field.align && size == field.size {
843 match field.backend_repr {
844 BackendRepr::Scalar(_) | BackendRepr::SimdVector { .. }
847 if optimize_abi =>
848 {
849 abi = field.backend_repr;
850 }
851 BackendRepr::ScalarPair { .. } => {
854 abi = field.backend_repr;
855 }
856 _ => {}
857 }
858 }
859 }
860
861 (Some((i, a)), Some((j, b)), None) => {
863 match (a.backend_repr, b.backend_repr) {
864 (BackendRepr::Scalar(a), BackendRepr::Scalar(b)) => {
865 let ((i, a), (j, b)) = if offsets[i] < offsets[j] {
867 ((i, a), (j, b))
868 } else {
869 ((j, b), (i, a))
870 };
871 let pair =
872 LayoutData::<FieldIdx, VariantIdx>::scalar_pair(&self.cx, a, b);
873 let pair_offsets = match pair.fields {
874 FieldsShape::Arbitrary { ref offsets, ref in_memory_order } => {
875 {
match (&in_memory_order.raw, &[FieldIdx::new(0), FieldIdx::new(1)]) {
(left_val, right_val) => {
if !(*left_val == *right_val) {
let kind = ::core::panicking::AssertKind::Eq;
::core::panicking::assert_failed(kind, &*left_val,
&*right_val, ::core::option::Option::None);
}
}
}
};assert_eq!(
876 in_memory_order.raw,
877 [FieldIdx::new(0), FieldIdx::new(1)]
878 );
879 offsets
880 }
881 FieldsShape::Primitive
882 | FieldsShape::Array { .. }
883 | FieldsShape::Union(..) => {
884 {
::core::panicking::panic_fmt(format_args!("encountered a non-arbitrary layout during enum layout"));
}panic!("encountered a non-arbitrary layout during enum layout")
885 }
886 };
887 if offsets[i] == pair_offsets[FieldIdx::new(0)]
888 && offsets[j] == pair_offsets[FieldIdx::new(1)]
889 && align == pair.align
890 && size == pair.size
891 {
892 abi = pair.backend_repr;
895 }
896 }
897 _ => {}
898 }
899 }
900
901 _ => {}
902 }
903 }
904 let uninhabited = fields.iter().any(|f| f.is_uninhabited());
905
906 let unadjusted_abi_align = if repr.transparent() {
907 match layout_of_single_non_zst_field {
908 Some(l) => l.unadjusted_abi_align,
909 None => {
910 align
912 }
913 }
914 } else {
915 unadjusted_abi_align
916 };
917
918 let repr_c = repr.c() || (repr.transparent() && fields.iter().any(|field| field.repr_c));
920
921 let seed = field_seed.wrapping_add(repr.field_shuffle_seed);
922
923 Ok(LayoutData {
924 variants: Variants::Single { index: VariantIdx::new(0) },
925 fields: FieldsShape::Arbitrary { offsets, in_memory_order },
926 backend_repr: abi,
927 largest_niche,
928 uninhabited,
929 align,
930 size,
931 max_repr_align,
932 unadjusted_abi_align,
933 repr_c,
934 randomization_seed: seed,
935 })
936 }
937
938 fn format_field_niches<'a, FieldIdx, VariantIdx, F>(
939 &self,
940 layout: &LayoutData<FieldIdx, VariantIdx>,
941 fields: &IndexSlice<FieldIdx, F>,
942 ) -> String
943 where
944 FieldIdx: Idx,
945 VariantIdx: Idx,
946 F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,
947 {
948 let dl = self.cx.data_layout();
949 let mut s = String::new();
950 for i in layout.fields.index_by_increasing_offset() {
951 let offset = layout.fields.offset(i);
952 let f = &fields[FieldIdx::new(i)];
953 s.write_fmt(format_args!("[o{0}a{1}s{2}", offset.bytes(), f.align.bytes(),
f.size.bytes()))write!(s, "[o{}a{}s{}", offset.bytes(), f.align.bytes(), f.size.bytes()).unwrap();
954 if let Some(n) = f.largest_niche {
955 s.write_fmt(format_args!(" n{0}b{1}s{2}", n.offset.bytes(),
n.available(dl).ilog2(), n.value.size(dl).bytes()))write!(
956 s,
957 " n{}b{}s{}",
958 n.offset.bytes(),
959 n.available(dl).ilog2(),
960 n.value.size(dl).bytes()
961 )
962 .unwrap();
963 }
964 s.write_fmt(format_args!("] "))write!(s, "] ").unwrap();
965 }
966 s
967 }
968}
969
970enum SimdVectorKind {
971 Scalable(NumScalableVectors),
973 PackedFixed,
975 Fixed,
977}
978
979fn vector_type_layout<FieldIdx, VariantIdx, F>(
980 kind: SimdVectorKind,
981 dl: &TargetDataLayout,
982 element: F,
983 count: u64,
984) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
985where
986 FieldIdx: Idx,
987 VariantIdx: Idx,
988 F: AsRef<LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,
989{
990 let elt = element.as_ref();
991 let count = BackendLaneCount::new(count)?;
992
993 let BackendRepr::Scalar(element) = elt.backend_repr else {
994 return Err(LayoutCalculatorError::NonPrimitiveSimdType(element));
995 };
996
997 let size =
999 elt.size.checked_mul(count.as_u64(), dl).ok_or(LayoutCalculatorError::SizeOverflow)?;
1000 let (repr, size, align) = match kind {
1001 SimdVectorKind::Scalable(number_of_vectors) => (
1002 BackendRepr::SimdScalableVector { element, count, number_of_vectors },
1003 size.checked_mul(number_of_vectors.0 as u64, dl)
1004 .ok_or(LayoutCalculatorError::SizeOverflow)?,
1005 dl.rust_vector_align(size),
1006 ),
1007 SimdVectorKind::PackedFixed if !count.is_power_of_two() => {
1011 (BackendRepr::Memory { sized: true }, size, Align::max_aligned_factor(size))
1012 }
1013 SimdVectorKind::PackedFixed | SimdVectorKind::Fixed => {
1014 (BackendRepr::SimdVector { element, count }, size, dl.rust_vector_align(size))
1015 }
1016 };
1017 let size = size.align_to(align);
1018
1019 Ok(LayoutData {
1020 variants: Variants::Single { index: VariantIdx::new(0) },
1021 fields: FieldsShape::Arbitrary {
1022 offsets: [Size::ZERO].into(),
1023 in_memory_order: [FieldIdx::new(0)].into(),
1024 },
1025 backend_repr: repr,
1026 largest_niche: elt.largest_niche,
1027 uninhabited: false,
1028 size,
1029 align,
1030 max_repr_align: None,
1031 unadjusted_abi_align: elt.align,
1032 repr_c: false,
1033 randomization_seed: elt.randomization_seed.wrapping_add(Hash64::new(count.as_u64())),
1034 })
1035}