Skip to main content

rustc_abi/layout/
enum_layout.rs

1use std::ops::Deref;
2use std::range::{RangeFrom, RangeInclusive, RangeToInclusive};
3use std::{cmp, fmt, iter};
4
5use rustc_index::Idx;
6use tracing::trace;
7
8use super::{LayoutCalculator, LayoutCalculatorError, LayoutCalculatorResult, absent};
9use crate::{
10    Align, BackendRepr, FieldsShape, HasDataLayout, IndexSlice, IndexVec, Integer, LayoutData,
11    Niche, Primitive, ReprOptions, Scalar, Size, StructKind, TagEncoding, TargetDataLayout,
12    VariantLayout, Variants, WrappingRange,
13};
14
15pub(super) fn layout_of_enum<'a, Cx: HasDataLayout, FieldIdx, VariantIdx, F>(
16    calculator: &LayoutCalculator<Cx>,
17    repr: &ReprOptions,
18    variants: &IndexSlice<VariantIdx, IndexVec<FieldIdx, F>>,
19    discr_range_of_repr: impl Fn(RangeFrom<i128>, RangeToInclusive<u128>) -> (Integer, bool),
20    discriminants: impl Iterator<Item = (VariantIdx, u128)>,
21) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
22where
23    FieldIdx: Idx,
24    VariantIdx: Idx,
25    F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,
26{
27    let dl = calculator.cx.data_layout();
28    // bail if the enum has an incoherent repr that cannot be computed
29    if repr.packed() {
30        return Err(LayoutCalculatorError::ReprConflict);
31    }
32
33    let niche_filling_layout = calculate_niche_filling_layout(calculator, repr, variants);
34
35    let tagged_layout =
36        calculate_tagged_layout(calculator, repr, variants, discr_range_of_repr, discriminants)?;
37
38    let Some(nl) = niche_filling_layout else {
39        return Ok(tagged_layout);
40    };
41
42    // Pick the smaller layout; otherwise,
43    // pick the layout with the larger niche; otherwise,
44    // pick tagged as it has simpler codegen.
45    use cmp::Ordering::*;
46    let niche_size =
47        |l: &LayoutData<FieldIdx, VariantIdx>| l.largest_niche.map_or(0, |n| n.available(dl));
48    let best_layout = match (
49        tagged_layout.size.cmp(&nl.size),
50        niche_size(&tagged_layout).cmp(&niche_size(&nl)),
51    ) {
52        (Greater, _) => nl,
53        (Equal, Less) => nl,
54        _ => tagged_layout,
55    };
56
57    Ok(best_layout)
58}
59
60struct CommonPrimitive {
61    primitive: Primitive,
62    offset: Size,
63    initialized_in_all_variants: bool,
64}
65
66fn calculate_common_primitive<'a, VariantIdx, FieldIdx, F>(
67    dl: &TargetDataLayout,
68    layout_variants: &IndexSlice<VariantIdx, VariantLayout<FieldIdx>>,
69    variants: &IndexSlice<VariantIdx, IndexVec<FieldIdx, F>>,
70) -> Option<CommonPrimitive>
71where
72    FieldIdx: Idx,
73    VariantIdx: Idx,
74    F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,
75{
76    // Try to use a ScalarPair for all tagged enums.
77    // That's possible only if we can find a common primitive type for all variants.
78    let mut common_prim = None;
79    let mut initialized_in_all_variants = true;
80    for (field_layouts, layout_variant) in iter::zip(variants, layout_variants) {
81        // We skip *all* ZST here and later check if we are good in terms of alignment.
82        // This lets us handle some cases involving aligned ZST.
83        let mut fields =
84            iter::zip(field_layouts, &layout_variant.field_offsets).filter(|p| !p.0.is_zst());
85        let (field, offset) = match (fields.next(), fields.next()) {
86            (None, None) => {
87                initialized_in_all_variants = false;
88                continue;
89            }
90            (Some(pair), None) => pair,
91            _ => {
92                return None;
93            }
94        };
95        let prim = match field.backend_repr {
96            BackendRepr::Scalar(scalar) => {
97                initialized_in_all_variants &= #[allow(non_exhaustive_omitted_patterns)] match scalar {
    Scalar::Initialized { .. } => true,
    _ => false,
}matches!(scalar, Scalar::Initialized { .. });
98                scalar.primitive()
99            }
100            _ => {
101                return None;
102            }
103        };
104        if let Some((old_prim, common_offset)) = common_prim {
105            // All variants must be at the same offset
106            if offset != common_offset {
107                return None;
108            }
109            // This is pretty conservative. We could go fancier
110            // by realising that (u8, u8) could just cohabit with
111            // u16 or even u32.
112            let new_prim = match (old_prim, prim) {
113                // Allow all identical primitives.
114                (x, y) if x == y => x,
115                // Allow integers of the same size with differing signedness.
116                // We arbitrarily choose the signedness of the first variant.
117                (p @ Primitive::Int(x, _), Primitive::Int(y, _)) if x == y => p,
118                // Allow integers mixed with pointers of the same layout.
119                // We must represent this using a pointer, to avoid
120                // roundtripping pointers through ptrtoint/inttoptr.
121                (p @ Primitive::Pointer(_), i @ Primitive::Int(..))
122                | (i @ Primitive::Int(..), p @ Primitive::Pointer(_))
123                    if p.size(dl) == i.size(dl) && p.default_align(dl) == i.default_align(dl) =>
124                {
125                    p
126                }
127                _ => {
128                    return None;
129                }
130            };
131            // We may be updating the primitive here, for example from int->ptr.
132            common_prim = Some((new_prim, common_offset));
133        } else {
134            common_prim = Some((prim, offset));
135        }
136    }
137    common_prim.map(|(primitive, offset)| CommonPrimitive {
138        primitive,
139        offset: *offset,
140        initialized_in_all_variants,
141    })
142}
143
144fn scalar_pair_repr<'a, Cx, VariantIdx, FieldIdx, F>(
145    calculator: &LayoutCalculator<Cx>,
146    layout_variants: &IndexSlice<VariantIdx, VariantLayout<FieldIdx>>,
147    variants: &IndexSlice<VariantIdx, IndexVec<FieldIdx, F>>,
148    size: Size,
149    align: Align,
150    tag: Scalar,
151) -> Option<BackendRepr>
152where
153    Cx: HasDataLayout,
154    FieldIdx: Idx,
155    VariantIdx: Idx,
156    F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,
157{
158    let CommonPrimitive { primitive, offset, initialized_in_all_variants } =
159        calculate_common_primitive(calculator.cx.data_layout(), layout_variants, variants)?;
160    let prim_scalar = if initialized_in_all_variants {
161        let size = primitive.size(calculator.cx.data_layout());
162        if !(size.bits() <= 128) {
    ::core::panicking::panic("assertion failed: size.bits() <= 128")
};assert!(size.bits() <= 128);
163        Scalar::Initialized { value: primitive, valid_range: WrappingRange::full(size) }
164    } else {
165        // Common prim might be uninit.
166        Scalar::Union { value: primitive }
167    };
168    let pair = LayoutData::<FieldIdx, VariantIdx>::scalar_pair(&calculator.cx, tag, prim_scalar);
169    let pair_offsets = match pair.fields {
170        FieldsShape::Arbitrary { ref offsets, ref in_memory_order } => {
171            {
    match (&in_memory_order.raw, &[FieldIdx::new(0), FieldIdx::new(1)]) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val, ::core::option::Option::None);
            }
        }
    }
};assert_eq!(in_memory_order.raw, [FieldIdx::new(0), FieldIdx::new(1)]);
172            offsets
173        }
174        _ => {
    ::core::panicking::panic_fmt(format_args!("encountered a non-arbitrary layout during enum layout"));
}panic!("encountered a non-arbitrary layout during enum layout"),
175    };
176    if pair_offsets[FieldIdx::new(0)] == Size::ZERO
177        && pair_offsets[FieldIdx::new(1)] == offset
178        && align == pair.align
179        && size == pair.size
180    {
181        // We can use `ScalarPair` only when it matches our
182        // already computed layout (including `#[repr(C)]`).
183        Some(pair.backend_repr)
184    } else {
185        None
186    }
187}
188
189fn calculate_tagged_abi<'a, Cx, VariantIdx, FieldIdx, F>(
190    calculator: &LayoutCalculator<Cx>,
191    layout_variants: &IndexSlice<VariantIdx, VariantLayout<FieldIdx>>,
192    variants: &IndexSlice<VariantIdx, IndexVec<FieldIdx, F>>,
193    size: Size,
194    align: Align,
195    tag: Scalar,
196) -> BackendRepr
197where
198    Cx: HasDataLayout,
199    FieldIdx: Idx,
200    VariantIdx: Idx,
201    F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,
202{
203    if tag.size(calculator.cx.data_layout()) == size {
204        // Make sure we only use scalar layout when the enum is entirely its
205        // own tag (i.e. it has no padding nor any non-ZST variant fields).
206        BackendRepr::Scalar(tag)
207    } else if let Some(repr) =
208        scalar_pair_repr(calculator, layout_variants, variants, size, align, tag)
209    {
210        repr
211    } else {
212        BackendRepr::Memory { sized: true }
213    }
214}
215
216fn sanity_check_ity(dl: &TargetDataLayout, repr: &ReprOptions, min_ity: Integer) {
217    let typeck_ity = Integer::from_attr(dl, repr.discr_type());
218    if typeck_ity < min_ity {
219        // It is a bug if Layout decided on a greater discriminant size than typeck for
220        // some reason at this point (based on values discriminant can take on). Mostly
221        // because this discriminant will be loaded, and then stored into variable of
222        // type calculated by typeck. Consider such case (a bug): typeck decided on
223        // byte-sized discriminant, but layout thinks we need a 16-bit to store all
224        // discriminant values. That would be a bug, because then, in codegen, in order
225        // to store this 16-bit discriminant into 8-bit sized temporary some of the
226        // space necessary to represent would have to be discarded (or layout is wrong
227        // on thinking it needs 16 bits)
228        {
    ::core::panicking::panic_fmt(format_args!("layout decided on a larger discriminant type ({0:?}) than typeck ({1:?})",
            min_ity, typeck_ity));
};panic!(
229            "layout decided on a larger discriminant type ({min_ity:?}) than typeck ({typeck_ity:?})"
230        );
231    }
232}
233
234fn calculate_tagged_min_ity(
235    repr: &ReprOptions,
236    necessary_discriminants: &[u128],
237    discr_size: Size,
238    discr_range_of_repr: impl Fn(RangeFrom<i128>, RangeToInclusive<u128>) -> (Integer, bool),
239) -> (Integer, bool) {
240    // When picking the integer to use, we respect how the discriminants were written
241    // in the original rust code, rather than looking only at the bit pattern.
242    let (min_negative, max_positive): (i128, u128) = if repr.discr_type().is_signed() {
243        necessary_discriminants.iter().copied().map(|val| discr_size.sign_extend(val)).fold(
244            (0_i128, 0_u128),
245            |(min, max), val| {
246                if let Ok(val) = u128::try_from(val) {
247                    (min, max.max(val))
248                } else {
249                    (min.min(val), max)
250                }
251            },
252        )
253    } else {
254        // We might have no inhabited variants, so pretend there's at least one.
255        (0, necessary_discriminants.iter().copied().max().unwrap_or(0))
256    };
257    {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event /rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout/enum_layout.rs:257",
                        "rustc_abi::layout::enum_layout", ::tracing::Level::TRACE,
                        ::tracing_core::__macro_support::Option::Some("/rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout/enum_layout.rs"),
                        ::tracing_core::__macro_support::Option::Some(257u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_abi::layout::enum_layout"),
                        ::tracing_core::field::FieldSet::new(&[{
                                            const NAME:
                                                ::tracing::__macro_support::FieldName<{
                                                    ::tracing::__macro_support::FieldName::len("min_negative")
                                                }> =
                                                ::tracing::__macro_support::FieldName::new("min_negative");
                                            NAME.as_str()
                                        },
                                        {
                                            const NAME:
                                                ::tracing::__macro_support::FieldName<{
                                                    ::tracing::__macro_support::FieldName::len("max_positive")
                                                }> =
                                                ::tracing::__macro_support::FieldName::new("max_positive");
                                            NAME.as_str()
                                        }], ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::TRACE <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::TRACE <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                __CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&::tracing::field::debug(&min_negative)
                                            as &dyn ::tracing::field::Value)),
                                (::tracing::__macro_support::Option::Some(&::tracing::field::debug(&max_positive)
                                            as &dyn ::tracing::field::Value))])
            });
    } else { ; }
};trace!(?min_negative, ?max_positive);
258
259    // Integer::discr_range_of_repr(tcx, ty, &repr, min, max);
260    discr_range_of_repr(RangeFrom { start: min_negative }, RangeToInclusive { last: max_positive })
261}
262
263fn finalize_tag_and_patch_variants<FieldIdx, VariantIdx>(
264    dl: &TargetDataLayout,
265    repr: &ReprOptions,
266    min_ity: Integer,
267    start_align: Align,
268    signed: bool,
269    necessary_discriminants: Vec<u128>,
270    layout_variants: &mut IndexSlice<VariantIdx, VariantLayout<FieldIdx>>,
271) -> Scalar
272where
273    FieldIdx: Idx,
274    VariantIdx: Idx,
275{
276    sanity_check_ity(dl, repr, min_ity);
277    // It is fine to make discr type however large (as an optimisation)
278    // after this point – we’ll just truncate the value we load in codegen.
279
280    // Check to see if we should use a different type for the
281    // discriminant. We can safely use a type with the same size
282    // as the alignment of the first field of each variant.
283    // We increase the size of the discriminant to avoid LLVM copying
284    // padding when it doesn't need to. This normally causes unaligned
285    // load/stores and excessive memcpy/memset operations. By using a
286    // bigger integer size, LLVM can be sure about its contents and
287    // won't be so conservative.
288
289    // Use the initial field alignment
290    let ity = if repr.c() || repr.int.is_some() {
291        min_ity
292    } else {
293        Integer::for_align(dl, start_align)
294            .unwrap_or(min_ity)
295            // If the alignment is not larger than the chosen discriminant size,
296            // don't use the alignment as the final size.
297            .max(min_ity)
298    };
299
300    // If the alignment is not larger than the chosen discriminant size,
301    // don't use the alignment as the final size.
302    if ity > min_ity {
303        // Patch up the variants' first few fields.
304        let old_ity_size = min_ity.size();
305        let new_ity_size = ity.size();
306        for variant in layout_variants {
307            for i in &mut variant.field_offsets {
308                if *i <= old_ity_size {
309                    {
    match (&*i, &old_ity_size) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val, ::core::option::Option::None);
            }
        }
    }
};assert_eq!(*i, old_ity_size);
310                    *i = new_ity_size;
311                }
312            }
313            // We might be making the struct larger.
314            if variant.size <= old_ity_size {
315                variant.size = new_ity_size;
316            }
317        }
318    }
319
320    let tag_valid_range = {
321        let tag_size = ity.size();
322        let tags = necessary_discriminants.into_iter().map(|d| tag_size.truncate(d));
323        WrappingRange::smallest_range_containing(tags, tag_size)
324            // We might have no inhabited variants, so pretend there's at least one.
325            .unwrap_or(WrappingRange { start: 0, end: 0 })
326    };
327    Scalar::Initialized { value: Primitive::Int(ity, signed), valid_range: tag_valid_range }
328}
329
330fn variant_is_inhabited<'a, VariantIdx, FieldIdx, F>(variant: &IndexSlice<FieldIdx, F>) -> bool
331where
332    FieldIdx: Idx,
333    VariantIdx: Idx,
334    F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,
335{
336    variant.iter().all(|f| !f.is_uninhabited())
337}
338
339fn calculate_tagged_layout<'a, Cx: HasDataLayout, FieldIdx, VariantIdx, F>(
340    calculator: &LayoutCalculator<Cx>,
341    repr: &ReprOptions,
342    variants: &IndexSlice<VariantIdx, IndexVec<FieldIdx, F>>,
343    discr_range_of_repr: impl Fn(RangeFrom<i128>, RangeToInclusive<u128>) -> (Integer, bool),
344    discriminants: impl Iterator<Item = (VariantIdx, u128)>,
345) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
346where
347    FieldIdx: Idx,
348    VariantIdx: Idx,
349    F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,
350{
351    let dl = calculator.cx.data_layout();
352    let discr_size = Integer::from_attr(dl, repr.discr_type()).size();
353
354    let necessary_discriminants: Vec<u128> = discriminants
355        .filter(|&(i, _)| repr.c() || variant_is_inhabited(&variants[i]))
356        .map(|(_, val)| val)
357        .collect();
358
359    let (min_ity, signed) =
360        calculate_tagged_min_ity(repr, &necessary_discriminants, discr_size, discr_range_of_repr);
361
362    let mut align = dl.aggregate_align;
363    let mut max_repr_align = repr.align;
364    let mut unadjusted_abi_align = align;
365    let mut combined_seed = repr.field_shuffle_seed;
366
367    let mut size = Size::ZERO;
368
369    // We're interested in the smallest alignment, so start large.
370    let mut start_align = Align::from_bytes(256).unwrap();
371    {
    match (&Integer::for_align(dl, start_align), &None) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val, ::core::option::Option::None);
            }
        }
    }
};assert_eq!(Integer::for_align(dl, start_align), None);
372
373    // repr(C) on an enum tells us to make a (tag, union) layout,
374    // so we need to grow the prefix alignment to be at least
375    // the alignment of the union. (This value is used both for
376    // determining the alignment of the overall enum, and the
377    // determining the alignment of the payload after the tag.)
378    let prefix_align = if repr.c() {
379        variants
380            .iter()
381            .flatten()
382            .fold(min_ity.align(dl), |prefix_align, field| prefix_align.max(field.align))
383    } else {
384        min_ity.align(dl)
385    };
386
387    // Create the set of structs that represent each variant.
388    let mut layout_variants = variants
389        .iter()
390        .map(|field_layouts| {
391            let st = calculator.layout_of_univariant(
392                field_layouts,
393                repr,
394                StructKind::Prefixed(min_ity.size(), prefix_align),
395            )?;
396            // Find the first field we can't move later
397            // to make room for a larger discriminant.
398            for field_idx in st.fields.index_by_increasing_offset() {
399                let field = &field_layouts[FieldIdx::new(field_idx)];
400                if !field.is_1zst() {
401                    start_align = start_align.min(field.align);
402                    break;
403                }
404            }
405            size = cmp::max(size, st.size);
406            align = align.max(st.align);
407            max_repr_align = max_repr_align.max(st.max_repr_align);
408            unadjusted_abi_align = unadjusted_abi_align.max(st.unadjusted_abi_align);
409            combined_seed = combined_seed.wrapping_add(st.randomization_seed);
410            Ok(VariantLayout::from_layout(st))
411        })
412        .collect::<Result<IndexVec<VariantIdx, _>, _>>()?;
413
414    // Align the maximum variant size to the largest alignment.
415    let size = size.align_to(align);
416
417    // FIXME(oli-obk): deduplicate and harden these checks
418    if size.bytes() >= dl.obj_size_bound() {
419        return Err(LayoutCalculatorError::SizeOverflow);
420    }
421
422    let tag = finalize_tag_and_patch_variants(
423        dl,
424        repr,
425        min_ity,
426        start_align,
427        signed,
428        necessary_discriminants,
429        &mut layout_variants,
430    );
431    let abi = calculate_tagged_abi(calculator, &layout_variants, variants, size, align, tag);
432
433    // If we pick a "clever" (by-value) ABI, we might have to adjust the ABI of the
434    // variants to ensure they are consistent. This is because a downcast is
435    // semantically a NOP, and thus should not affect layout.
436    if #[allow(non_exhaustive_omitted_patterns)] match abi {
    BackendRepr::Scalar(..) | BackendRepr::ScalarPair { .. } => true,
    _ => false,
}matches!(abi, BackendRepr::Scalar(..) | BackendRepr::ScalarPair { .. }) {
437        for variant in &mut layout_variants {
438            // We only do this for variants with fields; the others are not accessed anyway.
439            // Also do not overwrite any already existing "clever" ABIs.
440            if #[allow(non_exhaustive_omitted_patterns)] match variant.backend_repr {
    BackendRepr::Memory { .. } if variant.has_fields() => true,
    _ => false,
}matches!(variant.backend_repr, BackendRepr::Memory { .. } if variant.has_fields()) {
441                variant.backend_repr = abi;
442                // Also need to bump up the size, so that the entire value fits in here.
443                variant.size = cmp::max(variant.size, size);
444            }
445        }
446    }
447
448    let largest_niche = Niche::from_scalar(dl, Size::ZERO, tag);
449
450    Ok(LayoutData {
451        uninhabited: layout_variants.iter().all(|v| v.is_uninhabited()),
452        variants: Variants::Multiple {
453            tag,
454            tag_encoding: TagEncoding::Direct,
455            tag_field: FieldIdx::new(0),
456            variants: layout_variants,
457        },
458        fields: FieldsShape::Arbitrary {
459            offsets: [Size::ZERO].into(),
460            in_memory_order: [FieldIdx::new(0)].into(),
461        },
462        largest_niche,
463        backend_repr: abi,
464        align,
465        size,
466        max_repr_align,
467        unadjusted_abi_align,
468        repr_c: repr.c(),
469        randomization_seed: combined_seed,
470    })
471}
472
473struct VariantLayoutInfo {
474    align_abi: Align,
475}
476
477/// We are trying to add the tag at `niche_offset` with `niche_size` size in variants
478/// that are not `largest_variant_index`. The layout in these variants must be changed
479/// such that the data stored does not overlap with the tag.
480fn try_fit_variants_around_niche_tag<VariantIdx: Idx, FieldIdx: Idx>(
481    mut variant_layouts: IndexVec<VariantIdx, VariantLayout<FieldIdx>>,
482    variants_info: &IndexSlice<VariantIdx, VariantLayoutInfo>,
483    largest_variant_index: VariantIdx,
484    niche_offset: Size,
485    niche_size: Size,
486    size: Size,
487) -> Option<IndexVec<VariantIdx, VariantLayout<FieldIdx>>> {
488    for (i, layout) in variant_layouts.iter_enumerated_mut() {
489        if i == largest_variant_index {
490            continue;
491        }
492
493        layout.largest_niche = None;
494
495        if layout.size <= niche_offset {
496            // This variant will fit before the niche.
497            continue;
498        }
499
500        // Determine if it'll fit after the niche.
501        let this_align = variants_info[i].align_abi;
502        let this_offset = (niche_offset + niche_size).align_to(this_align);
503
504        if this_offset + layout.size > size {
505            return None;
506        }
507
508        // It'll fit, but we need to make some adjustments.
509        for offset in layout.field_offsets.iter_mut() {
510            *offset += this_offset;
511        }
512
513        // It can't be a Scalar or ScalarPair because the offset isn't 0.
514        if !layout.is_uninhabited() {
515            layout.backend_repr = BackendRepr::Memory { sized: true };
516        }
517        layout.size += this_offset;
518    }
519    Some(variant_layouts)
520}
521
522fn calculate_niche_abi<FieldIdx: Idx, VariantIdx: Idx>(
523    variant_layouts: &IndexSlice<VariantIdx, VariantLayout<FieldIdx>>,
524    variants_info: &IndexSlice<VariantIdx, VariantLayoutInfo>,
525    niche_scalar: Scalar,
526    largest_variant_index: VariantIdx,
527    size: Size,
528    align: Align,
529    niche_offset: Size,
530) -> BackendRepr {
531    let others_zst = variant_layouts
532        .iter_enumerated()
533        .all(|(i, layout)| i == largest_variant_index || layout.size == Size::ZERO);
534    let same_size = size == variant_layouts[largest_variant_index].size;
535    let same_align = align == variants_info[largest_variant_index].align_abi;
536
537    if same_size && same_align && others_zst {
538        match variant_layouts[largest_variant_index].backend_repr {
539            // When the total alignment and size match, we can use the
540            // same ABI as the scalar variant with the reserved niche.
541            BackendRepr::Scalar(_) => BackendRepr::Scalar(niche_scalar),
542            BackendRepr::ScalarPair { a: first, b: second, b_offset } => {
543                // Only the niche is guaranteed to be initialised,
544                // so use union layouts for the other primitive.
545                if niche_offset == Size::ZERO {
546                    BackendRepr::ScalarPair { a: niche_scalar, b: second.to_union(), b_offset }
547                } else {
548                    BackendRepr::ScalarPair { a: first.to_union(), b: niche_scalar, b_offset }
549                }
550            }
551            _ => BackendRepr::Memory { sized: true },
552        }
553    } else {
554        BackendRepr::Memory { sized: true }
555    }
556}
557
558fn calculate_niche_filling_layout<'a, Cx: HasDataLayout, FieldIdx, VariantIdx, F>(
559    calculator: &LayoutCalculator<Cx>,
560    repr: &ReprOptions,
561    variants: &IndexSlice<VariantIdx, IndexVec<FieldIdx, F>>,
562) -> Option<LayoutData<FieldIdx, VariantIdx>>
563where
564    FieldIdx: Idx,
565    VariantIdx: Idx,
566    F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,
567{
568    let dl = calculator.cx.data_layout();
569    if repr.inhibit_enum_layout_opt() {
570        return None;
571    }
572
573    if variants.len() < 2 {
574        return None;
575    }
576
577    let mut align = dl.aggregate_align;
578    let mut max_repr_align = repr.align;
579    let mut unadjusted_abi_align = align;
580    let mut combined_seed = repr.field_shuffle_seed;
581
582    let mut variants_info = IndexVec::<VariantIdx, _>::with_capacity(variants.len());
583    let variant_layouts = variants
584        .iter()
585        .map(|v| {
586            let st = calculator.layout_of_univariant(v, repr, StructKind::AlwaysSized).ok()?;
587
588            variants_info.push(VariantLayoutInfo { align_abi: st.align });
589
590            align = align.max(st.align);
591            max_repr_align = max_repr_align.max(st.max_repr_align);
592            unadjusted_abi_align = unadjusted_abi_align.max(st.unadjusted_abi_align);
593            combined_seed = combined_seed.wrapping_add(st.randomization_seed);
594
595            Some(VariantLayout::from_layout(st))
596        })
597        .collect::<Option<IndexVec<VariantIdx, _>>>()?;
598
599    let largest_variant_index = variant_layouts
600        .iter_enumerated()
601        .max_by_key(|(_i, layout)| layout.size.bytes())
602        .map(|(i, _layout)| i)?;
603
604    let needs_disc =
605        |index: VariantIdx| index != largest_variant_index && !absent(&variants[index]);
606    let niche_variants = RangeInclusive {
607        start: variants.indices().find(|v| needs_disc(*v)).unwrap(),
608        last: variants.indices().rev().find(|v| needs_disc(*v)).unwrap(),
609    };
610
611    let count = (niche_variants.last.index() as u128 - niche_variants.start.index() as u128) + 1;
612
613    // Use the largest niche in the largest variant.
614    let niche = variant_layouts[largest_variant_index].largest_niche?;
615    let (niche_start, niche_scalar) = niche.reserve(dl, count)?;
616    let size = variant_layouts[largest_variant_index].size.align_to(align);
617
618    let variant_layouts = try_fit_variants_around_niche_tag(
619        variant_layouts,
620        &variants_info,
621        largest_variant_index,
622        niche.offset,
623        niche.value.size(dl),
624        size,
625    )?;
626
627    let abi = calculate_niche_abi(
628        &variant_layouts,
629        &variants_info,
630        niche_scalar,
631        largest_variant_index,
632        size,
633        align,
634        niche.offset,
635    );
636
637    let layout = LayoutData {
638        uninhabited: variant_layouts.iter().all(|v| v.is_uninhabited()),
639        variants: Variants::Multiple {
640            tag: niche_scalar,
641            tag_encoding: TagEncoding::Niche {
642                untagged_variant: largest_variant_index,
643                niche_variants,
644                niche_start,
645            },
646            tag_field: FieldIdx::new(0),
647            variants: variant_layouts,
648        },
649        fields: FieldsShape::Arbitrary {
650            offsets: [niche.offset].into(),
651            in_memory_order: [FieldIdx::new(0)].into(),
652        },
653        backend_repr: abi,
654        largest_niche: Niche::from_scalar(dl, niche.offset, niche_scalar),
655        size,
656        align,
657        max_repr_align,
658        unadjusted_abi_align,
659        repr_c: repr.c(),
660        randomization_seed: combined_seed,
661    };
662
663    Some(layout)
664}