Skip to main content

rustc_abi/layout/
coroutine.rs

1//! Coroutine layout logic.
2//!
3//! When laying out coroutines, we divide our saved local fields into two
4//! categories: overlap-eligible and overlap-ineligible.
5//!
6//! Those fields which are ineligible for overlap go in a "prefix" at the
7//! beginning of the layout, and always have space reserved for them.
8//!
9//! Overlap-eligible fields are only assigned to one variant, so we lay
10//! those fields out for each variant and put them right after the
11//! prefix.
12//!
13//! Finally, in the layout details, we point to the fields from the
14//! variants they are assigned to. It is possible for some fields to be
15//! included in multiple variants. No field ever "moves around" in the
16//! layout; its offset is always the same.
17//!
18//! Also included in the layout are the upvars and the discriminant.
19//! These are included as fields on the "outer" layout; they are not part
20//! of any variant.
21
22use std::iter;
23
24use rustc_index::bit_set::{BitMatrix, DenseBitSet};
25use rustc_index::{Idx, IndexSlice, IndexVec};
26use tracing::{debug, trace};
27
28use crate::{
29    BackendRepr, FieldsShape, HasDataLayout, Integer, LayoutData, Primitive, ReprOptions, Scalar,
30    StructKind, TagEncoding, VariantLayout, Variants, WrappingRange,
31};
32
33/// This option controls how coroutine saved locals are packed
34/// into the coroutine state data
35#[derive(#[automatically_derived]#[doc(hidden)]unsafe impl ::core::clone::TrivialClone
    for PackCoroutineLayout {
}
#[automatically_derived]impl ::core::clone::Clone for PackCoroutineLayout {
    #[inline]
    #[rustc_trivial_field_reads]
    fn clone(&self) -> Self { *self }
}Clone, #[automatically_derived]impl ::core::marker::Copy for PackCoroutineLayout { }Copy, #[automatically_derived]impl ::core::fmt::Debug for PackCoroutineLayout {
    #[inline]
    #[rustc_trivial_field_reads]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        ::core::fmt::Formatter::write_str(f,
            match self {
                PackCoroutineLayout::Classic => "Classic",
                PackCoroutineLayout::CapturesOnly => "CapturesOnly",
            })
    }
}Debug, #[automatically_derived]impl ::core::default::Default for PackCoroutineLayout
    {
    #[inline]
    fn default() -> Self { Self::Classic }
}Default, #[automatically_derived]impl ::core::marker::StructuralPartialEq for
    PackCoroutineLayout {
}
#[automatically_derived]impl ::core::cmp::PartialEq for PackCoroutineLayout {
    #[inline]
    fn eq(&self, other: &Self) -> bool {
        ::core::intrinsics::discriminant_value(self) ==
            ::core::intrinsics::discriminant_value(other)
    }
}PartialEq, #[automatically_derived]impl ::core::cmp::Eq for PackCoroutineLayout { }Eq, #[automatically_derived]impl ::core::hash::Hash for PackCoroutineLayout {
    #[inline]
    fn hash<__H: ::core::hash::Hasher>(&self, state: &mut __H) {
        ::core::hash::Hash::hash(&::core::intrinsics::discriminant_value(self),
            state)
    }
}Hash)]
36#[cfg_attr(
37    feature = "nightly",
38    derive(
39        const _: () =
    {
        impl<__E: ::rustc_serialize::Encoder>
            ::rustc_serialize::Encodable<__E> for PackCoroutineLayout {
            fn encode(&self, __encoder: &mut __E) {
                let disc =
                    match *self {
                        PackCoroutineLayout::Classic => { 0usize }
                        PackCoroutineLayout::CapturesOnly => { 1usize }
                    };
                ::rustc_serialize::Encoder::emit_u8(__encoder, disc as u8);
            }
        }
    };rustc_macros::Encodable_NoContext,
40        const _: () =
    {
        impl<__D: ::rustc_serialize::Decoder>
            ::rustc_serialize::Decodable<__D> for PackCoroutineLayout {
            fn decode(__decoder: &mut __D) -> Self {
                match ::rustc_serialize::Decoder::read_u8(__decoder) as usize
                    {
                    0usize => { PackCoroutineLayout::Classic }
                    1usize => { PackCoroutineLayout::CapturesOnly }
                    n => {
                        ::core::panicking::panic_fmt(format_args!("invalid enum variant tag while decoding `PackCoroutineLayout`, expected 0..2, actual {0}",
                                n));
                    }
                }
            }
        }
    };rustc_macros::Decodable_NoContext,
41        const _: () =
    {
        impl ::rustc_data_structures::stable_hash::StableHash for
            PackCoroutineLayout {
            #[inline]
            fn stable_hash<__Hcx: ::rustc_data_structures::stable_hash::StableHashCtxt>(&self,
                __hcx: &mut __Hcx,
                __hasher:
                    &mut ::rustc_data_structures::stable_hash::StableHasher) {
                match *self {
                    PackCoroutineLayout::Classic =>
                        0u8.stable_hash(__hcx, __hasher),
                    PackCoroutineLayout::CapturesOnly =>
                        1u8.stable_hash(__hcx, __hasher),
                }
                match *self {
                    PackCoroutineLayout::Classic => {}
                    PackCoroutineLayout::CapturesOnly => {}
                }
            }
        }
    };rustc_macros::StableHash,
42    )
43)]
44pub enum PackCoroutineLayout {
45    /// The classic layout where captures are always promoted to coroutine state prefix
46    #[default]
47    Classic,
48    /// Captures are first saved into the `UNRESUMED` state and promoted
49    /// when they are used across more than one suspension
50    CapturesOnly,
51}
52
53/// Overlap eligibility and variant assignment for each CoroutineSavedLocal.
54#[derive(#[automatically_derived]impl<VariantIdx: ::core::clone::Clone,
    FieldIdx: ::core::clone::Clone> ::core::clone::Clone for
    SavedLocalEligibility<VariantIdx, FieldIdx> {
    #[inline]
    #[rustc_trivial_field_reads]
    fn clone(&self) -> Self {
        match self {
            Self::Unassigned => SavedLocalEligibility::Unassigned,
            Self::Assigned(__self_0) =>
                SavedLocalEligibility::Assigned(::core::clone::Clone::clone(__self_0)),
            Self::Ineligible(__self_0) =>
                SavedLocalEligibility::Ineligible(::core::clone::Clone::clone(__self_0)),
        }
    }
}Clone, #[automatically_derived]impl<VariantIdx: ::core::fmt::Debug,
    FieldIdx: ::core::fmt::Debug> ::core::fmt::Debug for
    SavedLocalEligibility<VariantIdx, FieldIdx> {
    #[inline]
    #[rustc_trivial_field_reads]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        match self {
            Self::Unassigned =>
                ::core::fmt::Formatter::write_str(f, "Unassigned"),
            Self::Assigned(__self_0) =>
                ::core::fmt::Formatter::debug_struct_field1_finish(f,
                    "Assigned", &__self_0),
            Self::Ineligible(__self_0) =>
                ::core::fmt::Formatter::debug_struct_field1_finish(f,
                    "Ineligible", &__self_0),
        }
    }
}Debug, #[automatically_derived]impl<VariantIdx: ::core::cmp::PartialEq,
    FieldIdx: ::core::cmp::PartialEq> ::core::marker::StructuralPartialEq for
    SavedLocalEligibility<VariantIdx, FieldIdx> {
}
#[automatically_derived]impl<VariantIdx: ::core::cmp::PartialEq,
    FieldIdx: ::core::cmp::PartialEq> ::core::cmp::PartialEq for
    SavedLocalEligibility<VariantIdx, FieldIdx> {
    #[inline]
    fn eq(&self, other: &Self) -> bool {
        ::core::intrinsics::discriminant_value(self) ==
                ::core::intrinsics::discriminant_value(other) &&
            match (self, other) {
                (Self::Assigned(__self_0), Self::Assigned(__arg1_0)) =>
                    __self_0 == __arg1_0,
                (Self::Ineligible(__self_0), Self::Ineligible(__arg1_0)) =>
                    __self_0 == __arg1_0,
                _ => true,
            }
    }
}PartialEq)]
55enum SavedLocalEligibility<VariantIdx, FieldIdx> {
56    Unassigned,
57    Assigned(VariantIdx),
58    Ineligible(Option<FieldIdx>),
59}
60
61/// Compute the eligibility and assignment of each local.
62fn coroutine_saved_local_eligibility<VariantIdx: Idx, FieldIdx: Idx, LocalIdx: Idx>(
63    nb_locals: usize,
64    variant_fields: &IndexSlice<VariantIdx, IndexVec<FieldIdx, LocalIdx>>,
65    storage_conflicts: &BitMatrix<LocalIdx, LocalIdx>,
66) -> (DenseBitSet<LocalIdx>, IndexVec<LocalIdx, SavedLocalEligibility<VariantIdx, FieldIdx>>) {
67    use SavedLocalEligibility::*;
68
69    let mut assignments: IndexVec<LocalIdx, _> = IndexVec::from_elem_n(Unassigned, nb_locals);
70
71    // The saved locals not eligible for overlap. These will get
72    // "promoted" to the prefix of our coroutine.
73    let mut ineligible_locals = DenseBitSet::new_empty(nb_locals);
74
75    // Figure out which of our saved locals are fields in only
76    // one variant. The rest are deemed ineligible for overlap.
77    for (variant_index, fields) in variant_fields.iter_enumerated() {
78        for local in fields {
79            match assignments[*local] {
80                Unassigned => {
81                    assignments[*local] = Assigned(variant_index);
82                }
83                Assigned(idx) => {
84                    // We've already seen this local at another suspension
85                    // point, so it is no longer a candidate.
86                    {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event /rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout/coroutine.rs:86",
                        "rustc_abi::layout::coroutine", ::tracing::Level::TRACE,
                        ::tracing_core::__macro_support::Option::Some("/rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout/coroutine.rs"),
                        ::tracing_core::__macro_support::Option::Some(86u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_abi::layout::coroutine"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::TRACE <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::TRACE <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                __CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("removing local {0:?} in >1 variant ({1:?}, {2:?})",
                                                    local, variant_index, idx) as
                                            &dyn ::tracing::field::Value))])
            });
    } else { ; }
};trace!(
87                        "removing local {:?} in >1 variant ({:?}, {:?})",
88                        local, variant_index, idx
89                    );
90                    ineligible_locals.insert(*local);
91                    assignments[*local] = Ineligible(None);
92                }
93                Ineligible(_) => {}
94            }
95        }
96    }
97
98    // Next, check every pair of eligible locals to see if they
99    // conflict.
100    for local_a in storage_conflicts.rows() {
101        let conflicts_a = storage_conflicts.count(local_a);
102        if ineligible_locals.contains(local_a) {
103            continue;
104        }
105
106        for local_b in storage_conflicts.iter(local_a) {
107            // local_a and local_b are storage live at the same time, therefore they
108            // cannot overlap in the coroutine layout. The only way to guarantee
109            // this is if they are in the same variant, or one is ineligible
110            // (which means it is stored in every variant).
111            if ineligible_locals.contains(local_b) || assignments[local_a] == assignments[local_b] {
112                continue;
113            }
114
115            // If they conflict, we will choose one to make ineligible.
116            // This is not always optimal; it's just a greedy heuristic that
117            // seems to produce good results most of the time.
118            let conflicts_b = storage_conflicts.count(local_b);
119            let (remove, other) =
120                if conflicts_a > conflicts_b { (local_a, local_b) } else { (local_b, local_a) };
121            ineligible_locals.insert(remove);
122            assignments[remove] = Ineligible(None);
123            {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event /rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout/coroutine.rs:123",
                        "rustc_abi::layout::coroutine", ::tracing::Level::TRACE,
                        ::tracing_core::__macro_support::Option::Some("/rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout/coroutine.rs"),
                        ::tracing_core::__macro_support::Option::Some(123u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_abi::layout::coroutine"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::TRACE <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::TRACE <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                __CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("removing local {0:?} due to conflict with {1:?}",
                                                    remove, other) as &dyn ::tracing::field::Value))])
            });
    } else { ; }
};trace!("removing local {:?} due to conflict with {:?}", remove, other);
124        }
125    }
126
127    // Count the number of variants in use. If only one of them, then it is
128    // impossible to overlap any locals in our layout. In this case it's
129    // always better to make the remaining locals ineligible, so we can
130    // lay them out with the other locals in the prefix and eliminate
131    // unnecessary padding bytes.
132    {
133        let mut used_variants = DenseBitSet::new_empty(variant_fields.len());
134        for assignment in &assignments {
135            if let Assigned(idx) = assignment {
136                used_variants.insert(*idx);
137            }
138        }
139        if used_variants.count() < 2 {
140            for assignment in assignments.iter_mut() {
141                *assignment = Ineligible(None);
142            }
143            ineligible_locals.insert_all();
144        }
145    }
146
147    // Write down the order of our locals that will be promoted to the prefix.
148    {
149        for (idx, local) in ineligible_locals.iter().enumerate() {
150            assignments[local] = Ineligible(Some(FieldIdx::new(idx)));
151        }
152    }
153    {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event /rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout/coroutine.rs:153",
                        "rustc_abi::layout::coroutine", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("/rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout/coroutine.rs"),
                        ::tracing_core::__macro_support::Option::Some(153u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_abi::layout::coroutine"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                __CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("coroutine saved local assignments: {0:?}",
                                                    assignments) as &dyn ::tracing::field::Value))])
            });
    } else { ; }
};debug!("coroutine saved local assignments: {:?}", assignments);
154
155    (ineligible_locals, assignments)
156}
157
158/// Compute the full coroutine layout.
159pub(super) fn layout<'a, F, VariantIdx, FieldIdx, LocalIdx>(
160    calc: &super::LayoutCalculator<impl HasDataLayout>,
161    local_layouts: &IndexSlice<LocalIdx, F>,
162    mut prefix_layouts: IndexVec<FieldIdx, F>,
163    variant_fields: &IndexSlice<VariantIdx, IndexVec<FieldIdx, LocalIdx>>,
164    storage_conflicts: &BitMatrix<LocalIdx, LocalIdx>,
165    tag_to_layout: impl Fn(Scalar) -> F,
166) -> super::LayoutCalculatorResult<FieldIdx, VariantIdx, F>
167where
168    F: core::ops::Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + core::fmt::Debug + Copy,
169    VariantIdx: Idx,
170    FieldIdx: Idx,
171    LocalIdx: Idx,
172{
173    use SavedLocalEligibility::*;
174
175    let (ineligible_locals, assignments) =
176        coroutine_saved_local_eligibility(local_layouts.len(), variant_fields, storage_conflicts);
177
178    // Build a prefix layout, including "promoting" all ineligible
179    // locals as part of the prefix. We compute the layout of all of
180    // these fields at once to get optimal packing.
181    let tag_index = prefix_layouts.next_index();
182
183    // `variant_fields` already accounts for the reserved variants, so no need to add them.
184    let max_discr = (variant_fields.len() - 1) as u128;
185    let discr_int = Integer::fit_unsigned(max_discr);
186    let tag = Scalar::Initialized {
187        value: Primitive::Int(discr_int, /* signed = */ false),
188        valid_range: WrappingRange { start: 0, end: max_discr },
189    };
190
191    let promoted_layouts = ineligible_locals.iter().map(|local| local_layouts[local]);
192    prefix_layouts.push(tag_to_layout(tag));
193    prefix_layouts.extend(promoted_layouts);
194    let prefix = calc.layout_of_univariant(
195        &prefix_layouts,
196        &ReprOptions::default(),
197        StructKind::AlwaysSized,
198    )?;
199
200    let (prefix_size, prefix_align) = (prefix.size, prefix.align);
201
202    // Split the prefix layout into the "outer" fields (upvars and
203    // discriminant) and the "promoted" fields. Promoted fields will
204    // get included in each variant that requested them in
205    // CoroutineLayout.
206    {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event /rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout/coroutine.rs:206",
                        "rustc_abi::layout::coroutine", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("/rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout/coroutine.rs"),
                        ::tracing_core::__macro_support::Option::Some(206u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_abi::layout::coroutine"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                __CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("prefix = {0:#?}",
                                                    prefix) as &dyn ::tracing::field::Value))])
            });
    } else { ; }
};debug!("prefix = {:#?}", prefix);
207    let (outer_fields, promoted_offsets, promoted_memory_index) = match prefix.fields {
208        FieldsShape::Arbitrary { mut offsets, in_memory_order } => {
209            // "a" (`0..b_start`) and "b" (`b_start..`) correspond to
210            // "outer" and "promoted" fields respectively.
211            let b_start = tag_index.plus(1);
212            let offsets_b = IndexVec::from_raw(offsets.raw.split_off(b_start.index()));
213            let offsets_a = offsets;
214
215            // Disentangle the "a" and "b" components of `in_memory_order`
216            // by preserving the order but keeping only one disjoint "half" each.
217            // FIXME(eddyb) build a better abstraction for permutations, if possible.
218            let mut in_memory_order_a = IndexVec::<u32, FieldIdx>::new();
219            let mut in_memory_order_b = IndexVec::<u32, FieldIdx>::new();
220            for i in in_memory_order {
221                if let Some(j) = i.index().checked_sub(b_start.index()) {
222                    in_memory_order_b.push(FieldIdx::new(j));
223                } else {
224                    in_memory_order_a.push(i);
225                }
226            }
227
228            let outer_fields =
229                FieldsShape::Arbitrary { offsets: offsets_a, in_memory_order: in_memory_order_a };
230            (outer_fields, offsets_b, in_memory_order_b.invert_bijective_mapping())
231        }
232        _ => ::core::panicking::unreachable_default_msg()unreachable!(),
233    };
234
235    let mut size = prefix.size;
236    let mut align = prefix.align;
237    let variants = variant_fields
238        .iter_enumerated()
239        .map(|(index, variant_fields)| {
240            // Only include overlap-eligible fields when we compute our variant layout.
241            let variant_only_tys = variant_fields
242                .iter()
243                .filter(|local| match assignments[**local] {
244                    Unassigned => ::core::panicking::unreachable_default_msg()unreachable!(),
245                    Assigned(v) if v == index => true,
246                    Assigned(_) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("assignment does not match variant")));
}unreachable!("assignment does not match variant"),
247                    Ineligible(_) => false,
248                })
249                .map(|local| local_layouts[*local]);
250
251            let mut variant = calc.layout_of_univariant(
252                &variant_only_tys.collect::<IndexVec<_, _>>(),
253                &ReprOptions::default(),
254                StructKind::Prefixed(prefix_size, prefix_align),
255            )?;
256
257            let FieldsShape::Arbitrary { offsets, in_memory_order } = variant.fields else {
258                ::core::panicking::unreachable_default_msg();unreachable!();
259            };
260
261            // Now, stitch the promoted and variant-only fields back together in
262            // the order they are mentioned by our CoroutineLayout.
263            // Because we only use some subset (that can differ between variants)
264            // of the promoted fields, we can't just pick those elements of the
265            // `promoted_memory_index` (as we'd end up with gaps).
266            // So instead, we build an "inverse memory_index", as if all of the
267            // promoted fields were being used, but leave the elements not in the
268            // subset as `invalid_field_idx`, which we can filter out later to
269            // obtain a valid (bijective) mapping.
270            let memory_index = in_memory_order.invert_bijective_mapping();
271            let invalid_field_idx = promoted_memory_index.len() + memory_index.len();
272            let mut combined_in_memory_order =
273                IndexVec::from_elem_n(FieldIdx::new(invalid_field_idx), invalid_field_idx);
274
275            let mut offsets_and_memory_index = iter::zip(offsets, memory_index);
276            let combined_offsets = variant_fields
277                .iter_enumerated()
278                .map(|(i, local)| {
279                    let (offset, memory_index) = match assignments[*local] {
280                        Unassigned => ::core::panicking::unreachable_default_msg()unreachable!(),
281                        Assigned(_) => {
282                            let (offset, memory_index) = offsets_and_memory_index.next().unwrap();
283                            (offset, promoted_memory_index.len() as u32 + memory_index)
284                        }
285                        Ineligible(field_idx) => {
286                            let field_idx = field_idx.unwrap();
287                            (promoted_offsets[field_idx], promoted_memory_index[field_idx])
288                        }
289                    };
290                    combined_in_memory_order[memory_index] = i;
291                    offset
292                })
293                .collect();
294
295            // Remove the unused slots to obtain the combined `in_memory_order`
296            // (also see previous comment).
297            combined_in_memory_order.raw.retain(|&i| i.index() != invalid_field_idx);
298
299            variant.fields = FieldsShape::Arbitrary {
300                offsets: combined_offsets,
301                in_memory_order: combined_in_memory_order,
302            };
303
304            size = size.max(variant.size);
305            align = align.max(variant.align);
306            Ok(VariantLayout::from_layout(variant))
307        })
308        .collect::<Result<IndexVec<VariantIdx, _>, _>>()?;
309
310    size = size.align_to(align);
311
312    let uninhabited = prefix.uninhabited || variants.iter().all(|v| v.is_uninhabited());
313    let abi = BackendRepr::Memory { sized: true };
314
315    Ok(LayoutData {
316        variants: Variants::Multiple {
317            tag,
318            tag_encoding: TagEncoding::Direct,
319            tag_field: tag_index,
320            variants,
321        },
322        fields: outer_fields,
323        backend_repr: abi,
324        // Suppress niches inside coroutines. If the niche is inside a field that is aliased (due to
325        // self-referentiality), getting the discriminant can cause aliasing violations.
326        // `UnsafeCell` blocks niches for the same reason, but we don't yet have `UnsafePinned` that
327        // would do the same for us here.
328        // See <https://github.com/rust-lang/rust/issues/63818>, <https://github.com/rust-lang/miri/issues/3780>.
329        // FIXME: Remove when <https://github.com/rust-lang/rust/issues/125735> is implemented and aliased coroutine fields are wrapped in `UnsafePinned`.
330        largest_niche: None,
331        uninhabited,
332        size,
333        align,
334        max_repr_align: None,
335        unadjusted_abi_align: align,
336        repr_c: false,
337        randomization_seed: Default::default(),
338    })
339}