1//! Coroutine layout logic.
2//!
3//! When laying out coroutines, we divide our saved local fields into two
4//! categories: overlap-eligible and overlap-ineligible.
5//!
6//! Those fields which are ineligible for overlap go in a "prefix" at the
7//! beginning of the layout, and always have space reserved for them.
8//!
9//! Overlap-eligible fields are only assigned to one variant, so we lay
10//! those fields out for each variant and put them right after the
11//! prefix.
12//!
13//! Finally, in the layout details, we point to the fields from the
14//! variants they are assigned to. It is possible for some fields to be
15//! included in multiple variants. No field ever "moves around" in the
16//! layout; its offset is always the same.
17//!
18//! Also included in the layout are the upvars and the discriminant.
19//! These are included as fields on the "outer" layout; they are not part
20//! of any variant.
2122use std::iter;
2324use rustc_index::bit_set::{BitMatrix, DenseBitSet};
25use rustc_index::{Idx, IndexSlice, IndexVec};
26use tracing::{debug, trace};
2728use crate::{
29BackendRepr, FieldsShape, HasDataLayout, Integer, LayoutData, Primitive, ReprOptions, Scalar,
30StructKind, TagEncoding, VariantLayout, Variants, WrappingRange,
31};
3233/// This option controls how coroutine saved locals are packed
34/// into the coroutine state data
35#[derive(#[automatically_derived]#[doc(hidden)]unsafe impl ::core::clone::TrivialClone
for PackCoroutineLayout {
}
#[automatically_derived]impl ::core::clone::Clone for PackCoroutineLayout {
#[inline]
#[rustc_trivial_field_reads]
fn clone(&self) -> Self { *self }
}Clone, #[automatically_derived]impl ::core::marker::Copy for PackCoroutineLayout { }Copy, #[automatically_derived]impl ::core::fmt::Debug for PackCoroutineLayout {
#[inline]
#[rustc_trivial_field_reads]
fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
::core::fmt::Formatter::write_str(f,
match self {
PackCoroutineLayout::Classic => "Classic",
PackCoroutineLayout::CapturesOnly => "CapturesOnly",
})
}
}Debug, #[automatically_derived]impl ::core::default::Default for PackCoroutineLayout
{
#[inline]
fn default() -> Self { Self::Classic }
}Default, #[automatically_derived]impl ::core::marker::StructuralPartialEq for
PackCoroutineLayout {
}
#[automatically_derived]impl ::core::cmp::PartialEq for PackCoroutineLayout {
#[inline]
fn eq(&self, other: &Self) -> bool {
::core::intrinsics::discriminant_value(self) ==
::core::intrinsics::discriminant_value(other)
}
}PartialEq, #[automatically_derived]impl ::core::cmp::Eq for PackCoroutineLayout { }Eq, #[automatically_derived]impl ::core::hash::Hash for PackCoroutineLayout {
#[inline]
fn hash<__H: ::core::hash::Hasher>(&self, state: &mut __H) {
::core::hash::Hash::hash(&::core::intrinsics::discriminant_value(self),
state)
}
}Hash)]
36#[cfg_attr(
37 feature = "nightly",
38 derive(
39const _: () =
{
impl<__E: ::rustc_serialize::Encoder>
::rustc_serialize::Encodable<__E> for PackCoroutineLayout {
fn encode(&self, __encoder: &mut __E) {
let disc =
match *self {
PackCoroutineLayout::Classic => { 0usize }
PackCoroutineLayout::CapturesOnly => { 1usize }
};
::rustc_serialize::Encoder::emit_u8(__encoder, disc as u8);
}
}
};rustc_macros::Encodable_NoContext,
40const _: () =
{
impl<__D: ::rustc_serialize::Decoder>
::rustc_serialize::Decodable<__D> for PackCoroutineLayout {
fn decode(__decoder: &mut __D) -> Self {
match ::rustc_serialize::Decoder::read_u8(__decoder) as usize
{
0usize => { PackCoroutineLayout::Classic }
1usize => { PackCoroutineLayout::CapturesOnly }
n => {
::core::panicking::panic_fmt(format_args!("invalid enum variant tag while decoding `PackCoroutineLayout`, expected 0..2, actual {0}",
n));
}
}
}
}
};rustc_macros::Decodable_NoContext,
41const _: () =
{
impl ::rustc_data_structures::stable_hash::StableHash for
PackCoroutineLayout {
#[inline]
fn stable_hash<__Hcx: ::rustc_data_structures::stable_hash::StableHashCtxt>(&self,
__hcx: &mut __Hcx,
__hasher:
&mut ::rustc_data_structures::stable_hash::StableHasher) {
match *self {
PackCoroutineLayout::Classic =>
0u8.stable_hash(__hcx, __hasher),
PackCoroutineLayout::CapturesOnly =>
1u8.stable_hash(__hcx, __hasher),
}
match *self {
PackCoroutineLayout::Classic => {}
PackCoroutineLayout::CapturesOnly => {}
}
}
}
};rustc_macros::StableHash,
42 )
43)]
44pub enum PackCoroutineLayout {
45/// The classic layout where captures are always promoted to coroutine state prefix
46#[default]
47Classic,
48/// Captures are first saved into the `UNRESUMED` state and promoted
49 /// when they are used across more than one suspension
50CapturesOnly,
51}
5253/// Overlap eligibility and variant assignment for each CoroutineSavedLocal.
54#[derive(#[automatically_derived]impl<VariantIdx: ::core::clone::Clone,
FieldIdx: ::core::clone::Clone> ::core::clone::Clone for
SavedLocalEligibility<VariantIdx, FieldIdx> {
#[inline]
#[rustc_trivial_field_reads]
fn clone(&self) -> Self {
match self {
Self::Unassigned => SavedLocalEligibility::Unassigned,
Self::Assigned(__self_0) =>
SavedLocalEligibility::Assigned(::core::clone::Clone::clone(__self_0)),
Self::Ineligible(__self_0) =>
SavedLocalEligibility::Ineligible(::core::clone::Clone::clone(__self_0)),
}
}
}Clone, #[automatically_derived]impl<VariantIdx: ::core::fmt::Debug,
FieldIdx: ::core::fmt::Debug> ::core::fmt::Debug for
SavedLocalEligibility<VariantIdx, FieldIdx> {
#[inline]
#[rustc_trivial_field_reads]
fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
match self {
Self::Unassigned =>
::core::fmt::Formatter::write_str(f, "Unassigned"),
Self::Assigned(__self_0) =>
::core::fmt::Formatter::debug_struct_field1_finish(f,
"Assigned", &__self_0),
Self::Ineligible(__self_0) =>
::core::fmt::Formatter::debug_struct_field1_finish(f,
"Ineligible", &__self_0),
}
}
}Debug, #[automatically_derived]impl<VariantIdx: ::core::cmp::PartialEq,
FieldIdx: ::core::cmp::PartialEq> ::core::marker::StructuralPartialEq for
SavedLocalEligibility<VariantIdx, FieldIdx> {
}
#[automatically_derived]impl<VariantIdx: ::core::cmp::PartialEq,
FieldIdx: ::core::cmp::PartialEq> ::core::cmp::PartialEq for
SavedLocalEligibility<VariantIdx, FieldIdx> {
#[inline]
fn eq(&self, other: &Self) -> bool {
::core::intrinsics::discriminant_value(self) ==
::core::intrinsics::discriminant_value(other) &&
match (self, other) {
(Self::Assigned(__self_0), Self::Assigned(__arg1_0)) =>
__self_0 == __arg1_0,
(Self::Ineligible(__self_0), Self::Ineligible(__arg1_0)) =>
__self_0 == __arg1_0,
_ => true,
}
}
}PartialEq)]
55enum SavedLocalEligibility<VariantIdx, FieldIdx> {
56 Unassigned,
57 Assigned(VariantIdx),
58 Ineligible(Option<FieldIdx>),
59}
6061/// Compute the eligibility and assignment of each local.
62fn coroutine_saved_local_eligibility<VariantIdx: Idx, FieldIdx: Idx, LocalIdx: Idx>(
63 nb_locals: usize,
64 variant_fields: &IndexSlice<VariantIdx, IndexVec<FieldIdx, LocalIdx>>,
65 storage_conflicts: &BitMatrix<LocalIdx, LocalIdx>,
66) -> (DenseBitSet<LocalIdx>, IndexVec<LocalIdx, SavedLocalEligibility<VariantIdx, FieldIdx>>) {
67use SavedLocalEligibility::*;
6869let mut assignments: IndexVec<LocalIdx, _> = IndexVec::from_elem_n(Unassigned, nb_locals);
7071// The saved locals not eligible for overlap. These will get
72 // "promoted" to the prefix of our coroutine.
73let mut ineligible_locals = DenseBitSet::new_empty(nb_locals);
7475// Figure out which of our saved locals are fields in only
76 // one variant. The rest are deemed ineligible for overlap.
77for (variant_index, fields) in variant_fields.iter_enumerated() {
78for local in fields {
79match assignments[*local] {
80 Unassigned => {
81 assignments[*local] = Assigned(variant_index);
82 }
83 Assigned(idx) => {
84// We've already seen this local at another suspension
85 // point, so it is no longer a candidate.
86{
use ::tracing::__macro_support::Callsite as _;
static __CALLSITE: ::tracing::callsite::DefaultCallsite =
{
static META: ::tracing::Metadata<'static> =
{
::tracing_core::metadata::Metadata::new("event /rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout/coroutine.rs:86",
"rustc_abi::layout::coroutine", ::tracing::Level::TRACE,
::tracing_core::__macro_support::Option::Some("/rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout/coroutine.rs"),
::tracing_core::__macro_support::Option::Some(86u32),
::tracing_core::__macro_support::Option::Some("rustc_abi::layout::coroutine"),
::tracing_core::field::FieldSet::new(&["message"],
::tracing_core::callsite::Identifier(&__CALLSITE)),
::tracing::metadata::Kind::EVENT)
};
::tracing::callsite::DefaultCallsite::new(&META)
};
let enabled =
::tracing::Level::TRACE <= ::tracing::level_filters::STATIC_MAX_LEVEL
&&
::tracing::Level::TRACE <=
::tracing::level_filters::LevelFilter::current() &&
{
let interest = __CALLSITE.interest();
!interest.is_never() &&
::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
interest)
};
if enabled {
(|value_set: ::tracing::field::ValueSet|
{
let meta = __CALLSITE.metadata();
::tracing::Event::dispatch(meta, &value_set);
;
})({
#[allow(unused_imports)]
use ::tracing::field::{debug, display, Value};
__CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("removing local {0:?} in >1 variant ({1:?}, {2:?})",
local, variant_index, idx) as
&dyn ::tracing::field::Value))])
});
} else { ; }
};trace!(
87"removing local {:?} in >1 variant ({:?}, {:?})",
88 local, variant_index, idx
89 );
90 ineligible_locals.insert(*local);
91 assignments[*local] = Ineligible(None);
92 }
93 Ineligible(_) => {}
94 }
95 }
96 }
9798// Next, check every pair of eligible locals to see if they
99 // conflict.
100for local_a in storage_conflicts.rows() {
101let conflicts_a = storage_conflicts.count(local_a);
102if ineligible_locals.contains(local_a) {
103continue;
104 }
105106for local_b in storage_conflicts.iter(local_a) {
107// local_a and local_b are storage live at the same time, therefore they
108 // cannot overlap in the coroutine layout. The only way to guarantee
109 // this is if they are in the same variant, or one is ineligible
110 // (which means it is stored in every variant).
111if ineligible_locals.contains(local_b) || assignments[local_a] == assignments[local_b] {
112continue;
113 }
114115// If they conflict, we will choose one to make ineligible.
116 // This is not always optimal; it's just a greedy heuristic that
117 // seems to produce good results most of the time.
118let conflicts_b = storage_conflicts.count(local_b);
119let (remove, other) =
120if conflicts_a > conflicts_b { (local_a, local_b) } else { (local_b, local_a) };
121 ineligible_locals.insert(remove);
122 assignments[remove] = Ineligible(None);
123{
use ::tracing::__macro_support::Callsite as _;
static __CALLSITE: ::tracing::callsite::DefaultCallsite =
{
static META: ::tracing::Metadata<'static> =
{
::tracing_core::metadata::Metadata::new("event /rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout/coroutine.rs:123",
"rustc_abi::layout::coroutine", ::tracing::Level::TRACE,
::tracing_core::__macro_support::Option::Some("/rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout/coroutine.rs"),
::tracing_core::__macro_support::Option::Some(123u32),
::tracing_core::__macro_support::Option::Some("rustc_abi::layout::coroutine"),
::tracing_core::field::FieldSet::new(&["message"],
::tracing_core::callsite::Identifier(&__CALLSITE)),
::tracing::metadata::Kind::EVENT)
};
::tracing::callsite::DefaultCallsite::new(&META)
};
let enabled =
::tracing::Level::TRACE <= ::tracing::level_filters::STATIC_MAX_LEVEL
&&
::tracing::Level::TRACE <=
::tracing::level_filters::LevelFilter::current() &&
{
let interest = __CALLSITE.interest();
!interest.is_never() &&
::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
interest)
};
if enabled {
(|value_set: ::tracing::field::ValueSet|
{
let meta = __CALLSITE.metadata();
::tracing::Event::dispatch(meta, &value_set);
;
})({
#[allow(unused_imports)]
use ::tracing::field::{debug, display, Value};
__CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("removing local {0:?} due to conflict with {1:?}",
remove, other) as &dyn ::tracing::field::Value))])
});
} else { ; }
};trace!("removing local {:?} due to conflict with {:?}", remove, other);
124 }
125 }
126127// Count the number of variants in use. If only one of them, then it is
128 // impossible to overlap any locals in our layout. In this case it's
129 // always better to make the remaining locals ineligible, so we can
130 // lay them out with the other locals in the prefix and eliminate
131 // unnecessary padding bytes.
132{
133let mut used_variants = DenseBitSet::new_empty(variant_fields.len());
134for assignment in &assignments {
135if let Assigned(idx) = assignment {
136 used_variants.insert(*idx);
137 }
138 }
139if used_variants.count() < 2 {
140for assignment in assignments.iter_mut() {
141*assignment = Ineligible(None);
142 }
143ineligible_locals.insert_all();
144 }
145 }
146147// Write down the order of our locals that will be promoted to the prefix.
148{
149for (idx, local) in ineligible_locals.iter().enumerate() {
150 assignments[local] = Ineligible(Some(FieldIdx::new(idx)));
151 }
152 }
153{
use ::tracing::__macro_support::Callsite as _;
static __CALLSITE: ::tracing::callsite::DefaultCallsite =
{
static META: ::tracing::Metadata<'static> =
{
::tracing_core::metadata::Metadata::new("event /rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout/coroutine.rs:153",
"rustc_abi::layout::coroutine", ::tracing::Level::DEBUG,
::tracing_core::__macro_support::Option::Some("/rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout/coroutine.rs"),
::tracing_core::__macro_support::Option::Some(153u32),
::tracing_core::__macro_support::Option::Some("rustc_abi::layout::coroutine"),
::tracing_core::field::FieldSet::new(&["message"],
::tracing_core::callsite::Identifier(&__CALLSITE)),
::tracing::metadata::Kind::EVENT)
};
::tracing::callsite::DefaultCallsite::new(&META)
};
let enabled =
::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
&&
::tracing::Level::DEBUG <=
::tracing::level_filters::LevelFilter::current() &&
{
let interest = __CALLSITE.interest();
!interest.is_never() &&
::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
interest)
};
if enabled {
(|value_set: ::tracing::field::ValueSet|
{
let meta = __CALLSITE.metadata();
::tracing::Event::dispatch(meta, &value_set);
;
})({
#[allow(unused_imports)]
use ::tracing::field::{debug, display, Value};
__CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("coroutine saved local assignments: {0:?}",
assignments) as &dyn ::tracing::field::Value))])
});
} else { ; }
};debug!("coroutine saved local assignments: {:?}", assignments);
154155 (ineligible_locals, assignments)
156}
157158/// Compute the full coroutine layout.
159pub(super) fn layout<'a, F, VariantIdx, FieldIdx, LocalIdx>(
160 calc: &super::LayoutCalculator<impl HasDataLayout>,
161 local_layouts: &IndexSlice<LocalIdx, F>,
162mut prefix_layouts: IndexVec<FieldIdx, F>,
163 variant_fields: &IndexSlice<VariantIdx, IndexVec<FieldIdx, LocalIdx>>,
164 storage_conflicts: &BitMatrix<LocalIdx, LocalIdx>,
165 tag_to_layout: impl Fn(Scalar) -> F,
166) -> super::LayoutCalculatorResult<FieldIdx, VariantIdx, F>
167where
168F: core::ops::Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + core::fmt::Debug + Copy,
169 VariantIdx: Idx,
170 FieldIdx: Idx,
171 LocalIdx: Idx,
172{
173use SavedLocalEligibility::*;
174175let (ineligible_locals, assignments) =
176coroutine_saved_local_eligibility(local_layouts.len(), variant_fields, storage_conflicts);
177178// Build a prefix layout, including "promoting" all ineligible
179 // locals as part of the prefix. We compute the layout of all of
180 // these fields at once to get optimal packing.
181let tag_index = prefix_layouts.next_index();
182183// `variant_fields` already accounts for the reserved variants, so no need to add them.
184let max_discr = (variant_fields.len() - 1) as u128;
185let discr_int = Integer::fit_unsigned(max_discr);
186let tag = Scalar::Initialized {
187 value: Primitive::Int(discr_int, /* signed = */ false),
188 valid_range: WrappingRange { start: 0, end: max_discr },
189 };
190191let promoted_layouts = ineligible_locals.iter().map(|local| local_layouts[local]);
192prefix_layouts.push(tag_to_layout(tag));
193prefix_layouts.extend(promoted_layouts);
194let prefix = calc.layout_of_univariant(
195&prefix_layouts,
196&ReprOptions::default(),
197 StructKind::AlwaysSized,
198 )?;
199200let (prefix_size, prefix_align) = (prefix.size, prefix.align);
201202// Split the prefix layout into the "outer" fields (upvars and
203 // discriminant) and the "promoted" fields. Promoted fields will
204 // get included in each variant that requested them in
205 // CoroutineLayout.
206{
use ::tracing::__macro_support::Callsite as _;
static __CALLSITE: ::tracing::callsite::DefaultCallsite =
{
static META: ::tracing::Metadata<'static> =
{
::tracing_core::metadata::Metadata::new("event /rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout/coroutine.rs:206",
"rustc_abi::layout::coroutine", ::tracing::Level::DEBUG,
::tracing_core::__macro_support::Option::Some("/rustc-dev/c7d5ba59d8e11ee2e2204b173458eec4cb41c973/compiler/rustc_abi/src/layout/coroutine.rs"),
::tracing_core::__macro_support::Option::Some(206u32),
::tracing_core::__macro_support::Option::Some("rustc_abi::layout::coroutine"),
::tracing_core::field::FieldSet::new(&["message"],
::tracing_core::callsite::Identifier(&__CALLSITE)),
::tracing::metadata::Kind::EVENT)
};
::tracing::callsite::DefaultCallsite::new(&META)
};
let enabled =
::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
&&
::tracing::Level::DEBUG <=
::tracing::level_filters::LevelFilter::current() &&
{
let interest = __CALLSITE.interest();
!interest.is_never() &&
::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
interest)
};
if enabled {
(|value_set: ::tracing::field::ValueSet|
{
let meta = __CALLSITE.metadata();
::tracing::Event::dispatch(meta, &value_set);
;
})({
#[allow(unused_imports)]
use ::tracing::field::{debug, display, Value};
__CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("prefix = {0:#?}",
prefix) as &dyn ::tracing::field::Value))])
});
} else { ; }
};debug!("prefix = {:#?}", prefix);
207let (outer_fields, promoted_offsets, promoted_memory_index) = match prefix.fields {
208 FieldsShape::Arbitrary { mut offsets, in_memory_order } => {
209// "a" (`0..b_start`) and "b" (`b_start..`) correspond to
210 // "outer" and "promoted" fields respectively.
211let b_start = tag_index.plus(1);
212let offsets_b = IndexVec::from_raw(offsets.raw.split_off(b_start.index()));
213let offsets_a = offsets;
214215// Disentangle the "a" and "b" components of `in_memory_order`
216 // by preserving the order but keeping only one disjoint "half" each.
217 // FIXME(eddyb) build a better abstraction for permutations, if possible.
218let mut in_memory_order_a = IndexVec::<u32, FieldIdx>::new();
219let mut in_memory_order_b = IndexVec::<u32, FieldIdx>::new();
220for i in in_memory_order {
221if let Some(j) = i.index().checked_sub(b_start.index()) {
222 in_memory_order_b.push(FieldIdx::new(j));
223 } else {
224 in_memory_order_a.push(i);
225 }
226 }
227228let outer_fields =
229 FieldsShape::Arbitrary { offsets: offsets_a, in_memory_order: in_memory_order_a };
230 (outer_fields, offsets_b, in_memory_order_b.invert_bijective_mapping())
231 }
232_ => ::core::panicking::unreachable_default_msg()unreachable!(),
233 };
234235let mut size = prefix.size;
236let mut align = prefix.align;
237let variants = variant_fields
238 .iter_enumerated()
239 .map(|(index, variant_fields)| {
240// Only include overlap-eligible fields when we compute our variant layout.
241let variant_only_tys = variant_fields
242 .iter()
243 .filter(|local| match assignments[**local] {
244 Unassigned => ::core::panicking::unreachable_default_msg()unreachable!(),
245 Assigned(v) if v == index => true,
246 Assigned(_) => {
::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
format_args!("assignment does not match variant")));
}unreachable!("assignment does not match variant"),
247 Ineligible(_) => false,
248 })
249 .map(|local| local_layouts[*local]);
250251let mut variant = calc.layout_of_univariant(
252&variant_only_tys.collect::<IndexVec<_, _>>(),
253&ReprOptions::default(),
254 StructKind::Prefixed(prefix_size, prefix_align),
255 )?;
256257let FieldsShape::Arbitrary { offsets, in_memory_order } = variant.fields else {
258::core::panicking::unreachable_default_msg();unreachable!();
259 };
260261// Now, stitch the promoted and variant-only fields back together in
262 // the order they are mentioned by our CoroutineLayout.
263 // Because we only use some subset (that can differ between variants)
264 // of the promoted fields, we can't just pick those elements of the
265 // `promoted_memory_index` (as we'd end up with gaps).
266 // So instead, we build an "inverse memory_index", as if all of the
267 // promoted fields were being used, but leave the elements not in the
268 // subset as `invalid_field_idx`, which we can filter out later to
269 // obtain a valid (bijective) mapping.
270let memory_index = in_memory_order.invert_bijective_mapping();
271let invalid_field_idx = promoted_memory_index.len() + memory_index.len();
272let mut combined_in_memory_order =
273 IndexVec::from_elem_n(FieldIdx::new(invalid_field_idx), invalid_field_idx);
274275let mut offsets_and_memory_index = iter::zip(offsets, memory_index);
276let combined_offsets = variant_fields
277 .iter_enumerated()
278 .map(|(i, local)| {
279let (offset, memory_index) = match assignments[*local] {
280 Unassigned => ::core::panicking::unreachable_default_msg()unreachable!(),
281 Assigned(_) => {
282let (offset, memory_index) = offsets_and_memory_index.next().unwrap();
283 (offset, promoted_memory_index.len() as u32 + memory_index)
284 }
285 Ineligible(field_idx) => {
286let field_idx = field_idx.unwrap();
287 (promoted_offsets[field_idx], promoted_memory_index[field_idx])
288 }
289 };
290 combined_in_memory_order[memory_index] = i;
291 offset
292 })
293 .collect();
294295// Remove the unused slots to obtain the combined `in_memory_order`
296 // (also see previous comment).
297combined_in_memory_order.raw.retain(|&i| i.index() != invalid_field_idx);
298299 variant.fields = FieldsShape::Arbitrary {
300 offsets: combined_offsets,
301 in_memory_order: combined_in_memory_order,
302 };
303304 size = size.max(variant.size);
305 align = align.max(variant.align);
306Ok(VariantLayout::from_layout(variant))
307 })
308 .collect::<Result<IndexVec<VariantIdx, _>, _>>()?;
309310size = size.align_to(align);
311312let uninhabited = prefix.uninhabited || variants.iter().all(|v| v.is_uninhabited());
313let abi = BackendRepr::Memory { sized: true };
314315Ok(LayoutData {
316 variants: Variants::Multiple {
317tag,
318 tag_encoding: TagEncoding::Direct,
319 tag_field: tag_index,
320variants,
321 },
322 fields: outer_fields,
323 backend_repr: abi,
324// Suppress niches inside coroutines. If the niche is inside a field that is aliased (due to
325 // self-referentiality), getting the discriminant can cause aliasing violations.
326 // `UnsafeCell` blocks niches for the same reason, but we don't yet have `UnsafePinned` that
327 // would do the same for us here.
328 // See <https://github.com/rust-lang/rust/issues/63818>, <https://github.com/rust-lang/miri/issues/3780>.
329 // FIXME: Remove when <https://github.com/rust-lang/rust/issues/125735> is implemented and aliased coroutine fields are wrapped in `UnsafePinned`.
330largest_niche: None,
331uninhabited,
332size,
333align,
334 max_repr_align: None,
335 unadjusted_abi_align: align,
336 repr_c: false,
337 randomization_seed: Default::default(),
338 })
339}