rustc_ty_utils/

layout.rs

use std::range::{RangeFrom, RangeToInclusive};

use hir::def_id::DefId;
use rustc_abi as abi;
use rustc_abi::Integer::{I8, I32};
use rustc_abi::Primitive::{self, Float, Int, Pointer};
use rustc_abi::{
    AddressSpace, BackendRepr, FIRST_VARIANT, FieldIdx, FieldsShape, HasDataLayout, Layout,
    LayoutCalculatorError, LayoutData, Niche, ReprOptions, Scalar, Size, StructKind, TagEncoding,
    VariantIdx, Variants, WrappingRange,
};
use rustc_data_structures::Limit;
use rustc_hashes::Hash64;
use rustc_hir as hir;
use rustc_hir::find_attr;
use rustc_index::{Idx as _, IndexVec};
use rustc_middle::bug;
use rustc_middle::query::Providers;
use rustc_middle::traits::ObligationCause;
use rustc_middle::ty::layout::{
    FloatExt, HasTyCtxt, IntegerExt, LayoutCx, LayoutError, LayoutOf, SimdLayoutError, TyAndLayout,
};
use rustc_middle::ty::print::with_no_trimmed_paths;
use rustc_middle::ty::{
    self, AdtDef, CoroutineArgsExt, EarlyBinder, PseudoCanonicalInput, Ty, TyCtxt,
    TypeVisitableExt, Unnormalized,
};
use rustc_session::{DataTypeKind, FieldInfo, FieldKind, SizeKind, VariantInfo};
use rustc_span::{Symbol, sym};
use tracing::{debug, instrument};

use crate::diagnostics::NonPrimitiveSimdType;

mod invariant;

pub(crate) fn provide(providers: &mut Providers) {
    *providers = Providers { layout_of, ..*providers };
}

#[instrument(skip(tcx, query), level = "debug")]
fn layout_of<'tcx>(
    tcx: TyCtxt<'tcx>,
    query: ty::PseudoCanonicalInput<'tcx, Ty<'tcx>>,
) -> Result<TyAndLayout<'tcx>, &'tcx LayoutError<'tcx>> {
    let PseudoCanonicalInput { typing_env: original_typing_env, value: original_ty } = query;
    debug!(?original_ty);

    // Optimization: We convert to TypingMode::PostAnalysis and convert opaque types in
    // the where bounds to their hidden types. This reduces overall uncached invocations
    // of `layout_of` and is thus a small performance improvement.
    let typing_env = original_typing_env.with_post_analysis_normalized(tcx);
    // Switching to `PostAnalysis` typing mode will reveal opaque types that's marked
    // as rigid in the original typing env.
    let unnormalized_ty = if typing_env != original_typing_env {
        ty::set_aliases_to_non_rigid(tcx, original_ty)
    } else {
        ty::Unnormalized::new_wip(original_ty)
    };

    // FIXME: We might want to have two different versions of `layout_of`:
    // One that can be called after typecheck has completed and can use
    // `normalize_erasing_regions` here and another one that can be called
    // before typecheck has completed and uses `try_normalize_erasing_regions`.
    let normalized_ty = match tcx.try_normalize_erasing_regions(typing_env, unnormalized_ty) {
        Ok(t) => t,
        Err(normalization_error) => {
            return Err(tcx.arena.alloc(LayoutError::NormalizationFailure(
                unnormalized_ty.skip_normalization(),
                normalization_error,
            )));
        }
    };

    if normalized_ty != original_ty {
        // Ensure this layout is also cached for the normalized type.
        return tcx.layout_of(typing_env.as_query_input(normalized_ty));
    }

    match typing_env.typing_mode() {
        ty::TypingMode::Codegen => {
            let with_postanalysis =
                ty::TypingEnv::new(typing_env.param_env, ty::TypingMode::PostAnalysis);
            let res = tcx.layout_of(with_postanalysis.as_query_input(normalized_ty));
            match res {
                Err(LayoutError::TooGeneric(_)) => {}
                _ => return res,
            };
        }
        ty::TypingMode::Coherence
        | ty::TypingMode::Typeck { .. }
        | ty::TypingMode::PostTypeckUntilBorrowck { .. }
        | ty::TypingMode::PostBorrowck { .. }
        | ty::TypingMode::Reflection
        | ty::TypingMode::ErasedNotCoherence(_)
        | ty::TypingMode::PostAnalysis => {}
    }

    let cx = LayoutCx::new(tcx, typing_env);

    let layout = layout_of_uncached(&cx, normalized_ty)?;
    let layout = TyAndLayout { ty: normalized_ty, layout };

    // If we are running with `-Zprint-type-sizes`, maybe record layouts
    // for dumping later.
    if cx.tcx().sess.opts.unstable_opts.print_type_sizes {
        record_layout_for_printing(&cx, layout);
    }

    invariant::layout_sanity_check(&cx, &layout);

    Ok(layout)
}

fn error<'tcx>(cx: &LayoutCx<'tcx>, err: LayoutError<'tcx>) -> &'tcx LayoutError<'tcx> {
    cx.tcx().arena.alloc(err)
}

fn map_error<'tcx>(
    cx: &LayoutCx<'tcx>,
    ty: Ty<'tcx>,
    err: LayoutCalculatorError<TyAndLayout<'tcx>>,
) -> &'tcx LayoutError<'tcx> {
    let err = match err {
        LayoutCalculatorError::SizeOverflow => {
            // This is sometimes not a compile error in `check` builds.
            // See `tests/ui/limits/huge-enum.rs` for an example.
            LayoutError::SizeOverflow(ty)
        }
        LayoutCalculatorError::UnexpectedUnsized(field) => {
            // This is sometimes not a compile error if there are trivially false where clauses.
            // See `tests/ui/layout/trivial-bounds-sized.rs` for an example.
            assert!(field.layout.is_unsized(), "invalid layout error {err:#?}");
            if cx.typing_env.param_env.caller_bounds().is_empty() {
                cx.tcx().dcx().delayed_bug(format!(
                    "encountered unexpected unsized field in layout of {ty:?}: {field:#?}"
                ));
            }
            LayoutError::Unknown(ty)
        }
        LayoutCalculatorError::EmptyUnion => {
            // This is always a compile error.
            let guar =
                cx.tcx().dcx().delayed_bug(format!("computed layout of empty union: {ty:?}"));
            LayoutError::ReferencesError(guar)
        }
        LayoutCalculatorError::ReprConflict => {
            // packed enums are the only known trigger of this, but others might arise
            let guar = cx
                .tcx()
                .dcx()
                .delayed_bug(format!("computed impossible repr (packed enum?): {ty:?}"));
            LayoutError::ReferencesError(guar)
        }
        LayoutCalculatorError::ZeroLengthSimdType => {
            // Can't be caught in typeck if the array length is generic.
            LayoutError::InvalidSimd { ty, kind: SimdLayoutError::ZeroLength }
        }
        LayoutCalculatorError::OversizedSimdType { max_lanes } => {
            // Can't be caught in typeck if the array length is generic.
            LayoutError::InvalidSimd { ty, kind: SimdLayoutError::TooManyLanes(Limit(max_lanes)) }
        }
        LayoutCalculatorError::NonPrimitiveSimdType(field) => {
            // This error isn't caught in typeck, e.g., if
            // the element type of the vector is generic.
            cx.tcx().dcx().emit_fatal(NonPrimitiveSimdType { ty, e_ty: field.ty })
        }
    };
    error(cx, err)
}

fn extract_const_value<'tcx>(
    cx: &LayoutCx<'tcx>,
    ty: Ty<'tcx>,
    ct: ty::Const<'tcx>,
) -> Result<ty::Value<'tcx>, &'tcx LayoutError<'tcx>> {
    match ct.kind() {
        ty::ConstKind::Value(cv) => Ok(cv),
        ty::ConstKind::Param(_) | ty::ConstKind::Expr(_) => {
            if !ct.has_param() {
                bug!("failed to normalize const, but it is not generic: {ct:?}");
            }
            Err(error(cx, LayoutError::TooGeneric(ty)))
        }
        ty::ConstKind::Alias(_, _) => {
            let err = if ct.has_param() {
                LayoutError::TooGeneric(ty)
            } else {
                // This case is reachable with unsatisfiable predicates and GCE (which will
                // cause anon consts to inherit the unsatisfiable predicates). For example
                // if we have an unsatisfiable `u8: Trait` bound, then it's not a compile
                // error to mention `[u8; <u8 as Trait>::CONST]`, but we can't compute its
                // layout.
                LayoutError::Unknown(ty)
            };
            Err(error(cx, err))
        }
        ty::ConstKind::Infer(_)
        | ty::ConstKind::Bound(..)
        | ty::ConstKind::Placeholder(_)
        | ty::ConstKind::Error(_) => {
            // `ty::ConstKind::Error` is handled at the top of `layout_of_uncached`
            // (via `ty.error_reported()`).
            bug!("layout_of: unexpected const: {ct:?}");
        }
    }
}

fn layout_of_uncached<'tcx>(
    cx: &LayoutCx<'tcx>,
    ty: Ty<'tcx>,
) -> Result<Layout<'tcx>, &'tcx LayoutError<'tcx>> {
    // Types that reference `ty::Error` pessimistically don't have a meaningful layout.
    // The only side-effect of this is possibly worse diagnostics in case the layout
    // was actually computable (like if the `ty::Error` showed up only in a `PhantomData`).
    if let Err(guar) = ty.error_reported() {
        return Err(error(cx, LayoutError::ReferencesError(guar)));
    }

    let tcx = cx.tcx();

    // layout of `async_drop_in_place<T>::{closure}` in case,
    // when T is a coroutine, contains this internal coroutine's ref

    let dl = cx.data_layout();
    let map_layout = |result: Result<_, _>| match result {
        Ok(layout) => Ok(tcx.mk_layout(layout)),
        Err(err) => Err(map_error(cx, ty, err)),
    };
    let scalar_unit = |value: Primitive| {
        let size = value.size(dl);
        assert!(size.bits() <= 128);
        Scalar::Initialized { value, valid_range: WrappingRange::full(size) }
    };
    let scalar = |value: Primitive| tcx.mk_layout(LayoutData::scalar(cx, scalar_unit(value)));

    let univariant = |tys: &[Ty<'tcx>], kind| {
        let fields = tys.iter().map(|ty| cx.layout_of(*ty)).try_collect::<IndexVec<_, _>>()?;
        let repr = ReprOptions::default();
        map_layout(cx.calc.univariant(&fields, &repr, kind))
    };
    debug_assert!(!ty.has_non_region_infer());

    Ok(match *ty.kind() {
        ty::Pat(ty, pat) => {
            let layout = cx.layout_of(ty)?.layout;
            let mut layout = LayoutData::clone(&layout.0);
            match *pat {
                ty::PatternKind::Range { start, end } => {
                    if let BackendRepr::Scalar(scalar) = &mut layout.backend_repr {
                        scalar.valid_range_mut().start = extract_const_value(cx, ty, start)?
                            .try_to_bits(tcx, cx.typing_env)
                            .ok_or_else(|| error(cx, LayoutError::Unknown(ty)))?;

                        scalar.valid_range_mut().end = extract_const_value(cx, ty, end)?
                            .try_to_bits(tcx, cx.typing_env)
                            .ok_or_else(|| error(cx, LayoutError::Unknown(ty)))?;

                        // FIXME(pattern_types): create implied bounds from pattern types in signatures
                        // that require that the range end is >= the range start so that we can't hit
                        // this error anymore without first having hit a trait solver error.
                        // Very fuzzy on the details here, but pattern types are an internal impl detail,
                        // so we can just go with this for now
                        if scalar.is_signed() {
                            let range = scalar.valid_range_mut();
                            let start = layout.size.sign_extend(range.start);
                            let end = layout.size.sign_extend(range.end);
                            if end < start {
                                let guar = tcx.dcx().err(format!(
                                    "pattern type ranges cannot wrap: {start}..={end}"
                                ));

                                return Err(error(cx, LayoutError::ReferencesError(guar)));
                            }
                        } else {
                            let range = scalar.valid_range_mut();
                            if range.end < range.start {
                                let guar = tcx.dcx().err(format!(
                                    "pattern type ranges cannot wrap: {}..={}",
                                    range.start, range.end
                                ));

                                return Err(error(cx, LayoutError::ReferencesError(guar)));
                            }
                        };

                        let niche = Niche {
                            offset: Size::ZERO,
                            value: scalar.primitive(),
                            valid_range: scalar.valid_range(cx),
                        };

                        layout.largest_niche = Some(niche);
                    } else {
                        bug!("pattern type with range but not scalar layout: {ty:?}, {layout:?}")
                    }
                }
                ty::PatternKind::NotNull => {
                    if let BackendRepr::Scalar(scalar)
                    | BackendRepr::ScalarPair { a: scalar, b: _, b_offset: _ } =
                        &mut layout.backend_repr
                    {
                        scalar.valid_range_mut().start = 1;
                        let niche = Niche {
                            offset: Size::ZERO,
                            value: scalar.primitive(),
                            valid_range: scalar.valid_range(cx),
                        };

                        layout.largest_niche = Some(niche);
                    } else {
                        bug!(
                            "pattern type with `!null` pattern but not scalar/pair layout: {ty:?}, {layout:?}"
                        )
                    }
                }

                ty::PatternKind::Or(variants) => match *variants[0] {
                    ty::PatternKind::Range { .. } => {
                        if let BackendRepr::Scalar(scalar) = &mut layout.backend_repr {
                            let variants: Result<Vec<_>, _> = variants
                                .iter()
                                .map(|pat| match *pat {
                                    ty::PatternKind::Range { start, end } => Ok((
                                        extract_const_value(cx, ty, start)
                                            .unwrap()
                                            .try_to_bits(tcx, cx.typing_env)
                                            .ok_or_else(|| error(cx, LayoutError::Unknown(ty)))?,
                                        extract_const_value(cx, ty, end)
                                            .unwrap()
                                            .try_to_bits(tcx, cx.typing_env)
                                            .ok_or_else(|| error(cx, LayoutError::Unknown(ty)))?,
                                    )),
                                    ty::PatternKind::NotNull | ty::PatternKind::Or(_) => {
                                        unreachable!("mixed or patterns are not allowed")
                                    }
                                })
                                .collect();
                            let mut variants = variants?;
                            if !scalar.is_signed() {
                                let guar = tcx.dcx().err(format!(
                                    "only signed integer base types are allowed for or-pattern pattern types at present"
                                ));

                                return Err(error(cx, LayoutError::ReferencesError(guar)));
                            }
                            variants.sort();
                            if variants.len() != 2 {
                                let guar = tcx
                                .dcx()
                                .err(format!("the only or-pattern types allowed are two range patterns that are directly connected at their overflow site"));

                                return Err(error(cx, LayoutError::ReferencesError(guar)));
                            }

                            // first is the one starting at the signed in range min
                            let mut first = variants[0];
                            let mut second = variants[1];
                            if second.0
                                == layout.size.truncate(layout.size.signed_int_min() as u128)
                            {
                                (second, first) = (first, second);
                            }

                            if layout.size.sign_extend(first.1) >= layout.size.sign_extend(second.0)
                            {
                                let guar = tcx.dcx().err(format!(
                                    "only non-overlapping pattern type ranges are allowed at present"
                                ));

                                return Err(error(cx, LayoutError::ReferencesError(guar)));
                            }
                            if layout.size.signed_int_max() as u128 != second.1 {
                                let guar = tcx.dcx().err(format!(
                                    "one pattern needs to end at `{ty}::MAX`, but was {} instead",
                                    second.1
                                ));

                                return Err(error(cx, LayoutError::ReferencesError(guar)));
                            }

                            // Now generate a wrapping range (which aren't allowed in surface syntax).
                            scalar.valid_range_mut().start = second.0;
                            scalar.valid_range_mut().end = first.1;

                            let niche = Niche {
                                offset: Size::ZERO,
                                value: scalar.primitive(),
                                valid_range: scalar.valid_range(cx),
                            };

                            layout.largest_niche = Some(niche);
                        } else {
                            bug!(
                                "pattern type with range but not scalar layout: {ty:?}, {layout:?}"
                            )
                        }
                    }
                    ty::PatternKind::NotNull => bug!("or patterns can't contain `!null` patterns"),
                    ty::PatternKind::Or(..) => bug!("patterns cannot have nested or patterns"),
                },
            }
            // Pattern types contain their base as their sole field.
            // This allows the rest of the compiler to process pattern types just like
            // single field transparent Adts, and only the parts of the compiler that
            // specifically care about pattern types will have to handle it.
            layout.fields = FieldsShape::Arbitrary {
                offsets: [Size::ZERO].into_iter().collect(),
                in_memory_order: [FieldIdx::new(0)].into_iter().collect(),
            };
            tcx.mk_layout(layout)
        }

        // Basic scalars.
        ty::Bool => tcx.mk_layout(LayoutData::scalar(
            cx,
            Scalar::Initialized {
                value: Int(I8, false),
                valid_range: WrappingRange { start: 0, end: 1 },
            },
        )),
        ty::Char => tcx.mk_layout(LayoutData::scalar(
            cx,
            Scalar::Initialized {
                value: Int(I32, false),
                valid_range: WrappingRange { start: 0, end: 0x10FFFF },
            },
        )),
        ty::Int(ity) => scalar(Int(abi::Integer::from_int_ty(dl, ity), true)),
        ty::Uint(ity) => scalar(Int(abi::Integer::from_uint_ty(dl, ity), false)),
        ty::Float(fty) => scalar(Float(abi::Float::from_float_ty(fty))),
        ty::FnPtr(..) => {
            let mut ptr = scalar_unit(Pointer(dl.instruction_address_space));
            ptr.valid_range_mut().start = 1;
            tcx.mk_layout(LayoutData::scalar(cx, ptr))
        }

        // The never type.
        ty::Never => tcx.mk_layout(LayoutData::never_type(cx)),

        // Potentially-wide pointers.
        ty::Ref(_, pointee, _) | ty::RawPtr(pointee, _) => {
            let mut data_ptr = scalar_unit(Pointer(AddressSpace::ZERO));
            if !ty.is_raw_ptr() {
                data_ptr.valid_range_mut().start = 1;
            }

            if pointee.is_sized(tcx, cx.typing_env) {
                return Ok(tcx.mk_layout(LayoutData::scalar(cx, data_ptr)));
            }

            let metadata = if let Some(metadata_def_id) = tcx.lang_items().metadata_type() {
                let pointee_metadata =
                    Ty::new_projection(tcx, ty::IsRigid::No, metadata_def_id, [pointee]);
                let metadata_ty = match tcx.try_normalize_erasing_regions(
                    cx.typing_env,
                    Unnormalized::new_wip(pointee_metadata),
                ) {
                    Ok(metadata_ty) => metadata_ty,
                    Err(mut err) => {
                        // Usually `<Ty as Pointee>::Metadata` can't be normalized because
                        // its struct tail cannot be normalized either, so try to get a
                        // more descriptive layout error here, which will lead to less confusing
                        // diagnostics.
                        //
                        // We use the raw struct tail function here to get the first tail
                        // that is an alias, which is likely the cause of the normalization
                        // error.
                        match tcx.try_normalize_erasing_regions(
                            cx.typing_env,
                            Unnormalized::new_wip(tcx.struct_tail_raw(
                                pointee,
                                &ObligationCause::dummy(),
                                |ty| ty.skip_norm_wip(),
                                || {},
                            )),
                        ) {
                            Ok(_) => {}
                            Err(better_err) => {
                                err = better_err;
                            }
                        }
                        return Err(error(cx, LayoutError::NormalizationFailure(pointee, err)));
                    }
                };

                let metadata_layout = cx.layout_of(metadata_ty)?;
                // If the metadata is a 1-zst, then the pointer is thin.
                if metadata_layout.is_1zst() {
                    return Ok(tcx.mk_layout(LayoutData::scalar(cx, data_ptr)));
                }

                let BackendRepr::Scalar(metadata) = metadata_layout.backend_repr else {
                    return Err(error(cx, LayoutError::Unknown(pointee)));
                };

                metadata
            } else {
                let unsized_part = tcx.struct_tail_for_codegen(pointee, cx.typing_env);

                match unsized_part.kind() {
                    ty::Foreign(..) => {
                        return Ok(tcx.mk_layout(LayoutData::scalar(cx, data_ptr)));
                    }
                    ty::Slice(_) | ty::Str => scalar_unit(Int(dl.ptr_sized_integer(), false)),
                    ty::Dynamic(..) => {
                        let mut vtable = scalar_unit(Pointer(AddressSpace::ZERO));
                        vtable.valid_range_mut().start = 1;
                        vtable
                    }
                    _ => {
                        return Err(error(cx, LayoutError::Unknown(pointee)));
                    }
                }
            };

            // Effectively a (ptr, meta) tuple.
            tcx.mk_layout(LayoutData::scalar_pair(cx, data_ptr, metadata))
        }

        // Arrays and slices.
        ty::Array(element, count) => {
            let count = extract_const_value(cx, ty, count)?
                .try_to_target_usize(tcx)
                .ok_or_else(|| error(cx, LayoutError::Unknown(ty)))?;

            let element = cx.layout_of(element)?;
            map_layout(cx.calc.array_like(&element, Some(count)))?
        }
        ty::Slice(element) => {
            let element = cx.layout_of(element)?;
            map_layout(cx.calc.array_like(&element, None).map(|mut layout| {
                // a randomly chosen value to distinguish slices
                layout.randomization_seed = Hash64::new(0x2dcba99c39784102);
                layout
            }))?
        }
        ty::Str => {
            let element = scalar(Int(I8, false));
            map_layout(cx.calc.array_like(&element, None).map(|mut layout| {
                // another random value
                layout.randomization_seed = Hash64::new(0xc1325f37d127be22);
                layout
            }))?
        }

        // Odd unit types.
        ty::FnDef(..) | ty::Dynamic(_, _) | ty::Foreign(..) => {
            let sized = matches!(ty.kind(), ty::FnDef(..));
            tcx.mk_layout(LayoutData::unit(cx, sized))
        }

        ty::Coroutine(def_id, args) => {
            match cx.typing_env.typing_mode() {
                ty::TypingMode::Codegen => {}
                ty::TypingMode::Coherence
                | ty::TypingMode::Typeck { .. }
                | ty::TypingMode::PostTypeckUntilBorrowck { .. }
                | ty::TypingMode::PostBorrowck { .. }
                | ty::TypingMode::Reflection
                | ty::TypingMode::ErasedNotCoherence(_)
                | ty::TypingMode::PostAnalysis => {
                    return Err(error(cx, LayoutError::TooGeneric(ty)));
                }
            }

            use rustc_middle::ty::layout::PrimitiveExt as _;

            let info = tcx.coroutine_layout(def_id, args)?;

            let local_layouts = info
                .field_tys
                .iter()
                .map(|local| {
                    let field_ty = EarlyBinder::bind(tcx, local.ty);
                    let uninit_ty =
                        Ty::new_maybe_uninit(tcx, field_ty.instantiate(tcx, args).skip_norm_wip());
                    cx.spanned_layout_of(uninit_ty, local.source_info.span)
                })
                .try_collect::<IndexVec<_, _>>()?;

            let prefix_layouts = args
                .as_coroutine()
                .upvar_tys()
                .iter()
                .map(|ty| cx.layout_of(ty))
                .try_collect::<IndexVec<_, _>>()?;

            let layout = cx
                .calc
                .coroutine(
                    &local_layouts,
                    prefix_layouts,
                    &info.variant_fields,
                    &info.storage_conflicts,
                    |tag| TyAndLayout {
                        ty: tag.primitive().to_ty(tcx),
                        layout: tcx.mk_layout(LayoutData::scalar(cx, tag)),
                    },
                )
                .map(|mut layout| {
                    // this is similar to how ReprOptions populates its field_shuffle_seed
                    layout.randomization_seed = tcx.def_path_hash(def_id).0.to_smaller_hash();
                    debug!("coroutine layout ({:?}): {:#?}", ty, layout);
                    layout
                });
            map_layout(layout)?
        }

        ty::Closure(_, args) => univariant(args.as_closure().upvar_tys(), StructKind::AlwaysSized)?,

        ty::CoroutineClosure(_, args) => {
            univariant(args.as_coroutine_closure().upvar_tys(), StructKind::AlwaysSized)?
        }

        ty::Tuple(tys) => {
            let kind =
                if tys.len() == 0 { StructKind::AlwaysSized } else { StructKind::MaybeUnsized };

            univariant(tys, kind)?
        }

        // Scalable vector types
        //
        // ```rust (ignore, example)
        // #[rustc_scalable_vector(3)]
        // struct svuint32_t(u32);
        //
        // #[rustc_scalable_vector]
        // struct svuint32x2_t(svuint32_t, svuint32_t);
        // ```
        ty::Adt(def, _args) if def.repr().scalable() => {
            let Some((element_count, element_ty, number_of_vectors)) =
                ty.scalable_vector_parts(tcx)
            else {
                let guar = tcx
                    .dcx()
                    .delayed_bug("`#[rustc_scalable_vector]` was applied to an invalid type");
                return Err(error(cx, LayoutError::ReferencesError(guar)));
            };

            let element_layout = cx.layout_of(element_ty)?;
            map_layout(cx.calc.scalable_vector_type(
                element_layout,
                element_count as u64,
                number_of_vectors,
            ))?
        }

        // SIMD vector types.
        ty::Adt(def, args) if def.repr().simd() => {
            // Supported SIMD vectors are ADTs with a single array field:
            //
            // * #[repr(simd)] struct S([T; 4])
            //
            // where T is a primitive scalar (integer/float/pointer).
            let Some(ty::Array(e_ty, e_len)) = def
                .is_struct()
                .then(|| &def.variant(FIRST_VARIANT).fields)
                .filter(|fields| fields.len() == 1)
                .map(|fields| *fields[FieldIdx::ZERO].ty(tcx, args).skip_norm_wip().kind())
            else {
                // Invalid SIMD types should have been caught by typeck by now.
                let guar = tcx.dcx().delayed_bug("#[repr(simd)] was applied to an invalid ADT");
                return Err(error(cx, LayoutError::ReferencesError(guar)));
            };

            let e_len = extract_const_value(cx, ty, e_len)?
                .try_to_target_usize(tcx)
                .ok_or_else(|| error(cx, LayoutError::Unknown(ty)))?;

            let e_ly = cx.layout_of(e_ty)?;

            // Check for the rustc_simd_monomorphize_lane_limit attribute and check the lane limit
            if let Some(limit) = find_attr!(
                tcx, def.did(),
                RustcSimdMonomorphizeLaneLimit(limit) => limit
            ) {
                if !limit.value_within_limit(e_len as usize) {
                    return Err(map_error(
                        &cx,
                        ty,
                        rustc_abi::LayoutCalculatorError::OversizedSimdType { max_lanes: limit.0 },
                    ));
                }
            }

            map_layout(cx.calc.simd_type(e_ly, e_len, def.repr().packed()))?
        }

        // ADTs.
        ty::Adt(def, args) => {
            // Cache the field layouts.
            let variants = def
                .variants()
                .iter()
                .map(|v| {
                    v.fields
                        .iter()
                        .map(|field| cx.layout_of(field.ty(tcx, args).skip_norm_wip()))
                        .try_collect::<IndexVec<_, _>>()
                })
                .try_collect::<IndexVec<VariantIdx, _>>()?;

            if def.is_union() {
                if def.repr().pack.is_some() && def.repr().align.is_some() {
                    let guar = tcx.dcx().span_delayed_bug(
                        tcx.def_span(def.did()),
                        "union cannot be packed and aligned",
                    );
                    return Err(error(cx, LayoutError::ReferencesError(guar)));
                }

                return map_layout(cx.calc.layout_of_union(&def.repr(), &variants));
            }

            // UnsafeCell and UnsafePinned both disable niche optimizations
            let is_special_no_niche = def.is_unsafe_cell() || def.is_unsafe_pinned();

            let discr_range_of_repr = |min: RangeFrom<i128>, max: RangeToInclusive<u128>| {
                abi::Integer::discr_range_of_repr(tcx, ty, &def.repr(), min.start, max.last)
            };

            let discriminants_iter = || {
                def.is_enum()
                    .then(|| def.discriminants(tcx).map(|(v, d)| (v, d.val)))
                    .into_flat_iter()
            };

            let maybe_unsized = def.is_struct()
                && def.non_enum_variant().tail_opt().is_some_and(|last_field| {
                    let typing_env = ty::TypingEnv::new(
                        tcx.param_env_normalized_for_post_analysis(def.did()),
                        cx.typing_env.typing_mode(),
                    );
                    !tcx.type_of(last_field.did)
                        .instantiate_identity()
                        .skip_norm_wip()
                        .is_sized(tcx, typing_env)
                });

            let layout = cx
                .calc
                .layout_of_struct_or_enum(
                    &def.repr(),
                    &variants,
                    def.is_enum(),
                    is_special_no_niche,
                    discr_range_of_repr,
                    discriminants_iter(),
                    !maybe_unsized,
                )
                .map_err(|err| map_error(cx, ty, err))?;

            if !maybe_unsized && layout.is_unsized() {
                bug!("got unsized layout for type that cannot be unsized {ty:?}: {layout:#?}");
            }

            // If the struct tail is sized and can be unsized, check that unsizing doesn't move the fields around.
            if cfg!(debug_assertions)
                && maybe_unsized
                && def
                    .non_enum_variant()
                    .tail()
                    .ty(tcx, args)
                    .skip_norm_wip()
                    .is_sized(tcx, cx.typing_env)
            {
                let mut variants = variants;
                let tail_replacement = cx.layout_of(Ty::new_slice(tcx, tcx.types.u8)).unwrap();
                *variants[FIRST_VARIANT].raw.last_mut().unwrap() = tail_replacement;

                let Ok(unsized_layout) = cx.calc.layout_of_struct_or_enum(
                    &def.repr(),
                    &variants,
                    def.is_enum(),
                    is_special_no_niche,
                    discr_range_of_repr,
                    discriminants_iter(),
                    !maybe_unsized,
                ) else {
                    bug!("failed to compute unsized layout of {ty:?}");
                };

                let FieldsShape::Arbitrary { offsets: sized_offsets, .. } = &layout.fields else {
                    bug!("unexpected FieldsShape for sized layout of {ty:?}: {:?}", layout.fields);
                };
                let FieldsShape::Arbitrary { offsets: unsized_offsets, .. } =
                    &unsized_layout.fields
                else {
                    bug!(
                        "unexpected FieldsShape for unsized layout of {ty:?}: {:?}",
                        unsized_layout.fields
                    );
                };

                let (sized_tail, sized_fields) = sized_offsets.raw.split_last().unwrap();
                let (unsized_tail, unsized_fields) = unsized_offsets.raw.split_last().unwrap();

                if sized_fields != unsized_fields {
                    bug!("unsizing {ty:?} changed field order!\n{layout:?}\n{unsized_layout:?}");
                }

                if sized_tail < unsized_tail {
                    bug!("unsizing {ty:?} moved tail backwards!\n{layout:?}\n{unsized_layout:?}");
                }
            }

            tcx.mk_layout(layout)
        }

        ty::UnsafeBinder(bound_ty) => {
            let ty = tcx.instantiate_bound_regions_with_erased(bound_ty.into());
            cx.layout_of(ty)?.layout
        }

        // Types with no meaningful known layout.
        ty::Param(_) | ty::Placeholder(..) => {
            return Err(error(cx, LayoutError::TooGeneric(ty)));
        }

        ty::Alias(..) => {
            // In case we're still in a generic context, aliases might be rigid. E.g.
            // if we've got a `T: Trait` where-bound, `T::Assoc` cannot be normalized
            // in the current context.
            //
            // For some builtin traits, generic aliases can be rigid even in an empty environment,
            // e.g. `<T as Pointee>::Metadata`.
            //
            // Due to trivial bounds, this can even be the case if the alias does not reference
            // any generic parameters, e.g. a `for<'a> u32: Trait<'a>` where-bound means that
            // `<u32 as Trait<'static>>::Assoc` is rigid.
            let err = if ty.has_param() || !cx.typing_env.param_env.caller_bounds().is_empty() {
                LayoutError::TooGeneric(ty)
            } else {
                LayoutError::ReferencesError(cx.tcx().dcx().delayed_bug(format!(
                    "unexpected rigid alias in layout_of after normalization: {ty:?}"
                )))
            };
            return Err(error(cx, err));
        }

        ty::Bound(..) | ty::CoroutineWitness(..) | ty::Infer(_) | ty::Error(_) => {
            // `ty::Error` is handled at the top of this function.
            bug!("layout_of: unexpected type `{ty}`")
        }
    })
}

fn record_layout_for_printing<'tcx>(cx: &LayoutCx<'tcx>, layout: TyAndLayout<'tcx>) {
    // Ignore layouts that are done with non-empty environments or
    // non-monomorphic layouts, as the user only wants to see the stuff
    // resulting from the final codegen session.
    if layout.ty.has_non_region_param() || !cx.typing_env.param_env.caller_bounds().is_empty() {
        return;
    }

    // (delay format until we actually need it)
    let record = |kind, packed, opt_discr_size, variants| {
        let type_desc = with_no_trimmed_paths!(format!("{}", layout.ty));
        cx.tcx().sess.code_stats.record_type_size(
            kind,
            type_desc,
            layout.align.abi,
            layout.size,
            packed,
            opt_discr_size,
            variants,
        );
    };

    match *layout.ty.kind() {
        ty::Adt(adt_def, _) => {
            debug!("print-type-size t: `{:?}` process adt", layout.ty);
            let adt_kind = adt_def.adt_kind();
            let adt_packed = adt_def.repr().pack.is_some();
            let (variant_infos, opt_discr_size) = variant_info_for_adt(cx, layout, adt_def);
            record(adt_kind.into(), adt_packed, opt_discr_size, variant_infos);
        }

        ty::Coroutine(def_id, args) => {
            debug!("print-type-size t: `{:?}` record coroutine", layout.ty);
            // Coroutines always have a begin/poisoned/end state with additional suspend points
            let (variant_infos, opt_discr_size) =
                variant_info_for_coroutine(cx, layout, def_id, args);
            record(DataTypeKind::Coroutine, false, opt_discr_size, variant_infos);
        }

        ty::Closure(..) => {
            debug!("print-type-size t: `{:?}` record closure", layout.ty);
            record(DataTypeKind::Closure, false, None, vec![]);
        }

        _ => {
            debug!("print-type-size t: `{:?}` skip non-nominal", layout.ty);
        }
    };
}

fn variant_info_for_adt<'tcx>(
    cx: &LayoutCx<'tcx>,
    layout: TyAndLayout<'tcx>,
    adt_def: AdtDef<'tcx>,
) -> (Vec<VariantInfo>, Option<Size>) {
    let build_variant_info = |n: Option<Symbol>, flds: &[Symbol], layout: TyAndLayout<'tcx>| {
        let mut min_size = Size::ZERO;
        let field_info: Vec<_> = flds
            .iter()
            .enumerate()
            .map(|(i, &name)| {
                let field_layout = layout.field(cx, i);
                let offset = layout.fields.offset(i);
                min_size = min_size.max(offset + field_layout.size);
                FieldInfo {
                    kind: FieldKind::AdtField,
                    name,
                    offset: offset.bytes(),
                    size: field_layout.size.bytes(),
                    align: field_layout.align.bytes(),
                    type_name: None,
                }
            })
            .collect();

        VariantInfo {
            name: n,
            kind: if layout.is_unsized() { SizeKind::Min } else { SizeKind::Exact },
            align: layout.align.bytes(),
            size: if min_size.bytes() == 0 { layout.size.bytes() } else { min_size.bytes() },
            fields: field_info,
        }
    };

    match layout.variants {
        Variants::Empty => (vec![], None),

        Variants::Single { index } => {
            debug!("print-type-size `{:#?}` variant {}", layout, adt_def.variant(index).name);
            let variant_def = &adt_def.variant(index);
            let fields: Vec<_> = variant_def.fields.iter().map(|f| f.name).collect();
            (vec![build_variant_info(Some(variant_def.name), &fields, layout)], None)
        }

        Variants::Multiple { tag, ref tag_encoding, .. } => {
            debug!(
                "print-type-size `{:#?}` adt general variants def {}",
                layout.ty,
                adt_def.variants().len()
            );
            let variant_infos: Vec<_> = adt_def
                .variants()
                .iter_enumerated()
                .map(|(i, variant_def)| {
                    let fields: Vec<_> = variant_def.fields.iter().map(|f| f.name).collect();
                    build_variant_info(Some(variant_def.name), &fields, layout.for_variant(cx, i))
                })
                .collect();

            (
                variant_infos,
                match tag_encoding {
                    TagEncoding::Direct => Some(tag.size(cx)),
                    _ => None,
                },
            )
        }
    }
}

fn variant_info_for_coroutine<'tcx>(
    cx: &LayoutCx<'tcx>,
    layout: TyAndLayout<'tcx>,
    def_id: DefId,
    args: ty::GenericArgsRef<'tcx>,
) -> (Vec<VariantInfo>, Option<Size>) {
    use itertools::Itertools;

    let Variants::Multiple { tag, ref tag_encoding, tag_field, .. } = layout.variants else {
        return (vec![], None);
    };

    let coroutine = cx.tcx().coroutine_layout(def_id, args).unwrap();
    let upvar_names = cx.tcx().closure_saved_names_of_captured_variables(def_id);

    let mut upvars_size = Size::ZERO;
    let upvar_fields: Vec<_> = args
        .as_coroutine()
        .upvar_tys()
        .iter()
        .zip_eq(upvar_names)
        .enumerate()
        .map(|(field_idx, (_, name))| {
            let field_layout = layout.field(cx, field_idx);
            let offset = layout.fields.offset(field_idx);
            upvars_size = upvars_size.max(offset + field_layout.size);
            FieldInfo {
                kind: FieldKind::Upvar,
                name: *name,
                offset: offset.bytes(),
                size: field_layout.size.bytes(),
                align: field_layout.align.bytes(),
                type_name: None,
            }
        })
        .collect();

    let mut variant_infos: Vec<_> = coroutine
        .variant_fields
        .iter_enumerated()
        .map(|(variant_idx, variant_def)| {
            let variant_layout = layout.for_variant(cx, variant_idx);
            let mut variant_size = Size::ZERO;
            let fields = variant_def
                .iter()
                .enumerate()
                .map(|(field_idx, local)| {
                    let field_name = coroutine.field_tys[*local].debuginfo_name;
                    let field_layout = variant_layout.field(cx, field_idx);
                    let offset = variant_layout.fields.offset(field_idx);
                    // The struct is as large as the last field's end
                    variant_size = variant_size.max(offset + field_layout.size);
                    FieldInfo {
                        kind: FieldKind::CoroutineLocal,
                        name: field_name.unwrap_or_else(|| {
                            Symbol::intern(&format!(".coroutine_field{}", local.as_usize()))
                        }),
                        offset: offset.bytes(),
                        size: field_layout.size.bytes(),
                        align: field_layout.align.bytes(),
                        // Include the type name if there is no field name, or if the name is the
                        // __awaitee placeholder symbol which means a child future being `.await`ed.
                        type_name: (field_name.is_none() || field_name == Some(sym::__awaitee))
                            .then(|| Symbol::intern(&field_layout.ty.to_string())),
                    }
                })
                .chain(upvar_fields.iter().copied())
                .collect();

            // If the variant has no state-specific fields, then it's the size of the upvars.
            if variant_size == Size::ZERO {
                variant_size = upvars_size;
            }

            // This `if` deserves some explanation.
            //
            // The layout code has a choice of where to place the discriminant of this coroutine.
            // If the discriminant of the coroutine is placed early in the layout (before the
            // variant's own fields), then it'll implicitly be counted towards the size of the
            // variant, since we use the maximum offset to calculate size.
            //    (side-note: I know this is a bit problematic given upvars placement, etc).
            //
            // This is important, since the layout printing code always subtracts this discriminant
            // size from the variant size if the struct is "enum"-like, so failing to account for it
            // will either lead to numerical underflow, or an underreported variant size...
            //
            // However, if the discriminant is placed past the end of the variant, then we need
            // to factor in the size of the discriminant manually. This really should be refactored
            // better, but this "works" for now.
            if layout.fields.offset(tag_field.as_usize()) >= variant_size {
                variant_size += match tag_encoding {
                    TagEncoding::Direct => tag.size(cx),
                    _ => Size::ZERO,
                };
            }

            VariantInfo {
                name: Some(Symbol::intern(&ty::CoroutineArgs::variant_name(variant_idx))),
                kind: SizeKind::Exact,
                size: variant_size.bytes(),
                align: variant_layout.align.bytes(),
                fields,
            }
        })
        .collect();

    // The first three variants are hardcoded to be `UNRESUMED`, `RETURNED` and `POISONED`.
    // We will move the `RETURNED` and `POISONED` elements to the end so we
    // are left with a sorting order according to the coroutines yield points:
    // First `Unresumed`, then the `SuspendN` followed by `Returned` and `Panicked` (POISONED).
    let end_states = variant_infos.drain(1..=2);
    let end_states: Vec<_> = end_states.collect();
    variant_infos.extend(end_states);

    (
        variant_infos,
        match tag_encoding {
            TagEncoding::Direct => Some(tag.size(cx)),
            _ => None,
        },
    )
}