1use std::ops::Bound;
2use std::{cmp, fmt};
3
4use rustc_abi::{
5 AddressSpace, Align, ExternAbi, FieldIdx, FieldsShape, HasDataLayout, LayoutData, PointeeInfo,
6 PointerKind, Primitive, ReprOptions, Scalar, Size, TagEncoding, TargetDataLayout,
7 TyAbiInterface, VariantIdx, Variants,
8};
9use rustc_error_messages::DiagMessage;
10use rustc_errors::{
11 Diag, DiagArgValue, DiagCtxtHandle, Diagnostic, EmissionGuarantee, IntoDiagArg, Level,
12};
13use rustc_hir::LangItem;
14use rustc_hir::def_id::DefId;
15use rustc_macros::{HashStable, TyDecodable, TyEncodable, extension};
16use rustc_session::config::OptLevel;
17use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span, Symbol, sym};
18use rustc_target::callconv::FnAbi;
19use rustc_target::spec::{HasTargetSpec, HasX86AbiOpt, Target, X86Abi};
20use tracing::debug;
21use {rustc_abi as abi, rustc_hir as hir};
22
23use crate::middle::codegen_fn_attrs::CodegenFnAttrFlags;
24use crate::query::TyCtxtAt;
25use crate::traits::ObligationCause;
26use crate::ty::normalize_erasing_regions::NormalizationError;
27use crate::ty::{self, CoroutineArgsExt, Ty, TyCtxt, TypeVisitableExt};
28
29#[extension(pub trait IntegerExt)]
30impl abi::Integer {
31 #[inline]
32 fn to_ty<'tcx>(&self, tcx: TyCtxt<'tcx>, signed: bool) -> Ty<'tcx> {
33 use abi::Integer::{I8, I16, I32, I64, I128};
34 match (*self, signed) {
35 (I8, false) => tcx.types.u8,
36 (I16, false) => tcx.types.u16,
37 (I32, false) => tcx.types.u32,
38 (I64, false) => tcx.types.u64,
39 (I128, false) => tcx.types.u128,
40 (I8, true) => tcx.types.i8,
41 (I16, true) => tcx.types.i16,
42 (I32, true) => tcx.types.i32,
43 (I64, true) => tcx.types.i64,
44 (I128, true) => tcx.types.i128,
45 }
46 }
47
48 fn from_int_ty<C: HasDataLayout>(cx: &C, ity: ty::IntTy) -> abi::Integer {
49 use abi::Integer::{I8, I16, I32, I64, I128};
50 match ity {
51 ty::IntTy::I8 => I8,
52 ty::IntTy::I16 => I16,
53 ty::IntTy::I32 => I32,
54 ty::IntTy::I64 => I64,
55 ty::IntTy::I128 => I128,
56 ty::IntTy::Isize => cx.data_layout().ptr_sized_integer(),
57 }
58 }
59 fn from_uint_ty<C: HasDataLayout>(cx: &C, ity: ty::UintTy) -> abi::Integer {
60 use abi::Integer::{I8, I16, I32, I64, I128};
61 match ity {
62 ty::UintTy::U8 => I8,
63 ty::UintTy::U16 => I16,
64 ty::UintTy::U32 => I32,
65 ty::UintTy::U64 => I64,
66 ty::UintTy::U128 => I128,
67 ty::UintTy::Usize => cx.data_layout().ptr_sized_integer(),
68 }
69 }
70
71 fn repr_discr<'tcx>(
76 tcx: TyCtxt<'tcx>,
77 ty: Ty<'tcx>,
78 repr: &ReprOptions,
79 min: i128,
80 max: i128,
81 ) -> (abi::Integer, bool) {
82 let unsigned_fit = abi::Integer::fit_unsigned(cmp::max(min as u128, max as u128));
87 let signed_fit = cmp::max(abi::Integer::fit_signed(min), abi::Integer::fit_signed(max));
88
89 if let Some(ity) = repr.int {
90 let discr = abi::Integer::from_attr(&tcx, ity);
91 let fit = if ity.is_signed() { signed_fit } else { unsigned_fit };
92 if discr < fit {
93 bug!(
94 "Integer::repr_discr: `#[repr]` hint too small for \
95 discriminant range of enum `{}`",
96 ty
97 )
98 }
99 return (discr, ity.is_signed());
100 }
101
102 let at_least = if repr.c() {
103 tcx.data_layout().c_enum_min_size
106 } else {
107 abi::Integer::I8
109 };
110
111 if unsigned_fit <= signed_fit {
113 (cmp::max(unsigned_fit, at_least), false)
114 } else {
115 (cmp::max(signed_fit, at_least), true)
116 }
117 }
118}
119
120#[extension(pub trait FloatExt)]
121impl abi::Float {
122 #[inline]
123 fn to_ty<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Ty<'tcx> {
124 use abi::Float::*;
125 match *self {
126 F16 => tcx.types.f16,
127 F32 => tcx.types.f32,
128 F64 => tcx.types.f64,
129 F128 => tcx.types.f128,
130 }
131 }
132
133 fn from_float_ty(fty: ty::FloatTy) -> Self {
134 use abi::Float::*;
135 match fty {
136 ty::FloatTy::F16 => F16,
137 ty::FloatTy::F32 => F32,
138 ty::FloatTy::F64 => F64,
139 ty::FloatTy::F128 => F128,
140 }
141 }
142}
143
144#[extension(pub trait PrimitiveExt)]
145impl Primitive {
146 #[inline]
147 fn to_ty<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Ty<'tcx> {
148 match *self {
149 Primitive::Int(i, signed) => i.to_ty(tcx, signed),
150 Primitive::Float(f) => f.to_ty(tcx),
151 Primitive::Pointer(_) => Ty::new_mut_ptr(tcx, tcx.types.unit),
153 }
154 }
155
156 #[inline]
159 fn to_int_ty<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Ty<'tcx> {
160 match *self {
161 Primitive::Int(i, signed) => i.to_ty(tcx, signed),
162 Primitive::Pointer(_) => {
164 let signed = false;
165 tcx.data_layout().ptr_sized_integer().to_ty(tcx, signed)
166 }
167 Primitive::Float(_) => bug!("floats do not have an int type"),
168 }
169 }
170}
171
172pub const WIDE_PTR_ADDR: usize = 0;
177
178pub const WIDE_PTR_EXTRA: usize = 1;
183
184pub const MAX_SIMD_LANES: u64 = rustc_abi::MAX_SIMD_LANES;
185
186#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, HashStable)]
189pub enum ValidityRequirement {
190 Inhabited,
191 Zero,
192 UninitMitigated0x01Fill,
195 Uninit,
197}
198
199impl ValidityRequirement {
200 pub fn from_intrinsic(intrinsic: Symbol) -> Option<Self> {
201 match intrinsic {
202 sym::assert_inhabited => Some(Self::Inhabited),
203 sym::assert_zero_valid => Some(Self::Zero),
204 sym::assert_mem_uninitialized_valid => Some(Self::UninitMitigated0x01Fill),
205 _ => None,
206 }
207 }
208}
209
210impl fmt::Display for ValidityRequirement {
211 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
212 match self {
213 Self::Inhabited => f.write_str("is inhabited"),
214 Self::Zero => f.write_str("allows being left zeroed"),
215 Self::UninitMitigated0x01Fill => f.write_str("allows being filled with 0x01"),
216 Self::Uninit => f.write_str("allows being left uninitialized"),
217 }
218 }
219}
220
221#[derive(Copy, Clone, Debug, HashStable, TyEncodable, TyDecodable)]
222pub enum SimdLayoutError {
223 ZeroLength,
225 TooManyLanes(u64),
228}
229
230#[derive(Copy, Clone, Debug, HashStable, TyEncodable, TyDecodable)]
231pub enum LayoutError<'tcx> {
232 Unknown(Ty<'tcx>),
240 SizeOverflow(Ty<'tcx>),
242 InvalidSimd { ty: Ty<'tcx>, kind: SimdLayoutError },
244 TooGeneric(Ty<'tcx>),
249 NormalizationFailure(Ty<'tcx>, NormalizationError<'tcx>),
257 ReferencesError(ErrorGuaranteed),
259 Cycle(ErrorGuaranteed),
261}
262
263impl<'tcx> LayoutError<'tcx> {
264 pub fn diagnostic_message(&self) -> DiagMessage {
265 use LayoutError::*;
266
267 use crate::fluent_generated::*;
268 match self {
269 Unknown(_) => middle_layout_unknown,
270 SizeOverflow(_) => middle_layout_size_overflow,
271 InvalidSimd { kind: SimdLayoutError::TooManyLanes(_), .. } => {
272 middle_layout_simd_too_many
273 }
274 InvalidSimd { kind: SimdLayoutError::ZeroLength, .. } => middle_layout_simd_zero_length,
275 TooGeneric(_) => middle_layout_too_generic,
276 NormalizationFailure(_, _) => middle_layout_normalization_failure,
277 Cycle(_) => middle_layout_cycle,
278 ReferencesError(_) => middle_layout_references_error,
279 }
280 }
281
282 pub fn into_diagnostic(self) -> crate::error::LayoutError<'tcx> {
283 use LayoutError::*;
284
285 use crate::error::LayoutError as E;
286 match self {
287 Unknown(ty) => E::Unknown { ty },
288 SizeOverflow(ty) => E::Overflow { ty },
289 InvalidSimd { ty, kind: SimdLayoutError::TooManyLanes(max_lanes) } => {
290 E::SimdTooManyLanes { ty, max_lanes }
291 }
292 InvalidSimd { ty, kind: SimdLayoutError::ZeroLength } => E::SimdZeroLength { ty },
293 TooGeneric(ty) => E::TooGeneric { ty },
294 NormalizationFailure(ty, e) => {
295 E::NormalizationFailure { ty, failure_ty: e.get_type_for_failure() }
296 }
297 Cycle(_) => E::Cycle,
298 ReferencesError(_) => E::ReferencesError,
299 }
300 }
301}
302
303impl<'tcx> fmt::Display for LayoutError<'tcx> {
306 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
307 match *self {
308 LayoutError::Unknown(ty) => write!(f, "the type `{ty}` has an unknown layout"),
309 LayoutError::TooGeneric(ty) => {
310 write!(f, "the type `{ty}` does not have a fixed layout")
311 }
312 LayoutError::SizeOverflow(ty) => {
313 write!(f, "values of the type `{ty}` are too big for the target architecture")
314 }
315 LayoutError::InvalidSimd { ty, kind: SimdLayoutError::TooManyLanes(max_lanes) } => {
316 write!(f, "the SIMD type `{ty}` has more elements than the limit {max_lanes}")
317 }
318 LayoutError::InvalidSimd { ty, kind: SimdLayoutError::ZeroLength } => {
319 write!(f, "the SIMD type `{ty}` has zero elements")
320 }
321 LayoutError::NormalizationFailure(t, e) => write!(
322 f,
323 "unable to determine layout for `{}` because `{}` cannot be normalized",
324 t,
325 e.get_type_for_failure()
326 ),
327 LayoutError::Cycle(_) => write!(f, "a cycle occurred during layout computation"),
328 LayoutError::ReferencesError(_) => write!(f, "the type has an unknown layout"),
329 }
330 }
331}
332
333impl<'tcx> IntoDiagArg for LayoutError<'tcx> {
334 fn into_diag_arg(self, _: &mut Option<std::path::PathBuf>) -> DiagArgValue {
335 self.to_string().into_diag_arg(&mut None)
336 }
337}
338
339#[derive(Clone, Copy)]
340pub struct LayoutCx<'tcx> {
341 pub calc: abi::LayoutCalculator<TyCtxt<'tcx>>,
342 pub typing_env: ty::TypingEnv<'tcx>,
343}
344
345impl<'tcx> LayoutCx<'tcx> {
346 pub fn new(tcx: TyCtxt<'tcx>, typing_env: ty::TypingEnv<'tcx>) -> Self {
347 Self { calc: abi::LayoutCalculator::new(tcx), typing_env }
348 }
349}
350
351#[derive(Copy, Clone, Debug)]
356pub enum SizeSkeleton<'tcx> {
357 Known(Size, Option<Align>),
360
361 Generic(ty::Const<'tcx>),
366
367 Pointer {
369 non_zero: bool,
371 tail: Ty<'tcx>,
375 },
376}
377
378impl<'tcx> SizeSkeleton<'tcx> {
379 pub fn compute(
380 ty: Ty<'tcx>,
381 tcx: TyCtxt<'tcx>,
382 typing_env: ty::TypingEnv<'tcx>,
383 ) -> Result<SizeSkeleton<'tcx>, &'tcx LayoutError<'tcx>> {
384 debug_assert!(!ty.has_non_region_infer());
385
386 let err = match tcx.layout_of(typing_env.as_query_input(ty)) {
388 Ok(layout) => {
389 if layout.is_sized() {
390 return Ok(SizeSkeleton::Known(layout.size, Some(layout.align.abi)));
391 } else {
392 return Err(tcx.arena.alloc(LayoutError::Unknown(ty)));
394 }
395 }
396 Err(err @ LayoutError::TooGeneric(_)) => err,
397 Err(
399 e @ LayoutError::Cycle(_)
400 | e @ LayoutError::Unknown(_)
401 | e @ LayoutError::SizeOverflow(_)
402 | e @ LayoutError::InvalidSimd { .. }
403 | e @ LayoutError::NormalizationFailure(..)
404 | e @ LayoutError::ReferencesError(_),
405 ) => return Err(e),
406 };
407
408 match *ty.kind() {
409 ty::Ref(_, pointee, _) | ty::RawPtr(pointee, _) => {
410 let non_zero = !ty.is_raw_ptr();
411
412 let tail = tcx.struct_tail_raw(
413 pointee,
414 &ObligationCause::dummy(),
415 |ty| match tcx.try_normalize_erasing_regions(typing_env, ty) {
416 Ok(ty) => ty,
417 Err(e) => Ty::new_error_with_message(
418 tcx,
419 DUMMY_SP,
420 format!(
421 "normalization failed for {} but no errors reported",
422 e.get_type_for_failure()
423 ),
424 ),
425 },
426 || {},
427 );
428
429 match tail.kind() {
430 ty::Param(_) | ty::Alias(ty::Projection | ty::Inherent, _) => {
431 debug_assert!(tail.has_non_region_param());
432 Ok(SizeSkeleton::Pointer {
433 non_zero,
434 tail: tcx.erase_and_anonymize_regions(tail),
435 })
436 }
437 ty::Error(guar) => {
438 return Err(tcx.arena.alloc(LayoutError::ReferencesError(*guar)));
440 }
441 _ => bug!(
442 "SizeSkeleton::compute({ty}): layout errored ({err:?}), yet \
443 tail `{tail}` is not a type parameter or a projection",
444 ),
445 }
446 }
447 ty::Array(inner, len) if tcx.features().transmute_generic_consts() => {
448 let len_eval = len.try_to_target_usize(tcx);
449 if len_eval == Some(0) {
450 return Ok(SizeSkeleton::Known(Size::from_bytes(0), None));
451 }
452
453 match SizeSkeleton::compute(inner, tcx, typing_env)? {
454 SizeSkeleton::Known(s, a) => {
457 if let Some(c) = len_eval {
458 let size = s
459 .bytes()
460 .checked_mul(c)
461 .ok_or_else(|| &*tcx.arena.alloc(LayoutError::SizeOverflow(ty)))?;
462 return Ok(SizeSkeleton::Known(Size::from_bytes(size), a));
464 }
465 Err(err)
466 }
467 SizeSkeleton::Pointer { .. } | SizeSkeleton::Generic(_) => Err(err),
468 }
469 }
470
471 ty::Adt(def, args) => {
472 if def.is_union() || def.variants().is_empty() || def.variants().len() > 2 {
474 return Err(err);
475 }
476
477 let zero_or_ptr_variant = |i| {
479 let i = VariantIdx::from_usize(i);
480 let fields =
481 def.variant(i).fields.iter().map(|field| {
482 SizeSkeleton::compute(field.ty(tcx, args), tcx, typing_env)
483 });
484 let mut ptr = None;
485 for field in fields {
486 let field = field?;
487 match field {
488 SizeSkeleton::Known(size, align) => {
489 let is_1zst = size.bytes() == 0
490 && align.is_some_and(|align| align.bytes() == 1);
491 if !is_1zst {
492 return Err(err);
493 }
494 }
495 SizeSkeleton::Pointer { .. } => {
496 if ptr.is_some() {
497 return Err(err);
498 }
499 ptr = Some(field);
500 }
501 SizeSkeleton::Generic(_) => {
502 return Err(err);
503 }
504 }
505 }
506 Ok(ptr)
507 };
508
509 let v0 = zero_or_ptr_variant(0)?;
510 if def.variants().len() == 1 {
512 if let Some(SizeSkeleton::Pointer { non_zero, tail }) = v0 {
513 return Ok(SizeSkeleton::Pointer {
514 non_zero: non_zero
515 || match tcx.layout_scalar_valid_range(def.did()) {
516 (Bound::Included(start), Bound::Unbounded) => start > 0,
517 (Bound::Included(start), Bound::Included(end)) => {
518 0 < start && start < end
519 }
520 _ => false,
521 },
522 tail,
523 });
524 } else {
525 return Err(err);
526 }
527 }
528
529 let v1 = zero_or_ptr_variant(1)?;
530 match (v0, v1) {
532 (Some(SizeSkeleton::Pointer { non_zero: true, tail }), None)
533 | (None, Some(SizeSkeleton::Pointer { non_zero: true, tail })) => {
534 Ok(SizeSkeleton::Pointer { non_zero: false, tail })
535 }
536 _ => Err(err),
537 }
538 }
539
540 ty::Alias(..) => {
541 let normalized = tcx.normalize_erasing_regions(typing_env, ty);
542 if ty == normalized {
543 Err(err)
544 } else {
545 SizeSkeleton::compute(normalized, tcx, typing_env)
546 }
547 }
548
549 ty::Pat(base, _) => SizeSkeleton::compute(base, tcx, typing_env),
551
552 _ => Err(err),
553 }
554 }
555
556 pub fn same_size(self, other: SizeSkeleton<'tcx>) -> bool {
557 match (self, other) {
558 (SizeSkeleton::Known(a, _), SizeSkeleton::Known(b, _)) => a == b,
559 (SizeSkeleton::Pointer { tail: a, .. }, SizeSkeleton::Pointer { tail: b, .. }) => {
560 a == b
561 }
562 (SizeSkeleton::Generic(a), SizeSkeleton::Generic(b)) => a == b,
565 _ => false,
566 }
567 }
568}
569
570pub trait HasTyCtxt<'tcx>: HasDataLayout {
571 fn tcx(&self) -> TyCtxt<'tcx>;
572}
573
574pub trait HasTypingEnv<'tcx> {
575 fn typing_env(&self) -> ty::TypingEnv<'tcx>;
576
577 fn param_env(&self) -> ty::ParamEnv<'tcx> {
580 self.typing_env().param_env
581 }
582}
583
584impl<'tcx> HasDataLayout for TyCtxt<'tcx> {
585 #[inline]
586 fn data_layout(&self) -> &TargetDataLayout {
587 &self.data_layout
588 }
589}
590
591impl<'tcx> HasTargetSpec for TyCtxt<'tcx> {
592 fn target_spec(&self) -> &Target {
593 &self.sess.target
594 }
595}
596
597impl<'tcx> HasX86AbiOpt for TyCtxt<'tcx> {
598 fn x86_abi_opt(&self) -> X86Abi {
599 X86Abi {
600 regparm: self.sess.opts.unstable_opts.regparm,
601 reg_struct_return: self.sess.opts.unstable_opts.reg_struct_return,
602 }
603 }
604}
605
606impl<'tcx> HasTyCtxt<'tcx> for TyCtxt<'tcx> {
607 #[inline]
608 fn tcx(&self) -> TyCtxt<'tcx> {
609 *self
610 }
611}
612
613impl<'tcx> HasDataLayout for TyCtxtAt<'tcx> {
614 #[inline]
615 fn data_layout(&self) -> &TargetDataLayout {
616 &self.data_layout
617 }
618}
619
620impl<'tcx> HasTargetSpec for TyCtxtAt<'tcx> {
621 fn target_spec(&self) -> &Target {
622 &self.sess.target
623 }
624}
625
626impl<'tcx> HasTyCtxt<'tcx> for TyCtxtAt<'tcx> {
627 #[inline]
628 fn tcx(&self) -> TyCtxt<'tcx> {
629 **self
630 }
631}
632
633impl<'tcx> HasTypingEnv<'tcx> for LayoutCx<'tcx> {
634 fn typing_env(&self) -> ty::TypingEnv<'tcx> {
635 self.typing_env
636 }
637}
638
639impl<'tcx> HasDataLayout for LayoutCx<'tcx> {
640 fn data_layout(&self) -> &TargetDataLayout {
641 self.calc.cx.data_layout()
642 }
643}
644
645impl<'tcx> HasTargetSpec for LayoutCx<'tcx> {
646 fn target_spec(&self) -> &Target {
647 self.calc.cx.target_spec()
648 }
649}
650
651impl<'tcx> HasX86AbiOpt for LayoutCx<'tcx> {
652 fn x86_abi_opt(&self) -> X86Abi {
653 self.calc.cx.x86_abi_opt()
654 }
655}
656
657impl<'tcx> HasTyCtxt<'tcx> for LayoutCx<'tcx> {
658 fn tcx(&self) -> TyCtxt<'tcx> {
659 self.calc.cx
660 }
661}
662
663pub trait MaybeResult<T> {
664 type Error;
665
666 fn from(x: Result<T, Self::Error>) -> Self;
667 fn to_result(self) -> Result<T, Self::Error>;
668}
669
670impl<T> MaybeResult<T> for T {
671 type Error = !;
672
673 fn from(Ok(x): Result<T, Self::Error>) -> Self {
674 x
675 }
676 fn to_result(self) -> Result<T, Self::Error> {
677 Ok(self)
678 }
679}
680
681impl<T, E> MaybeResult<T> for Result<T, E> {
682 type Error = E;
683
684 fn from(x: Result<T, Self::Error>) -> Self {
685 x
686 }
687 fn to_result(self) -> Result<T, Self::Error> {
688 self
689 }
690}
691
692pub type TyAndLayout<'tcx> = rustc_abi::TyAndLayout<'tcx, Ty<'tcx>>;
693
694pub trait LayoutOfHelpers<'tcx>: HasDataLayout + HasTyCtxt<'tcx> + HasTypingEnv<'tcx> {
697 type LayoutOfResult: MaybeResult<TyAndLayout<'tcx>> = TyAndLayout<'tcx>;
700
701 #[inline]
704 fn layout_tcx_at_span(&self) -> Span {
705 DUMMY_SP
706 }
707
708 fn handle_layout_err(
716 &self,
717 err: LayoutError<'tcx>,
718 span: Span,
719 ty: Ty<'tcx>,
720 ) -> <Self::LayoutOfResult as MaybeResult<TyAndLayout<'tcx>>>::Error;
721}
722
723pub trait LayoutOf<'tcx>: LayoutOfHelpers<'tcx> {
725 #[inline]
728 fn layout_of(&self, ty: Ty<'tcx>) -> Self::LayoutOfResult {
729 self.spanned_layout_of(ty, DUMMY_SP)
730 }
731
732 #[inline]
737 fn spanned_layout_of(&self, ty: Ty<'tcx>, span: Span) -> Self::LayoutOfResult {
738 let span = if !span.is_dummy() { span } else { self.layout_tcx_at_span() };
739 let tcx = self.tcx().at(span);
740
741 MaybeResult::from(
742 tcx.layout_of(self.typing_env().as_query_input(ty))
743 .map_err(|err| self.handle_layout_err(*err, span, ty)),
744 )
745 }
746}
747
748impl<'tcx, C: LayoutOfHelpers<'tcx>> LayoutOf<'tcx> for C {}
749
750impl<'tcx> LayoutOfHelpers<'tcx> for LayoutCx<'tcx> {
751 type LayoutOfResult = Result<TyAndLayout<'tcx>, &'tcx LayoutError<'tcx>>;
752
753 #[inline]
754 fn handle_layout_err(
755 &self,
756 err: LayoutError<'tcx>,
757 _: Span,
758 _: Ty<'tcx>,
759 ) -> &'tcx LayoutError<'tcx> {
760 self.tcx().arena.alloc(err)
761 }
762}
763
764impl<'tcx, C> TyAbiInterface<'tcx, C> for Ty<'tcx>
765where
766 C: HasTyCtxt<'tcx> + HasTypingEnv<'tcx>,
767{
768 fn ty_and_layout_for_variant(
769 this: TyAndLayout<'tcx>,
770 cx: &C,
771 variant_index: VariantIdx,
772 ) -> TyAndLayout<'tcx> {
773 let layout = match this.variants {
774 Variants::Single { index } if index == variant_index => {
776 return this;
777 }
778
779 Variants::Single { .. } | Variants::Empty => {
780 let tcx = cx.tcx();
785 let typing_env = cx.typing_env();
786
787 if let Ok(original_layout) = tcx.layout_of(typing_env.as_query_input(this.ty)) {
789 assert_eq!(original_layout.variants, this.variants);
790 }
791
792 let fields = match this.ty.kind() {
793 ty::Adt(def, _) if def.variants().is_empty() => {
794 bug!("for_variant called on zero-variant enum {}", this.ty)
795 }
796 ty::Adt(def, _) => def.variant(variant_index).fields.len(),
797 _ => bug!("`ty_and_layout_for_variant` on unexpected type {}", this.ty),
798 };
799 tcx.mk_layout(LayoutData::uninhabited_variant(cx, variant_index, fields))
800 }
801
802 Variants::Multiple { ref variants, .. } => {
803 cx.tcx().mk_layout(variants[variant_index].clone())
804 }
805 };
806
807 assert_eq!(*layout.variants(), Variants::Single { index: variant_index });
808
809 TyAndLayout { ty: this.ty, layout }
810 }
811
812 fn ty_and_layout_field(this: TyAndLayout<'tcx>, cx: &C, i: usize) -> TyAndLayout<'tcx> {
813 enum TyMaybeWithLayout<'tcx> {
814 Ty(Ty<'tcx>),
815 TyAndLayout(TyAndLayout<'tcx>),
816 }
817
818 fn field_ty_or_layout<'tcx>(
819 this: TyAndLayout<'tcx>,
820 cx: &(impl HasTyCtxt<'tcx> + HasTypingEnv<'tcx>),
821 i: usize,
822 ) -> TyMaybeWithLayout<'tcx> {
823 let tcx = cx.tcx();
824 let tag_layout = |tag: Scalar| -> TyAndLayout<'tcx> {
825 TyAndLayout {
826 layout: tcx.mk_layout(LayoutData::scalar(cx, tag)),
827 ty: tag.primitive().to_ty(tcx),
828 }
829 };
830
831 match *this.ty.kind() {
832 ty::Bool
833 | ty::Char
834 | ty::Int(_)
835 | ty::Uint(_)
836 | ty::Float(_)
837 | ty::FnPtr(..)
838 | ty::Never
839 | ty::FnDef(..)
840 | ty::CoroutineWitness(..)
841 | ty::Foreign(..)
842 | ty::Pat(_, _)
843 | ty::Dynamic(_, _) => {
844 bug!("TyAndLayout::field({:?}): not applicable", this)
845 }
846
847 ty::UnsafeBinder(bound_ty) => {
848 let ty = tcx.instantiate_bound_regions_with_erased(bound_ty.into());
849 field_ty_or_layout(TyAndLayout { ty, ..this }, cx, i)
850 }
851
852 ty::Ref(_, pointee, _) | ty::RawPtr(pointee, _) => {
854 assert!(i < this.fields.count());
855
856 if i == 0 {
861 let nil = tcx.types.unit;
862 let unit_ptr_ty = if this.ty.is_raw_ptr() {
863 Ty::new_mut_ptr(tcx, nil)
864 } else {
865 Ty::new_mut_ref(tcx, tcx.lifetimes.re_static, nil)
866 };
867
868 let typing_env = ty::TypingEnv::fully_monomorphized();
872 return TyMaybeWithLayout::TyAndLayout(TyAndLayout {
873 ty: this.ty,
874 ..tcx.layout_of(typing_env.as_query_input(unit_ptr_ty)).unwrap()
875 });
876 }
877
878 let mk_dyn_vtable = |principal: Option<ty::PolyExistentialTraitRef<'tcx>>| {
879 let min_count = ty::vtable_min_entries(
880 tcx,
881 principal.map(|principal| {
882 tcx.instantiate_bound_regions_with_erased(principal)
883 }),
884 );
885 Ty::new_imm_ref(
886 tcx,
887 tcx.lifetimes.re_static,
888 Ty::new_array(tcx, tcx.types.usize, min_count.try_into().unwrap()),
890 )
891 };
892
893 let metadata = if let Some(metadata_def_id) = tcx.lang_items().metadata_type()
894 && !pointee.references_error()
897 {
898 let metadata = tcx.normalize_erasing_regions(
899 cx.typing_env(),
900 Ty::new_projection(tcx, metadata_def_id, [pointee]),
901 );
902
903 if let ty::Adt(def, args) = metadata.kind()
908 && tcx.is_lang_item(def.did(), LangItem::DynMetadata)
909 && let ty::Dynamic(data, _) = args.type_at(0).kind()
910 {
911 mk_dyn_vtable(data.principal())
912 } else {
913 metadata
914 }
915 } else {
916 match tcx.struct_tail_for_codegen(pointee, cx.typing_env()).kind() {
917 ty::Slice(_) | ty::Str => tcx.types.usize,
918 ty::Dynamic(data, _) => mk_dyn_vtable(data.principal()),
919 _ => bug!("TyAndLayout::field({:?}): not applicable", this),
920 }
921 };
922
923 TyMaybeWithLayout::Ty(metadata)
924 }
925
926 ty::Array(element, _) | ty::Slice(element) => TyMaybeWithLayout::Ty(element),
928 ty::Str => TyMaybeWithLayout::Ty(tcx.types.u8),
929
930 ty::Closure(_, args) => field_ty_or_layout(
932 TyAndLayout { ty: args.as_closure().tupled_upvars_ty(), ..this },
933 cx,
934 i,
935 ),
936
937 ty::CoroutineClosure(_, args) => field_ty_or_layout(
938 TyAndLayout { ty: args.as_coroutine_closure().tupled_upvars_ty(), ..this },
939 cx,
940 i,
941 ),
942
943 ty::Coroutine(def_id, args) => match this.variants {
944 Variants::Empty => unreachable!(),
945 Variants::Single { index } => TyMaybeWithLayout::Ty(
946 args.as_coroutine()
947 .state_tys(def_id, tcx)
948 .nth(index.as_usize())
949 .unwrap()
950 .nth(i)
951 .unwrap(),
952 ),
953 Variants::Multiple { tag, tag_field, .. } => {
954 if FieldIdx::from_usize(i) == tag_field {
955 return TyMaybeWithLayout::TyAndLayout(tag_layout(tag));
956 }
957 TyMaybeWithLayout::Ty(args.as_coroutine().prefix_tys()[i])
958 }
959 },
960
961 ty::Tuple(tys) => TyMaybeWithLayout::Ty(tys[i]),
962
963 ty::Adt(def, args) => {
965 match this.variants {
966 Variants::Single { index } => {
967 let field = &def.variant(index).fields[FieldIdx::from_usize(i)];
968 TyMaybeWithLayout::Ty(field.ty(tcx, args))
969 }
970 Variants::Empty => panic!("there is no field in Variants::Empty types"),
971
972 Variants::Multiple { tag, .. } => {
974 assert_eq!(i, 0);
975 return TyMaybeWithLayout::TyAndLayout(tag_layout(tag));
976 }
977 }
978 }
979
980 ty::Alias(..)
981 | ty::Bound(..)
982 | ty::Placeholder(..)
983 | ty::Param(_)
984 | ty::Infer(_)
985 | ty::Error(_) => bug!("TyAndLayout::field: unexpected type `{}`", this.ty),
986 }
987 }
988
989 match field_ty_or_layout(this, cx, i) {
990 TyMaybeWithLayout::Ty(field_ty) => {
991 cx.tcx().layout_of(cx.typing_env().as_query_input(field_ty)).unwrap_or_else(|e| {
992 bug!(
993 "failed to get layout for `{field_ty}`: {e:?},\n\
994 despite it being a field (#{i}) of an existing layout: {this:#?}",
995 )
996 })
997 }
998 TyMaybeWithLayout::TyAndLayout(field_layout) => field_layout,
999 }
1000 }
1001
1002 fn ty_and_layout_pointee_info_at(
1005 this: TyAndLayout<'tcx>,
1006 cx: &C,
1007 offset: Size,
1008 ) -> Option<PointeeInfo> {
1009 let tcx = cx.tcx();
1010 let typing_env = cx.typing_env();
1011
1012 let pointee_info = match *this.ty.kind() {
1013 ty::RawPtr(p_ty, _) if offset.bytes() == 0 => {
1014 tcx.layout_of(typing_env.as_query_input(p_ty)).ok().map(|layout| PointeeInfo {
1015 size: layout.size,
1016 align: layout.align.abi,
1017 safe: None,
1018 })
1019 }
1020 ty::FnPtr(..) if offset.bytes() == 0 => {
1021 tcx.layout_of(typing_env.as_query_input(this.ty)).ok().map(|layout| PointeeInfo {
1022 size: layout.size,
1023 align: layout.align.abi,
1024 safe: None,
1025 })
1026 }
1027 ty::Ref(_, ty, mt) if offset.bytes() == 0 => {
1028 let optimize = tcx.sess.opts.optimize != OptLevel::No;
1032 let kind = match mt {
1033 hir::Mutability::Not => {
1034 PointerKind::SharedRef { frozen: optimize && ty.is_freeze(tcx, typing_env) }
1035 }
1036 hir::Mutability::Mut => {
1037 PointerKind::MutableRef { unpin: optimize && ty.is_unpin(tcx, typing_env) }
1038 }
1039 };
1040
1041 tcx.layout_of(typing_env.as_query_input(ty)).ok().map(|layout| PointeeInfo {
1042 size: layout.size,
1043 align: layout.align.abi,
1044 safe: Some(kind),
1045 })
1046 }
1047
1048 _ => {
1049 let mut data_variant = match &this.variants {
1050 Variants::Multiple {
1060 tag_encoding:
1061 TagEncoding::Niche { untagged_variant, niche_variants, niche_start },
1062 tag_field,
1063 variants,
1064 ..
1065 } if variants.len() == 2
1066 && this.fields.offset(tag_field.as_usize()) == offset =>
1067 {
1068 let tagged_variant = if *untagged_variant == VariantIdx::ZERO {
1069 VariantIdx::from_u32(1)
1070 } else {
1071 VariantIdx::from_u32(0)
1072 };
1073 assert_eq!(tagged_variant, *niche_variants.start());
1074 if *niche_start == 0 {
1075 Some(this.for_variant(cx, *untagged_variant))
1081 } else {
1082 None
1083 }
1084 }
1085 Variants::Multiple { .. } => None,
1086 _ => Some(this),
1087 };
1088
1089 if let Some(variant) = data_variant
1090 && let FieldsShape::Union(_) = variant.fields
1092 {
1093 data_variant = None;
1094 }
1095
1096 let mut result = None;
1097
1098 if let Some(variant) = data_variant {
1099 let ptr_end = offset + Primitive::Pointer(AddressSpace::ZERO).size(cx);
1102 for i in 0..variant.fields.count() {
1103 let field_start = variant.fields.offset(i);
1104 if field_start <= offset {
1105 let field = variant.field(cx, i);
1106 result = field.to_result().ok().and_then(|field| {
1107 if ptr_end <= field_start + field.size {
1108 let field_info =
1110 field.pointee_info_at(cx, offset - field_start);
1111 field_info
1112 } else {
1113 None
1114 }
1115 });
1116 if result.is_some() {
1117 break;
1118 }
1119 }
1120 }
1121 }
1122
1123 if let Some(ref mut pointee) = result {
1127 if offset.bytes() == 0
1128 && let Some(boxed_ty) = this.ty.boxed_ty()
1129 {
1130 debug_assert!(pointee.safe.is_none());
1131 let optimize = tcx.sess.opts.optimize != OptLevel::No;
1132 pointee.safe = Some(PointerKind::Box {
1133 unpin: optimize && boxed_ty.is_unpin(tcx, typing_env),
1134 global: this.ty.is_box_global(tcx),
1135 });
1136 }
1137 }
1138
1139 result
1140 }
1141 };
1142
1143 debug!(
1144 "pointee_info_at (offset={:?}, type kind: {:?}) => {:?}",
1145 offset,
1146 this.ty.kind(),
1147 pointee_info
1148 );
1149
1150 pointee_info
1151 }
1152
1153 fn is_adt(this: TyAndLayout<'tcx>) -> bool {
1154 matches!(this.ty.kind(), ty::Adt(..))
1155 }
1156
1157 fn is_never(this: TyAndLayout<'tcx>) -> bool {
1158 matches!(this.ty.kind(), ty::Never)
1159 }
1160
1161 fn is_tuple(this: TyAndLayout<'tcx>) -> bool {
1162 matches!(this.ty.kind(), ty::Tuple(..))
1163 }
1164
1165 fn is_unit(this: TyAndLayout<'tcx>) -> bool {
1166 matches!(this.ty.kind(), ty::Tuple(list) if list.len() == 0)
1167 }
1168
1169 fn is_transparent(this: TyAndLayout<'tcx>) -> bool {
1170 matches!(this.ty.kind(), ty::Adt(def, _) if def.repr().transparent())
1171 }
1172}
1173
1174#[inline]
1215#[tracing::instrument(level = "debug", skip(tcx))]
1216pub fn fn_can_unwind(tcx: TyCtxt<'_>, fn_def_id: Option<DefId>, abi: ExternAbi) -> bool {
1217 if let Some(did) = fn_def_id {
1218 if tcx.codegen_fn_attrs(did).flags.contains(CodegenFnAttrFlags::NEVER_UNWIND) {
1220 return false;
1221 }
1222
1223 if !tcx.sess.panic_strategy().unwinds() && !tcx.is_foreign_item(did) {
1228 return false;
1229 }
1230
1231 if !tcx.sess.opts.unstable_opts.panic_in_drop.unwinds()
1236 && tcx.is_lang_item(did, LangItem::DropInPlace)
1237 {
1238 return false;
1239 }
1240 }
1241
1242 use ExternAbi::*;
1249 match abi {
1250 C { unwind }
1251 | System { unwind }
1252 | Cdecl { unwind }
1253 | Stdcall { unwind }
1254 | Fastcall { unwind }
1255 | Vectorcall { unwind }
1256 | Thiscall { unwind }
1257 | Aapcs { unwind }
1258 | Win64 { unwind }
1259 | SysV64 { unwind } => unwind,
1260 PtxKernel
1261 | Msp430Interrupt
1262 | X86Interrupt
1263 | GpuKernel
1264 | EfiApi
1265 | AvrInterrupt
1266 | AvrNonBlockingInterrupt
1267 | CmseNonSecureCall
1268 | CmseNonSecureEntry
1269 | Custom
1270 | RiscvInterruptM
1271 | RiscvInterruptS
1272 | RustInvalid
1273 | Unadjusted => false,
1274 Rust | RustCall | RustCold => tcx.sess.panic_strategy().unwinds(),
1275 }
1276}
1277
1278#[derive(Copy, Clone, Debug, HashStable)]
1280pub enum FnAbiError<'tcx> {
1281 Layout(LayoutError<'tcx>),
1283}
1284
1285impl<'a, 'b, G: EmissionGuarantee> Diagnostic<'a, G> for FnAbiError<'b> {
1286 fn into_diag(self, dcx: DiagCtxtHandle<'a>, level: Level) -> Diag<'a, G> {
1287 match self {
1288 Self::Layout(e) => e.into_diagnostic().into_diag(dcx, level),
1289 }
1290 }
1291}
1292
1293#[derive(Debug)]
1296pub enum FnAbiRequest<'tcx> {
1297 OfFnPtr { sig: ty::PolyFnSig<'tcx>, extra_args: &'tcx ty::List<Ty<'tcx>> },
1298 OfInstance { instance: ty::Instance<'tcx>, extra_args: &'tcx ty::List<Ty<'tcx>> },
1299}
1300
1301pub trait FnAbiOfHelpers<'tcx>: LayoutOfHelpers<'tcx> {
1304 type FnAbiOfResult: MaybeResult<&'tcx FnAbi<'tcx, Ty<'tcx>>> = &'tcx FnAbi<'tcx, Ty<'tcx>>;
1307
1308 fn handle_fn_abi_err(
1316 &self,
1317 err: FnAbiError<'tcx>,
1318 span: Span,
1319 fn_abi_request: FnAbiRequest<'tcx>,
1320 ) -> <Self::FnAbiOfResult as MaybeResult<&'tcx FnAbi<'tcx, Ty<'tcx>>>>::Error;
1321}
1322
1323pub trait FnAbiOf<'tcx>: FnAbiOfHelpers<'tcx> {
1325 #[inline]
1330 fn fn_abi_of_fn_ptr(
1331 &self,
1332 sig: ty::PolyFnSig<'tcx>,
1333 extra_args: &'tcx ty::List<Ty<'tcx>>,
1334 ) -> Self::FnAbiOfResult {
1335 let span = self.layout_tcx_at_span();
1337 let tcx = self.tcx().at(span);
1338
1339 MaybeResult::from(
1340 tcx.fn_abi_of_fn_ptr(self.typing_env().as_query_input((sig, extra_args))).map_err(
1341 |err| self.handle_fn_abi_err(*err, span, FnAbiRequest::OfFnPtr { sig, extra_args }),
1342 ),
1343 )
1344 }
1345
1346 #[inline]
1352 #[tracing::instrument(level = "debug", skip(self))]
1353 fn fn_abi_of_instance(
1354 &self,
1355 instance: ty::Instance<'tcx>,
1356 extra_args: &'tcx ty::List<Ty<'tcx>>,
1357 ) -> Self::FnAbiOfResult {
1358 let span = self.layout_tcx_at_span();
1360 let tcx = self.tcx().at(span);
1361
1362 MaybeResult::from(
1363 tcx.fn_abi_of_instance(self.typing_env().as_query_input((instance, extra_args)))
1364 .map_err(|err| {
1365 let span =
1370 if !span.is_dummy() { span } else { tcx.def_span(instance.def_id()) };
1371 self.handle_fn_abi_err(
1372 *err,
1373 span,
1374 FnAbiRequest::OfInstance { instance, extra_args },
1375 )
1376 }),
1377 )
1378 }
1379}
1380
1381impl<'tcx, C: FnAbiOfHelpers<'tcx>> FnAbiOf<'tcx> for C {}
1382
1383impl<'tcx> TyCtxt<'tcx> {
1384 pub fn offset_of_subfield<I>(
1385 self,
1386 typing_env: ty::TypingEnv<'tcx>,
1387 mut layout: TyAndLayout<'tcx>,
1388 indices: I,
1389 ) -> Size
1390 where
1391 I: Iterator<Item = (VariantIdx, FieldIdx)>,
1392 {
1393 let cx = LayoutCx::new(self, typing_env);
1394 let mut offset = Size::ZERO;
1395
1396 for (variant, field) in indices {
1397 layout = layout.for_variant(&cx, variant);
1398 let index = field.index();
1399 offset += layout.fields.offset(index);
1400 layout = layout.field(&cx, index);
1401 if !layout.is_sized() {
1402 let tail = self.struct_tail_for_codegen(layout.ty, typing_env);
1404 if !matches!(tail.kind(), ty::Slice(..)) {
1405 bug!(
1406 "offset of not-statically-aligned field (type {:?}) cannot be computed statically",
1407 layout.ty
1408 );
1409 }
1410 }
1411 }
1412
1413 offset
1414 }
1415}